Split planner

load.sh

@@ -0,0 +1,119 @@
+#!/bin/bash
+
+if [ $# -lt 6 ]
+then
+        echo "Load benchmark in the cluster (EStore project)"
+        echo "Parameters"
+        echo "1) benchmark type (e.g. ycsb, voter)"
+        echo "2) skew (uniform, high-skew, low-skew)"
+        echo "3) number of tuples"
+        echo "4) number of partitions per server"
+        echo "5) first server"
+        echo "6) initial number of servers"
+	echo "7) maximum number of servers"
+        echo "8) servers to skip (optional)"
+        echo
+        echo "Type DEF to continue with default parameters"
+        read command
+	if [ "$command" != "DEF" ]
+	then
+        	exit
+	fi
+fi
+
+if [ -z "$1" ]
+then
+        bench="ycsb"
+else
+        bench=$1
+fi
+
+if [ -z "$2" ]
+then
+        skew="high-skew"
+else
+        skew=$2
+fi
+
+if [[ $2 =~ '(uniform|high-skew|low-skew)' ]]
+then
+        echo "incorrect skew"
+        exit
+fi
+
+if [[ $2 == "uniform" ]]
+then
+        sed -i".bak" '/requestdistribution/d' properties/benchmarks/ycsb.properties
+        sed -i".bak" '/skew_factor/d' properties/benchmarks/ycsb.properties
+        sed -i".bak" '/num_hot_spots/d' properties/benchmarks/ycsb.properties
+        sed -i".bak" '/percent_accesses_to_hot_spots/d' properties/benchmarks/ycsb.properties
+        echo "requestdistribution=uniform" >> properties/benchmarks/ycsb.properties
+fi
+
+if [[ $2 == "low-skew" ]]
+then
+        sed -i".bak" '/requestdistribution/d' properties/benchmarks/ycsb.properties
+        sed -i".bak" '/skew_factor/d' properties/benchmarks/ycsb.properties
+        sed -i".bak" '/num_hot_spots/d' properties/benchmarks/ycsb.properties
+        sed -i".bak" '/percent_accesses_to_hot_spots/d' properties/benchmarks/ycsb.properties
+        echo "requestdistribution=zipfian" >> properties/benchmarks/ycsb.properties
+        echo "skew_factor = 0.65" >> properties/benchmarks/ycsb.properties
+fi
+
+if [[ $2 == "high-skew" ]]
+then
+        sed -i".bak" '/requestdistribution/d' properties/benchmarks/ycsb.properties
+        sed -i".bak" '/skew_factor/d' properties/benchmarks/ycsb.properties
+        sed -i".bak" '/num_hot_spots/d' properties/benchmarks/ycsb.properties
+        sed -i".bak" '/percent_accesses_to_hot_spots/d' properties/benchmarks/ycsb.properties
+        echo "requestdistribution=zipfian" >> properties/benchmarks/ycsb.properties
+        echo "skew_factor = 0.65" >> properties/benchmarks/ycsb.properties
+        echo "num_hot_spots=30" >> properties/benchmarks/ycsb.properties
+        echo "percent_accesses_to_hot_spots=0.80" >> properties/benchmarks/ycsb.properties
+fi
+
+if [ -z "$3" ]
+then
+        tuples=60000000
+else
+        tuples=$3
+fi
+sed -i".bak" '/num_records/d' properties/benchmarks/ycsb.properties
+echo "num_records = $tuples" >> properties/benchmarks/ycsb.properties
+
+if [ -z "$4" ]
+then
+        part_per_server=6
+else
+        part_per_server=$4
+fi
+
+if [ -z "$5" ]
+then
+        init_server=5
+else
+        init_server=$5
+fi
+
+if [ -z "$6" ]
+then
+        num_servers=5
+else
+        num_servers=$6
+fi
+
+if [ -z "$7" ]
+then
+        max_servers=5
+else
+        max_servers=$7
+fi
+
+skip_list="${@:8}"
+partitions=$(( $part_per_server * $num_servers ))
+
+python scripts/reconfiguration/plan-generator.py -t ycsb -s $tuples -p $partitions > plan.json
+echo "./prepare.sh $bench $init_server $max_servers $part_per_server $skip_list"
+./prepare.sh $bench $init_server $max_servers $part_per_server $skip_list
+
+ant hstore-benchmark -Dproject=$bench -Dglobal.hasher_plan=plan.json -Dglobal.hasher_class=edu.brown.hashing.TwoTieredRangeHasher -Dnoshutdown=true -Dnoexecute=true -Dsite.txn_restart_limit_sysproc=100 -Dsite.jvm_asserts=false -Dsite.reconfig_live=false | tee load.log

prepare.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+
+
+if [ $# -lt 4 ]
+then
+	echo "Allocates partitions starting from server da${init_server}"
+	echo "Parameters"
+	echo "1) benchmark type (e.g. tpcc, ycsb)"
+	echo "2) initial server"
+	echo "3) number of servers"
+	echo "4) number of partitions per server"
+	echo "5) servers to skip (optional)"
+	exit
+fi
+
+cmd="ant hstore-prepare -Dproject=$1 -Dhosts=\""
+init_server=$2
+num_servers=$3
+part_per_server=$4
+skip_list="${@:5}"
+
+curr_host=0
+curr_part=0
+first_loop=""
+
+for (( var=1; var<=$num_servers; var++ ))
+do
+	end_part=`expr $curr_part + $part_per_server - 1`
+	curr_server=`expr $init_server + $curr_host`
+	if [[ $skip_list =~ $curr_server ]]
+	then
+		var=`expr $var - 1`
+		curr_host=`expr $curr_host + 1`
+	else
+		if [ "$curr_server" -lt "10" ]
+		then
+			cmd="${cmd}${first_loop}da0${curr_server}:${curr_host}:${curr_part}-${end_part}"
+		else
+			cmd="${cmd}${first_loop}da${curr_server}:${curr_host}:${curr_part}-${end_part}"
+		fi
+		curr_part=`expr $curr_part + $part_per_server`
+		curr_host=`expr $curr_host + 1`
+		first_loop=";"
+	fi
+done
+
+echo "$cmd\""
+set -x
+eval "$cmd\""

properties/benchmarks/ycsb.properties

@@ -5,33 +5,28 @@
 builder = edu.brown.benchmark.ycsb.YCSBProjectBuilder
 
 # If 'fixed_size' is set to true, then then the YCSBLoader will
-# only add the number of tuples defined in 'num_records'
 
 # Specify the distribution for key selection
-#requestdistribution=custom
-#requestdistribution=uniform
-requestdistribution=zipfian
 
 # Override operation weights
 ReadRecordProportion=0.85
 UpdateRecordProportion=0.15
 #ScanRecordProportion=0.30
 
 fixed_size = true
-num_records = 60000000
-loadthreads = 1
+#loadthreads = 1
 
 # Zipfian skew factor for tuple access
 # for 95% of accesses -> 5% of tuples
-#skew_factor = 1.27
 # something in the middle
-#skew_factor = 0.96
 # for 2/3 of accesses -> 1/3 of tuples
-skew_factor = 0.65
 #scrambled=true
 #mirrored=true
 #interval=3000
 #shift=5
 #random_shift=false
-num_hot_spots=30
-percent_accesses_to_hot_spots=0.80
+requestdistribution=zipfian
+skew_factor = 0.65
+num_records = 600000
+num_hot_spots=40
+percent_accesses_to_hot_spots=0.60

scripts/partitioning/cpu_partition_monitor.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+if [ $# -eq 0 ]
+then
+	echo ""
+	echo "enter site ID (two digits) and partition ID (three digits)"
+	echo "the site and the partitions must be local"
+	echo "example: ${0} 00 001 checks site 0 and partition 1"
+	echo ""
+	exit
+fi
+
+PID=$(top -n1 | grep -m1 java | perl -pe 's/\e\[?.*?[\@-~] ? ?//g' 2> /dev/null | cut -f1 -d' ')
+#PID=$(top -n1 | grep -m1 java | cut -f1 -d' ')
+NID=$(printf '%d' $(jstack $PID | grep -m1 "H${1}-${2}" | cut -d '=' -f 4 | cut -d ' ' -f 1))
+top -H -n1 | grep java | grep -m1 $NID

scripts/partitioning/verticalskew.py

@@ -25,10 +25,13 @@ def sinewave(steps, max_factor, frequency = 2.0):
 		print "ERROR: frequency must be greater than one"
 		exit(1)
 
-	for current in range(steps):
+	lastRate = 1.0
+	for current in range(1, steps+1):
 		ampl = (max_factor - 1.0) / 2.0
 		radians = math.pi * 2 * (frequency * float(current)/float(steps))
-		res.append(ampl * math.sin(radians + 1.5 * math.pi) + ampl + 1.0)
+		currentRate = ampl * math.sin(radians + 1.5 * math.pi) + ampl + 1.0
+		res.append(currentRate/lastRate)
+		lastRate = currentRate
 
 	return res
 
@@ -50,9 +53,12 @@ def linear(steps, max_factor):
 		print "ERROR: steps must be greater than one"
 		exit(1)
 
-	increment = (max_factor - 1.0) / float(steps)
-	for current in range(steps):
-		res.append(1.0 + float(current) * increment);
+	lastRate = 1.0
+	stepIncrement = (max_factor - 1.0) / (float(steps) - 1)
+	for current in range(1, steps+1):
+		currentRate = 1.0 + float(current) * stepIncrement
+		res.append(currentRate / lastRate);
+		lastRate = currentRate	
 
 	return res
 
@@ -85,16 +91,16 @@ def spike(steps, max_factor, period, duration):
 		exit(1)
 
 	in_spike = False
+	last_spike = 0
 	for current in range(1,steps+1):
-		if not in_spike and current % period == 0:
+		if not in_spike and (current - last_spike) % period == 0:
 			in_spike = True
 			spike_started = current
 			res.append(max_factor)
-		elif in_spike and (current - spike_started) < duration:
-			res.append(max_factor)
 		elif in_spike and (current - spike_started) >= duration:
 			in_spike = False
-			res.append(1.0)
+			last_spike = current
+			res.append(1.0 / max_factor)
 		else:
 			res.append(1.0)
 

scripts/reconfiguration/slurp-hevent.sh

@@ -0,0 +1,32 @@
+#! /bin/bash
+#The file to pull
+hevent_file=~/git/h-store/hevent.log
+
+#clean up
+rm -rf err
+while read line
+do
+    name=$line
+    rm -rf $name
+done < hosts.txt
+
+#get files
+touch hevent-merge.log
+parallel-slurp -h hosts.txt -e err -o out $hevent_file  hevent.log 
+while read line
+do
+    name=$line
+    cat $name/hevent.log >> hevent-merge.log 
+done < hosts.txt
+
+#merge
+sort hevent-merge.log > hevent.log
+rm hevent-merge.log
+
+#clean up downloaded individual files
+rm -rf out
+while read line
+do
+    name=$line
+    rm -rf $name
+done < hosts.txt

src/benchmarks/edu/brown/benchmark/voter/voter-ddl.sql

@@ -64,4 +64,5 @@ AS
      FROM votes
  GROUP BY contestant_number
         , state
-;
+;
+

src/benchmarks/edu/brown/benchmark/ycsb/YCSBClient.java

@@ -262,7 +262,7 @@ public void runLoop() {
     }
 
     @Override
-    protected boolean runOnce() throws IOException {
+    public boolean runOnce() throws IOException {
         // pick random transaction to call, weighted by txnWeights
         final Transaction target = this.txnWeights.nextValue();
 

src/benchmarks/org/voltdb/benchmark/tpcc/TPCCClient.java

@@ -565,7 +565,7 @@ protected int getExpectedOutgoingMessageSize() {
     }
 
     @Override
-    protected boolean runOnce() throws NoConnectionsException {
+    public boolean runOnce() throws NoConnectionsException {
         m_blockOnBackpressure = false;
         // will send procedures to first connection w/o backpressure
         // if all connections have backpressure, will round robin across

src/ee/indexes/indexkey.h

@@ -158,7 +158,7 @@ class IntsKey {
              * in this next uint64_t.
              */
             if (intraKeyOffset < 0) {
-                intraKeyOffset = sizeof(uint64_t) - 1;
+                intraKeyOffset = (int)sizeof(uint64_t) - 1;
                 keyOffset++;
             }
         }
@@ -174,7 +174,7 @@ class IntsKey {
             retval |= (0xFF & (data[keyOffset] >> (intraKeyOffset * 8))) << (ii * 8);
             intraKeyOffset--;
             if (intraKeyOffset < 0) {
-                intraKeyOffset = sizeof(uint64_t) - 1;
+                intraKeyOffset = (int)sizeof(uint64_t) - 1;
                 keyOffset++;
             }
         }
@@ -184,7 +184,7 @@ class IntsKey {
     std::string debug( const voltdb::TupleSchema *keySchema) const {
         std::ostringstream buffer;
         int keyOffset = 0;
-        int intraKeyOffset = sizeof(uint64_t) - 1;
+        int intraKeyOffset = (int)sizeof(uint64_t) - 1;
         const int columnCount = keySchema->columnCount();
         for (int ii = 0; ii < columnCount; ii++) {
             switch(keySchema->columnType(ii)) {
@@ -222,7 +222,7 @@ class IntsKey {
         const TupleSchema *keySchema = tuple->getSchema();
         const int columnCount = keySchema->columnCount();
         int keyOffset = 0;
-        int intraKeyOffset = sizeof(uint64_t) - 1;
+        int intraKeyOffset = (int)sizeof(uint64_t) - 1;
         for (int ii = 0; ii < columnCount; ii++) {
             switch(keySchema->columnType(ii)) {
             case voltdb::VALUE_TYPE_BIGINT: {
@@ -260,7 +260,7 @@ class IntsKey {
         ::memset(data, 0, keySize * sizeof(uint64_t));
         const int columnCount = keySchema->columnCount();
         int keyOffset = 0;
-        int intraKeyOffset = sizeof(uint64_t) - 1;
+        int intraKeyOffset = (int)sizeof(uint64_t) - 1;
         for (int ii = 0; ii < columnCount; ii++) {
             switch(keySchema->columnType(ii)) {
             case voltdb::VALUE_TYPE_BIGINT: {