implement primary and secondary role, for replicationv0.5.0

6 files changed, 597 insertions, 227 deletions

diff --git a/integration/conf/logc.config b/integration/conf/logc.config
new file mode 100644
index 0000000..3cc31f3
--- /dev/null
+++ b/integration/conf/logc.config
@@ -0,0 +1,14 @@
+node_name=logc
+
+tsrv_rpc=localhost:7162
+tseq_rpc=localhost:7163
+
+tsrv_http=localhost:7164
+tseq_http=localhost:7165
+
+ssrv_role=secondary
+ssrv_interval_sec=2
+ssrv_endpoint=localhost:7166
+ssrv_internal=localhost:7167
+ssrv_prefix=testonly
+ssrv_shard_start=2009
diff --git a/integration/conf/primary.config b/integration/conf/primary.config
new file mode 100644
index 0000000..4126651
--- /dev/null
+++ b/integration/conf/primary.config
@@ -0,0 +1,14 @@
+node_name=loga
+
+tsrv_rpc=localhost:6962
+tseq_rpc=localhost:6963
+
+tsrv_http=localhost:6964
+tseq_http=localhost:6965
+
+ssrv_role=primary
+ssrv_interval_sec=5
+ssrv_endpoint=localhost:6966
+ssrv_internal=localhost:6967
+ssrv_prefix=testonly
+ssrv_shard_start=2009
diff --git a/integration/conf/secondary.config b/integration/conf/secondary.config
new file mode 100644
index 0000000..d00d11e
--- /dev/null
+++ b/integration/conf/secondary.config
@@ -0,0 +1,14 @@
+node_name=logb
+
+tsrv_rpc=localhost:7062
+tseq_rpc=localhost:7063
+
+tsrv_http=localhost:7064
+tseq_http=localhost:7065
+
+ssrv_role=secondary
+ssrv_interval_sec=2
+ssrv_endpoint=localhost:7066
+ssrv_internal=localhost:7067
+ssrv_prefix=testonly
+ssrv_shard_start=2009
diff --git a/integration/conf/sigsum.config b/integration/conf/sigsum.config
deleted file mode 100644
index a28e854..0000000
--- a/integration/conf/sigsum.config
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/bin/bash
-
-ssrv_endpoint=localhost:6966
-ssrv_prefix=testonly
-ssrv_shard_start=2009
-ssrv_interval=5s
diff --git a/integration/conf/trillian.config b/integration/conf/trillian.config
deleted file mode 100644
index eaa6f6d..0000000
--- a/integration/conf/trillian.config
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/bash
-
-tsrv_rpc=localhost:6962
-tseq_rpc=localhost:6963
-
-tsrv_http=localhost:6964
-tseq_http=localhost:6965
diff --git a/integration/test.sh b/integration/test.sh
index 25de7a6..6442704 100755
--- a/integration/test.sh
+++ b/integration/test.sh
@@ -4,50 +4,108 @@
 # Requirements to run
 #
 #   - Install required dependencies, see check_go_deps()
-#   - Add the empty values in conf/client.config
+#   - Fill in the empty values in conf/client.config
 #
-# Usage:
+# Example usage:
 #
 #     $ ./test.sh
 #
 
 set -eu
+shopt -s nullglob
 trap cleanup EXIT
 
+declare g_offline_mode=1
+
+declare -A nvars
+declare nodes="loga logb"
+declare -r loga=conf/primary.config
+declare -r logb=conf/secondary.config
+declare -r logc=conf/logc.config
+declare -r client=conf/client.config
+
 function main() {
-	log_dir=$(mktemp -d)
+	local testflavour=basic
+	[[ $# > 0 ]] && { testflavour=$1; shift; }
 
 	check_go_deps
-	trillian_setup conf/trillian.config
-	sigsum_setup   conf/sigsum.config
-	client_setup   conf/client.config
-	check_setup
 
-	run_tests
+	node_setup $loga $logb
+
+	# Primary
+	nvars[$loga:ssrv_extra_args]="-secondary-url=http://${nvars[$logb:int_url]}"
+	nvars[$loga:ssrv_extra_args]+=" -secondary-pubkey=${nvars[$logb:ssrv_pub]}"
+	node_start $loga
+
+	# Secondary
+	nvars[$logb:ssrv_extra_args]="-primary-url=http://${nvars[$loga:int_url]}"
+	nvars[$logb:ssrv_extra_args]+=" -primary-pubkey=${nvars[$loga:ssrv_pub]}"
+	node_start $logb
+
+	client_setup $client
+	check_setup $loga $logb
+	run_tests $loga $logb 0 5
+	run_tests $loga $logb 5 1
+
+	if [[ $testflavour == extended ]]; then
+		# for tree equality tests later on; FIXME: remove
+		test_signed_tree_head $loga 6
+		cp ${nvars[$loga:log_dir]}/rsp ${nvars[$loga:log_dir]}/last_sth
+
+		node_stop_fe $loga $logb
+		node_destroy $loga; node_stop_be $loga
+		node_setup $logc
+
+		node_promote $logb $loga
+		nvars[$logb:ssrv_extra_args]="-secondary-url=http://${nvars[$logc:int_url]}"
+		nvars[$logb:ssrv_extra_args]+=" -secondary-pubkey=${nvars[$logc:ssrv_pub]}"
+		node_start_fe $logb
+
+		nvars[$logc:ssrv_extra_args]="-primary-url=http://${nvars[$logb:int_url]}"
+		nvars[$logc:ssrv_extra_args]+=" -primary-pubkey=${nvars[$logb:ssrv_pub]}"
+		nodes+=" logc"
+		node_start $logc
+
+		check_setup $logb $logc
+		run_tests_extended $logb $logc 6 ${nvars[$loga:log_dir]}/last_sth
+	fi
 }
 
 function check_go_deps() {
-	[[ $(command -v trillian_log_signer) ]] || die "Hint: go install github.com/google/trillian/cmd/trillian_log_signer@v1.3.13"
-	[[ $(command -v trillian_log_server) ]] || die "Hint: go install github.com/google/trillian/cmd/trillian_log_server@v1.3.13"
-	[[ $(command -v createtree)          ]] || die "Hint: go install github.com/google/trillian/cmd/createtree@v1.3.13"
-	[[ $(command -v deletetree)          ]] || die "Hint: go install github.com/google/trillian/cmd/deletetree@v1.3.13"
-	[[ $(command -v sigsum_log_go)       ]] || die "Hint: go install git.sigsum.org/log-go/cmd/sigsum_log_go@latest"
-	[[ $(command -v sigsum-debug)        ]] || die "Hint: install sigsum-debug from sigsum-go, branch merge/sigsum-debug"
+	[[ $(command -v trillian_log_signer)  ]] || die "Hint: go install github.com/google/trillian/cmd/trillian_log_signer@v1.3.13"
+	[[ $(command -v trillian_log_server)  ]] || die "Hint: go install github.com/google/trillian/cmd/trillian_log_server@v1.3.13"
+	[[ $(command -v createtree)           ]] || die "Hint: go install github.com/google/trillian/cmd/createtree@v1.3.13"
+	[[ $(command -v deletetree)           ]] || die "Hint: go install github.com/google/trillian/cmd/deletetree@v1.3.13"
+	[[ $(command -v updatetree)           ]] || die "Hint: go install github.com/google/trillian/cmd/updatetree@v1.3.13"
+	[[ $(command -v sigsum-log-primary)   ]] || die "Hint: go install git.sigsum.org/log-go/cmd/sigsum-log-primary@latest"
+	[[ $(command -v sigsum-log-secondary) ]] || die "Hint: go install git.sigsum.org/log-go/cmd/sigsum-log-secondary@latest"
+	[[ $(command -v sigsum-debug)         ]] || die "Hint: go install git.sigsum.org/sigsum-go/cmd/sigsum-debug@latest"
 }
 
 function client_setup() {
-	info "setting up client"
-	source $1
+	for i in $@; do
+		info "setting up client ($i)"
+		source $1 # NOTE: not ready for multiple clients --  stomping on everything
+
+		cli_pub=$(echo $cli_priv | sigsum-debug key public)
+		cli_key_hash=$(echo $cli_pub | sigsum-debug key hash)
 
-	cli_pub=$(echo $cli_priv | sigsum-debug key public)
-	cli_key_hash=$(echo $cli_pub | sigsum-debug key hash)
+		[[ $cli_domain_hint =~ ^_sigsum_v0..+ ]] ||
+			die "must have a valid domain hint"
+
+		if [[ $g_offline_mode -ne 1 ]]; then
+			verify_domain_hint_in_dns $cli_domain_hint $cli_key_hash
+		fi
+	done
+}
 
-	[[ $cli_domain_hint =~ ^_sigsum_v0..+ ]] ||
-		die "must have a valid domain hint"
+function verify_domain_hint_in_dns() {
+	local domain_hint=$1; shift
+	local key_hash=$1; shift
 
-	for line in $(dig +short -t txt $cli_domain_hint); do
+	for line in $(dig +short -t txt $domain_hint); do
 		key_hash=${line:1:${#line}-2}
-		if [[ $key_hash == $cli_key_hash ]]; then
+		if [[ $key_hash == $key_hash ]]; then
 			return
 		fi
 	done
@@ -55,179 +113,358 @@ function client_setup() {
 	die "must have a properly configured domain hint"
 }
 
+function node_setup() {
+	for i in $@; do
+		local dir=$(mktemp -d /tmp/sigsum-log-test.XXXXXXXXXX)
+		info "$i: logging to $dir"
+		nvars[$i:log_dir]=$dir
+		trillian_setup $i
+		sigsum_setup $i
+	done
+}
+
+# node_start starts trillian and sigsum and creates new trees
+function node_start() {
+	for i in $@; do
+		trillian_start $i
+		sigsum_start $i
+	done
+}
+
+# node_start_* starts sequencer and sigsum but does not create new trees
+function node_start_fe() {
+	trillian_start_sequencer $@
+	sigsum_start $@
+}
+
+function node_start_be() {
+	trillian_start_server $@
+}
+
+function node_promote() {
+	local new_primary=$1; shift
+	local prev_primary=$1; shift
+	[[ ${nvars[$new_primary:ssrv_role]} == secondary ]] || die "$new_primary: not a secondary node"
+	[[ ${nvars[$prev_primary:ssrv_role]} == primary ]] || die "$prev_primary: not the primary node"
+
+	info "promoting secondary node to primary ($new_primary)"
+	local srv=${nvars[$new_primary:tsrv_rpc]}
+	local tree_id=${nvars[$new_primary:ssrv_tree_id]}
+
+	# NOTE: updatetree doesn't seem to exit with !=0 when failing
+	# TODO: try combining the first two invocations into one
+	[[ $(updatetree --admin_server $srv -tree_id $tree_id -tree_state FROZEN -logtostderr 2>/dev/null) == FROZEN ]] || \
+		die "unable to freeze tree $tree_id"
+	[[ $(updatetree --admin_server $srv -tree_id $tree_id -tree_type LOG     -logtostderr 2>/dev/null) == FROZEN ]] || \
+		die "unable to change tree type to LOG for tree $tree_id"
+	[[ $(updatetree --admin_server $srv -tree_id $tree_id -tree_state ACTIVE -logtostderr 2>/dev/null) == ACTIVE ]] || \
+		die "unable to unfreeze tree $tree_id"
+	info "tree $tree_id type changed from PREORDERED_LOG to LOG"
+
+	nvars[$new_primary:ssrv_role]=primary
+	nvars[$new_primary:ssrv_interval]=5 # FIXME: parameterize
+	nvars[$new_primary:ssrv_priv]=${nvars[$prev_primary:ssrv_priv]}
+	nvars[$new_primary:ssrv_pub]=${nvars[$prev_primary:ssrv_pub]}
+	nvars[$new_primary:ssrv_key_hash]=${nvars[$prev_primary:ssrv_key_hash]}
+}
+
 function trillian_setup() {
-	info "setting up Trillian"
-	source $1
+	for i in $@; do
+		info "setting up Trillian ($i)"
+
+		source $i
+		nvars[$i:tsrv_rpc]=$tsrv_rpc
+		nvars[$i:tsrv_http]=$tsrv_http
+		nvars[$i:tseq_rpc]=$tseq_rpc
+		nvars[$i:tseq_http]=$tseq_http
+	done
+}
 
-	trillian_log_server\
-		-rpc_endpoint=$tsrv_rpc\
-		-http_endpoint=$tsrv_http\
-		-log_dir=$log_dir 2>/dev/null &
-	tsrv_pid=$!
-	info "started Trillian log server (pid $tsrv_pid)"
+# trillian_start starts trillian components and creates new trees
+function trillian_start() {
+	trillian_start_server $@
+	trillian_start_sequencer $@
+	trillian_createtree $@
+}
 
-	trillian_log_signer\
-		-force_master\
-		-rpc_endpoint=$tseq_rpc\
-		-http_endpoint=$tseq_http\
-		-log_dir=$log_dir 2>/dev/null &
+function trillian_start_server() {
+	for i in $@; do
+		info "starting up Trillian server ($i)"
 
-	tseq_pid=$!
-	info "started Trillian log sequencer (pid $tseq_pid)"
+		trillian_log_server\
+			-rpc_endpoint=${nvars[$i:tsrv_rpc]}\
+			-http_endpoint=${nvars[$i:tsrv_http]}\
+			-log_dir=${nvars[$i:log_dir]} 2>/dev/null &
+		nvars[$i:tsrv_pid]=$!
+		info "started Trillian log server (pid ${nvars[$i:tsrv_pid]})"
+	done
+}
+
+function trillian_start_sequencer() {
+	for i in $@; do
+		# no sequencer needed for secondaries
+		[[ ${nvars[$i:ssrv_role]} == secondary ]] && continue
+
+		info "starting up Trillian sequencer ($i)"
+		trillian_log_signer\
+			-force_master\
+			-rpc_endpoint=${nvars[$i:tseq_rpc]}\
+			-http_endpoint=${nvars[$i:tseq_http]}\
+			-log_dir=${nvars[$i:log_dir]} 2>/dev/null &
+		nvars[$i:tseq_pid]=$!
+		info "started Trillian log sequencer (pid ${nvars[$i:tseq_pid]})"
+	done
+}
+
+function trillian_createtree() {
+	for i in $@; do
+		local createtree_extra_args=""
 
-	ssrv_tree_id=$(createtree --admin_server $tsrv_rpc 2>/dev/null)
-	[[ $? -eq 0 ]] ||
-		die "must provision a new Merkle tree"
+		[[ ${nvars[$i:ssrv_role]} == secondary ]] && createtree_extra_args=" -tree_type PREORDERED_LOG"
+		nvars[$i:ssrv_tree_id]=$(createtree --admin_server ${nvars[$i:tsrv_rpc]} $createtree_extra_args -logtostderr 2>/dev/null)
+		[[ $? -eq 0 ]] || die "must provision a new Merkle tree"
 
-	info "provisioned Merkle tree with id $ssrv_tree_id"
+		info "provisioned Merkle tree with id ${nvars[$i:ssrv_tree_id]}"
+	done
 }
 
 function sigsum_setup() {
-	info "setting up Sigsum server"
-	source $1
-
-	wit1_priv=$(sigsum-debug key private)
-	wit1_pub=$(echo $wit1_priv | sigsum-debug key public)
-	wit1_key_hash=$(echo $wit1_pub | sigsum-debug key hash)
-
-	wit2_priv=$(sigsum-debug key private)
-	wit2_pub=$(echo $wit2_priv | sigsum-debug key public)
-	wit2_key_hash=$(echo $wit2_pub | sigsum-debug key hash)
-
-	ssrv_witnesses=$wit1_pub,$wit2_pub
-	ssrv_priv=$(sigsum-debug key private)
-	ssrv_pub=$(echo $ssrv_priv | sigsum-debug key public)
-	ssrv_key_hash=$(echo $ssrv_pub | sigsum-debug key hash)
-
-	sigsum_log_go\
-		-prefix=$ssrv_prefix\
-		-trillian_id=$ssrv_tree_id\
-		-shard_interval_start=$ssrv_shard_start\
-		-key=<(echo $ssrv_priv)\
-		-witnesses=$ssrv_witnesses\
-		-interval=$ssrv_interval\
-		-http_endpoint=$ssrv_endpoint\
-		-log-color="true"\
-		-log-level="debug"\
-		-log-file=$log_dir/sigsum-log.log 2>/dev/null &
-	ssrv_pid=$!
-
-	log_url=$ssrv_endpoint/$ssrv_prefix/sigsum/v0
-	info "started Sigsum log server on $ssrv_endpoint (pid $ssrv_pid)"
+	for i in $@; do
+		info "setting up Sigsum server ($i)"
+		source $i
+
+		nvars[$i:ssrv_role]=$ssrv_role
+		nvars[$i:ssrv_endpoint]=$ssrv_endpoint
+		nvars[$i:ssrv_internal]=$ssrv_internal
+		nvars[$i:ssrv_prefix]=$ssrv_prefix
+		nvars[$i:ssrv_shard_start]=$ssrv_shard_start
+		nvars[$i:ssrv_interval]=$ssrv_interval_sec
+
+
+		nvars[$i:log_url]=${nvars[$i:ssrv_endpoint]}/${nvars[$i:ssrv_prefix]}/sigsum/v0
+		nvars[$i:int_url]=${nvars[$i:ssrv_internal]}/${nvars[$i:ssrv_prefix]}/sigsum/v0
+
+		nvars[$i:wit1_priv]=$(sigsum-debug key private)
+		nvars[$i:wit1_pub]=$(echo ${nvars[$i:wit1_priv]} | sigsum-debug key public)
+		nvars[$i:wit1_key_hash]=$(echo ${nvars[$i:wit1_pub]} | sigsum-debug key hash)
+		nvars[$i:wit2_priv]=$(sigsum-debug key private)
+		nvars[$i:wit2_pub]=$(echo ${nvars[$i:wit2_priv]} | sigsum-debug key public)
+		nvars[$i:wit2_key_hash]=$(echo ${nvars[$i:wit2_pub]} | sigsum-debug key hash)
+		nvars[$i:ssrv_witnesses]=${nvars[$i:wit1_pub]},${nvars[$i:wit2_pub]}
+
+		nvars[$i:ssrv_priv]=$(sigsum-debug key private)
+		nvars[$i:ssrv_pub]=$(echo ${nvars[$i:ssrv_priv]} | sigsum-debug key public)
+		nvars[$i:ssrv_key_hash]=$(echo ${nvars[$i:ssrv_pub]} | sigsum-debug key hash)
+	done
 }
 
-function cleanup() {
-	set +e
+function sigsum_start() {
+	for i in $@; do
+		local role=${nvars[$i:ssrv_role]}
+		local binary=sigsum-log-primary;
+		local extra_args="${nvars[$i:ssrv_extra_args]}"
+
+		if [[ $role = primary ]]; then
+			extra_args+=" -witnesses=${nvars[$i:ssrv_witnesses]}"
+			extra_args+=" -shard-interval-start=${nvars[$i:ssrv_shard_start]}"
+		else
+			binary=sigsum-log-secondary
+		fi
+		info "starting Sigsum log $role node ($i)"
+
+		args="$extra_args \
+                      -url-prefix=${nvars[$i:ssrv_prefix]} \
+		      -tree-id=${nvars[$i:ssrv_tree_id]} \
+		      -trillian-rpc-server=${nvars[$i:tsrv_rpc]} \
+		      -interval=${nvars[$i:ssrv_interval]}s \
+		      -external-endpoint=${nvars[$i:ssrv_endpoint]} \
+		      -internal-endpoint=${nvars[$i:ssrv_internal]} \
+		      -test-mode=true \
+		      -log-color=false \
+		      -log-level=debug \
+		      -log-file=${nvars[$i:log_dir]}/sigsum-log.log"
+		$binary $args -key=<(echo ${nvars[$i:ssrv_priv]}) \
+			2>${nvars[$i:log_dir]}/sigsum-log.$(date +%s).stderr &
+		nvars[$i:ssrv_pid]=$!
+
+		info "started Sigsum log server on ${nvars[$i:ssrv_endpoint]} / ${nvars[$i:ssrv_internal]} (pid ${nvars[$i:ssrv_pid]})"
+	done
+}
 
-	info "cleaning up, please wait..."
-	sleep 1
+function node_stop() {
+	node_stop_fe $@
+	node_stop_be $@
+}
 
-	kill -2 $ssrv_pid
-	kill -2 $tseq_pid
-	while :; do
-		sleep 1
+# Delete log tree for, requires trillian server ("backend") to be running
+function node_destroy() {
+	for i in $@; do
+		if ! deletetree -admin_server=$tsrv_rpc -log_id=${nvars[$i:ssrv_tree_id]} -logtostderr 2>/dev/null; then
+			warn "failed deleting provisioned Merkle tree ${nvars[$i:ssrv_tree_id]}"
+		else
+			info "deleted provisioned Merkle tree ${nvars[$i:ssrv_tree_id]}"
+		fi
+	done
+}
 
-		ps -p $tseq_pid >/dev/null && continue
-		ps -p $ssrv_pid >/dev/null && continue
+function node_stop_fe() {
+	for i in $@; do
+
+		[[ -v nvars[$i:ssrv_pid] ]] && pp ${nvars[$i:ssrv_pid]} && kill ${nvars[$i:ssrv_pid]} # FIXME: why is SIGINT (often) not enough?
+		[[ -v nvars[$i:tseq_pid] ]] && pp ${nvars[$i:tseq_pid]} && kill -2 ${nvars[$i:tseq_pid]}
+		while :; do
+			sleep 1
+
+			[[ -v nvars[$i:tseq_pid] ]] && pp ${nvars[$i:tseq_pid]} && continue
+			[[ -v nvars[$i:ssrv_pid] ]] && pp ${nvars[$i:ssrv_pid]} && continue
+
+			break
+		done
+		info "stopped Trillian log sequencer ($i)"
+		info "stopped Sigsum log server ($i)"
 
-		break
 	done
+}
 
-	info "stopped Trillian log sequencer"
-	info "stopped Sigsum log server"
+function node_stop_be() {
+	for i in $@; do
+		pp ${nvars[$i:tsrv_pid]} && kill -2 ${nvars[$i:tsrv_pid]}
+		while :; do
+			sleep 1
 
-	if ! deletetree -admin_server=$tsrv_rpc -log_id=$ssrv_tree_id; then
-		warn "failed deleting provisioned Merkle tree"
-	else
-		info "deleteted provisioned Merkle tree"
-	fi
+			pp ${nvars[$i:tsrv_pid]} && continue
 
-	kill -2 $tsrv_pid
-	while :; do
-		sleep 1
+			break
+		done
+		info "stopped Trillian log server ($i)"
+	done
+}
+
+function cleanup() {
+	set +e
+
+	info "cleaning up, please wait..."
 
-		ps -p $tsrv_pid >/dev/null && continue
+	for var in $nodes; do
+		declare -n cleanup_i=$var # Using unique iterator name, bc leaking
+		node_stop_fe $cleanup_i
+	done
 
-		break
+	for var in $nodes; do
+		declare -n cleanup_i=$var # Using unique iterator name, bc leaking
+		node_destroy $cleanup_i
 	done
 
-	info "stopped Trillian log server"
+	for var in $nodes; do
+		declare -n cleanup_i=$var # Using unique iterator name, bc leaking
+		node_stop_be $cleanup_i
+	done
 
-	printf "\n  Press any key to delete logs in $log_dir"
-	read dummy
+	for var in $nodes; do
+		declare -n cleanup_i=$var # Using unique iterator name, bc leaking
+		printf "\n  Press enter to delete logs in ${nvars[$cleanup_i:log_dir]}"
+		read dummy
 
-	rm -rf $log_dir
+		rm -rf ${nvars[$cleanup_i:log_dir]}
+	done
 }
 
 function check_setup() {
 	sleep 3
-
-	ps -p $tseq_pid >/dev/null || die "must have Trillian log sequencer"
-	ps -p $tsrv_pid >/dev/null || die "must have Trillian log server"
-	ps -p $ssrv_pid >/dev/null || die "must have Sigsum log server"
+	for i in $@; do
+		info "checking setup for $i"
+		if [[ ${nvars[$i:ssrv_role]} == primary ]]; then
+			[[ -v nvars[$i:tseq_pid] ]] && pp ${nvars[$i:tseq_pid]} || die "must have Trillian log sequencer ($i)"
+		fi
+		[[ -v nvars[$i:tsrv_pid] ]] && pp ${nvars[$i:tsrv_pid]} || die "must have Trillian log server ($i)"
+		[[ -v nvars[$i:ssrv_pid] ]] && pp ${nvars[$i:ssrv_pid]} || die "must have Sigsum log server ($i)"
+	done
 }
 
 function run_tests() {
-	num_leaf=5
+	local pri=$1; shift
+	local sec=$1; shift
+	local start_leaf=$1; shift # 0-based
+	local num_leaf=$1; shift
 
-	test_signed_tree_head 0
-	for i in $(seq 1 $num_leaf); do
-		test_add_leaf $i
-	done
+	info "running ordinary tests, pri=$pri, start_leaf=$start_leaf, num_leaf=$num_leaf"
+
+	test_signed_tree_head $pri $start_leaf
 
-	info "waiting for $num_leaf leaves to be merged..."
-	sleep ${ssrv_interval::-1}
+	info "adding $num_leaf leaves"
+	test_add_leaves $pri $(( $start_leaf + 1 )) $num_leaf
+	num_leaf=$(( $num_leaf + $start_leaf ))
 
-	test_signed_tree_head $num_leaf
-	for i in $(seq 1 $(( $num_leaf - 1 ))); do
-		test_consistency_proof $i $num_leaf
+	test_signed_tree_head $pri $num_leaf
+	for i in $(seq $(( $start_leaf + 1 )) $(( $num_leaf - 1 ))); do
+		test_consistency_proof $pri $i $num_leaf
 	done
 
-	test_cosignature $wit1_key_hash $wit1_priv
-	test_cosignature $wit2_key_hash $wit2_priv
+	test_cosignature $pri ${nvars[$pri:wit1_key_hash]} ${nvars[$pri:wit1_priv]}
+	test_cosignature $pri ${nvars[$pri:wit2_key_hash]} ${nvars[$pri:wit2_priv]}
 
-	info "waiting for cosignature to be available..."
-	sleep ${ssrv_interval::-1}
+	info "waiting for cosignature(s) to be available..."
+	sleep ${nvars[$pri:ssrv_interval]}
 
-	test_cosigned_tree_head $num_leaf
-	for i in $(seq 1 $num_leaf); do
-		test_inclusion_proof $num_leaf $i $(( $i - 1 ))
+	test_cosigned_tree_head $pri $num_leaf
+	for i in $(seq  $(( $start_leaf + 1 )) $num_leaf); do
+		test_inclusion_proof $pri $num_leaf $i $(( $i - 1 ))
 	done
 
-	for i in $(seq 1 $num_leaf); do
-		test_get_leaf $i $(( $i - 1 ))
+	for i in $(seq  $(( $start_leaf + 1 )) $num_leaf); do
+		test_get_leaf $pri $i $(( $i - 1 ))
 	done
 
 	warn "no signatures and merkle proofs were verified"
 }
 
+run_tests_extended() {
+	local pri=$1; shift
+	local sec=$1; shift
+	local current_tree_size=$1; shift
+	local old_pri_sth_rsp=$1; shift
+	info "running extended tests"
+
+	info "wait for new primary and secondary to catch up and merge"
+	sleep $(( ${nvars[$pri:ssrv_interval]} + ${nvars[$sec:ssrv_interval]} + 1 ))
+
+	test_signed_tree_head $pri $current_tree_size
+	test_tree_heads_equal ${nvars[$pri:log_dir]}/rsp $old_pri_sth_rsp
+
+	run_tests $pri $sec $current_tree_size 5
+}
+
 function test_signed_tree_head() {
-	desc="GET tree-head-to-cosign (tree size $1)"
-	curl -s -w "%{http_code}" $log_url/get-tree-head-to-cosign \
-		>$log_dir/rsp
+	local pri=$1; shift
+	local tree_size=$1; shift
+	local log_dir=${nvars[$pri:log_dir]}
+	local desc="GET get-tree-head-to-cosign (tree size $tree_size)"
+
+	curl -s -w "%{http_code}" ${nvars[$pri:log_url]}/get-tree-head-to-cosign \
+	     >$log_dir/rsp
 
-	if [[ $(status_code) != 200 ]]; then
-		fail "$desc: http status code $(status_code)"
+	if [[ $(status_code $pri) != 200 ]]; then
+		fail "$desc: http status code $(status_code $pri)"
 		return
 	fi
 
-	if ! keys "timestamp" "tree_size" "root_hash" "signature"; then
-		fail "$desc: ascii keys in response $(debug_response)"
+	if ! keys $pri "timestamp" "tree_size" "root_hash" "signature"; then
+		fail "$desc: ascii keys in response $(debug_response $pri)"
 		return
 	fi
 
 	now=$(date +%s)
-	if [[ $(value_of "timestamp") -gt $now ]]; then
-		fail "$desc: timestamp $(value_of "timestamp") is too large"
+	if [[ $(value_of $pri "timestamp") -gt $now ]]; then
+		fail "$desc: timestamp $(value_of $pri "timestamp") is too high"
 		return
 	fi
-	if [[ $(value_of "timestamp") -lt $(( $now - ${ssrv_interval::-1} )) ]]; then
-		fail "$desc: timestamp $(value_of "timestamp") is too small"
+	if [[ $(value_of $pri "timestamp") -lt $(( $now - ${nvars[$pri:ssrv_interval]} - 1 )) ]]; then
+		fail "$desc: timestamp $(value_of $pri "timestamp") is too low"
 		return
 	fi
 
-	if [[ $(value_of "tree_size") != $1 ]]; then
-		fail "$desc: tree size $(value_of "tree_size")"
+	if [[ $(value_of $pri "tree_size") != $tree_size ]]; then
+		fail "$desc: tree size $(value_of $pri "tree_size")"
 		return
 	fi
 
@@ -235,39 +472,65 @@ function test_signed_tree_head() {
 	pass $desc
 }
 
+function test_tree_heads_equal() {
+	local rsp1=$1; shift
+	local rsp2=$1; shift
+	local desc="comparing tree heads ($rsp1, $rsp2)"
+
+	n1_tree_size=$(value_of_file $rsp1 "tree_size")
+	n2_tree_size=$(value_of_file $rsp2 "tree_size")
+	if [[ $n1_tree_size -ne $n2_tree_size ]]; then
+		fail "$desc: tree_size: $n1_tree_size != $n2_tree_size"
+		return
+	fi
+
+	n1_root_hash=$(value_of_file $rsp1 "root_hash")
+	n2_root_hash=$(value_of_file $rsp2 "root_hash")
+	if [[ $n1_root_hash != $n2_root_hash ]]; then
+		fail "$desc: root_hash: $n1_root_hash != $n2_root_hash"
+		return
+	fi
+
+	pass $desc
+}
+
 function test_cosigned_tree_head() {
-	desc="GET get-tree-head-cosigned (all witnesses)"
-	curl -s -w "%{http_code}" $log_url/get-tree-head-cosigned \
-		>$log_dir/rsp
+	local pri=$1; shift
+	local tree_size=$1; shift
+	local log_dir=${nvars[$pri:log_dir]}
+	local desc="GET get-tree-head-cosigned (all witnesses), tree_size $tree_size"
 
-	if [[ $(status_code) != 200 ]]; then
-		fail "$desc: http status code $(status_code)"
+	curl -s -w "%{http_code}" ${nvars[$pri:log_url]}/get-tree-head-cosigned \
+	     >$log_dir/rsp
+
+	if [[ $(status_code $pri) != 200 ]]; then
+		fail "$desc: http status code $(status_code $pri)"
 		return
 	fi
 
-	if ! keys "timestamp" "tree_size" "root_hash" "signature" "cosignature" "key_hash"; then
-		fail "$desc: ascii keys in response $(debug_response)"
+	if ! keys $pri "timestamp" "tree_size" "root_hash" "signature" "cosignature" "key_hash"; then
+		fail "$desc: ascii keys in response $(debug_response $pri)"
 		return
 	fi
 
 	now=$(date +%s)
-	if [[ $(value_of "timestamp") -gt $now ]]; then
-		fail "$desc: timestamp $(value_of "timestamp") is too large"
+	if [[ $(value_of $pri "timestamp") -gt $now ]]; then
+		fail "$desc: timestamp $(value_of $pri "timestamp") is too large"
 		return
 	fi
-	if [[ $(value_of "timestamp") -lt $(( $now - ${ssrv_interval::-1} * 2 )) ]]; then
-		fail "$desc: timestamp $(value_of "timestamp") is too small"
+	if [[ $(value_of $pri "timestamp") -lt $(( $now - ${nvars[$pri:ssrv_interval]} * 2 )) ]]; then
+		fail "$desc: timestamp $(value_of $pri "timestamp") is too small"
 		return
 	fi
 
-	if [[ $(value_of "tree_size") != $1 ]]; then
-		fail "$desc: tree size $(value_of "tree_size")"
+	if [[ $(value_of $pri "tree_size") != $tree_size ]]; then
+		fail "$desc: tree size $(value_of $pri "tree_size")"
 		return
 	fi
 
-	for got in $(value_of key_hash); do
+	for got in $(value_of $pri key_hash); do
 		found=""
-		for want in $wit1_key_hash $wit2_key_hash; do
+		for want in ${nvars[$pri:wit1_key_hash]} ${nvars[$pri:wit2_key_hash]}; do
 			if [[ $got == $want ]]; then
 				found=true
 			fi
@@ -285,23 +548,29 @@ function test_cosigned_tree_head() {
 }
 
 function test_inclusion_proof() {
-	desc="GET get-inclusion-proof (tree_size $1, data \"$2\", index $3)"
-	signature=$(echo $2 | sigsum-debug leaf sign -k $cli_priv -h $ssrv_shard_start)
-	leaf_hash=$(echo $2 | sigsum-debug leaf hash -k $cli_key_hash -s $signature -h $ssrv_shard_start)
-	curl -s -w "%{http_code}" $log_url/get-inclusion-proof/$1/$leaf_hash >$log_dir/rsp
-
-	if [[ $(status_code) != 200 ]]; then
-		fail "$desc: http status code $(status_code)"
+	local pri=$1; shift
+	local tree_size=$1; shift
+	local data=$1; shift
+	local index=$1; shift
+	local log_dir=${nvars[$pri:log_dir]}
+	local desc="GET get-inclusion-proof (tree_size $tree_size, data \"$data\", index $index)"
+
+	local signature=$(echo ${data} | sigsum-debug leaf sign -k $cli_priv -h ${nvars[$pri:ssrv_shard_start]})
+	local leaf_hash=$(echo ${data} | sigsum-debug leaf hash -k $cli_key_hash -s $signature -h ${nvars[$pri:ssrv_shard_start]})
+	curl -s -w "%{http_code}" ${nvars[$pri:log_url]}/get-inclusion-proof/${tree_size}/${leaf_hash} >${log_dir}/rsp
+
+	if [[ $(status_code $pri) != 200 ]]; then
+		fail "$desc: http status code $(status_code $pri)"
 		return
 	fi
 
-	if ! keys "leaf_index" "inclusion_path"; then
-		fail "$desc: ascii keys in response $(debug_response)"
+	if ! keys $pri "leaf_index" "inclusion_path"; then
+		fail "$desc: ascii keys in response $(debug_response $pri)"
 		return
 	fi
 
-	if [[ $(value_of leaf_index) != $3 ]]; then
-		fail "$desc: wrong leaf index $(value_of leaf_index)"
+	if [[ $(value_of $pri leaf_index) != ${index} ]]; then
+		fail "$desc: wrong leaf index $(value_of $pri leaf_index)"
 		return
 	fi
 
@@ -310,16 +579,19 @@ function test_inclusion_proof() {
 }
 
 function test_consistency_proof() {
-	desc="GET get-consistency-proof (old_size $1, new_size $2)"
-	curl -s -w "%{http_code}" $log_url/get-consistency-proof/$1/$2 >$log_dir/rsp
+	local pri=$1; shift
+	local log_dir=${nvars[$pri:log_dir]}
+	local desc="GET get-consistency-proof (old_size $1, new_size $2)"
+
+	curl -s -w "%{http_code}" ${nvars[$pri:log_url]}/get-consistency-proof/$1/$2 >$log_dir/rsp
 
-	if [[ $(status_code) != 200 ]]; then
-		fail "$desc: http status code $(status_code)"
+	if [[ $(status_code $pri) != 200 ]]; then
+		fail "$desc: http status code $(status_code $pri)"
 		return
 	fi
 
-	if ! keys "consistency_path"; then
-		fail "$desc: ascii keys in response $(debug_response)"
+	if ! keys $pri "consistency_path"; then
+		fail "$desc: ascii keys in response $(debug_response $pri)"
 		return
 	fi
 
@@ -328,80 +600,130 @@ function test_consistency_proof() {
 }
 
 function test_get_leaf() {
-	desc="GET get-leaves (data \"$1\", index $2)"
-	curl -s -w "%{http_code}" $log_url/get-leaves/$2/$2 >$log_dir/rsp
+	local pri=$1; shift
+	local data="$1"; shift
+	local index="$1"; shift
+	local log_dir=${nvars[$pri:log_dir]}
+	local desc="GET get-leaves (data \"$data\", index $index)"
 
-	if [[ $(status_code) != 200 ]]; then
-		fail "$desc: http status code $(status_code)"
+	curl -s -w "%{http_code}" ${nvars[$pri:log_url]}/get-leaves/$index/$index >$log_dir/rsp
+
+	if [[ $(status_code $pri) != 200 ]]; then
+		fail "$desc: http status code $(status_code $pri)"
 		return
 	fi
 
-	if ! keys "shard_hint" "checksum" "signature" "key_hash"; then
-		fail "$desc: ascii keys in response $(debug_response)"
+	if ! keys $pri "shard_hint" "checksum" "signature" "key_hash"; then
+		fail "$desc: ascii keys in response $(debug_response $pri)"
 		return
 	fi
 
-	if [[ $(value_of shard_hint) != $ssrv_shard_start ]]; then
-		fail "$desc: wrong shard hint $(value_of shard_hint)"
+	if [[ $(value_of $pri shard_hint) != ${nvars[$pri:ssrv_shard_start]} ]]; then
+		fail "$desc: wrong shard hint $(value_of $pri shard_hint)"
 		return
 	fi
 
-	message=$(openssl dgst -binary <(echo $1) | base16)
-	checksum=$(openssl dgst -binary <(echo $message | base16 -d) | base16)
-	if [[ $(value_of checksum) != $checksum ]]; then
-		fail "$desc: wrong checksum $(value_of checksum)"
+	local message=$(openssl dgst -binary <(echo $data) | base16)
+	local checksum=$(openssl dgst -binary <(echo $message | base16 -d) | base16)
+	if [[ $(value_of $pri checksum) != $checksum ]]; then
+		fail "$desc: wrong checksum $(value_of $pri checksum)"
 		return
 	fi
 
-	if [[ $(value_of key_hash) != $cli_key_hash ]]; then
-		fail "$desc: wrong key hash $(value_of key_hash)"
+	if [[ $(value_of $pri key_hash) != $cli_key_hash ]]; then
+		fail "$desc: wrong key hash $(value_of $pri key_hash)"
 	fi
 
 	# TODO: check leaf signature
 	pass $desc
 }
 
-function test_add_leaf() {
-	desc="POST add-leaf (data \"$1\")"
-	echo "shard_hint=$ssrv_shard_start" > $log_dir/req
-	echo "message=$(openssl dgst -binary <(echo $1) | base16)" >> $log_dir/req
-	echo "signature=$(echo $1 |
-		sigsum-debug leaf sign -k $cli_priv -h $ssrv_shard_start)" >> $log_dir/req
+function test_add_leaves() {
+	local s=$1; shift
+	local start=$1; shift	# integer, used as data and filename under subs/
+	local end=$(( $start + $1 - 1 )); shift # number of leaves to add
+	local desc="add leaves"
+	local log_dir=${nvars[$s:log_dir]}
+	[[ -d $log_dir/subs/$s ]] || mkdir -p $log_dir/subs/$s
+
+	local -a rc
+	for i in $(seq $start $end); do
+		rc[$i]=$(add_leaf $s $i)
+	done
+
+	# TODO: bail out and fail after $timeout seconds
+	while true; do
+		local keep_going=0
+		for i in $(seq $start $end); do
+			if [[ ${rc[$i]} -eq 202 ]]; then
+				keep_going=1
+				break
+			fi
+		done
+		[[ $keep_going -eq 0 ]] && break
+
+		sleep 1
+		for i in $(seq $start $end); do
+			if [[ ${rc[$i]} -eq 202 ]]; then
+				rc[$i]=$(add_leaf $s $i)
+				if [[ ${rc[$i]} -eq 200 ]]; then
+					if ! keys $s; then
+						fail "$desc (data \"$i\"): ascii keys in response $(debug_response $s)"
+					fi
+				fi
+			fi
+		done
+	done
+
+	local all_good=1
+	for i in $(seq $start $end); do
+		if [[ ${rc[$i]} -ne 200 ]]; then
+			fail "$desc (data \"$i\") HTTP status code: ${rc[$i]}"
+			all_good=0
+		fi
+		echo ${rc[$i]} > "$log_dir/subs/$s/$i"
+	done
+	[[ $all_good -eq 1 ]] && pass $desc
+}
+
+function add_leaf() {
+	local s=$1; shift
+	local data="$1"; shift
+	local log_dir=${nvars[$s:log_dir]}
+
+	echo "shard_hint=${nvars[$s:ssrv_shard_start]}" > $log_dir/req
+	echo "message=$(openssl dgst -binary <(echo $data) | base16)" >> $log_dir/req
+	echo "signature=$(echo $data |
+		sigsum-debug leaf sign -k $cli_priv -h ${nvars[$s:ssrv_shard_start]})" >> $log_dir/req
 	echo "public_key=$cli_pub" >> $log_dir/req
 	echo "domain_hint=$cli_domain_hint" >> $log_dir/req
-	cat $log_dir/req |
-		curl -s -w "%{http_code}" --data-binary @- $log_url/add-leaf \
-		>$log_dir/rsp
-
-	if [[ $(status_code) != 200 ]]; then
-		fail "$desc: http status code $(status_code)"
-		return
-	fi
 
-	if ! keys; then
-		fail "$desc: ascii keys in response $(debug_response)"
-		return
-	fi
+	cat $log_dir/req |
+		curl -s -w "%{http_code}" --data-binary @- ${nvars[$s:log_url]}/add-leaf \
+		     >$log_dir/rsp
 
-	pass $desc
+	echo $(status_code $s)
 }
 
 function test_cosignature() {
-	desc="POST add-cosignature (witness $1)"
+	local pri=$1; shift
+	local log_dir=${nvars[$pri:log_dir]}
+	local desc="POST add-cosignature (witness $1)"
+
 	echo "key_hash=$1" > $log_dir/req
-	echo "cosignature=$(curl -s $log_url/get-tree-head-to-cosign |
-		sigsum-debug head sign -k $2 -h $ssrv_key_hash)" >> $log_dir/req
+	echo "cosignature=$(curl -s ${nvars[$pri:log_url]}/get-tree-head-to-cosign |
+		sigsum-debug head sign -k $2 -h ${nvars[$pri:ssrv_key_hash]})" >> $log_dir/req
 	cat $log_dir/req |
-		curl -s -w "%{http_code}" --data-binary @- $log_url/add-cosignature \
-		>$log_dir/rsp
+		curl -s -w "%{http_code}" --data-binary @- ${nvars[$pri:log_url]}/add-cosignature \
+		     >$log_dir/rsp
 
-	if [[ $(status_code) != 200 ]]; then
-		fail "$desc: http status code $(status_code)"
+	if [[ $(status_code $pri) != 200 ]]; then
+		fail "$desc: http status code $(status_code $pri)"
 		return
 	fi
 
-	if ! keys; then
-		fail "$desc: ascii keys in response $(debug_response)"
+	if ! keys $pri; then
+		fail "$desc: ascii keys in response $(debug_response $pri)"
 		return
 	fi
 
@@ -409,15 +731,23 @@ function test_cosignature() {
 }
 
 function debug_response() {
+	local i=$1; shift
 	echo ""
-	cat $log_dir/rsp
+	cat ${nvars[$i:log_dir]}/rsp
 }
 
 function status_code() {
-	tail -n1 $log_dir/rsp
+	local i=$1; shift
+	tail -n1 ${nvars[$i:log_dir]}/rsp
 }
 
 function value_of() {
+	local s=$1; shift
+	value_of_file ${nvars[$s:log_dir]}/rsp $@
+}
+
+function value_of_file() {
+	local rsp=$1; shift
 	while read line; do
 		key=$(echo $line | cut -d"=" -f1)
 		if [[ $key != $1 ]]; then
@@ -426,16 +756,17 @@ function value_of() {
 
 		value=$(echo $line | cut -d"=" -f2)
 		echo $value
-	done < <(head --lines=-1 $log_dir/rsp)
+	done < <(head --lines=-1 $rsp)
 }
 
 function keys() {
+        local s=$1; shift
 	declare -A map
 	map[thedummystring]=to_avoid_error_on_size_zero
 	while read line; do
 		key=$(echo $line | cut -d"=" -f1)
 		map[$key]=ok
-	done < <(head --lines=-1 $log_dir/rsp)
+	done < <(head --lines=-1 ${nvars[$s:log_dir]}/rsp)
 
 	if [[ $# != $(( ${#map[@]} - 1 )) ]]; then
 		return 1
@@ -448,6 +779,12 @@ function keys() {
 	return 0
 }
 
+# Is proces with PID $1 running or not?
+function pp() {
+	[[ $1 == -p ]] && shift
+	[[ -d /proc/$1 ]]
+}
+
 function die() {
 	echo -e "\e[37m$(date +"%y-%m-%d %H:%M:%S %Z")\e[0m [\e[31mFATA\e[0m] $@" >&2
 	exit 1
@@ -469,4 +806,8 @@ function fail() {
 	echo -e "\e[37m$(date +"%y-%m-%d %H:%M:%S %Z")\e[0m [\e[91mFAIL\e[0m] $@" >&2
 }
 
-main
+main $@
+
+# Local Variables:
+# sh-basic-offset: 8
+# End: