wip

11 files changed, 512 insertions, 182 deletions

diff --git a/cmd/sigsum_log_go/main.go b/cmd/sigsum_log_go/main.go
index b64da1a..a1aaad6 100644
--- a/cmd/sigsum_log_go/main.go
+++ b/cmd/sigsum_log_go/main.go
@@ -43,6 +43,11 @@ var (
 	logFile      = flag.String("log-file", "", "file to write logs to (Default: stderr)")
 	logLevel     = flag.String("log-level", "info", "log level (Available options: debug, info, warning, error. Default: info)")
 	logColor     = flag.Bool("log-color", false, "colored logging output (Default: off)")
+	role         = flag.String("role", "primary", "log role: primary (default) or secondary")
+	secondaryURL = flag.String("secondary-url", "", "secondary node endpoint for fetching latest replicated tree head")
+	secondaryPubkey = flag.String("secondary-pubkey", "", "hex-encoded Ed25519 public key for secondary node")
+	primaryURL   = flag.String("primary-url", "", "primary node endpoint for fetching leafs")
+	primaryPubkey = flag.String("primary-pubkey", "", "hex-encoded Ed25519 public key for primary node")
 
 	gitCommit = "unknown"
 )
@@ -130,6 +135,7 @@ func setupInstanceFromFlags() (*instance.Instance, error) {
 	if err != nil {
 		return nil, fmt.Errorf("newLogIdentity: %v", err)
 	}
+
 	i.TreeID = *trillianID
 	i.Prefix = *prefix
 	i.MaxRange = *maxRange
@@ -156,13 +162,52 @@ func setupInstanceFromFlags() (*instance.Instance, error) {
 	}
 
 	// Setup state manager
-	i.Stateman, err = state.NewStateManagerSingle(i.Client, i.Signer, i.Interval, i.Deadline)
-	if err != nil {
-		return nil, fmt.Errorf("NewStateManagerSingle: %v", err)
+	switch *role {
+	case "primary":
+		if *primaryURL != "" {
+			return nil, fmt.Errorf("a primary node must not configure primary-url")
+		}
+		if *primaryPubkey != "" {
+			return nil, fmt.Errorf("a primary node must not configure primary-pubkey")
+		}
+		if *secondaryURL != "" && *secondaryPubkey != "" {
+			p, err := newServiceEndpoint(*secondaryURL, *secondaryPubkey)
+			if err != nil {
+				return nil, fmt.Errorf("newServiceEndpoint: %v", err)
+			}
+			i.Peer = *p
+		}
+		i.Role = instance.Primary
+		i.Stateman, err = state.NewStateManagerSingle(i.Client, i.Signer, i.Interval, i.Deadline, i.Peer.URL, i.Peer.Pubkey)
+		if err != nil {
+			return nil, fmt.Errorf("NewStateManagerSingle: %v", err)
+		}
+		i.DNS = dns.NewDefaultResolver()
+
+
+	case "secondary":
+		if *secondaryURL != "" {
+			return nil, fmt.Errorf("a secondary node must not configure secondary-url")
+		}
+		if *secondaryPubkey != "" {
+			return nil, fmt.Errorf("a secondary node must not configure secondary-pubkey")
+		}
+		p, err := newServiceEndpoint(*primaryURL, *primaryPubkey)
+		if err != nil {
+			return nil, fmt.Errorf("newServiceEndpoint: %v", err)
+		}
+		i.Peer = *p
+		i.Role = instance.Secondary
+		// TODO: verify that GRPC.TreeType() == PREORDERED_LOG
+		i.Stateman, err = state.NewStateManagerSingleSecondary(i.Client, i.Signer, i.Interval, i.Deadline, i.Peer.URL, i.Peer.Pubkey)
+		if err != nil {
+			return nil, fmt.Errorf("NewStateManagerSingleSecondary: %v", err)
+		}
+
+	default:
+		return nil, fmt.Errorf("invalid role: %s", *role)
 	}
 
-	// Setup DNS verifier
-	i.DNS = dns.NewDefaultResolver()
 
 	// Register HTTP endpoints
 	mux := http.NewServeMux()
@@ -210,6 +255,21 @@ func newWitnessMap(witnesses string) (map[types.Hash]types.PublicKey, error) {
 	return w, nil
 }
 
+func newServiceEndpoint(url string, pkhex string) (*instance.ServiceEndpoint, error) {
+	pkbuf, err := hex.DecodeString(pkhex)
+	if err != nil {
+		return nil, fmt.Errorf("DecodeString: %v", err)
+	}
+
+	var ep instance.ServiceEndpoint
+	ep.URL = url
+	if n := copy(ep.Pubkey[:], pkbuf); n != types.PublicKeySize {
+		return nil, fmt.Errorf("invalid pubkey size: %v", n)
+	}
+
+	return &ep, nil
+}
+
 // await waits for a shutdown signal and then runs a clean-up function
 func await(ctx context.Context, done func()) {
 	sigs := make(chan os.Signal, 1)
diff --git a/go.mod b/go.mod
index fdb3f09..cf35ecd 100644
--- a/go.mod
+++ b/go.mod
@@ -2,6 +2,8 @@ module git.sigsum.org/log-go
 
 go 1.15
 
+replace git.sigsum.org/sigsum-go => /home/linus/p/sigsum/src/sigsum-go
+
 require (
 	git.sigsum.org/sigsum-go v0.0.8
 	github.com/golang/mock v1.4.4
diff --git a/integration/conf/client.config b/integration/conf/client.config
index fe89790..ab14150 100644
--- a/integration/conf/client.config
+++ b/integration/conf/client.config
@@ -9,5 +9,5 @@
 #     key hash.  See `sigsum-debug pubkey` and `sigsum-debug keyhash`.
 #
 
-cli_priv=
-cli_domain_hint=
+cli_priv=97cacf277d874e4b4b626a3f6663c5fd1995c64b2f07e952ad988061fa66db411da859316863410010ba487a098a4b45d7862a7c89235d0350b6b6d21f182576
+cli_domain_hint=_sigsum_v0.lntest.sigsum.org
diff --git a/integration/conf/primary.config b/integration/conf/primary.config
new file mode 100644
index 0000000..ba598a9
--- /dev/null
+++ b/integration/conf/primary.config
@@ -0,0 +1,11 @@
+tsrv_rpc=localhost:6962
+tseq_rpc=localhost:6963
+
+tsrv_http=localhost:6964
+tseq_http=localhost:6965
+
+ssrv_role=primary
+ssrv_endpoint=localhost:6966
+ssrv_prefix=testonly
+ssrv_shard_start=2009
+ssrv_interval=5s
diff --git a/integration/conf/secondary.config b/integration/conf/secondary.config
new file mode 100644
index 0000000..5f04df2
--- /dev/null
+++ b/integration/conf/secondary.config
@@ -0,0 +1,11 @@
+tsrv_rpc=localhost:7062
+tseq_rpc=localhost:7063
+
+tsrv_http=localhost:7064
+tseq_http=localhost:7065
+
+ssrv_role=secondary
+ssrv_endpoint=localhost:7066
+ssrv_prefix=testonly
+ssrv_shard_start=2009
+ssrv_interval=5s
diff --git a/integration/conf/sigsum.config b/integration/conf/sigsum.config
deleted file mode 100644
index a28e854..0000000
--- a/integration/conf/sigsum.config
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/bin/bash
-
-ssrv_endpoint=localhost:6966
-ssrv_prefix=testonly
-ssrv_shard_start=2009
-ssrv_interval=5s
diff --git a/integration/conf/trillian.config b/integration/conf/trillian.config
deleted file mode 100644
index eaa6f6d..0000000
--- a/integration/conf/trillian.config
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/bash
-
-tsrv_rpc=localhost:6962
-tseq_rpc=localhost:6963
-
-tsrv_http=localhost:6964
-tseq_http=localhost:6965
diff --git a/integration/test.sh b/integration/test.sh
index 25de7a6..2016c48 100755
--- a/integration/test.sh
+++ b/integration/test.sh
@@ -12,17 +12,32 @@
 #
 
 set -eu
+shopt -s nullglob
 trap cleanup EXIT
 
-function main() {
-	log_dir=$(mktemp -d)
+declare -A nodes
+declare -A nodes
+
+pri=conf/primary.config
+sec=conf/secondary.config
 
+function main() {
 	check_go_deps
-	trillian_setup conf/trillian.config
-	sigsum_setup   conf/sigsum.config
-	client_setup   conf/client.config
-	check_setup
 
+	node_setup $pri
+	node_setup $sec
+
+	nodes[$pri:ssrv_extra_args]="-secondary-url=${nodes[$sec:ssrv_endpoint]}"
+	nodes[$pri:ssrv_extra_args]+=" -secondary-pubkey=${nodes[$sec:ssrv_pub]}"
+	node_start $pri
+
+	nodes[$sec:ssrv_extra_args]="-primary-url=${nodes[$pri:ssrv_endpoint]}"
+	nodes[$sec:ssrv_extra_args]+=" -primary-pubkey=${nodes[$pri:ssrv_pub]}"
+	node_start $sec
+
+	client_setup conf/client.config
+
+	check_setup
 	run_tests
 }
 
@@ -55,65 +70,100 @@ function client_setup() {
 	die "must have a properly configured domain hint"
 }
 
+function node_setup() {
+	local i=$1; shift
+	nodes[$i:log_dir]=$(mktemp -d)
+	trillian_setup $i
+	sigsum_setup $i
+}
+
+function node_start() {
+	local i=$1; shift
+	trillian_start $i
+	sigsum_start $i
+}
+
 function trillian_setup() {
-	info "setting up Trillian"
-	source $1
+	local i=$1; shift
+	info "setting up Trillian ($i)"
+
+	source $i
+	nodes[$i:tsrv_rpc]=$tsrv_rpc
+	nodes[$i:tsrv_http]=$tsrv_http
+	nodes[$i:tseq_rpc]=$tseq_rpc
+	nodes[$i:tseq_http]=$tseq_http
+}
+
+function trillian_start() {
+	local i=$1; shift
+	info "starting up Trillian ($i)"
 
 	trillian_log_server\
-		-rpc_endpoint=$tsrv_rpc\
-		-http_endpoint=$tsrv_http\
-		-log_dir=$log_dir 2>/dev/null &
-	tsrv_pid=$!
-	info "started Trillian log server (pid $tsrv_pid)"
+		-rpc_endpoint=${nodes[$i:tsrv_rpc]}\
+		-http_endpoint=${nodes[$i:tsrv_http]}\
+		-log_dir=${nodes[$i:log_dir]} 2>/dev/null &
+	nodes[$i:tsrv_pid]=$!
+	info "started Trillian log server (pid ${nodes[$i:tsrv_pid]})"
 
 	trillian_log_signer\
 		-force_master\
-		-rpc_endpoint=$tseq_rpc\
-		-http_endpoint=$tseq_http\
-		-log_dir=$log_dir 2>/dev/null &
-
-	tseq_pid=$!
-	info "started Trillian log sequencer (pid $tseq_pid)"
+		-rpc_endpoint=${nodes[$i:tseq_rpc]}\
+		-http_endpoint=${nodes[$i:tseq_http]}\
+		-log_dir=${nodes[$i:log_dir]} 2>/dev/null &
+	nodes[$i:tseq_pid]=$!
+	info "started Trillian log sequencer (pid ${nodes[$i:tseq_pid]})"
 
-	ssrv_tree_id=$(createtree --admin_server $tsrv_rpc 2>/dev/null)
+	nodes[$i:ssrv_tree_id]=$(createtree --admin_server ${nodes[$i:tsrv_rpc]} 2>/dev/null)
 	[[ $? -eq 0 ]] ||
 		die "must provision a new Merkle tree"
 
-	info "provisioned Merkle tree with id $ssrv_tree_id"
+	info "provisioned Merkle tree with id ${nodes[$i:ssrv_tree_id]}"
 }
 
 function sigsum_setup() {
-	info "setting up Sigsum server"
-	source $1
-
-	wit1_priv=$(sigsum-debug key private)
-	wit1_pub=$(echo $wit1_priv | sigsum-debug key public)
-	wit1_key_hash=$(echo $wit1_pub | sigsum-debug key hash)
-
-	wit2_priv=$(sigsum-debug key private)
-	wit2_pub=$(echo $wit2_priv | sigsum-debug key public)
-	wit2_key_hash=$(echo $wit2_pub | sigsum-debug key hash)
+	local i=$1; shift
+	info "setting up Sigsum server ($i)"
+	source $i
+
+	nodes[$i:ssrv_role]=$ssrv_role
+	nodes[$i:ssrv_endpoint]=$ssrv_endpoint
+	nodes[$i:ssrv_prefix]=$ssrv_prefix
+	nodes[$i:ssrv_shard_start]=$ssrv_shard_start
+	nodes[$i:ssrv_interval]=$ssrv_interval
+	nodes[$i:log_url]=${nodes[$i:ssrv_endpoint]}/${nodes[$i:ssrv_prefix]}/sigsum/v0
+
+	nodes[$i:wit1_priv]=$(sigsum-debug key private)
+	nodes[$i:wit1_pub]=$(echo ${nodes[$i:wit1_priv]} | sigsum-debug key public)
+	nodes[$i:wit1_key_hash]=$(echo ${nodes[$i:wit1_pub]} | sigsum-debug key hash)
+	nodes[$i:wit2_priv]=$(sigsum-debug key private)
+	nodes[$i:wit2_pub]=$(echo ${nodes[$i:wit2_priv]} | sigsum-debug key public)
+	nodes[$i:wit2_key_hash]=$(echo ${nodes[$i:wit2_pub]} | sigsum-debug key hash)
+	nodes[$i:ssrv_witnesses]=${nodes[$i:wit1_pub]},${nodes[$i:wit2_pub]}
+
+	nodes[$i:ssrv_priv]=$(sigsum-debug key private)
+	nodes[$i:ssrv_pub]=$(echo ${nodes[$i:ssrv_priv]} | sigsum-debug key public)
+	nodes[$i:ssrv_key_hash]=$(echo ${nodes[$i:ssrv_pub]} | sigsum-debug key hash)
+}
 
-	ssrv_witnesses=$wit1_pub,$wit2_pub
-	ssrv_priv=$(sigsum-debug key private)
-	ssrv_pub=$(echo $ssrv_priv | sigsum-debug key public)
-	ssrv_key_hash=$(echo $ssrv_pub | sigsum-debug key hash)
+function sigsum_start() {
+	local i=$1; shift
+	info "starting Sigsum log server ($i)"
 
 	sigsum_log_go\
-		-prefix=$ssrv_prefix\
-		-trillian_id=$ssrv_tree_id\
-		-shard_interval_start=$ssrv_shard_start\
-		-key=<(echo $ssrv_priv)\
-		-witnesses=$ssrv_witnesses\
-		-interval=$ssrv_interval\
-		-http_endpoint=$ssrv_endpoint\
+		-prefix=${nodes[$i:ssrv_prefix]}\
+		-trillian_id=${nodes[$i:ssrv_tree_id]}\
+		-shard_interval_start=${nodes[$i:ssrv_shard_start]}\
+		-key=<(echo ${nodes[$i:ssrv_priv]})\
+		-witnesses=${nodes[$i:ssrv_witnesses]}\
+		-interval=${nodes[$i:ssrv_interval]}\
+		-http_endpoint=${nodes[$i:ssrv_endpoint]}\
 		-log-color="true"\
 		-log-level="debug"\
-		-log-file=$log_dir/sigsum-log.log 2>/dev/null &
-	ssrv_pid=$!
+		-role=${nodes[$i:ssrv_role]} ${nodes[$i:ssrv_extra_args]} \
+		-log-file=${nodes[$i:log_dir]}/sigsum-log.log 2>/dev/null &
+	nodes[$i:ssrv_pid]=$!
 
-	log_url=$ssrv_endpoint/$ssrv_prefix/sigsum/v0
-	info "started Sigsum log server on $ssrv_endpoint (pid $ssrv_pid)"
+	info "started Sigsum log server on ${nodes[$i:ssrv_endpoint]} (pid ${nodes[$i:ssrv_pid]})"
 }
 
 function cleanup() {
@@ -122,49 +172,58 @@ function cleanup() {
 	info "cleaning up, please wait..."
 	sleep 1
 
-	kill -2 $ssrv_pid
-	kill -2 $tseq_pid
-	while :; do
-		sleep 1
+	for i in $pri $sec; do
+		boundp $i:ssrv_pid && kill -2 ${nodes[$i:ssrv_pid]}
+		boundp $i:tseq_pid && kill -2 ${nodes[$i:tseq_pid]}
+		while :; do
+			sleep 1
 
-		ps -p $tseq_pid >/dev/null && continue
-		ps -p $ssrv_pid >/dev/null && continue
+			boundp $i:tseq_pid && ps -p ${nodes[$i:tseq_pid]} >/dev/null && continue
+			boundp $i:ssrv_pid && ps -p ${nodes[$i:$ssrv_pid]} >/dev/null && continue
 
-		break
+			break
+		done
+	done
+	info "stopped Trillian log sequencer(s)"
+	info "stopped Sigsum log server(s)"
+
+	for i in $pri $sec; do
+		if ! deletetree -admin_server=$tsrv_rpc -log_id=${nodes[$i:ssrv_tree_id]}; then
+			warn "failed deleting provisioned Merkle tree ${nodes[$i:ssrv_tree_id]}"
+		else
+			info "deleted provisioned Merkle tree ${nodes[$i:ssrv_tree_id]}"
+		fi
 	done
 
-	info "stopped Trillian log sequencer"
-	info "stopped Sigsum log server"
-
-	if ! deletetree -admin_server=$tsrv_rpc -log_id=$ssrv_tree_id; then
-		warn "failed deleting provisioned Merkle tree"
-	else
-		info "deleteted provisioned Merkle tree"
-	fi
-
-	kill -2 $tsrv_pid
-	while :; do
-		sleep 1
+	for i in $pri $sec; do
+		boundp $i:tsrv_pid || continue
+		kill -2 ${nodes[$i:tsrv_pid]}
+		while :; do
+			sleep 1
 
-		ps -p $tsrv_pid >/dev/null && continue
+			ps -p ${nodes[$i:tsrv_pid]} >/dev/null && continue
 
-		break
+			break
+		done
 	done
+	info "stopped Trillian log server(s)"
 
-	info "stopped Trillian log server"
-
-	printf "\n  Press any key to delete logs in $log_dir"
-	read dummy
+	for i in $pri $sec; do
+		printf "\n  Press any key to delete logs in ${nodes[$i:log_dir]}"
+		read dummy
 
-	rm -rf $log_dir
+		rm -rf ${nodes[$i:log_dir]}
+	done
 }
 
 function check_setup() {
-	sleep 3
+	for i in $pri $sec; do
+		sleep 3
 
-	ps -p $tseq_pid >/dev/null || die "must have Trillian log sequencer"
-	ps -p $tsrv_pid >/dev/null || die "must have Trillian log server"
-	ps -p $ssrv_pid >/dev/null || die "must have Sigsum log server"
+		ps -p ${nodes[$i:tseq_pid]} >/dev/null || die "must have Trillian log sequencer ($i)"
+		ps -p ${nodes[$i:tsrv_pid]} >/dev/null || die "must have Trillian log server ($i)"
+		ps -p ${nodes[$i:ssrv_pid]} >/dev/null || die "must have Sigsum log server ($i)"
+	done
 }
 
 function run_tests() {
@@ -176,18 +235,18 @@ function run_tests() {
 	done
 
 	info "waiting for $num_leaf leaves to be merged..."
-	sleep ${ssrv_interval::-1}
+	sleep ${nodes[$pri:ssrv_interval]::-1}
 
 	test_signed_tree_head $num_leaf
 	for i in $(seq 1 $(( $num_leaf - 1 ))); do
 		test_consistency_proof $i $num_leaf
 	done
 
-	test_cosignature $wit1_key_hash $wit1_priv
-	test_cosignature $wit2_key_hash $wit2_priv
+	test_cosignature ${nodes[$pri:wit1_key_hash]} ${nodes[$pri:wit1_priv]}
+	test_cosignature ${nodes[$pri:wit2_key_hash]} ${nodes[$pri:wit2_priv]}
 
 	info "waiting for cosignature to be available..."
-	sleep ${ssrv_interval::-1}
+	sleep ${nodes[$pri:ssrv_interval]::-1}
 
 	test_cosigned_tree_head $num_leaf
 	for i in $(seq 1 $num_leaf); do
@@ -202,32 +261,33 @@ function run_tests() {
 }
 
 function test_signed_tree_head() {
+	local log_dir=${nodes[$pri:log_dir]}
 	desc="GET tree-head-to-cosign (tree size $1)"
-	curl -s -w "%{http_code}" $log_url/get-tree-head-to-cosign \
+	curl -s -w "%{http_code}" ${nodes[$pri:log_url]}/get-tree-head-to-cosign \
 		>$log_dir/rsp
 
-	if [[ $(status_code) != 200 ]]; then
-		fail "$desc: http status code $(status_code)"
+	if [[ $(status_code $pri) != 200 ]]; then
+		fail "$desc: http status code $(status_code $pri)"
 		return
 	fi
 
-	if ! keys "timestamp" "tree_size" "root_hash" "signature"; then
-		fail "$desc: ascii keys in response $(debug_response)"
+	if ! keys $pri "timestamp" "tree_size" "root_hash" "signature"; then
+		fail "$desc: ascii keys in response $(debug_response $pri)"
 		return
 	fi
 
 	now=$(date +%s)
-	if [[ $(value_of "timestamp") -gt $now ]]; then
-		fail "$desc: timestamp $(value_of "timestamp") is too large"
+	if [[ $(value_of $pri "timestamp") -gt $now ]]; then
+		fail "$desc: timestamp $(value_of $pri "timestamp") is too large"
 		return
 	fi
-	if [[ $(value_of "timestamp") -lt $(( $now - ${ssrv_interval::-1} )) ]]; then
-		fail "$desc: timestamp $(value_of "timestamp") is too small"
+	if [[ $(value_of $pri "timestamp") -lt $(( $now - ${nodes[$pri:ssrv_interval]::-1} )) ]]; then
+		fail "$desc: timestamp $(value_of $pri "timestamp") is too small"
 		return
 	fi
 
-	if [[ $(value_of "tree_size") != $1 ]]; then
-		fail "$desc: tree size $(value_of "tree_size")"
+	if [[ $(value_of $pri "tree_size") != $1 ]]; then
+		fail "$desc: tree size $(value_of $pri "tree_size")"
 		return
 	fi
 
@@ -236,38 +296,39 @@ function test_signed_tree_head() {
 }
 
 function test_cosigned_tree_head() {
+	local log_dir=${nodes[$pri:log_dir]}
 	desc="GET get-tree-head-cosigned (all witnesses)"
-	curl -s -w "%{http_code}" $log_url/get-tree-head-cosigned \
+	curl -s -w "%{http_code}" ${nodes[$pri:log_url]}/get-tree-head-cosigned \
 		>$log_dir/rsp
 
-	if [[ $(status_code) != 200 ]]; then
-		fail "$desc: http status code $(status_code)"
+	if [[ $(status_code $pri) != 200 ]]; then
+		fail "$desc: http status code $(status_code $pri)"
 		return
 	fi
 
-	if ! keys "timestamp" "tree_size" "root_hash" "signature" "cosignature" "key_hash"; then
-		fail "$desc: ascii keys in response $(debug_response)"
+	if ! keys $pri "timestamp" "tree_size" "root_hash" "signature" "cosignature" "key_hash"; then
+		fail "$desc: ascii keys in response $(debug_response $pri)"
 		return
 	fi
 
 	now=$(date +%s)
-	if [[ $(value_of "timestamp") -gt $now ]]; then
-		fail "$desc: timestamp $(value_of "timestamp") is too large"
+	if [[ $(value_of $pri "timestamp") -gt $now ]]; then
+		fail "$desc: timestamp $(value_of $pri "timestamp") is too large"
 		return
 	fi
-	if [[ $(value_of "timestamp") -lt $(( $now - ${ssrv_interval::-1} * 2 )) ]]; then
-		fail "$desc: timestamp $(value_of "timestamp") is too small"
+	if [[ $(value_of $pri "timestamp") -lt $(( $now - ${nodes[$pri:ssrv_interval]::-1} * 2 )) ]]; then
+		fail "$desc: timestamp $(value_of $pri "timestamp") is too small"
 		return
 	fi
 
-	if [[ $(value_of "tree_size") != $1 ]]; then
-		fail "$desc: tree size $(value_of "tree_size")"
+	if [[ $(value_of $pri "tree_size") != $1 ]]; then
+		fail "$desc: tree size $(value_of $pri "tree_size")"
 		return
 	fi
 
-	for got in $(value_of key_hash); do
+	for got in $(value_of $pri key_hash); do
 		found=""
-		for want in $wit1_key_hash $wit2_key_hash; do
+		for want in ${nodes[$pri:wit1_key_hash]} ${nodes[$pri:wit2_key_hash]}; do
 			if [[ $got == $want ]]; then
 				found=true
 			fi
@@ -285,23 +346,24 @@ function test_cosigned_tree_head() {
 }
 
 function test_inclusion_proof() {
+	local log_dir=${nodes[$pri:log_dir]}
 	desc="GET get-inclusion-proof (tree_size $1, data \"$2\", index $3)"
-	signature=$(echo $2 | sigsum-debug leaf sign -k $cli_priv -h $ssrv_shard_start)
-	leaf_hash=$(echo $2 | sigsum-debug leaf hash -k $cli_key_hash -s $signature -h $ssrv_shard_start)
-	curl -s -w "%{http_code}" $log_url/get-inclusion-proof/$1/$leaf_hash >$log_dir/rsp
+	signature=$(echo $2 | sigsum-debug leaf sign -k $cli_priv -h ${nodes[$pri:ssrv_shard_start]})
+	leaf_hash=$(echo $2 | sigsum-debug leaf hash -k $cli_key_hash -s $signature -h ${nodes[$pri:ssrv_shard_start]})
+	curl -s -w "%{http_code}" ${nodes[$pri:log_url]}/get-inclusion-proof/$1/$leaf_hash >$log_dir/rsp
 
-	if [[ $(status_code) != 200 ]]; then
-		fail "$desc: http status code $(status_code)"
+	if [[ $(status_code $pri) != 200 ]]; then
+		fail "$desc: http status code $(status_code $pri)"
 		return
 	fi
 
-	if ! keys "leaf_index" "inclusion_path"; then
-		fail "$desc: ascii keys in response $(debug_response)"
+	if ! keys $pri "leaf_index" "inclusion_path"; then
+		fail "$desc: ascii keys in response $(debug_response $pri)"
 		return
 	fi
 
-	if [[ $(value_of leaf_index) != $3 ]]; then
-		fail "$desc: wrong leaf index $(value_of leaf_index)"
+	if [[ $(value_of $pri leaf_index) != $3 ]]; then
+		fail "$desc: wrong leaf index $(value_of $pri leaf_index)"
 		return
 	fi
 
@@ -310,16 +372,17 @@ function test_inclusion_proof() {
 }
 
 function test_consistency_proof() {
+	local log_dir=${nodes[$pri:log_dir]}
 	desc="GET get-consistency-proof (old_size $1, new_size $2)"
-	curl -s -w "%{http_code}" $log_url/get-consistency-proof/$1/$2 >$log_dir/rsp
+	curl -s -w "%{http_code}" ${nodes[$pri:log_url]}/get-consistency-proof/$1/$2 >$log_dir/rsp
 
-	if [[ $(status_code) != 200 ]]; then
-		fail "$desc: http status code $(status_code)"
+	if [[ $(status_code $pri) != 200 ]]; then
+		fail "$desc: http status code $(status_code $pri)"
 		return
 	fi
 
-	if ! keys "consistency_path"; then
-		fail "$desc: ascii keys in response $(debug_response)"
+	if ! keys $pri "consistency_path"; then
+		fail "$desc: ascii keys in response $(debug_response $pri)"
 		return
 	fi
 
@@ -328,33 +391,34 @@ function test_consistency_proof() {
 }
 
 function test_get_leaf() {
+	local log_dir=${nodes[$pri:log_dir]}
 	desc="GET get-leaves (data \"$1\", index $2)"
-	curl -s -w "%{http_code}" $log_url/get-leaves/$2/$2 >$log_dir/rsp
+	curl -s -w "%{http_code}" ${nodes[$pri:log_url]}/get-leaves/$2/$2 >$log_dir/rsp
 
-	if [[ $(status_code) != 200 ]]; then
-		fail "$desc: http status code $(status_code)"
+	if [[ $(status_code $pri) != 200 ]]; then
+		fail "$desc: http status code $(status_code $pri)"
 		return
 	fi
 
-	if ! keys "shard_hint" "checksum" "signature" "key_hash"; then
-		fail "$desc: ascii keys in response $(debug_response)"
+	if ! keys $pri "shard_hint" "checksum" "signature" "key_hash"; then
+		fail "$desc: ascii keys in response $(debug_response $pri)"
 		return
 	fi
 
-	if [[ $(value_of shard_hint) != $ssrv_shard_start ]]; then
-		fail "$desc: wrong shard hint $(value_of shard_hint)"
+	if [[ $(value_of $pri shard_hint) != ${nodes[$pri:ssrv_shard_start]} ]]; then
+		fail "$desc: wrong shard hint $(value_of $pri shard_hint)"
 		return
 	fi
 
 	message=$(openssl dgst -binary <(echo $1) | base16)
 	checksum=$(openssl dgst -binary <(echo $message | base16 -d) | base16)
-	if [[ $(value_of checksum) != $checksum ]]; then
-		fail "$desc: wrong checksum $(value_of checksum)"
+	if [[ $(value_of $pri checksum) != $checksum ]]; then
+		fail "$desc: wrong checksum $(value_of $pri checksum)"
 		return
 	fi
 
-	if [[ $(value_of key_hash) != $cli_key_hash ]]; then
-		fail "$desc: wrong key hash $(value_of key_hash)"
+	if [[ $(value_of $pri key_hash) != $cli_key_hash ]]; then
+		fail "$desc: wrong key hash $(value_of $pri key_hash)"
 	fi
 
 	# TODO: check leaf signature
@@ -362,24 +426,25 @@ function test_get_leaf() {
 }
 
 function test_add_leaf() {
+	local log_dir=${nodes[$pri:log_dir]}
 	desc="POST add-leaf (data \"$1\")"
-	echo "shard_hint=$ssrv_shard_start" > $log_dir/req
+	echo "shard_hint=${nodes[$pri:ssrv_shard_start]}" > $log_dir/req
 	echo "message=$(openssl dgst -binary <(echo $1) | base16)" >> $log_dir/req
 	echo "signature=$(echo $1 |
-		sigsum-debug leaf sign -k $cli_priv -h $ssrv_shard_start)" >> $log_dir/req
+		sigsum-debug leaf sign -k $cli_priv -h ${nodes[$pri:ssrv_shard_start]})" >> $log_dir/req
 	echo "public_key=$cli_pub" >> $log_dir/req
 	echo "domain_hint=$cli_domain_hint" >> $log_dir/req
 	cat $log_dir/req |
-		curl -s -w "%{http_code}" --data-binary @- $log_url/add-leaf \
+		curl -s -w "%{http_code}" --data-binary @- ${nodes[$pri:log_url]}/add-leaf \
 		>$log_dir/rsp
 
-	if [[ $(status_code) != 200 ]]; then
-		fail "$desc: http status code $(status_code)"
+	if [[ $(status_code $pri) != 200 ]]; then
+		fail "$desc: http status code $(status_code $pri)"
 		return
 	fi
 
-	if ! keys; then
-		fail "$desc: ascii keys in response $(debug_response)"
+	if ! keys $pri; then
+		fail "$desc: ascii keys in response $(debug_response $pri)"
 		return
 	fi
 
@@ -387,21 +452,24 @@ function test_add_leaf() {
 }
 
 function test_cosignature() {
+	local log_dir=${nodes[$pri:log_dir]}
+	#local log_url=${nodes[$pri:log_url]}
+	#local ssrv_key_hash=${nodes[$pri:ssrv_key_hash]}
 	desc="POST add-cosignature (witness $1)"
 	echo "key_hash=$1" > $log_dir/req
-	echo "cosignature=$(curl -s $log_url/get-tree-head-to-cosign |
-		sigsum-debug head sign -k $2 -h $ssrv_key_hash)" >> $log_dir/req
+	echo "cosignature=$(curl -s ${nodes[$pri:log_url]}/get-tree-head-to-cosign |
+		sigsum-debug head sign -k $2 -h ${nodes[$pri:ssrv_key_hash]})" >> $log_dir/req
 	cat $log_dir/req |
-		curl -s -w "%{http_code}" --data-binary @- $log_url/add-cosignature \
+		curl -s -w "%{http_code}" --data-binary @- ${nodes[$pri:log_url]}/add-cosignature \
 		>$log_dir/rsp
 
-	if [[ $(status_code) != 200 ]]; then
-		fail "$desc: http status code $(status_code)"
+	if [[ $(status_code $pri) != 200 ]]; then
+		fail "$desc: http status code $(status_code $pri)"
 		return
 	fi
 
-	if ! keys; then
-		fail "$desc: ascii keys in response $(debug_response)"
+	if ! keys $pri; then
+		fail "$desc: ascii keys in response $(debug_response $pri)"
 		return
 	fi
 
@@ -409,15 +477,18 @@ function test_cosignature() {
 }
 
 function debug_response() {
+	local i=$1; shift
 	echo ""
-	cat $log_dir/rsp
+	cat ${nodes[$i:log_dir]}/rsp
 }
 
 function status_code() {
-	tail -n1 $log_dir/rsp
+	local i=$1; shift
+	tail -n1 ${nodes[$i:log_dir]}/rsp
 }
 
 function value_of() {
+	local i=$1; shift
 	while read line; do
 		key=$(echo $line | cut -d"=" -f1)
 		if [[ $key != $1 ]]; then
@@ -426,16 +497,17 @@ function value_of() {
 
 		value=$(echo $line | cut -d"=" -f2)
 		echo $value
-	done < <(head --lines=-1 $log_dir/rsp)
+	done < <(head --lines=-1 ${nodes[$i:log_dir]}/rsp)
 }
 
 function keys() {
+        local i=$1; shift
 	declare -A map
 	map[thedummystring]=to_avoid_error_on_size_zero
 	while read line; do
 		key=$(echo $line | cut -d"=" -f1)
 		map[$key]=ok
-	done < <(head --lines=-1 $log_dir/rsp)
+	done < <(head --lines=-1 ${nodes[$i:log_dir]}/rsp)
 
 	if [[ $# != $(( ${#map[@]} - 1 )) ]]; then
 		return 1
@@ -448,6 +520,11 @@ function keys() {
 	return 0
 }
 
+function boundp {
+    [[ ${!nodes[@]} == *$1* ]] && return 1
+    return 0
+}
+
 function die() {
 	echo -e "\e[37m$(date +"%y-%m-%d %H:%M:%S %Z")\e[0m [\e[31mFATA\e[0m] $@" >&2
 	exit 1
diff --git a/pkg/instance/instance.go b/pkg/instance/instance.go
index f4c0089..78b5d81 100644
--- a/pkg/instance/instance.go
+++ b/pkg/instance/instance.go
@@ -35,22 +35,44 @@ type Instance struct {
 	Signer   crypto.Signer      // provides access to Ed25519 private key
 	Stateman state.StateManager // coordinates access to (co)signed tree heads
 	DNS      dns.Verifier       // checks if domain name knows a public key
+	Role     Role
+	Peer     ServiceEndpoint
+}
+
+type Role int64
+const (
+	Primary Role = iota
+	Secondary
+)
+
+type ServiceEndpoint struct {
+	URL string
+	Pubkey types.PublicKey
 }
 
-// Handlers returns a list of sigsum handlers
 func (i *Instance) Handlers() []Handler {
-	return []Handler{
-		Handler{Instance: i, Handler: addLeaf, Endpoint: types.EndpointAddLeaf, Method: http.MethodPost},
-		Handler{Instance: i, Handler: addCosignature, Endpoint: types.EndpointAddCosignature, Method: http.MethodPost},
-		Handler{Instance: i, Handler: getTreeHeadToCosign, Endpoint: types.EndpointGetTreeHeadToCosign, Method: http.MethodGet},
-		Handler{Instance: i, Handler: getTreeHeadCosigned, Endpoint: types.EndpointGetTreeHeadCosigned, Method: http.MethodGet},
-		Handler{Instance: i, Handler: getCheckpoint, Endpoint: types.Endpoint("get-checkpoint"), Method: http.MethodGet},
-		Handler{Instance: i, Handler: getConsistencyProof, Endpoint: types.EndpointGetConsistencyProof, Method: http.MethodGet},
-		Handler{Instance: i, Handler: getInclusionProof, Endpoint: types.EndpointGetInclusionProof, Method: http.MethodGet},
-		Handler{Instance: i, Handler: getLeaves, Endpoint: types.EndpointGetLeaves, Method: http.MethodGet},
+	switch i.Role {
+	case Primary:
+		return []Handler{
+			Handler{Instance: i, Handler: addLeaf, Endpoint: types.EndpointAddLeaf, Method: http.MethodPost},
+			Handler{Instance: i, Handler: addCosignature, Endpoint: types.EndpointAddCosignature, Method: http.MethodPost},
+			Handler{Instance: i, Handler: getTreeHeadToCosign, Endpoint: types.EndpointGetTreeHeadToCosign, Method: http.MethodGet}, // ToSign -> ToCoSign
+			Handler{Instance: i, Handler: getTreeHeadCosigned, Endpoint: types.EndpointGetTreeHeadCosigned, Method: http.MethodGet},
+			Handler{Instance: i, Handler: getCheckpoint, Endpoint: types.Endpoint("get-checkpoint"), Method: http.MethodGet},
+			Handler{Instance: i, Handler: getConsistencyProof, Endpoint: types.EndpointGetConsistencyProof, Method: http.MethodGet},
+			Handler{Instance: i, Handler: getInclusionProof, Endpoint: types.EndpointGetInclusionProof, Method: http.MethodGet},
+			Handler{Instance: i, Handler: getLeaves, Endpoint: types.EndpointGetLeaves, Method: http.MethodGet},
+		}
+	case Secondary:
+		return []Handler{
+			Handler{Instance: i, Handler: getTreeHeadToCosign, Endpoint: types.EndpointGetSecondaryTreeHead, Method: http.MethodGet},
+		}
+	default:
+		return []Handler{}
 	}
 }
 
+
 // checkHTTPMethod checks if an HTTP method is supported
 func (i *Instance) checkHTTPMethod(m string) bool {
 	return m == http.MethodGet || m == http.MethodPost
@@ -95,7 +117,7 @@ func (i *Instance) cosignatureRequestFromHTTP(r *http.Request) (*requests.Cosign
 func (i *Instance) consistencyProofRequestFromHTTP(r *http.Request) (*requests.ConsistencyProof, error) {
 	var req requests.ConsistencyProof
 	if err := req.FromURL(r.URL.Path); err != nil {
-		return nil, fmt.Errorf("FromASCII: %v", err)
+		return nil, fmt.Errorf("FromURL: %v", err)
 	}
 	if req.OldSize < 1 {
 		return nil, fmt.Errorf("OldSize(%d) must be larger than zero", req.OldSize)
diff --git a/pkg/state/single.go b/pkg/state/single.go
index 695f0e3..2e44fee 100644
--- a/pkg/state/single.go
+++ b/pkg/state/single.go
@@ -8,18 +8,21 @@ import (
 	"sync"
 	"time"
 
+	"git.sigsum.org/log-go/pkg/client"
 	"git.sigsum.org/log-go/pkg/db"
 	"git.sigsum.org/sigsum-go/pkg/log"
+	//"git.sigsum.org/sigsum-go/pkg/requests"
 	"git.sigsum.org/sigsum-go/pkg/types"
 )
 
-// StateManagerSingle implements a single-instance StateManager
+// StateManagerSingle implements a single-instance StateManager for primary nodes
 type StateManagerSingle struct {
 	client    db.Client
 	signer    crypto.Signer
 	namespace types.Hash
 	interval  time.Duration
 	deadline  time.Duration
+	secondary *client.Client
 
 	// Lock-protected access to pointers.  A write lock is only obtained once
 	// per interval when doing pointer rotation.  All endpoints are readers.
@@ -32,13 +35,14 @@ type StateManagerSingle struct {
 	cosignatures map[types.Hash]*types.Signature
 }
 
-func NewStateManagerSingle(client db.Client, signer crypto.Signer, interval, deadline time.Duration) (*StateManagerSingle, error) {
+func NewStateManagerSingle(dbcli db.Client, signer crypto.Signer, interval, deadline time.Duration, securl string, secpk types.PublicKey) (*StateManagerSingle, error) {
 	sm := &StateManagerSingle{
-		client:    client,
+		client:    dbcli,
 		signer:    signer,
 		namespace: *types.HashFn(signer.Public().(ed25519.PublicKey)),
 		interval:  interval,
 		deadline:  deadline,
+		secondary: client.NewClient(securl, secpk),
 	}
 	sth, err := sm.latestSTH(context.Background())
 	sm.setCosignedTreeHead()
@@ -157,9 +161,57 @@ func (sm *StateManagerSingle) latestSTH(ctx context.Context) (*types.SignedTreeH
 	if err != nil {
 		return nil, fmt.Errorf("failed fetching tree head: %v", err)
 	}
-	sth, err := th.Sign(sm.signer, &sm.namespace)
+
+	//pth, err := choseTree(ctx, sm.deadline, sm.secondary, th)
+	pth, err := th, nil	// DEBUG
+	if err != nil {
+		return nil, fmt.Errorf("failed chosing tree head: %v", err)
+	}
+
+	sth, err := pth.Sign(sm.signer, &sm.namespace)
 	if err != nil {
 		return nil, fmt.Errorf("failed signing tree head: %v", err)
 	}
+
 	return sth, nil
 }
+
+func choseTree(ctx context.Context, deadline time.Duration, secondary *client.Client, th *types.TreeHead) (*types.TreeHead, error) {
+	// TODO: handle multiple secondaries and not just one
+
+	if !secondary.Configured {
+		return th, nil
+	}
+
+	sctx, cancel := context.WithTimeout(ctx, deadline) // FIXME: use a separate timeout value for secondaries?
+	defer cancel()
+	secsth, err := secondary.GetCurrentTreeHead(sctx)
+	if err != nil {
+		return nil, fmt.Errorf("failed getting the latest tree head from all secondaries: %v", err)
+	}
+
+	if secsth.TreeSize < th.TreeSize {
+		// We're stuck at secsth.size so let's verify
+		// consistency since secsth and sign that
+
+		// TODO: get and verify consinstency proof
+
+		// req := &requests.ConsistencyProof{
+		// 	OldSize: secsth.TreeSize,
+		// 	NewSize: th.TreeSize,
+		// }
+
+		// proof, err := sm.client.GetConsistencyProof(ctx, req)
+		// if err != nil {
+		// 	return nil, fmt.Errorf("unable to get consistency proof from %d to %d: %v", req.OldSize, req.NewSize, err)
+		// }
+
+		// if !proof.Verify() {
+		// 	return nil, fmt.Errorf("invalid consistency proof from %d to %d", req.OldSize, req.NewSize)
+		// }
+
+		th = &secsth.TreeHead // FIXME: need to copy?
+	}
+
+	return th, nil
+}
diff --git a/pkg/state/single_sec.go b/pkg/state/single_sec.go
new file mode 100644
index 0000000..9c69fa8
--- /dev/null
+++ b/pkg/state/single_sec.go
@@ -0,0 +1,108 @@
+package state
+
+import (
+	"context"
+	"crypto"
+	"crypto/ed25519"
+	"fmt"
+	"sync"
+	"time"
+
+	"git.sigsum.org/log-go/pkg/client"
+	"git.sigsum.org/log-go/pkg/db"
+	"git.sigsum.org/sigsum-go/pkg/log"
+	//"git.sigsum.org/sigsum-go/pkg/requests"
+	"git.sigsum.org/sigsum-go/pkg/types"
+)
+
+// StateManagerSingleSecondary implements a single-instance StateManager for secondary nodes
+type StateManagerSingleSecondary struct {
+	client    db.Client
+	signer    crypto.Signer
+	namespace types.Hash
+	interval  time.Duration
+	deadline  time.Duration
+	primary   *client.Client
+
+	// Lock-protected access to pointers.  A write lock is only obtained once
+	// per interval when doing pointer rotation.  All endpoints are readers.
+	sync.RWMutex
+	signedTreeHead *types.SignedTreeHead
+}
+
+func NewStateManagerSingleSecondary(dbcli db.Client, signer crypto.Signer, interval, deadline time.Duration, primurl string, primpk types.PublicKey) (*StateManagerSingleSecondary, error) {
+	sm := &StateManagerSingleSecondary{
+		client:    dbcli,
+		signer:    signer,
+		namespace: *types.HashFn(signer.Public().(ed25519.PublicKey)),
+		interval:  interval,
+		deadline:  deadline,
+		primary:   client.NewClient(primurl, primpk),
+	}
+	sth, err := sm.latestSTH(context.Background())
+	sm.setSignedTreeHead(sth)
+	return sm, err
+}
+
+func (sm *StateManagerSingleSecondary) Run(ctx context.Context) {
+	rotation := func() {
+		nextSTH, err := sm.latestSTH(ctx)
+		if err != nil {
+			log.Warning("cannot rotate without tree head: %v", err)
+			return
+		}
+		sm.rotate(nextSTH)
+	}
+
+	ticker := time.NewTicker(sm.interval)
+	defer ticker.Stop()
+
+	// TODO: fetch leaves from primary
+
+	rotation()
+	for {
+		select {
+		case <-ticker.C:
+			rotation()
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+func (sm *StateManagerSingleSecondary) AddCosignature(ctx context.Context, pub *types.PublicKey, sig *types.Signature) error {
+	return fmt.Errorf("internal error: AddCosignature() called in secondary node")
+}
+func (sm *StateManagerSingleSecondary) CosignedTreeHead(_ context.Context) (*types.CosignedTreeHead, error) {
+	return nil, fmt.Errorf("internal error: AddCosignature() called in secondary node")
+}
+func (sm *StateManagerSingleSecondary) ToCosignTreeHead(_ context.Context) (*types.SignedTreeHead, error) {
+	return nil, fmt.Errorf("internal error: AddCosignature() called in secondary node")
+}
+
+func (sm *StateManagerSingleSecondary) setSignedTreeHead(nextSTH *types.SignedTreeHead) {
+	sm.signedTreeHead = nextSTH
+}
+
+func (sm *StateManagerSingleSecondary) latestSTH(ctx context.Context) (*types.SignedTreeHead, error) {
+	ictx, cancel := context.WithTimeout(ctx, sm.deadline)
+	defer cancel()
+
+	th, err := sm.client.GetTreeHead(ictx)
+	if err != nil {
+		return nil, fmt.Errorf("failed fetching tree head: %v", err)
+	}
+	sth, err := th.Sign(sm.signer, &sm.namespace)
+	if err != nil {
+		return nil, fmt.Errorf("failed signing tree head: %v", err)
+	}
+	return sth, nil
+}
+
+func (sm *StateManagerSingleSecondary) rotate(nextSTH *types.SignedTreeHead) {
+	sm.Lock()
+	defer sm.Unlock()
+
+	log.Debug("rotating tree heads")
+	sm.setSignedTreeHead(nextSTH)
+}