diff options
Diffstat (limited to 'integration')
| -rw-r--r-- | integration/conf/client.config | 4 | ||||
| -rw-r--r-- | integration/conf/primary.config | 11 | ||||
| -rw-r--r-- | integration/conf/secondary.config | 11 | ||||
| -rw-r--r-- | integration/conf/sigsum.config | 6 | ||||
| -rw-r--r-- | integration/conf/trillian.config | 7 | ||||
| -rwxr-xr-x | integration/test.sh | 371 | 
6 files changed, 248 insertions, 162 deletions
| diff --git a/integration/conf/client.config b/integration/conf/client.config index fe89790..ab14150 100644 --- a/integration/conf/client.config +++ b/integration/conf/client.config @@ -9,5 +9,5 @@  #     key hash.  See `sigsum-debug pubkey` and `sigsum-debug keyhash`.  # -cli_priv= -cli_domain_hint= +cli_priv=97cacf277d874e4b4b626a3f6663c5fd1995c64b2f07e952ad988061fa66db411da859316863410010ba487a098a4b45d7862a7c89235d0350b6b6d21f182576 +cli_domain_hint=_sigsum_v0.lntest.sigsum.org diff --git a/integration/conf/primary.config b/integration/conf/primary.config new file mode 100644 index 0000000..ba598a9 --- /dev/null +++ b/integration/conf/primary.config @@ -0,0 +1,11 @@ +tsrv_rpc=localhost:6962 +tseq_rpc=localhost:6963 + +tsrv_http=localhost:6964 +tseq_http=localhost:6965 + +ssrv_role=primary +ssrv_endpoint=localhost:6966 +ssrv_prefix=testonly +ssrv_shard_start=2009 +ssrv_interval=5s diff --git a/integration/conf/secondary.config b/integration/conf/secondary.config new file mode 100644 index 0000000..5f04df2 --- /dev/null +++ b/integration/conf/secondary.config @@ -0,0 +1,11 @@ +tsrv_rpc=localhost:7062 +tseq_rpc=localhost:7063 + +tsrv_http=localhost:7064 +tseq_http=localhost:7065 + +ssrv_role=secondary +ssrv_endpoint=localhost:7066 +ssrv_prefix=testonly +ssrv_shard_start=2009 +ssrv_interval=5s diff --git a/integration/conf/sigsum.config b/integration/conf/sigsum.config deleted file mode 100644 index a28e854..0000000 --- a/integration/conf/sigsum.config +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -ssrv_endpoint=localhost:6966 -ssrv_prefix=testonly -ssrv_shard_start=2009 -ssrv_interval=5s diff --git a/integration/conf/trillian.config b/integration/conf/trillian.config deleted file mode 100644 index eaa6f6d..0000000 --- a/integration/conf/trillian.config +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -tsrv_rpc=localhost:6962 -tseq_rpc=localhost:6963 - -tsrv_http=localhost:6964 -tseq_http=localhost:6965 diff --git a/integration/test.sh b/integration/test.sh index 25de7a6..2016c48 100755 --- a/integration/test.sh +++ b/integration/test.sh @@ -12,17 +12,32 @@  #  set -eu +shopt -s nullglob  trap cleanup EXIT -function main() { -	log_dir=$(mktemp -d) +declare -A nodes +declare -A nodes + +pri=conf/primary.config +sec=conf/secondary.config +function main() {  	check_go_deps -	trillian_setup conf/trillian.config -	sigsum_setup   conf/sigsum.config -	client_setup   conf/client.config -	check_setup +	node_setup $pri +	node_setup $sec + +	nodes[$pri:ssrv_extra_args]="-secondary-url=${nodes[$sec:ssrv_endpoint]}" +	nodes[$pri:ssrv_extra_args]+=" -secondary-pubkey=${nodes[$sec:ssrv_pub]}" +	node_start $pri + +	nodes[$sec:ssrv_extra_args]="-primary-url=${nodes[$pri:ssrv_endpoint]}" +	nodes[$sec:ssrv_extra_args]+=" -primary-pubkey=${nodes[$pri:ssrv_pub]}" +	node_start $sec + +	client_setup conf/client.config + +	check_setup  	run_tests  } @@ -55,65 +70,100 @@ function client_setup() {  	die "must have a properly configured domain hint"  } +function node_setup() { +	local i=$1; shift +	nodes[$i:log_dir]=$(mktemp -d) +	trillian_setup $i +	sigsum_setup $i +} + +function node_start() { +	local i=$1; shift +	trillian_start $i +	sigsum_start $i +} +  function trillian_setup() { -	info "setting up Trillian" -	source $1 +	local i=$1; shift +	info "setting up Trillian ($i)" + +	source $i +	nodes[$i:tsrv_rpc]=$tsrv_rpc +	nodes[$i:tsrv_http]=$tsrv_http +	nodes[$i:tseq_rpc]=$tseq_rpc +	nodes[$i:tseq_http]=$tseq_http +} + +function trillian_start() { +	local i=$1; shift +	info "starting up Trillian ($i)"  	trillian_log_server\ -		-rpc_endpoint=$tsrv_rpc\ -		-http_endpoint=$tsrv_http\ -		-log_dir=$log_dir 2>/dev/null & -	tsrv_pid=$! -	info "started Trillian log server (pid $tsrv_pid)" +		-rpc_endpoint=${nodes[$i:tsrv_rpc]}\ +		-http_endpoint=${nodes[$i:tsrv_http]}\ +		-log_dir=${nodes[$i:log_dir]} 2>/dev/null & +	nodes[$i:tsrv_pid]=$! +	info "started Trillian log server (pid ${nodes[$i:tsrv_pid]})"  	trillian_log_signer\  		-force_master\ -		-rpc_endpoint=$tseq_rpc\ -		-http_endpoint=$tseq_http\ -		-log_dir=$log_dir 2>/dev/null & - -	tseq_pid=$! -	info "started Trillian log sequencer (pid $tseq_pid)" +		-rpc_endpoint=${nodes[$i:tseq_rpc]}\ +		-http_endpoint=${nodes[$i:tseq_http]}\ +		-log_dir=${nodes[$i:log_dir]} 2>/dev/null & +	nodes[$i:tseq_pid]=$! +	info "started Trillian log sequencer (pid ${nodes[$i:tseq_pid]})" -	ssrv_tree_id=$(createtree --admin_server $tsrv_rpc 2>/dev/null) +	nodes[$i:ssrv_tree_id]=$(createtree --admin_server ${nodes[$i:tsrv_rpc]} 2>/dev/null)  	[[ $? -eq 0 ]] ||  		die "must provision a new Merkle tree" -	info "provisioned Merkle tree with id $ssrv_tree_id" +	info "provisioned Merkle tree with id ${nodes[$i:ssrv_tree_id]}"  }  function sigsum_setup() { -	info "setting up Sigsum server" -	source $1 - -	wit1_priv=$(sigsum-debug key private) -	wit1_pub=$(echo $wit1_priv | sigsum-debug key public) -	wit1_key_hash=$(echo $wit1_pub | sigsum-debug key hash) - -	wit2_priv=$(sigsum-debug key private) -	wit2_pub=$(echo $wit2_priv | sigsum-debug key public) -	wit2_key_hash=$(echo $wit2_pub | sigsum-debug key hash) +	local i=$1; shift +	info "setting up Sigsum server ($i)" +	source $i + +	nodes[$i:ssrv_role]=$ssrv_role +	nodes[$i:ssrv_endpoint]=$ssrv_endpoint +	nodes[$i:ssrv_prefix]=$ssrv_prefix +	nodes[$i:ssrv_shard_start]=$ssrv_shard_start +	nodes[$i:ssrv_interval]=$ssrv_interval +	nodes[$i:log_url]=${nodes[$i:ssrv_endpoint]}/${nodes[$i:ssrv_prefix]}/sigsum/v0 + +	nodes[$i:wit1_priv]=$(sigsum-debug key private) +	nodes[$i:wit1_pub]=$(echo ${nodes[$i:wit1_priv]} | sigsum-debug key public) +	nodes[$i:wit1_key_hash]=$(echo ${nodes[$i:wit1_pub]} | sigsum-debug key hash) +	nodes[$i:wit2_priv]=$(sigsum-debug key private) +	nodes[$i:wit2_pub]=$(echo ${nodes[$i:wit2_priv]} | sigsum-debug key public) +	nodes[$i:wit2_key_hash]=$(echo ${nodes[$i:wit2_pub]} | sigsum-debug key hash) +	nodes[$i:ssrv_witnesses]=${nodes[$i:wit1_pub]},${nodes[$i:wit2_pub]} + +	nodes[$i:ssrv_priv]=$(sigsum-debug key private) +	nodes[$i:ssrv_pub]=$(echo ${nodes[$i:ssrv_priv]} | sigsum-debug key public) +	nodes[$i:ssrv_key_hash]=$(echo ${nodes[$i:ssrv_pub]} | sigsum-debug key hash) +} -	ssrv_witnesses=$wit1_pub,$wit2_pub -	ssrv_priv=$(sigsum-debug key private) -	ssrv_pub=$(echo $ssrv_priv | sigsum-debug key public) -	ssrv_key_hash=$(echo $ssrv_pub | sigsum-debug key hash) +function sigsum_start() { +	local i=$1; shift +	info "starting Sigsum log server ($i)"  	sigsum_log_go\ -		-prefix=$ssrv_prefix\ -		-trillian_id=$ssrv_tree_id\ -		-shard_interval_start=$ssrv_shard_start\ -		-key=<(echo $ssrv_priv)\ -		-witnesses=$ssrv_witnesses\ -		-interval=$ssrv_interval\ -		-http_endpoint=$ssrv_endpoint\ +		-prefix=${nodes[$i:ssrv_prefix]}\ +		-trillian_id=${nodes[$i:ssrv_tree_id]}\ +		-shard_interval_start=${nodes[$i:ssrv_shard_start]}\ +		-key=<(echo ${nodes[$i:ssrv_priv]})\ +		-witnesses=${nodes[$i:ssrv_witnesses]}\ +		-interval=${nodes[$i:ssrv_interval]}\ +		-http_endpoint=${nodes[$i:ssrv_endpoint]}\  		-log-color="true"\  		-log-level="debug"\ -		-log-file=$log_dir/sigsum-log.log 2>/dev/null & -	ssrv_pid=$! +		-role=${nodes[$i:ssrv_role]} ${nodes[$i:ssrv_extra_args]} \ +		-log-file=${nodes[$i:log_dir]}/sigsum-log.log 2>/dev/null & +	nodes[$i:ssrv_pid]=$! -	log_url=$ssrv_endpoint/$ssrv_prefix/sigsum/v0 -	info "started Sigsum log server on $ssrv_endpoint (pid $ssrv_pid)" +	info "started Sigsum log server on ${nodes[$i:ssrv_endpoint]} (pid ${nodes[$i:ssrv_pid]})"  }  function cleanup() { @@ -122,49 +172,58 @@ function cleanup() {  	info "cleaning up, please wait..."  	sleep 1 -	kill -2 $ssrv_pid -	kill -2 $tseq_pid -	while :; do -		sleep 1 +	for i in $pri $sec; do +		boundp $i:ssrv_pid && kill -2 ${nodes[$i:ssrv_pid]} +		boundp $i:tseq_pid && kill -2 ${nodes[$i:tseq_pid]} +		while :; do +			sleep 1 -		ps -p $tseq_pid >/dev/null && continue -		ps -p $ssrv_pid >/dev/null && continue +			boundp $i:tseq_pid && ps -p ${nodes[$i:tseq_pid]} >/dev/null && continue +			boundp $i:ssrv_pid && ps -p ${nodes[$i:$ssrv_pid]} >/dev/null && continue -		break +			break +		done +	done +	info "stopped Trillian log sequencer(s)" +	info "stopped Sigsum log server(s)" + +	for i in $pri $sec; do +		if ! deletetree -admin_server=$tsrv_rpc -log_id=${nodes[$i:ssrv_tree_id]}; then +			warn "failed deleting provisioned Merkle tree ${nodes[$i:ssrv_tree_id]}" +		else +			info "deleted provisioned Merkle tree ${nodes[$i:ssrv_tree_id]}" +		fi  	done -	info "stopped Trillian log sequencer" -	info "stopped Sigsum log server" - -	if ! deletetree -admin_server=$tsrv_rpc -log_id=$ssrv_tree_id; then -		warn "failed deleting provisioned Merkle tree" -	else -		info "deleteted provisioned Merkle tree" -	fi - -	kill -2 $tsrv_pid -	while :; do -		sleep 1 +	for i in $pri $sec; do +		boundp $i:tsrv_pid || continue +		kill -2 ${nodes[$i:tsrv_pid]} +		while :; do +			sleep 1 -		ps -p $tsrv_pid >/dev/null && continue +			ps -p ${nodes[$i:tsrv_pid]} >/dev/null && continue -		break +			break +		done  	done +	info "stopped Trillian log server(s)" -	info "stopped Trillian log server" - -	printf "\n  Press any key to delete logs in $log_dir" -	read dummy +	for i in $pri $sec; do +		printf "\n  Press any key to delete logs in ${nodes[$i:log_dir]}" +		read dummy -	rm -rf $log_dir +		rm -rf ${nodes[$i:log_dir]} +	done  }  function check_setup() { -	sleep 3 +	for i in $pri $sec; do +		sleep 3 -	ps -p $tseq_pid >/dev/null || die "must have Trillian log sequencer" -	ps -p $tsrv_pid >/dev/null || die "must have Trillian log server" -	ps -p $ssrv_pid >/dev/null || die "must have Sigsum log server" +		ps -p ${nodes[$i:tseq_pid]} >/dev/null || die "must have Trillian log sequencer ($i)" +		ps -p ${nodes[$i:tsrv_pid]} >/dev/null || die "must have Trillian log server ($i)" +		ps -p ${nodes[$i:ssrv_pid]} >/dev/null || die "must have Sigsum log server ($i)" +	done  }  function run_tests() { @@ -176,18 +235,18 @@ function run_tests() {  	done  	info "waiting for $num_leaf leaves to be merged..." -	sleep ${ssrv_interval::-1} +	sleep ${nodes[$pri:ssrv_interval]::-1}  	test_signed_tree_head $num_leaf  	for i in $(seq 1 $(( $num_leaf - 1 ))); do  		test_consistency_proof $i $num_leaf  	done -	test_cosignature $wit1_key_hash $wit1_priv -	test_cosignature $wit2_key_hash $wit2_priv +	test_cosignature ${nodes[$pri:wit1_key_hash]} ${nodes[$pri:wit1_priv]} +	test_cosignature ${nodes[$pri:wit2_key_hash]} ${nodes[$pri:wit2_priv]}  	info "waiting for cosignature to be available..." -	sleep ${ssrv_interval::-1} +	sleep ${nodes[$pri:ssrv_interval]::-1}  	test_cosigned_tree_head $num_leaf  	for i in $(seq 1 $num_leaf); do @@ -202,32 +261,33 @@ function run_tests() {  }  function test_signed_tree_head() { +	local log_dir=${nodes[$pri:log_dir]}  	desc="GET tree-head-to-cosign (tree size $1)" -	curl -s -w "%{http_code}" $log_url/get-tree-head-to-cosign \ +	curl -s -w "%{http_code}" ${nodes[$pri:log_url]}/get-tree-head-to-cosign \  		>$log_dir/rsp -	if [[ $(status_code) != 200 ]]; then -		fail "$desc: http status code $(status_code)" +	if [[ $(status_code $pri) != 200 ]]; then +		fail "$desc: http status code $(status_code $pri)"  		return  	fi -	if ! keys "timestamp" "tree_size" "root_hash" "signature"; then -		fail "$desc: ascii keys in response $(debug_response)" +	if ! keys $pri "timestamp" "tree_size" "root_hash" "signature"; then +		fail "$desc: ascii keys in response $(debug_response $pri)"  		return  	fi  	now=$(date +%s) -	if [[ $(value_of "timestamp") -gt $now ]]; then -		fail "$desc: timestamp $(value_of "timestamp") is too large" +	if [[ $(value_of $pri "timestamp") -gt $now ]]; then +		fail "$desc: timestamp $(value_of $pri "timestamp") is too large"  		return  	fi -	if [[ $(value_of "timestamp") -lt $(( $now - ${ssrv_interval::-1} )) ]]; then -		fail "$desc: timestamp $(value_of "timestamp") is too small" +	if [[ $(value_of $pri "timestamp") -lt $(( $now - ${nodes[$pri:ssrv_interval]::-1} )) ]]; then +		fail "$desc: timestamp $(value_of $pri "timestamp") is too small"  		return  	fi -	if [[ $(value_of "tree_size") != $1 ]]; then -		fail "$desc: tree size $(value_of "tree_size")" +	if [[ $(value_of $pri "tree_size") != $1 ]]; then +		fail "$desc: tree size $(value_of $pri "tree_size")"  		return  	fi @@ -236,38 +296,39 @@ function test_signed_tree_head() {  }  function test_cosigned_tree_head() { +	local log_dir=${nodes[$pri:log_dir]}  	desc="GET get-tree-head-cosigned (all witnesses)" -	curl -s -w "%{http_code}" $log_url/get-tree-head-cosigned \ +	curl -s -w "%{http_code}" ${nodes[$pri:log_url]}/get-tree-head-cosigned \  		>$log_dir/rsp -	if [[ $(status_code) != 200 ]]; then -		fail "$desc: http status code $(status_code)" +	if [[ $(status_code $pri) != 200 ]]; then +		fail "$desc: http status code $(status_code $pri)"  		return  	fi -	if ! keys "timestamp" "tree_size" "root_hash" "signature" "cosignature" "key_hash"; then -		fail "$desc: ascii keys in response $(debug_response)" +	if ! keys $pri "timestamp" "tree_size" "root_hash" "signature" "cosignature" "key_hash"; then +		fail "$desc: ascii keys in response $(debug_response $pri)"  		return  	fi  	now=$(date +%s) -	if [[ $(value_of "timestamp") -gt $now ]]; then -		fail "$desc: timestamp $(value_of "timestamp") is too large" +	if [[ $(value_of $pri "timestamp") -gt $now ]]; then +		fail "$desc: timestamp $(value_of $pri "timestamp") is too large"  		return  	fi -	if [[ $(value_of "timestamp") -lt $(( $now - ${ssrv_interval::-1} * 2 )) ]]; then -		fail "$desc: timestamp $(value_of "timestamp") is too small" +	if [[ $(value_of $pri "timestamp") -lt $(( $now - ${nodes[$pri:ssrv_interval]::-1} * 2 )) ]]; then +		fail "$desc: timestamp $(value_of $pri "timestamp") is too small"  		return  	fi -	if [[ $(value_of "tree_size") != $1 ]]; then -		fail "$desc: tree size $(value_of "tree_size")" +	if [[ $(value_of $pri "tree_size") != $1 ]]; then +		fail "$desc: tree size $(value_of $pri "tree_size")"  		return  	fi -	for got in $(value_of key_hash); do +	for got in $(value_of $pri key_hash); do  		found="" -		for want in $wit1_key_hash $wit2_key_hash; do +		for want in ${nodes[$pri:wit1_key_hash]} ${nodes[$pri:wit2_key_hash]}; do  			if [[ $got == $want ]]; then  				found=true  			fi @@ -285,23 +346,24 @@ function test_cosigned_tree_head() {  }  function test_inclusion_proof() { +	local log_dir=${nodes[$pri:log_dir]}  	desc="GET get-inclusion-proof (tree_size $1, data \"$2\", index $3)" -	signature=$(echo $2 | sigsum-debug leaf sign -k $cli_priv -h $ssrv_shard_start) -	leaf_hash=$(echo $2 | sigsum-debug leaf hash -k $cli_key_hash -s $signature -h $ssrv_shard_start) -	curl -s -w "%{http_code}" $log_url/get-inclusion-proof/$1/$leaf_hash >$log_dir/rsp +	signature=$(echo $2 | sigsum-debug leaf sign -k $cli_priv -h ${nodes[$pri:ssrv_shard_start]}) +	leaf_hash=$(echo $2 | sigsum-debug leaf hash -k $cli_key_hash -s $signature -h ${nodes[$pri:ssrv_shard_start]}) +	curl -s -w "%{http_code}" ${nodes[$pri:log_url]}/get-inclusion-proof/$1/$leaf_hash >$log_dir/rsp -	if [[ $(status_code) != 200 ]]; then -		fail "$desc: http status code $(status_code)" +	if [[ $(status_code $pri) != 200 ]]; then +		fail "$desc: http status code $(status_code $pri)"  		return  	fi -	if ! keys "leaf_index" "inclusion_path"; then -		fail "$desc: ascii keys in response $(debug_response)" +	if ! keys $pri "leaf_index" "inclusion_path"; then +		fail "$desc: ascii keys in response $(debug_response $pri)"  		return  	fi -	if [[ $(value_of leaf_index) != $3 ]]; then -		fail "$desc: wrong leaf index $(value_of leaf_index)" +	if [[ $(value_of $pri leaf_index) != $3 ]]; then +		fail "$desc: wrong leaf index $(value_of $pri leaf_index)"  		return  	fi @@ -310,16 +372,17 @@ function test_inclusion_proof() {  }  function test_consistency_proof() { +	local log_dir=${nodes[$pri:log_dir]}  	desc="GET get-consistency-proof (old_size $1, new_size $2)" -	curl -s -w "%{http_code}" $log_url/get-consistency-proof/$1/$2 >$log_dir/rsp +	curl -s -w "%{http_code}" ${nodes[$pri:log_url]}/get-consistency-proof/$1/$2 >$log_dir/rsp -	if [[ $(status_code) != 200 ]]; then -		fail "$desc: http status code $(status_code)" +	if [[ $(status_code $pri) != 200 ]]; then +		fail "$desc: http status code $(status_code $pri)"  		return  	fi -	if ! keys "consistency_path"; then -		fail "$desc: ascii keys in response $(debug_response)" +	if ! keys $pri "consistency_path"; then +		fail "$desc: ascii keys in response $(debug_response $pri)"  		return  	fi @@ -328,33 +391,34 @@ function test_consistency_proof() {  }  function test_get_leaf() { +	local log_dir=${nodes[$pri:log_dir]}  	desc="GET get-leaves (data \"$1\", index $2)" -	curl -s -w "%{http_code}" $log_url/get-leaves/$2/$2 >$log_dir/rsp +	curl -s -w "%{http_code}" ${nodes[$pri:log_url]}/get-leaves/$2/$2 >$log_dir/rsp -	if [[ $(status_code) != 200 ]]; then -		fail "$desc: http status code $(status_code)" +	if [[ $(status_code $pri) != 200 ]]; then +		fail "$desc: http status code $(status_code $pri)"  		return  	fi -	if ! keys "shard_hint" "checksum" "signature" "key_hash"; then -		fail "$desc: ascii keys in response $(debug_response)" +	if ! keys $pri "shard_hint" "checksum" "signature" "key_hash"; then +		fail "$desc: ascii keys in response $(debug_response $pri)"  		return  	fi -	if [[ $(value_of shard_hint) != $ssrv_shard_start ]]; then -		fail "$desc: wrong shard hint $(value_of shard_hint)" +	if [[ $(value_of $pri shard_hint) != ${nodes[$pri:ssrv_shard_start]} ]]; then +		fail "$desc: wrong shard hint $(value_of $pri shard_hint)"  		return  	fi  	message=$(openssl dgst -binary <(echo $1) | base16)  	checksum=$(openssl dgst -binary <(echo $message | base16 -d) | base16) -	if [[ $(value_of checksum) != $checksum ]]; then -		fail "$desc: wrong checksum $(value_of checksum)" +	if [[ $(value_of $pri checksum) != $checksum ]]; then +		fail "$desc: wrong checksum $(value_of $pri checksum)"  		return  	fi -	if [[ $(value_of key_hash) != $cli_key_hash ]]; then -		fail "$desc: wrong key hash $(value_of key_hash)" +	if [[ $(value_of $pri key_hash) != $cli_key_hash ]]; then +		fail "$desc: wrong key hash $(value_of $pri key_hash)"  	fi  	# TODO: check leaf signature @@ -362,24 +426,25 @@ function test_get_leaf() {  }  function test_add_leaf() { +	local log_dir=${nodes[$pri:log_dir]}  	desc="POST add-leaf (data \"$1\")" -	echo "shard_hint=$ssrv_shard_start" > $log_dir/req +	echo "shard_hint=${nodes[$pri:ssrv_shard_start]}" > $log_dir/req  	echo "message=$(openssl dgst -binary <(echo $1) | base16)" >> $log_dir/req  	echo "signature=$(echo $1 | -		sigsum-debug leaf sign -k $cli_priv -h $ssrv_shard_start)" >> $log_dir/req +		sigsum-debug leaf sign -k $cli_priv -h ${nodes[$pri:ssrv_shard_start]})" >> $log_dir/req  	echo "public_key=$cli_pub" >> $log_dir/req  	echo "domain_hint=$cli_domain_hint" >> $log_dir/req  	cat $log_dir/req | -		curl -s -w "%{http_code}" --data-binary @- $log_url/add-leaf \ +		curl -s -w "%{http_code}" --data-binary @- ${nodes[$pri:log_url]}/add-leaf \  		>$log_dir/rsp -	if [[ $(status_code) != 200 ]]; then -		fail "$desc: http status code $(status_code)" +	if [[ $(status_code $pri) != 200 ]]; then +		fail "$desc: http status code $(status_code $pri)"  		return  	fi -	if ! keys; then -		fail "$desc: ascii keys in response $(debug_response)" +	if ! keys $pri; then +		fail "$desc: ascii keys in response $(debug_response $pri)"  		return  	fi @@ -387,21 +452,24 @@ function test_add_leaf() {  }  function test_cosignature() { +	local log_dir=${nodes[$pri:log_dir]} +	#local log_url=${nodes[$pri:log_url]} +	#local ssrv_key_hash=${nodes[$pri:ssrv_key_hash]}  	desc="POST add-cosignature (witness $1)"  	echo "key_hash=$1" > $log_dir/req -	echo "cosignature=$(curl -s $log_url/get-tree-head-to-cosign | -		sigsum-debug head sign -k $2 -h $ssrv_key_hash)" >> $log_dir/req +	echo "cosignature=$(curl -s ${nodes[$pri:log_url]}/get-tree-head-to-cosign | +		sigsum-debug head sign -k $2 -h ${nodes[$pri:ssrv_key_hash]})" >> $log_dir/req  	cat $log_dir/req | -		curl -s -w "%{http_code}" --data-binary @- $log_url/add-cosignature \ +		curl -s -w "%{http_code}" --data-binary @- ${nodes[$pri:log_url]}/add-cosignature \  		>$log_dir/rsp -	if [[ $(status_code) != 200 ]]; then -		fail "$desc: http status code $(status_code)" +	if [[ $(status_code $pri) != 200 ]]; then +		fail "$desc: http status code $(status_code $pri)"  		return  	fi -	if ! keys; then -		fail "$desc: ascii keys in response $(debug_response)" +	if ! keys $pri; then +		fail "$desc: ascii keys in response $(debug_response $pri)"  		return  	fi @@ -409,15 +477,18 @@ function test_cosignature() {  }  function debug_response() { +	local i=$1; shift  	echo "" -	cat $log_dir/rsp +	cat ${nodes[$i:log_dir]}/rsp  }  function status_code() { -	tail -n1 $log_dir/rsp +	local i=$1; shift +	tail -n1 ${nodes[$i:log_dir]}/rsp  }  function value_of() { +	local i=$1; shift  	while read line; do  		key=$(echo $line | cut -d"=" -f1)  		if [[ $key != $1 ]]; then @@ -426,16 +497,17 @@ function value_of() {  		value=$(echo $line | cut -d"=" -f2)  		echo $value -	done < <(head --lines=-1 $log_dir/rsp) +	done < <(head --lines=-1 ${nodes[$i:log_dir]}/rsp)  }  function keys() { +        local i=$1; shift  	declare -A map  	map[thedummystring]=to_avoid_error_on_size_zero  	while read line; do  		key=$(echo $line | cut -d"=" -f1)  		map[$key]=ok -	done < <(head --lines=-1 $log_dir/rsp) +	done < <(head --lines=-1 ${nodes[$i:log_dir]}/rsp)  	if [[ $# != $(( ${#map[@]} - 1 )) ]]; then  		return 1 @@ -448,6 +520,11 @@ function keys() {  	return 0  } +function boundp { +    [[ ${!nodes[@]} == *$1* ]] && return 1 +    return 0 +} +  function die() {  	echo -e "\e[37m$(date +"%y-%m-%d %H:%M:%S %Z")\e[0m [\e[31mFATA\e[0m] $@" >&2  	exit 1 | 
