From 519784b5ee58370d6c1262b0eb6c72ee3580f293 Mon Sep 17 00:00:00 2001 From: Rasmus Dahlberg Date: Tue, 1 Jun 2021 00:21:30 +0200 Subject: started to update stfe server Work in progress. --- endpoint.go | 174 +++++++++++++++++++++++++++--------------------------- instance.go | 13 ++-- log_parameters.go | 80 ++++++++----------------- request.go | 160 +++++++++++++++++++++---------------------------- server/main.go | 90 +++++++++++++--------------- sth.go | 144 +++++++++++++++++++++----------------------- trillian.go | 12 ++-- util.go | 48 +++++++-------- 8 files changed, 326 insertions(+), 395 deletions(-) diff --git a/endpoint.go b/endpoint.go index d3da95e..98a5ce8 100644 --- a/endpoint.go +++ b/endpoint.go @@ -2,10 +2,10 @@ package stfe import ( "context" + "crypto/ed25519" "fmt" - "strings" - "net/http" + "strings" "github.com/golang/glog" "github.com/google/trillian" @@ -16,14 +16,14 @@ import ( type Endpoint string const ( - EndpointAddEntry = Endpoint("add-entry") + EndpointAddEntry = Endpoint("add-leaf") EndpointAddCosignature = Endpoint("add-cosignature") - EndpointGetLatestSth = Endpoint("get-latest-sth") - EndpointGetStableSth = Endpoint("get-stable-sth") - EndpointGetCosignedSth = Endpoint("get-cosigned-sth") + EndpointGetLatestSth = Endpoint("get-tree-head-latest") + EndpointGetStableSth = Endpoint("get-tree-head-to-sign") + EndpointGetCosignedSth = Endpoint("get-tree-head-cosigned") EndpointGetProofByHash = Endpoint("get-proof-by-hash") EndpointGetConsistencyProof = Endpoint("get-consistency-proof") - EndpointGetEntries = Endpoint("get-entries") + EndpointGetEntries = Endpoint("get-leaves") ) // Path joins a number of components to form a full endpoint path, e.g., base @@ -34,18 +34,14 @@ func (e Endpoint) Path(components ...string) string { func addEntry(ctx context.Context, i *Instance, w http.ResponseWriter, r *http.Request) (int, error) { glog.V(3).Info("handling add-entry request") - item, err := i.LogParameters.parseAddEntryV1Request(r) + leaf, err := i.LogParameters.parseAddEntryV1Request(r) if err != nil { return http.StatusBadRequest, fmt.Errorf("parseAddEntryV1Request: %v", err) } - leaf, err := types.Marshal(*item) - if err != nil { - return http.StatusInternalServerError, fmt.Errorf("Marshal: %v", err) // should never happen - } trsp, err := i.Client.QueueLeaf(ctx, &trillian.QueueLeafRequest{ LogId: i.LogParameters.TreeId, Leaf: &trillian.LogLeaf{ - LeafValue: leaf, + LeafValue: leaf.Marshal(), ExtraData: nil, }, }) @@ -57,12 +53,13 @@ func addEntry(ctx context.Context, i *Instance, w http.ResponseWriter, r *http.R func addCosignature(ctx context.Context, i *Instance, w http.ResponseWriter, r *http.Request) (int, error) { glog.V(3).Info("handling add-cosignature request") - costh, err := i.LogParameters.parseAddCosignatureV1Request(r) + req, err := i.LogParameters.parseAddCosignatureRequest(r) if err != nil { - return http.StatusBadRequest, err + return http.StatusBadRequest, fmt.Errorf("parseAddCosignatureRequest: %v", err) } - if err := i.SthSource.AddCosignature(ctx, costh); err != nil { - return http.StatusBadRequest, err + vk := i.LogParameters.Witnesses[*req.KeyHash] + if err := i.SthSource.AddCosignature(ctx, ed25519.PublicKey(vk[:]), req.Signature); err != nil { + return http.StatusBadRequest, fmt.Errorf("AddCosignature: %v", err) } return http.StatusOK, nil } @@ -73,8 +70,8 @@ func getLatestSth(ctx context.Context, i *Instance, w http.ResponseWriter, _ *ht if err != nil { return http.StatusInternalServerError, fmt.Errorf("Latest: %v", err) } - if err := writeOctetResponse(w, *sth); err != nil { - return http.StatusInternalServerError, fmt.Errorf("writeOctetResponse: %v", err) + if err := sth.MarshalASCII(w); err != nil { + return http.StatusInternalServerError, fmt.Errorf("MarshalASCII: %v", err) } return http.StatusOK, nil } @@ -85,101 +82,106 @@ func getStableSth(ctx context.Context, i *Instance, w http.ResponseWriter, _ *ht if err != nil { return http.StatusInternalServerError, fmt.Errorf("Latest: %v", err) } - if err := writeOctetResponse(w, *sth); err != nil { - return http.StatusInternalServerError, fmt.Errorf("writeOctetResponse: %v", err) + if err := sth.MarshalASCII(w); err != nil { + return http.StatusInternalServerError, fmt.Errorf("MarshalASCII: %v", err) } return http.StatusOK, nil } func getCosignedSth(ctx context.Context, i *Instance, w http.ResponseWriter, _ *http.Request) (int, error) { glog.V(3).Info("handling get-cosigned-sth request") - costh, err := i.SthSource.Cosigned(ctx) + sth, err := i.SthSource.Cosigned(ctx) if err != nil { return http.StatusInternalServerError, fmt.Errorf("Cosigned: %v", err) } - if err := writeOctetResponse(w, *costh); err != nil { - return http.StatusInternalServerError, fmt.Errorf("writeOctetResponse: %v", err) + if err := sth.MarshalASCII(w); err != nil { + return http.StatusInternalServerError, fmt.Errorf("MarshalASCII: %v", err) } return http.StatusOK, nil } func getConsistencyProof(ctx context.Context, i *Instance, w http.ResponseWriter, r *http.Request) (int, error) { glog.V(3).Info("handling get-consistency-proof request") - req, err := i.LogParameters.parseGetConsistencyProofV1Request(r) + req, err := i.LogParameters.parseGetConsistencyProofRequest(r) if err != nil { return http.StatusBadRequest, err } trsp, err := i.Client.GetConsistencyProof(ctx, &trillian.GetConsistencyProofRequest{ LogId: i.LogParameters.TreeId, - FirstTreeSize: int64(req.First), - SecondTreeSize: int64(req.Second), + FirstTreeSize: int64(req.OldSize), + SecondTreeSize: int64(req.NewSize), }) if errInner := checkGetConsistencyProof(i.LogParameters, trsp, err); errInner != nil { return http.StatusInternalServerError, fmt.Errorf("bad GetConsistencyProofResponse: %v", errInner) } - if err := writeOctetResponse(w, *types.NewConsistencyProofV1(i.LogParameters.LogId, req.First, req.Second, NewNodePathFromHashPath(trsp.Proof.Hashes))); err != nil { - return http.StatusInternalServerError, fmt.Errorf("writeOctetResponse: %v", err) - } - return http.StatusOK, nil -} - -func getProofByHash(ctx context.Context, i *Instance, w http.ResponseWriter, r *http.Request) (int, error) { - glog.V(3).Info("handling get-proof-by-hash request") - req, err := i.LogParameters.parseGetProofByHashV1Request(r) - if err != nil { - return http.StatusBadRequest, err - } - - trsp, err := i.Client.GetInclusionProofByHash(ctx, &trillian.GetInclusionProofByHashRequest{ - LogId: i.LogParameters.TreeId, - LeafHash: req.Hash[:], - TreeSize: int64(req.TreeSize), - OrderBySequence: true, - }) - if errInner := checkGetInclusionProofByHash(i.LogParameters, trsp, err); errInner != nil { - return http.StatusInternalServerError, fmt.Errorf("bad GetInclusionProofByHashResponse: %v", errInner) - } - - if err := writeOctetResponse(w, *types.NewInclusionProofV1(i.LogParameters.LogId, req.TreeSize, uint64(trsp.Proof[0].LeafIndex), NewNodePathFromHashPath(trsp.Proof[0].Hashes))); err != nil { - return http.StatusInternalServerError, fmt.Errorf("writeOctetResponse: %v", err) - } - return http.StatusOK, nil -} - -func getEntries(ctx context.Context, i *Instance, w http.ResponseWriter, r *http.Request) (int, error) { - glog.V(3).Info("handling get-entries request") - req, err := i.LogParameters.parseGetEntriesV1Request(r) - if err != nil { - return http.StatusBadRequest, err - } - - trsp, err := i.Client.GetLeavesByRange(ctx, &trillian.GetLeavesByRangeRequest{ - LogId: i.LogParameters.TreeId, - StartIndex: int64(req.Start), - Count: int64(req.End-req.Start) + 1, - }) - if errInner := checkGetLeavesByRange(req, trsp, err); errInner != nil { - return http.StatusInternalServerError, fmt.Errorf("checkGetLeavesByRangeResponse: %v", errInner) // there is one StatusBadRequest in here tho.. + proof := &types.ConsistencyProof{ + NewSize: req.NewSize, + OldSize: req.OldSize, + Path: NodePathFromHashes(trsp.Proof.Hashes), } - - if rsp, err := NewStItemListFromLeaves(trsp.Leaves); err != nil { - return http.StatusInternalServerError, fmt.Errorf("NewStItemListFromLeaves: %v", err) // should never happen - } else if err := writeOctetResponse(w, *rsp); err != nil { - return http.StatusInternalServerError, fmt.Errorf("writeOctetResponse: %v", err) + if err := proof.MarshalASCII(w); err != nil { + return http.StatusInternalServerError, fmt.Errorf("MarshalASCII: %v", err) } return http.StatusOK, nil } -func writeOctetResponse(w http.ResponseWriter, i interface{}) error { - b, err := types.Marshal(i) - if err != nil { - return fmt.Errorf("Marshal: %v", err) - } - w.Header().Set("Content-Type", "application/octet-stream") - if _, err := w.Write(b); err != nil { - return fmt.Errorf("Write: %v", err) - } - return nil -} +//func getProofByHash(ctx context.Context, i *Instance, w http.ResponseWriter, r *http.Request) (int, error) { +// glog.V(3).Info("handling get-proof-by-hash request") +// req, err := i.LogParameters.parseGetProofByHashV1Request(r) +// if err != nil { +// return http.StatusBadRequest, err +// } +// +// trsp, err := i.Client.GetInclusionProofByHash(ctx, &trillian.GetInclusionProofByHashRequest{ +// LogId: i.LogParameters.TreeId, +// LeafHash: req.Hash[:], +// TreeSize: int64(req.TreeSize), +// OrderBySequence: true, +// }) +// if errInner := checkGetInclusionProofByHash(i.LogParameters, trsp, err); errInner != nil { +// return http.StatusInternalServerError, fmt.Errorf("bad GetInclusionProofByHashResponse: %v", errInner) +// } +// +// if err := writeOctetResponse(w, *types.NewInclusionProofV1(i.LogParameters.LogId, req.TreeSize, uint64(trsp.Proof[0].LeafIndex), NewNodePathFromHashPath(trsp.Proof[0].Hashes))); err != nil { +// return http.StatusInternalServerError, fmt.Errorf("writeOctetResponse: %v", err) +// } +// return http.StatusOK, nil +//} +// +//func getEntries(ctx context.Context, i *Instance, w http.ResponseWriter, r *http.Request) (int, error) { +// glog.V(3).Info("handling get-entries request") +// req, err := i.LogParameters.parseGetEntriesV1Request(r) +// if err != nil { +// return http.StatusBadRequest, err +// } +// +// trsp, err := i.Client.GetLeavesByRange(ctx, &trillian.GetLeavesByRangeRequest{ +// LogId: i.LogParameters.TreeId, +// StartIndex: int64(req.Start), +// Count: int64(req.End-req.Start) + 1, +// }) +// if errInner := checkGetLeavesByRange(req, trsp, err); errInner != nil { +// return http.StatusInternalServerError, fmt.Errorf("checkGetLeavesByRangeResponse: %v", errInner) // there is one StatusBadRequest in here tho.. +// } +// +// if rsp, err := NewStItemListFromLeaves(trsp.Leaves); err != nil { +// return http.StatusInternalServerError, fmt.Errorf("NewStItemListFromLeaves: %v", err) // should never happen +// } else if err := writeOctetResponse(w, *rsp); err != nil { +// return http.StatusInternalServerError, fmt.Errorf("writeOctetResponse: %v", err) +// } +// return http.StatusOK, nil +//} +// +//func writeOctetResponse(w http.ResponseWriter, i interface{}) error { +// b, err := types.Marshal(i) +// if err != nil { +// return fmt.Errorf("Marshal: %v", err) +// } +// w.Header().Set("Content-Type", "application/octet-stream") +// if _, err := w.Write(b); err != nil { +// return fmt.Errorf("Write: %v", err) +// } +// return nil +//} diff --git a/instance.go b/instance.go index 67336f8..d11032e 100644 --- a/instance.go +++ b/instance.go @@ -9,6 +9,7 @@ import ( "github.com/golang/glog" "github.com/google/trillian" + "github.com/system-transparency/stfe/types" ) // Instance is an instance of the system transparency front-end @@ -26,9 +27,9 @@ func (i *Instance) Handlers() []Handler { Handler{Instance: i, Handler: getLatestSth, Endpoint: EndpointGetLatestSth, Method: http.MethodGet}, Handler{Instance: i, Handler: getStableSth, Endpoint: EndpointGetStableSth, Method: http.MethodGet}, Handler{Instance: i, Handler: getCosignedSth, Endpoint: EndpointGetCosignedSth, Method: http.MethodGet}, - Handler{Instance: i, Handler: getProofByHash, Endpoint: EndpointGetProofByHash, Method: http.MethodPost}, + //Handler{Instance: i, Handler: getProofByHash, Endpoint: EndpointGetProofByHash, Method: http.MethodPost}, Handler{Instance: i, Handler: getConsistencyProof, Endpoint: EndpointGetConsistencyProof, Method: http.MethodPost}, - Handler{Instance: i, Handler: getEntries, Endpoint: EndpointGetEntries, Method: http.MethodPost}, + //Handler{Instance: i, Handler: getEntries, Endpoint: EndpointGetEntries, Method: http.MethodPost}, } } @@ -52,10 +53,10 @@ func (a Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { var now time.Time = time.Now() var statusCode int defer func() { - rspcnt.Inc(a.Instance.LogParameters.LogIdStr, string(a.Endpoint), fmt.Sprintf("%d", statusCode)) - latency.Observe(time.Now().Sub(now).Seconds(), a.Instance.LogParameters.LogIdStr, string(a.Endpoint), fmt.Sprintf("%d", statusCode)) + rspcnt.Inc(a.Instance.LogParameters.LogId, string(a.Endpoint), fmt.Sprintf("%d", statusCode)) + latency.Observe(time.Now().Sub(now).Seconds(), a.Instance.LogParameters.LogId, string(a.Endpoint), fmt.Sprintf("%d", statusCode)) }() - reqcnt.Inc(a.Instance.LogParameters.LogIdStr, string(a.Endpoint)) + reqcnt.Inc(a.Instance.LogParameters.LogId, string(a.Endpoint)) ctx, cancel := context.WithDeadline(r.Context(), now.Add(a.Instance.LogParameters.Deadline)) defer cancel() @@ -69,6 +70,6 @@ func (a Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { statusCode, err := a.Handler(ctx, a.Instance, w, r) if err != nil { glog.Warningf("handler error %s/%s: %v", a.Instance.LogParameters.Prefix, a.Endpoint, err) - http.Error(w, "", statusCode) + http.Error(w, fmt.Sprintf("%s%s%s%s", "Error", types.Delim, err.Error(), types.EOL), statusCode) } } diff --git a/log_parameters.go b/log_parameters.go index a2a2d7a..aceff3e 100644 --- a/log_parameters.go +++ b/log_parameters.go @@ -2,6 +2,7 @@ package stfe import ( "crypto" + "crypto/ed25519" "fmt" "time" @@ -10,66 +11,37 @@ import ( // LogParameters is a collection of log parameters type LogParameters struct { - LogId *types.Namespace // log identifier - LogIdBytes []byte // serialized log id - LogIdStr string // serialized log id (hex) - TreeId int64 // used internally by Trillian - Prefix string // e.g., "test" for /test - MaxRange int64 // max entries per get-entries request - SubmitterPolicy bool // if we have a submitter policy (true means that namespaces must be registered) - WitnessPolicy bool // if we have a witness policy (true means that namespaces must be registered) - Submitters *types.NamespacePool // trusted submitters - Witnesses *types.NamespacePool // trusted witnesses - Deadline time.Duration // gRPC deadline - Interval time.Duration // cosigning sth frequency - HashType crypto.Hash // hash function used by Trillian - Signer crypto.Signer // access to Ed25519 private key -} + LogId string // serialized log id (hex) + TreeId int64 // used internally by Trillian + Prefix string // e.g., "test" for /test + MaxRange int64 // max entries per get-entries request + Deadline time.Duration // gRPC deadline + Interval time.Duration // cosigning sth frequency + HashType crypto.Hash // hash function used by Trillian + Signer crypto.Signer // access to Ed25519 private key -// NewLogParameters creates newly initialized log parameters -func NewLogParameters(signer crypto.Signer, logId *types.Namespace, treeId int64, prefix string, submitters, witnesses *types.NamespacePool, maxRange int64, interval, deadline time.Duration, submitterPolicy, witnessPolicy bool) (*LogParameters, error) { - logIdBytes, err := types.Marshal(*logId) - if err != nil { - return nil, fmt.Errorf("Marshal failed for log identifier: %v", err) - } - return &LogParameters{ - LogId: logId, - LogIdBytes: logIdBytes, - LogIdStr: fmt.Sprintf("%x", logIdBytes), - TreeId: treeId, - Prefix: prefix, - MaxRange: maxRange, - SubmitterPolicy: submitterPolicy, - WitnessPolicy: witnessPolicy, - Submitters: submitters, - Witnesses: witnesses, - Deadline: deadline, - Interval: interval, - HashType: crypto.SHA256, - Signer: signer, - }, nil + // Witnesses map trusted witness identifiers to public verification keys + Witnesses map[[types.HashSize]byte][types.VerificationKeySize]byte } -// SignTreeHeadV1 signs a TreeHeadV1 structure -func (lp *LogParameters) SignTreeHeadV1(th *types.TreeHeadV1) (*types.StItem, error) { - serialized, err := types.Marshal(*th) - if err != nil { - return nil, fmt.Errorf("Marshal failed for TreeHeadV1: %v", err) - } - sig, err := lp.Signer.Sign(nil, serialized, crypto.Hash(0)) +// Sign signs a tree head +func (lp *LogParameters) Sign(th *types.TreeHead) (*types.SignedTreeHead, error) { + sig, err := lp.Signer.Sign(nil, th.Marshal(), crypto.Hash(0)) if err != nil { return nil, fmt.Errorf("Sign failed: %v", err) } - lastSthTimestamp.Set(float64(time.Now().Unix()), lp.LogIdStr) - lastSthSize.Set(float64(th.TreeSize), lp.LogIdStr) - return &types.StItem{ - Format: types.StFormatSignedTreeHeadV1, - SignedTreeHeadV1: &types.SignedTreeHeadV1{ - TreeHead: *th, - Signature: types.SignatureV1{ - Namespace: *lp.LogId, - Signature: sig, - }, + lastSthTimestamp.Set(float64(time.Now().Unix()), lp.LogId) + lastSthSize.Set(float64(th.TreeSize), lp.LogId) + + sigident := types.SigIdent{ + KeyHash: types.Hash(lp.Signer.Public().(ed25519.PublicKey)[:]), + Signature: &[types.SignatureSize]byte{}, + } + copy(sigident.Signature[:], sig) + return &types.SignedTreeHead{ + TreeHead: *th, + SigIdent: []*types.SigIdent{ + &sigident, }, }, nil } diff --git a/request.go b/request.go index 7c95f34..3b57779 100644 --- a/request.go +++ b/request.go @@ -3,114 +3,90 @@ package stfe import ( "fmt" - "io/ioutil" + "crypto/ed25519" "net/http" "github.com/system-transparency/stfe/types" ) -func (lp *LogParameters) parseAddEntryV1Request(r *http.Request) (*types.StItem, error) { - var item types.StItem - if err := unpackOctetPost(r, &item); err != nil { - return nil, fmt.Errorf("unpackOctetPost: %v", err) - } - if item.Format != types.StFormatSignedChecksumV1 { - return nil, fmt.Errorf("invalid StItem format: %v", item.Format) - } - - // Check that submitter namespace is valid - namespace := &item.SignedChecksumV1.Signature.Namespace - if lp.SubmitterPolicy { - var ok bool - if namespace, ok = lp.Submitters.Find(namespace); !ok { - return nil, fmt.Errorf("unknown submitter namespace: %v", namespace) - } - } - // Check that namespace signed add-entry request - if msg, err := types.Marshal(item.SignedChecksumV1.Data); err != nil { - return nil, fmt.Errorf("Marshal: %v", err) // should never happen - } else if err := namespace.Verify(msg, item.SignedChecksumV1.Signature.Signature); err != nil { - return nil, fmt.Errorf("Verify: %v", err) - } - return &item, nil -} - -func (lp *LogParameters) parseAddCosignatureV1Request(r *http.Request) (*types.StItem, error) { - var item types.StItem - if err := unpackOctetPost(r, &item); err != nil { - return nil, fmt.Errorf("unpackOctetPost: %v", err) - } - if item.Format != types.StFormatCosignedTreeHeadV1 { - return nil, fmt.Errorf("invalid StItem format: %v", item.Format) - } - if got, want := len(item.CosignedTreeHeadV1.Cosignatures), 1; got != want { - return nil, fmt.Errorf("invalid number of cosignatures: %d", got) - } - - // Check that witness namespace is valid - namespace := &item.CosignedTreeHeadV1.Cosignatures[0].Namespace - if lp.WitnessPolicy { - var ok bool - if namespace, ok = lp.Witnesses.Find(namespace); !ok { - return nil, fmt.Errorf("unknown witness namespace: %v", namespace) - } - } - // Check that namespace signed add-cosignature request - if msg, err := types.Marshal(*types.NewSignedTreeHeadV1(&item.CosignedTreeHeadV1.SignedTreeHead.TreeHead, &item.CosignedTreeHeadV1.SignedTreeHead.Signature).SignedTreeHeadV1); err != nil { - return nil, fmt.Errorf("Marshal: %v", err) // should never happen - } else if err := namespace.Verify(msg, item.CosignedTreeHeadV1.Cosignatures[0].Signature); err != nil { - return nil, fmt.Errorf("Verify: %v", err) +func (lp *LogParameters) parseAddEntryV1Request(r *http.Request) (*types.Leaf, error) { + var req types.LeafRequest + if err := req.UnmarshalASCII(r.Body); err != nil { + return nil, fmt.Errorf("UnmarshalASCII: %v", err) } - return &item, nil -} -func (lp *LogParameters) parseGetConsistencyProofV1Request(r *http.Request) (*types.GetConsistencyProofV1, error) { - var item types.GetConsistencyProofV1 - if err := unpackOctetPost(r, &item); err != nil { - return nil, fmt.Errorf("unpackOctetPost: %v", err) - } - if item.First < 1 { - return nil, fmt.Errorf("first(%d) must be larger than zero", item.First) - } - if item.Second <= item.First { - return nil, fmt.Errorf("second(%d) must be larger than first(%d)", item.Second, item.First) - } - return &item, nil + if pub, msg, sig := ed25519.PublicKey(req.VerificationKey[:]), req.Message.Marshal(), req.Signature[:]; !ed25519.Verify(pub, msg, sig) { + return nil, fmt.Errorf("Invalid signature") + } + // TODO: check shard hint + // TODO: check domain hint + return &types.Leaf{ + Message: req.Message, + SigIdent: types.SigIdent{ + Signature: req.Signature, + KeyHash: types.Hash(req.VerificationKey[:]), + }, + }, nil } -func (lp *LogParameters) parseGetProofByHashV1Request(r *http.Request) (*types.GetProofByHashV1, error) { - var item types.GetProofByHashV1 - if err := unpackOctetPost(r, &item); err != nil { +func (lp *LogParameters) parseAddCosignatureRequest(r *http.Request) (*types.CosignatureRequest, error) { + var req types.CosignatureRequest + if err := req.UnmarshalASCII(r.Body); err != nil { return nil, fmt.Errorf("unpackOctetPost: %v", err) } - if item.TreeSize < 1 { - return nil, fmt.Errorf("TreeSize(%d) must be larger than zero", item.TreeSize) + if _, ok := lp.Witnesses[*req.KeyHash]; !ok { + return nil, fmt.Errorf("Unknown witness: %x", req.KeyHash) } - return &item, nil + return &req, nil } -func (lp *LogParameters) parseGetEntriesV1Request(r *http.Request) (*types.GetEntriesV1, error) { - var item types.GetEntriesV1 - if err := unpackOctetPost(r, &item); err != nil { - return nil, fmt.Errorf("unpackOctetPost: %v", err) +func (lp *LogParameters) parseGetConsistencyProofRequest(r *http.Request) (*types.ConsistencyProofRequest, error) { + var req types.ConsistencyProofRequest + if err := req.UnmarshalASCII(r.Body); err != nil { + return nil, fmt.Errorf("UnmarshalASCII: %v", err) } - - if item.Start > item.End { - return nil, fmt.Errorf("start(%v) must be less than or equal to end(%v)", item.Start, item.End) + if req.OldSize < 1 { + return nil, fmt.Errorf("OldSize(%d) must be larger than zero", req.OldSize) } - if item.End-item.Start+1 > uint64(lp.MaxRange) { - item.End = item.Start + uint64(lp.MaxRange) - 1 + if req.NewSize <= req.OldSize { + return nil, fmt.Errorf("NewSize(%d) must be larger than OldSize(%d)", req.NewSize, req.OldSize) } - return &item, nil + return &req, nil } -func unpackOctetPost(r *http.Request, out interface{}) error { - body, err := ioutil.ReadAll(r.Body) - if err != nil { - return fmt.Errorf("failed reading request body: %v", err) - } - if err := types.Unmarshal(body, out); err != nil { - return fmt.Errorf("Unmarshal: %v", err) - } - return nil -} +//func (lp *LogParameters) parseGetProofByHashV1Request(r *http.Request) (*types.GetProofByHashV1, error) { +// var item types.GetProofByHashV1 +// if err := unpackOctetPost(r, &item); err != nil { +// return nil, fmt.Errorf("unpackOctetPost: %v", err) +// } +// if item.TreeSize < 1 { +// return nil, fmt.Errorf("TreeSize(%d) must be larger than zero", item.TreeSize) +// } +// return &item, nil +//} +// +//func (lp *LogParameters) parseGetEntriesV1Request(r *http.Request) (*types.GetEntriesV1, error) { +// var item types.GetEntriesV1 +// if err := unpackOctetPost(r, &item); err != nil { +// return nil, fmt.Errorf("unpackOctetPost: %v", err) +// } +// +// if item.Start > item.End { +// return nil, fmt.Errorf("start(%v) must be less than or equal to end(%v)", item.Start, item.End) +// } +// if item.End-item.Start+1 > uint64(lp.MaxRange) { +// item.End = item.Start + uint64(lp.MaxRange) - 1 +// } +// return &item, nil +//} +// +//func unpackOctetPost(r *http.Request, out interface{}) error { +// body, err := ioutil.ReadAll(r.Body) +// if err != nil { +// return fmt.Errorf("failed reading request body: %v", err) +// } +// if err := types.Unmarshal(body, out); err != nil { +// return fmt.Errorf("Unmarshal: %v", err) +// } +// return nil +//} diff --git a/server/main.go b/server/main.go index 74e4ad3..1fecb43 100644 --- a/server/main.go +++ b/server/main.go @@ -3,19 +3,19 @@ package main import ( "context" + "crypto" + "crypto/ed25519" + "encoding/hex" "flag" "fmt" + "net/http" "os" + "os/signal" "strings" "sync" "syscall" "time" - "crypto/ed25519" - "encoding/base64" - "net/http" - "os/signal" - "github.com/golang/glog" "github.com/google/trillian" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -25,18 +25,15 @@ import ( ) var ( - httpEndpoint = flag.String("http_endpoint", "localhost:6965", "host:port specification of where stfe serves clients") - rpcBackend = flag.String("log_rpc_server", "localhost:6962", "host:port specification of where Trillian serves clients") - prefix = flag.String("prefix", "st/v1", "a prefix that proceeds each endpoint path") - trillianID = flag.Int64("trillian_id", 0, "log identifier in the Trillian database") - deadline = flag.Duration("deadline", time.Second*10, "deadline for backend requests") - key = flag.String("key", "", "base64-encoded Ed25519 signing key") - submitterPolicy = flag.Bool("submitter_policy", false, "whether there is any submitter namespace policy (default: none, accept unregistered submitter namespaces)") - witnessPolicy = flag.Bool("witness_policy", false, "whether there is any witness namespace policy (default: none, accept unregistered witness namespaces)") - submitters = flag.String("submitters", "", "comma-separated list of trusted submitter namespaces in base64 (default: none)") - witnesses = flag.String("witnesses", "", "comma-separated list of trusted submitter namespaces in base64 (default: none)") - maxRange = flag.Int64("max_range", 10, "maximum number of entries that can be retrived in a single request") - interval = flag.Duration("interval", time.Minute*10, "interval used to rotate the log's cosigned STH") + httpEndpoint = flag.String("http_endpoint", "localhost:6965", "host:port specification of where stfe serves clients") + rpcBackend = flag.String("log_rpc_server", "localhost:6962", "host:port specification of where Trillian serves clients") + prefix = flag.String("prefix", "st/v0", "a prefix that proceeds each endpoint path") + trillianID = flag.Int64("trillian_id", 0, "log identifier in the Trillian database") + deadline = flag.Duration("deadline", time.Second*10, "deadline for backend requests") + key = flag.String("key", "", "hex-encoded Ed25519 signing key") + witnesses = flag.String("witnesses", "", "comma-separated list of trusted witness verification keys in hex") + maxRange = flag.Int64("max_range", 10, "maximum number of entries that can be retrived in a single request") + interval = flag.Duration("interval", time.Second*30, "interval used to rotate the log's cosigned STH") ) func main() { @@ -99,30 +96,27 @@ func setupInstanceFromFlags() (*stfe.Instance, error) { // Prometheus metrics glog.V(3).Infof("Adding prometheus handler on path: /metrics") http.Handle("/metrics", promhttp.Handler()) - // Trusted submitters - submitters, err := newNamespacePoolFromString(*submitters) - if err != nil { - return nil, fmt.Errorf("submitters: newNamespacePoolFromString: %v", err) - } // Trusted witnesses - witnesses, err := newNamespacePoolFromString(*witnesses) + witnesses, err := newWitnessMap(*witnesses) if err != nil { - return nil, fmt.Errorf("witnesses: NewNamespacePool: %v", err) + return nil, fmt.Errorf("newWitnessMap: %v", err) } - // Log identity - sk, err := base64.StdEncoding.DecodeString(*key) + // Secret signing key + sk, err := hex.DecodeString(*key) if err != nil { return nil, fmt.Errorf("sk: DecodeString: %v", err) } - signer := ed25519.PrivateKey(sk) - logId, err := types.NewNamespaceEd25519V1([]byte(ed25519.PrivateKey(sk).Public().(ed25519.PublicKey))) - if err != nil { - return nil, fmt.Errorf("NewNamespaceEd25519V1: %v", err) - } // Setup log parameters - lp, err := stfe.NewLogParameters(signer, logId, *trillianID, *prefix, submitters, witnesses, *maxRange, *interval, *deadline, *submitterPolicy, *witnessPolicy) - if err != nil { - return nil, fmt.Errorf("NewLogParameters: %v", err) + lp := &stfe.LogParameters{ + LogId: hex.EncodeToString([]byte(ed25519.PrivateKey(sk).Public().(ed25519.PublicKey))), + TreeId: *trillianID, + Prefix: *prefix, + MaxRange: *maxRange, + Deadline: *deadline, + Interval: *interval, + HashType: crypto.SHA256, + Signer: ed25519.PrivateKey(sk), + Witnesses: witnesses, } // Setup STH source source, err := stfe.NewActiveSthSource(client, lp) @@ -138,28 +132,24 @@ func setupInstanceFromFlags() (*stfe.Instance, error) { return i, nil } -// newNamespacePoolFromString creates a new namespace pool from a -// comma-separated list of serialized and base64-encoded namespaces. -func newNamespacePoolFromString(str string) (*types.NamespacePool, error) { - var namespaces []*types.Namespace - if len(str) > 0 { - for _, b64 := range strings.Split(str, ",") { - b, err := base64.StdEncoding.DecodeString(b64) +// newWitnessMap creates a new map of trusted witnesses +func newWitnessMap(witnesses string) (map[[types.HashSize]byte][types.VerificationKeySize]byte, error) { + w := make(map[[types.HashSize]byte][types.VerificationKeySize]byte) + if len(witnesses) > 0 { + for _, witness := range strings.Split(witnesses, ",") { + b, err := hex.DecodeString(witness) if err != nil { return nil, fmt.Errorf("DecodeString: %v", err) } - var namespace types.Namespace - if err := types.Unmarshal(b, &namespace); err != nil { - return nil, fmt.Errorf("Unmarshal: %v", err) + + var vk [types.VerificationKeySize]byte + if n := copy(vk[:], b); n != types.VerificationKeySize { + return nil, fmt.Errorf("Invalid verification key size: %v", n) } - namespaces = append(namespaces, &namespace) + w[*types.Hash(vk[:])] = vk } } - pool, err := types.NewNamespacePool(namespaces) - if err != nil { - return nil, fmt.Errorf("NewNamespacePool: %v", err) - } - return pool, nil + return w, nil } // await waits for a shutdown signal and then runs a clean-up function diff --git a/sth.go b/sth.go index ee4dd2f..1399241 100644 --- a/sth.go +++ b/sth.go @@ -2,6 +2,7 @@ package stfe import ( "context" + "crypto/ed25519" "fmt" "reflect" "sync" @@ -13,34 +14,32 @@ import ( "github.com/system-transparency/stfe/types" ) -// SthSource provides access to the log's STHs. +// SthSource provides access to the log's (co)signed tree heads type SthSource interface { - // Latest returns the most reccent signed_tree_head_v*. - Latest(context.Context) (*types.StItem, error) - // Stable returns the most recent signed_tree_head_v* that is stable for - // some period of time, e.g., 10 minutes. - Stable(context.Context) (*types.StItem, error) - // Cosigned returns the most recent cosigned_tree_head_v*. - Cosigned(context.Context) (*types.StItem, error) - // AddCosignature attempts to add a cosignature to the stable STH. The - // passed cosigned_tree_head_v* must have a single verified cosignature. - AddCosignature(context.Context, *types.StItem) error - // Run keeps the STH source updated until cancelled + Latest(context.Context) (*types.SignedTreeHead, error) + Stable(context.Context) (*types.SignedTreeHead, error) + Cosigned(context.Context) (*types.SignedTreeHead, error) + AddCosignature(context.Context, ed25519.PublicKey, *[types.SignatureSize]byte) error Run(context.Context) } // ActiveSthSource implements the SthSource interface for an STFE instance that // accepts new logging requests, i.e., the log is running in read+write mode. type ActiveSthSource struct { - client trillian.TrillianLogClient - logParameters *LogParameters - currCosth *types.StItem // current cosigned_tree_head_v1 (already finalized) - nextCosth *types.StItem // next cosigned_tree_head_v1 (under preparation) - cosignatureFrom map[[types.NamespaceFingerprintSize]byte]bool // track who we got cosignatures from in nextCosth - mutex sync.RWMutex + client trillian.TrillianLogClient + logParameters *LogParameters + sync.RWMutex + + // cosigned is the current cosigned tree head that is served + cosigned types.SignedTreeHead + + // tosign is the current tree head that is being cosigned + tosign types.SignedTreeHead + + // cosignature keeps track of all collected cosignatures for tosign + cosignature map[[types.HashSize]byte]*types.SigIdent } -// NewActiveSthSource returns an initialized ActiveSthSource func NewActiveSthSource(cli trillian.TrillianLogClient, lp *LogParameters) (*ActiveSthSource, error) { s := ActiveSthSource{ client: cli, @@ -53,10 +52,9 @@ func NewActiveSthSource(cli trillian.TrillianLogClient, lp *LogParameters) (*Act return nil, fmt.Errorf("Latest: %v", err) } - // TODO: load persisted cosigned STH? - s.currCosth = types.NewCosignedTreeHeadV1(sth.SignedTreeHeadV1, nil) - s.nextCosth = types.NewCosignedTreeHeadV1(sth.SignedTreeHeadV1, nil) - s.cosignatureFrom = make(map[[types.NamespaceFingerprintSize]byte]bool) + s.cosigned = *sth + s.tosign = *sth + s.cosignature = make(map[[types.HashSize]byte]*types.SigIdent) return &s, nil } @@ -70,16 +68,13 @@ func (s *ActiveSthSource) Run(ctx context.Context) { return } // rotate - s.mutex.Lock() - defer s.mutex.Unlock() - if err := s.rotate(sth); err != nil { - glog.Warningf("rotate failed: %v", err) - } - // TODO: persist cosigned STH? + s.Lock() + defer s.Unlock() + s.rotate(sth) }) } -func (s *ActiveSthSource) Latest(ctx context.Context) (*types.StItem, error) { +func (s *ActiveSthSource) Latest(ctx context.Context) (*types.SignedTreeHead, error) { trsp, err := s.client.GetLatestSignedLogRoot(ctx, &trillian.GetLatestSignedLogRootRequest{ LogId: s.logParameters.TreeId, }) @@ -87,69 +82,62 @@ func (s *ActiveSthSource) Latest(ctx context.Context) (*types.StItem, error) { if errInner := checkGetLatestSignedLogRoot(s.logParameters, trsp, err, &lr); errInner != nil { return nil, fmt.Errorf("invalid signed log root response: %v", errInner) } - return s.logParameters.SignTreeHeadV1(NewTreeHeadV1FromLogRoot(&lr)) + return s.logParameters.Sign(NewTreeHeadFromLogRoot(&lr)) } -func (s *ActiveSthSource) Stable(_ context.Context) (*types.StItem, error) { - s.mutex.RLock() - defer s.mutex.RUnlock() - if s.nextCosth == nil { - return nil, fmt.Errorf("no stable sth available") - } - return types.NewSignedTreeHeadV1(&s.nextCosth.CosignedTreeHeadV1.SignedTreeHead.TreeHead, &s.nextCosth.CosignedTreeHeadV1.SignedTreeHead.Signature), nil +func (s *ActiveSthSource) Stable(_ context.Context) (*types.SignedTreeHead, error) { + s.RLock() + defer s.RUnlock() + return &s.tosign, nil } -func (s *ActiveSthSource) Cosigned(_ context.Context) (*types.StItem, error) { - s.mutex.RLock() - defer s.mutex.RUnlock() - if s.currCosth == nil { - return nil, fmt.Errorf("no cosigned sth available") - } - return s.currCosth, nil +func (s *ActiveSthSource) Cosigned(_ context.Context) (*types.SignedTreeHead, error) { + s.RLock() + defer s.RUnlock() + return &s.cosigned, nil } -func (s *ActiveSthSource) AddCosignature(_ context.Context, costh *types.StItem) error { - s.mutex.Lock() - defer s.mutex.Unlock() - if !reflect.DeepEqual(s.nextCosth.CosignedTreeHeadV1.SignedTreeHead, costh.CosignedTreeHeadV1.SignedTreeHead) { - return fmt.Errorf("cosignature covers a different tree head") - } - witness, err := costh.CosignedTreeHeadV1.Cosignatures[0].Namespace.Fingerprint() - if err != nil { - return fmt.Errorf("namespace without fingerprint: %v", err) +func (s *ActiveSthSource) AddCosignature(_ context.Context, vk ed25519.PublicKey, sig *[types.SignatureSize]byte) error { + s.Lock() + defer s.Unlock() + + if msg := s.tosign.TreeHead.Marshal(); !ed25519.Verify(vk, msg, sig[:]) { + return fmt.Errorf("Invalid signature for tree head with timestamp: %d", s.tosign.TreeHead.Timestamp) } - if _, ok := s.cosignatureFrom[*witness]; ok { + witness := types.Hash(vk[:]) + if _, ok := s.cosignature[*witness]; ok { + glog.V(3).Infof("received cosignature again (duplicate)") return nil // duplicate } - s.cosignatureFrom[*witness] = true - s.nextCosth.CosignedTreeHeadV1.Cosignatures = append(s.nextCosth.CosignedTreeHeadV1.Cosignatures, costh.CosignedTreeHeadV1.Cosignatures[0]) + s.cosignature[*witness] = &types.SigIdent{ + Signature: sig, + KeyHash: witness, + } + glog.V(3).Infof("accepted new cosignature") return nil } // rotate rotates the log's cosigned and stable STH. The caller must aquire the // source's read-write lock if there are concurrent reads and/or writes. -func (s *ActiveSthSource) rotate(fixedSth *types.StItem) error { - // rotate stable -> cosigned - if reflect.DeepEqual(&s.currCosth.CosignedTreeHeadV1.SignedTreeHead, &s.nextCosth.CosignedTreeHeadV1.SignedTreeHead) { - for _, sigv1 := range s.currCosth.CosignedTreeHeadV1.Cosignatures { - witness, err := sigv1.Namespace.Fingerprint() - if err != nil { - return fmt.Errorf("namespace without fingerprint: %v", err) - } - if _, ok := s.cosignatureFrom[*witness]; !ok { - s.cosignatureFrom[*witness] = true - s.nextCosth.CosignedTreeHeadV1.Cosignatures = append(s.nextCosth.CosignedTreeHeadV1.Cosignatures, sigv1) +func (s *ActiveSthSource) rotate(next *types.SignedTreeHead) { + if reflect.DeepEqual(s.cosigned.TreeHead, s.tosign.TreeHead) { + for _, sigident := range s.cosigned.SigIdent[1:] { // skip log sigident + if _, ok := s.cosignature[*sigident.KeyHash]; !ok { + s.cosignature[*sigident.KeyHash] = sigident } } } - s.currCosth.CosignedTreeHeadV1.SignedTreeHead = s.nextCosth.CosignedTreeHeadV1.SignedTreeHead - s.currCosth.CosignedTreeHeadV1.Cosignatures = make([]types.SignatureV1, len(s.nextCosth.CosignedTreeHeadV1.Cosignatures)) - copy(s.currCosth.CosignedTreeHeadV1.Cosignatures, s.nextCosth.CosignedTreeHeadV1.Cosignatures) - - // rotate new stable -> stable - if !reflect.DeepEqual(&s.nextCosth.CosignedTreeHeadV1.SignedTreeHead, fixedSth.SignedTreeHeadV1) { - s.nextCosth = types.NewCosignedTreeHeadV1(fixedSth.SignedTreeHeadV1, nil) - s.cosignatureFrom = make(map[[types.NamespaceFingerprintSize]byte]bool) - } - return nil + var cosignatures []*types.SigIdent + for _, sigident := range s.cosignature { + cosignatures = append(cosignatures, sigident) + } // cosignatures contains all cosignatures, even if repeated tree head + + // Update cosigned tree head + s.cosigned.TreeHead = s.tosign.TreeHead + s.cosigned.SigIdent = append(s.tosign.SigIdent, cosignatures...) + + // Update to-sign tree head + s.tosign = *next + s.cosignature = make(map[[types.HashSize]byte]*types.SigIdent) + glog.V(3).Infof("rotated sth") } diff --git a/trillian.go b/trillian.go index 2adf567..f358d4d 100644 --- a/trillian.go +++ b/trillian.go @@ -26,7 +26,7 @@ func checkQueueLeaf(rsp *trillian.QueueLeafResponse, err error) error { return nil } -func checkGetLeavesByRange(req *stfetypes.GetEntriesV1, rsp *trillian.GetLeavesByRangeResponse, err error) error { +func checkGetLeavesByRange(req *stfetypes.LeavesRequest, rsp *trillian.GetLeavesByRangeResponse, err error) error { if err != nil { return fmt.Errorf("Trillian Error: %v", err) } @@ -42,8 +42,8 @@ func checkGetLeavesByRange(req *stfetypes.GetEntriesV1, rsp *trillian.GetLeavesB if len(rsp.Leaves) == 0 { return fmt.Errorf("Trillian error: no leaves") } - if len(rsp.Leaves) > int(req.End-req.Start+1) { - return fmt.Errorf("too many leaves: %d for [%d,%d]", len(rsp.Leaves), req.Start, req.End) + if len(rsp.Leaves) > int(req.EndSize-req.StartSize+1) { + return fmt.Errorf("too many leaves: %d for [%d,%d]", len(rsp.Leaves), req.StartSize, req.EndSize) } // Ensure that a bad start parameter results in an error @@ -51,13 +51,13 @@ func checkGetLeavesByRange(req *stfetypes.GetEntriesV1, rsp *trillian.GetLeavesB if err := lr.UnmarshalBinary(rsp.SignedLogRoot.LogRoot); err != nil { return fmt.Errorf("cannot unmarshal log root: %v", err) } - if uint64(req.Start) >= lr.TreeSize { - return fmt.Errorf("invalid start(%d): tree size is %d", req.Start, lr.TreeSize) + if uint64(req.StartSize) >= lr.TreeSize { + return fmt.Errorf("invalid start(%d): tree size is %d", req.StartSize, lr.TreeSize) } // Ensure that we got and return expected leaf indices for i, leaf := range rsp.Leaves { - if got, want := leaf.LeafIndex, int64(req.Start+uint64(i)); got != want { + if got, want := leaf.LeafIndex, int64(req.StartSize+uint64(i)); got != want { return fmt.Errorf("invalid leaf index(%d): wanted %d", got, want) } } diff --git a/util.go b/util.go index 847c3f7..b5cca48 100644 --- a/util.go +++ b/util.go @@ -1,40 +1,42 @@ package stfe import ( - "fmt" + //"fmt" - "github.com/google/trillian" + //"github.com/google/trillian" ttypes "github.com/google/trillian/types" "github.com/system-transparency/stfe/types" ) -func NewTreeHeadV1FromLogRoot(lr *ttypes.LogRootV1) *types.TreeHeadV1 { - return &types.TreeHeadV1{ - Timestamp: uint64(lr.TimestampNanos / 1000 / 1000), +func NewTreeHeadFromLogRoot(lr *ttypes.LogRootV1) *types.TreeHead { + var hash [types.HashSize]byte + th := types.TreeHead{ + Timestamp: uint64(lr.TimestampNanos / 1000 / 1000 / 1000), TreeSize: uint64(lr.TreeSize), - RootHash: types.NodeHash{ - Data: lr.RootHash, - }, - Extension: make([]byte, 0), + RootHash: &hash, } + copy(th.RootHash[:], lr.RootHash) + return &th } -func NewNodePathFromHashPath(hashes [][]byte) []types.NodeHash { - path := make([]types.NodeHash, 0, len(hashes)) +func NodePathFromHashes(hashes [][]byte) []*[types.HashSize]byte { + var path []*[types.HashSize]byte for _, hash := range hashes { - path = append(path, types.NodeHash{hash}) + var h [types.HashSize]byte + copy(h[:], hash) + path = append(path, &h) } return path } -func NewStItemListFromLeaves(leaves []*trillian.LogLeaf) (*types.StItemList, error) { - items := make([]types.StItem, 0, len(leaves)) - for _, leaf := range leaves { - var item types.StItem - if err := types.Unmarshal(leaf.LeafValue, &item); err != nil { - return nil, fmt.Errorf("Unmarshal failed: %v", err) - } - items = append(items, item) - } - return &types.StItemList{items}, nil -} +//func NewStItemListFromLeaves(leaves []*trillian.LogLeaf) (*types.StItemList, error) { +// items := make([]types.StItem, 0, len(leaves)) +// for _, leaf := range leaves { +// var item types.StItem +// if err := types.Unmarshal(leaf.LeafValue, &item); err != nil { +// return nil, fmt.Errorf("Unmarshal failed: %v", err) +// } +// items = append(items, item) +// } +// return &types.StItemList{items}, nil +//} -- cgit v1.2.3