p2p: add message validation spec tests

p2p/spectest/README.md

@@ -0,0 +1,33 @@
+# P2P Spectests
+
+This package contains spec tests for the P2P validation surface.
+
+## Layout
+
+- `all_tests.go`: registry of all P2P spec tests.
+- `run_test.go`: Go test entrypoint.
+- `tests/`: concrete spec test types and grouped case files.
+- `testdoc/`: centralized test type and case documentation strings.
+- `generate/`: JSON generator for test vectors and future state-comparison outputs.
+
+## Current scope
+
+The current suite covers pubsub message validation in:
+
+- malformed payload handling
+- signed SSV message type filtering
+- consensus message validation
+- pre-consensus partial-signature validation
+- post-consensus partial-signature validation
+
+## Running
+
+```bash
+go test ./p2p/...
+```
+
+## Generating
+
+```bash
+go generate ./p2p/spectest/generate
+```

p2p/spectest/all_tests.go

@@ -0,0 +1,39 @@
+package spectest
+
+import (
+	"github.com/ssvlabs/ssv-spec/p2p/spectest/tests"
+	"github.com/ssvlabs/ssv-spec/p2p/spectest/tests/msgvalidation/consensus"
+	"github.com/ssvlabs/ssv-spec/p2p/spectest/tests/msgvalidation/general"
+	"github.com/ssvlabs/ssv-spec/p2p/spectest/tests/msgvalidation/postconsensus"
+	"github.com/ssvlabs/ssv-spec/p2p/spectest/tests/msgvalidation/preconsensus"
+)
+
+var AllTests = []tests.TestF{
+	general.MalformedPubsubPayload,
+	general.NoSigners,
+	general.NoSignatures,
+	general.EmptySignature,
+	general.SignatureCountMismatch,
+	general.ZeroSigner,
+	general.NonUniqueSigners,
+	general.UnsupportedSSVMessageType,
+	consensus.ValidExistingInstanceConsensus,
+	consensus.UnknownInstanceConsensus,
+	consensus.InvalidConsensusSignature,
+	consensus.WrongConsensusIdentifier,
+	consensus.FutureConsensusMultiSigner,
+	consensus.ValidFutureDecidedConsensus,
+	consensus.DecidedConsensusBadFullData,
+	preconsensus.ValidPreConsensusPartialSignature,
+	preconsensus.InvalidPreConsensusPartialSignatureSignature,
+	preconsensus.PreConsensusWithoutRunningDuty,
+	preconsensus.PreConsensusSignerMismatch,
+	preconsensus.PreConsensusPastSlot,
+	preconsensus.PreConsensusFutureSlot,
+	preconsensus.PreConsensusUnknownValidatorIndex,
+	preconsensus.MultiSignerPartialSignature,
+	postconsensus.ValidPostConsensusPartialSignature,
+	postconsensus.PostConsensusWithoutDecidedValue,
+	postconsensus.PostConsensusWithoutRunningInstance,
+	postconsensus.PostConsensusNotDecided,
+}

p2p/spectest/generate/main.go

@@ -0,0 +1,124 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strings"
+
+	"github.com/pkg/errors"
+	comparable2 "github.com/ssvlabs/ssv-spec/types/testingutils/comparable"
+
+	"github.com/ssvlabs/ssv-spec/p2p/spectest"
+	"github.com/ssvlabs/ssv-spec/p2p/spectest/tests"
+)
+
+//go:generate go run main.go
+
+var testsDir = "tests"
+var stateComparisonDir = "state_comparison"
+
+func main() {
+	clearStateComparisonFolder()
+	clearTestsFolder()
+
+	all := map[string]tests.SpecTest{}
+	for _, testF := range spectest.AllTests {
+		test := testF()
+		name := reflect.TypeOf(test).String() + "_" + test.TestName()
+		if all[name] != nil {
+			panic(fmt.Sprintf("duplicate test: %s\n", name))
+		}
+		all[name] = test
+	}
+
+	log.Printf("found %d tests\n", len(all))
+	if len(all) != len(spectest.AllTests) {
+		log.Fatalf("did not generate all tests\n")
+	}
+
+	if err := os.MkdirAll(testsDir, 0o700); err != nil && !os.IsExist(err) {
+		panic(err.Error())
+	}
+	for name, test := range all {
+		byts, err := json.MarshalIndent(test, "", "  ")
+		if err != nil {
+			panic(err.Error())
+		}
+		writeJSON(filepath.Join(testsDir, sanitize(name)), byts)
+	}
+
+	byts, err := json.MarshalIndent(all, "", "  ")
+	if err != nil {
+		panic(err.Error())
+	}
+	writeJSON(testsDir, byts)
+
+	for _, testF := range spectest.AllTests {
+		test := testF()
+		post, err := test.GetPostState()
+		if err != nil {
+			panic(errors.Wrapf(err, "failed to get post state for test: %s", test.TestName()).Error())
+		}
+		writeJSONStateComparison(test, post)
+	}
+}
+
+func clearStateComparisonFolder() {
+	if err := os.RemoveAll(stateComparisonDir); err != nil {
+		panic(err.Error())
+	}
+
+	if err := os.Mkdir(stateComparisonDir, 0o700); err != nil {
+		panic(err.Error())
+	}
+}
+
+func clearTestsFolder() {
+	if err := os.RemoveAll(testsDir); err != nil {
+		panic(err.Error())
+	}
+
+	if err := os.Mkdir(testsDir, 0o700); err != nil {
+		panic(err.Error())
+	}
+}
+
+func writeJSONStateComparison(test tests.SpecTest, post interface{}) {
+	if post == nil {
+		log.Printf("skipping state comparison json, not supported: %s\n", test.TestName())
+		return
+	}
+
+	byts, err := json.MarshalIndent(post, "", "  ")
+	if err != nil {
+		panic(err.Error())
+	}
+
+	scDir := comparable2.GetSCDir(".", reflect.TypeOf(test).String())
+	if err := os.MkdirAll(scDir, 0o700); err != nil && !os.IsExist(err) {
+		panic(err.Error())
+	}
+
+	file := filepath.Join(scDir, fmt.Sprintf("%s.json", sanitize(test.TestName())))
+	log.Printf("writing state comparison json: %s\n", file)
+	if err := os.WriteFile(file, byts, 0o664); err != nil {
+		panic(err.Error())
+	}
+}
+
+func writeJSON(name string, data []byte) {
+	file := name + ".json"
+	log.Printf("writing spec tests json to: %s\n", file)
+	if err := os.WriteFile(file, data, 0o664); err != nil {
+		panic(err.Error())
+	}
+}
+
+func sanitize(name string) string {
+	name = strings.ReplaceAll(name, " ", "_")
+	return strings.ReplaceAll(name, "*", "")
+}