diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..6313b56c --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +* text=auto eol=lf diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..7d45aecc --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,40 @@ +services: + registry: + image: docker.io/library/registry:2 + volumes: + - registry:/var/lib/registry/ + dind: + depends_on: + - registry + image: complement-dind + build: + dockerfile: dockerfiles/dind.Dockerfile + privileged: true + entrypoint: dockerd + command: + - --tls=false + - --host=tcp://0.0.0.0:2375 + - --storage-driver=overlay2 + - --insecure-registry=registry:5000 + volumes: + - dind-cache:/var/lib/docker + - complement:/root/complement + complement: + depends_on: + - dind + profiles: + - complement + image: complement + build: + dockerfile: dockerfiles/complement.Dockerfile + environment: + - DOCKER_HOST=tcp://dind:2375 + volumes: + - complement:/root/complement +networks: + default: + name: complement +volumes: + registry: + dind-cache: + complement: diff --git a/dockerfiles/README.md b/dockerfiles/README.md index 57bebd86..743cec73 100644 --- a/dockerfiles/README.md +++ b/dockerfiles/README.md @@ -7,3 +7,23 @@ duplication, we now point to dockerfiles in respective repositories rather than - Dendrite: https://github.com/matrix-org/dendrite/blob/v0.8.2/build/scripts/Complement.Dockerfile - Synapse: https://github.com/matrix-org/synapse/blob/develop/docker/complement/Dockerfile + +# MessageHub +Build: +``` +docker compose build +docker compose --profile complement build +``` + +Run: +``` +docker compose run --rm complement +``` + +Example `complement/.env` file: +``` +COMPLEMENT_BASE_IMAGE=registry:5000/complement-messagehub +COMPLEMENT_DEBUG=1 +COMPLEMENT_ALWAYS_PRINT_SERVER_LOGS=1 +COMPLEMENT_SPAWN_HS_TIMEOUT_SECS=5 +``` diff --git a/dockerfiles/complement.Dockerfile b/dockerfiles/complement.Dockerfile new file mode 100644 index 00000000..195208ee --- /dev/null +++ b/dockerfiles/complement.Dockerfile @@ -0,0 +1,15 @@ +FROM docker.io/library/golang:1.18-alpine as build +WORKDIR /root/project +COPY go.mod . +COPY go.sum . +RUN go mod download +COPY . . +RUN go build -v -o /bin/app + +FROM docker.io/library/docker +RUN apk update && apk add git +WORKDIR /root/project +COPY . . +WORKDIR /root/complement +COPY --from=build /bin/app /bin/app +ENTRYPOINT /bin/app diff --git a/dockerfiles/dind.Dockerfile b/dockerfiles/dind.Dockerfile new file mode 100644 index 00000000..6d32df5a --- /dev/null +++ b/dockerfiles/dind.Dockerfile @@ -0,0 +1,2 @@ +FROM docker.io/library/docker:dind +RUN apk update && apk add git diff --git a/dockerfiles/dind.complement.Dockerfile b/dockerfiles/dind.complement.Dockerfile new file mode 100644 index 00000000..a440a612 --- /dev/null +++ b/dockerfiles/dind.complement.Dockerfile @@ -0,0 +1,10 @@ +FROM docker.io/library/golang:1.18 +RUN echo "deb http://deb.debian.org/debian buster-backports main" > /etc/apt/sources.list.d/complement.list \ + && apt-get update \ + && apt-get install -y libolm3 libolm-dev/buster-backports + +WORKDIR /root/complement +COPY go.mod . +COPY go.sum . +RUN go mod download +CMD go test -v ./tests/... diff --git a/go.sum b/go.sum index 333697af..8b67aafe 100644 --- a/go.sum +++ b/go.sum @@ -76,12 +76,6 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/matrix-org/gomatrix v0.0.0-20190528120928-7df988a63f26/go.mod h1:3fxX6gUjWyI/2Bt7J1OLhpCzOfO/bB3AiX0cJtEKud0= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16 h1:ZtO5uywdd5dLDCud4r0r55eP4j9FuUNpl60Gmntcop4= github.com/matrix-org/gomatrix v0.0.0-20210324163249-be2af5ef2e16/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220519174812-8904a93b5a99 h1:JsIMDWZl2B9bReq/yNlLSsBk2Zvx+JO8Ci8B+eZDHzM= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220519174812-8904a93b5a99/go.mod h1:V5eO8rn/C3rcxig37A/BCeKerLFS+9Avg/77FIeTZ48= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220524100759-f98e737f8f9c h1:J9krMtVgo4mV/G+mRA1u3GL6nNxdNnuPcs891uIQGic= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220524100759-f98e737f8f9c/go.mod h1:V5eO8rn/C3rcxig37A/BCeKerLFS+9Avg/77FIeTZ48= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220526125151-b6f33bc40ed8 h1:5cdzTYzcwWxp7TvYtHi5WpMHarPddpFDxHnkLtgGgdE= -github.com/matrix-org/gomatrixserverlib v0.0.0-20220526125151-b6f33bc40ed8/go.mod h1:V5eO8rn/C3rcxig37A/BCeKerLFS+9Avg/77FIeTZ48= github.com/matrix-org/gomatrixserverlib v0.0.0-20220526140030-dcfbb70ff32d h1:IwyG/58rFn0/ugD0A/IdSIo7D/oLJ4+k3NznlYhzyHs= github.com/matrix-org/gomatrixserverlib v0.0.0-20220526140030-dcfbb70ff32d/go.mod h1:jX38yp3SSLJNftBg3PXU1ayd0PCLIiDHQ4xAc9DIixk= github.com/matrix-org/util v0.0.0-20190711121626-527ce5ddefc7/go.mod h1:vVQlW/emklohkZnOPwD3LrZUBqdfsbiyO3p1lNV8F6U= @@ -127,7 +121,6 @@ github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -github.com/tidwall/sjson v1.0.3/go.mod h1:bURseu1nuBkFpIES5cz6zBtjmYeOQmEESshn7VpF15Y= github.com/tidwall/sjson v1.2.4 h1:cuiLzLnaMeBhRmEv00Lpk3tkYrcxpmbU81tAY4Dw0tc= github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp19pSNM= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -142,8 +135,6 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20220513210258-46612604a0f9/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220518034528-6f7dac969898 h1:SLP7Q4Di66FONjDJbCYrCRrh97focO6sLogHO7/g8F0= -golang.org/x/crypto v0.0.0-20220518034528-6f7dac969898/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e h1:T8NU3HyQ8ClP4SEE+KbFlg6n0NhuTsN4MyznaarGsZM= golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= diff --git a/main.go b/main.go new file mode 100644 index 00000000..36331705 --- /dev/null +++ b/main.go @@ -0,0 +1,75 @@ +package main + +import ( + "context" + "fmt" + "log" + "os" + "os/exec" + "time" + + "github.com/docker/docker/client" +) + +func run(name string, arg ...string) error { + fmt.Println(name, arg) + cmd := exec.Command(name, arg...) + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} + +func main() { + ctx := context.Background() + + log.Println("Copying project files to volume...") + run("sh", "-c", "rm -r /root/complement/*") + err := run("sh", "-c", "cp -r /root/project/* /root/complement") + if err != nil { + panic(err) + } + + client, err := client.NewClientWithOpts(client.FromEnv) + if err != nil { + panic(err) + } + defer client.Close() + for { + _, err := client.Ping(ctx) + if err == nil { + break + } + log.Println("Waiting for dind...") + time.Sleep(3 * time.Second) + } + + log.Println("Building complement...") + err = run("docker", "build", + "--force-rm", + "--tag", "complement", + "--file", "dockerfiles/dind.complement.Dockerfile", + "/root/complement") + if err != nil { + panic(err) + } + + log.Println("Running complement...") + errs := make([]error, 0) + for _, path := range []string{"./tests/messagehub"} { + err = run("docker", "run", + "--rm", + "--env", fmt.Sprintf("DOCKER_HOST=%s", client.DaemonHost()), + "--env-file", "/root/complement/complement/.env", + "--volume", "/root/complement:/root/complement", + "--network", "host", + "complement", + "go", "test", "-v", path) + if err != nil { + errs = append(errs, fmt.Errorf("error running tests %s: %w", path, err)) + } + } + if len(errs) > 0 { + panic(fmt.Errorf("errors: %v", errs)) + } +} diff --git a/tests/messagehub/apidoc_profile_avatar_url_test.go b/tests/messagehub/apidoc_profile_avatar_url_test.go new file mode 100644 index 00000000..047e72ec --- /dev/null +++ b/tests/messagehub/apidoc_profile_avatar_url_test.go @@ -0,0 +1,42 @@ +package messagehub_tests + +import ( + "testing" + + "github.com/matrix-org/complement/internal/b" + "github.com/matrix-org/complement/internal/client" + "github.com/matrix-org/complement/internal/match" + "github.com/matrix-org/complement/internal/must" +) + +func TestProfileAvatarURL(t *testing.T) { + deployment := Deploy(t, b.BlueprintAlice) + defer deployment.Destroy(t) + unauthedClient := deployment.Client(t, "hs1", "") + authedClient := deployment.Client(t, "hs1", "@alice:hs1") + avatarURL := "mxc://example.com/SEsfnsuifSDFSSEF" + // sytest: PUT /profile/:user_id/avatar_url sets my avatar + t.Run("PUT /profile/:user_id/avatar_url sets my avatar", func(t *testing.T) { + reqBody := client.WithJSONBody(t, map[string]interface{}{ + "avatar_url": avatarURL, + }) + res := authedClient.MustDoFunc(t, "PUT", []string{"_matrix", "client", "r0", "profile", authedClient.UserID, "avatar_url"}, reqBody) + + must.MatchResponse(t, res, match.HTTPResponse{ + StatusCode: 200, + }) + }) + + // sytest: GET /profile/:user_id/avatar_url publicly accessible + t.Run("GET /profile/:user_id/avatar_url publicly accessible", func(t *testing.T) { + res := unauthedClient.DoFunc(t, "GET", []string{"_matrix", "client", "r0", "profile", authedClient.UserID, "avatar_url"}) + + must.MatchResponse(t, res, match.HTTPResponse{ + StatusCode: 200, + JSON: []match.JSON{ + match.JSONKeyPresent("avatar_url"), + match.JSONKeyEqual("avatar_url", avatarURL), + }, + }) + }) +} diff --git a/tests/messagehub/apidoc_profile_displayname_test.go b/tests/messagehub/apidoc_profile_displayname_test.go new file mode 100644 index 00000000..0e419eb4 --- /dev/null +++ b/tests/messagehub/apidoc_profile_displayname_test.go @@ -0,0 +1,35 @@ +package messagehub_tests + +import ( + "testing" + + "github.com/matrix-org/complement/internal/b" + "github.com/matrix-org/complement/internal/client" + "github.com/matrix-org/complement/internal/match" + "github.com/matrix-org/complement/internal/must" +) + +func TestProfileDisplayName(t *testing.T) { + deployment := Deploy(t, b.BlueprintAlice) + defer deployment.Destroy(t) + unauthedClient := deployment.Client(t, "hs1", "") + authedClient := deployment.Client(t, "hs1", "@alice:hs1") + displayName := "my_display_name" + // sytest: PUT /profile/:user_id/displayname sets my name + t.Run("PUT /profile/:user_id/displayname sets my name", func(t *testing.T) { + reqBody := client.WithJSONBody(t, map[string]interface{}{ + "displayname": displayName, + }) + _ = authedClient.MustDoFunc(t, "PUT", []string{"_matrix", "client", "r0", "profile", authedClient.UserID, "displayname"}, reqBody) + }) + // sytest: GET /profile/:user_id/displayname publicly accessible + t.Run("GET /profile/:user_id/displayname publicly accessible", func(t *testing.T) { + res := unauthedClient.DoFunc(t, "GET", []string{"_matrix", "client", "r0", "profile", authedClient.UserID, "displayname"}) + must.MatchResponse(t, res, match.HTTPResponse{ + StatusCode: 200, + JSON: []match.JSON{ + match.JSONKeyEqual("displayname", displayName), + }, + }) + }) +} diff --git a/tests/messagehub/apidoc_version_test.go b/tests/messagehub/apidoc_version_test.go new file mode 100644 index 00000000..89b2f744 --- /dev/null +++ b/tests/messagehub/apidoc_version_test.go @@ -0,0 +1,74 @@ +package messagehub_tests + +import ( + "fmt" + "regexp" + "testing" + + "github.com/tidwall/gjson" + + "github.com/matrix-org/complement/internal/b" + "github.com/matrix-org/complement/internal/match" + "github.com/matrix-org/complement/internal/must" +) + +// https://spec.matrix.org/v1.1/#specification-versions +// altered to limit to X = 1+, as v0 has never existed +const GlobalVersionRegex = `v[1-9]\d*\.\d+(?:-\S+)?` + +// https://github.com/matrix-org/matrix-doc/blob/client_server/r0.6.1/specification/index.rst#specification-versions +// altered to limit to X = 0 (r0), as r1+ will never exist. +const r0Regex = `r0\.\d+\.\d+` + +func TestVersionStructure(t *testing.T) { + deployment := Deploy(t, b.BlueprintAlice) + defer deployment.Destroy(t) + + client := deployment.Client(t, "hs1", "") + + // sytest: Version responds 200 OK with valid structure + t.Run("Version responds 200 OK with valid structure", func(t *testing.T) { + res := client.MustDoFunc(t, "GET", []string{"_matrix", "client", "versions"}) + + // Matches; + // - r0.?.? + // where ? is any single digit + // - v1^.*(-#) + // where 1^ is 1 through 9 for the first digit, then any digit thereafter, + // and * is any single or multiple of digits + // optionally with dash-separated metadata: (-#) + versionRegex, _ := regexp.Compile("^(" + r0Regex + "|" + GlobalVersionRegex + ")$") + + must.MatchResponse(t, res, match.HTTPResponse{ + JSON: []match.JSON{ + match.JSONKeyPresent("versions"), + match.JSONArrayEach("versions", func(val gjson.Result) error { + if val.Type != gjson.String { + return fmt.Errorf("'versions' value is not a string: %s", val.Raw) + } + if !versionRegex.MatchString(val.Str) { + return fmt.Errorf("value in 'versions' array did not match version regex: %s", val.Str) + } + return nil + }), + // Check when unstable_features is present if it's an object + func(body []byte) error { + res := gjson.GetBytes(body, "unstable_features") + if !res.Exists() { + return nil + } + if !res.IsObject() { + return fmt.Errorf("unstable_features was present, and wasn't an object") + } + for k, v := range res.Map() { + // gjson doesn't have a "boolean" type to check against + if v.Type != gjson.True && v.Type != gjson.False { + return fmt.Errorf("value for key 'unstable_features.%s' is of the wrong type, got %s want boolean", k, v.Type) + } + } + return nil + }, + }, + }) + }) +} diff --git a/tests/messagehub/main_test.go b/tests/messagehub/main_test.go new file mode 100644 index 00000000..13b4d17f --- /dev/null +++ b/tests/messagehub/main_test.go @@ -0,0 +1,117 @@ +package messagehub_tests + +import ( + "context" + "fmt" + "log" + "os" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/sirupsen/logrus" + + "github.com/matrix-org/complement/internal/b" + "github.com/matrix-org/complement/internal/config" + "github.com/matrix-org/complement/internal/docker" +) + +var namespaceCounter uint64 + +// persist the complement builder which is set when the tests start via TestMain +var complementBuilder *docker.Builder + +// TestMain is the main entry point for Complement. +// +// It will clean up any old containers/images/networks from the previous run, then run the tests, then clean up +// again. No blueprints are made at this point as they are lazily made on demand. +func TestMain(m *testing.M) { + cfg := config.NewConfigFromEnvVars("messagehub", "") + log.Printf("config: %+v", cfg) + builder, err := docker.NewBuilder(cfg) + if err != nil { + fmt.Printf("Error: %s", err) + os.Exit(1) + } + complementBuilder = builder + // remove any old images/containers/networks in case we died horribly before + builder.Cleanup() + + // we use GMSL which uses logrus by default. We don't want those logs in our test output unless they are Serious. + logrus.SetLevel(logrus.ErrorLevel) + + exitCode := m.Run() + builder.Cleanup() + os.Exit(exitCode) +} + +// Deploy will deploy the given blueprint or terminate the test. +// It will construct the blueprint if it doesn't already exist in the docker image cache. +// This function is the main setup function for all tests as it provides a deployment with +// which tests can interact with. +func Deploy(t *testing.T, blueprint b.Blueprint) *docker.Deployment { + t.Helper() + timeStartBlueprint := time.Now() + if complementBuilder == nil { + t.Fatalf("complementBuilder not set, did you forget to call TestMain?") + } + if err := complementBuilder.ConstructBlueprintIfNotExist(blueprint); err != nil { + t.Fatalf("Deploy: Failed to construct blueprint: %s", err) + } + namespace := fmt.Sprintf("%d", atomic.AddUint64(&namespaceCounter, 1)) + d, err := docker.NewDeployer(namespace, complementBuilder.Config) + if err != nil { + t.Fatalf("Deploy: NewDeployer returned error %s", err) + } + timeStartDeploy := time.Now() + dep, err := d.Deploy(context.Background(), blueprint.Name) + if err != nil { + t.Fatalf("Deploy: Deploy returned error %s", err) + } + t.Logf("Deploy times: %v blueprints, %v containers", timeStartDeploy.Sub(timeStartBlueprint), time.Since(timeStartDeploy)) + return dep +} + +// nolint:unused +type Waiter struct { + mu sync.Mutex + ch chan bool + closed bool +} + +// NewWaiter returns a generic struct which can be waited on until `Waiter.Finish` is called. +// A Waiter is similar to a `sync.WaitGroup` of size 1, but without the ability to underflow and +// with built-in timeouts. +// nolint:unused +func NewWaiter() *Waiter { + return &Waiter{ + ch: make(chan bool), + mu: sync.Mutex{}, + } +} + +// Wait blocks until Finish() is called or until the timeout is reached. +// If the timeout is reached, the test is failed. +func (w *Waiter) Wait(t *testing.T, timeout time.Duration) { + t.Helper() + select { + case <-w.ch: + return + case <-time.After(timeout): + t.Fatalf("Wait: timed out after %f seconds.", timeout.Seconds()) + } +} + +// Finish will cause all goroutines waiting via Wait to stop waiting and return. +// Once this function has been called, subsequent calls to Wait will return immediately. +// To begin waiting again, make a new Waiter. +func (w *Waiter) Finish() { + w.mu.Lock() + defer w.mu.Unlock() + if w.closed { + return + } + w.closed = true + close(w.ch) +}