Merge branch 'master' into implement-caddy

.github/workflows/test.yml

@@ -0,0 +1,40 @@
+name: 'test backend'
+on:   ['push', 'pull_request']
+
+jobs:
+  # Uncomment to get a debug shell.
+  # debug: {runs-on: 'ubuntu-latest', steps: [{uses: 'actions/checkout@v4'}, {uses: 'mxschmitt/action-tmate@v3'}]}
+
+  test:
+    name:    'test (linux)'
+    runs-on: 'ubuntu-latest'
+    steps:
+      - uses: 'actions/checkout@v4'
+      - name: 'test (linux)'
+        run: |
+          # Quick exit on compile errors.
+          go build -race ./cmd/goatcounter || exit 1
+
+          # Make sure it at least compiles on macOS, Windows, and arm64
+          GOARCH=arm64              go build ./cmd/goatcounter
+          GOARCH=arm64 GOOS=darwin  go build ./cmd/goatcounter
+          GOOS=windows              go build ./cmd/goatcounter
+
+          go test -race -timeout=3m ./...
+
+          docker compose up -d --wait
+          export PGHOST=localhost
+          export PGPORT=5433
+          export PGDATABASE=goatcounter
+          export PGUSER=goatcounter
+          export PGPASSWORD=goatcounter
+          export PGSSLMODE=disable
+          go test -race -timeout=3m -tags pgsql ./...
+
+  staticcheck:
+    name:    'staticcheck'
+    runs-on: 'ubuntu-latest'
+    steps:
+      - uses: 'actions/checkout@v4'
+      - uses: 'dominikh/[email protected]'
+        with: {version: '2024.1'}

README.md

@@ -57,7 +57,7 @@ Features
 
 [privacy]: https://www.goatcounter.com/privacy
 [gdpr]: https://www.goatcounter.com/gdpr
-[sessions]: https://github.com/arp242/goatcounter/blob/master/docs/sessions.md
+[sessions]: http://www.goatcounter.com/help/sessions
 
 
 Getting data in to GoatCounter

acme/acme_test.go

@@ -50,7 +50,7 @@ func TestSetup(t *testing.T) {
 			haveACME := acmeH != nil
 
 			if tlsC != nil {
-				t.Logf(zruntime.FuncName(tlsC.GetCertificate))
+				t.Log(zruntime.FuncName(tlsC.GetCertificate))
 			}
 			if haveTLS != tt.wantTLS {
 				t.Errorf("have TLS %t; want %t", haveTLS, tt.wantTLS)

acme/testdata/test.pem

@@ -0,0 +1,20 @@
+-----BEGIN PRIVATE KEY-----
+MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgZ+6CjBdA0CRpyZ7b
+uQDTAPO3mGuVslKgDJYBqQhsADWhRANCAARjVjMjeXYRvpQbHlGaxmQG5/zFMpyT
+R8AhaO2iYbniv79fig831YYcELEJ1aqdYSkkObSBkbxfbPe/0iXcvWfp
+-----END PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+MIICVDCCAfqgAwIBAgIRAP7F6nNH6HULuRZf7hMhJYgwCgYIKoZIzj0EAwIwgYsx
+HTAbBgNVBAoTFHpjZXJ0IGRldmVsb3BtZW50IENBMTEwLwYDVQQLDChtYXJ0aW5A
+ZTU4NS5hcnAyNDIubmV0IChNYXJ0aW4gVG91cm5vaWopMTcwNQYDVQQDDC56Y2Vy
+dCBtYXJ0aW5AZTU4NS5hcnAyNDIubmV0IChNYXJ0aW4gVG91cm5vaWopMB4XDTIx
+MDcyMDEwNTA0NFoXDTIyMDcyMDEwNTA0NFowWzEmMCQGA1UEChMdemNlcnQgZGV2
+ZWxvcG1lbnQgY2VydGlmaWNhdGUxMTAvBgNVBAsMKG1hcnRpbkBlNTg1LmFycDI0
+Mi5uZXQgKE1hcnRpbiBUb3Vybm9paikwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNC
+AARjVjMjeXYRvpQbHlGaxmQG5/zFMpyTR8AhaO2iYbniv79fig831YYcELEJ1aqd
+YSkkObSBkbxfbPe/0iXcvWfpo24wbDAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAww
+CgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAfBgNVHSMEGDAWgBRIhKNifk+PlpTR
+WaHGmtoEl5z4BzAWBgNVHREEDzANggtleGFtcGxlLmNvbTAKBggqhkjOPQQDAgNI
+ADBFAiEA+k4vdFwrZXskGCKZ5gwxpuiqqXqf5JDjDDmxT4r7S9gCIAxDStKT/SDg
+9NDcX1EZZ0h+Ay3jWg3Gggf2ccENAL/5
+-----END CERTIFICATE-----

api_token.go

@@ -153,7 +153,7 @@ func (t *APIToken) Insert(ctx context.Context) error {
 
 	t.ID, err = zdb.InsertID(ctx, "api_token_id",
 		`insert into api_tokens (site_id, user_id, name, token, permissions, created_at) values (?)`,
-		zdb.L{t.SiteID, GetUser(ctx).ID, t.Name, t.Token, t.Permissions, t.CreatedAt})
+		[]any{t.SiteID, GetUser(ctx).ID, t.Name, t.Token, t.Permissions, t.CreatedAt})
 	return errors.Wrap(err, "APIToken.Insert")
 }
 
@@ -226,7 +226,7 @@ func (t *APITokens) Find(ctx context.Context, ident []string) error {
 		{{:ids api_token_id in (:ids) or}}
 		{{:strs! 0=1}}
 		{{:strs token in (:strs)}}`,
-		zdb.P{"ids": ids, "strs": strs})
+		map[string]any{"ids": ids, "strs": strs})
 	return errors.Wrap(err, "APITokens.Find")
 }
 

cmd/goatcounter/dashboard.go

@@ -292,7 +292,7 @@ func dash(url, key string, rng ztime.Range) error {
 		"%s – %d of %d visits shown", rng.String(), data.hits.Total, data.total.Total),
 		78), zli.Bold))
 	fmt.Printf("┌%s┬%s┐\n", strings.Repeat("─", 48), strings.Repeat("─", 30))
-	m := zint.Max(len(left), len(right))
+	m := max(len(left), len(right))
 	for i := 0; i < m; i++ {
 		l, r := row{}, row{}
 		if i < len(right) {

cmd/goatcounter/db_test.go

@@ -191,18 +191,18 @@ func TestDBSite(t *testing.T) {
 			"-db="+dbc,
 			"-find=1",
 		)
-		wantExit(t, exit, out, 1)
+		wantExit(t, exit, out, 0)
 
 		have := zdb.DumpString(ctx, `select site_id, parent, cname, state from sites order by site_id`) +
 			zdb.DumpString(ctx, `select user_id, site_id, email from users order by user_id`)
 		want := `
 			site_id  parent  cname               state
-			1        NULL    gctest.localhost    a
-			2        1       update.example.com  a
-			3        1       stats2.stats        a
+			1        NULL    NULL                d
+			2        NULL    update.example.com  a
+			3        2       stats2.stats        a
 			user_id  site_id  email
-			1        1        [email protected]
-			2        1        [email protected]`
+			1        2        [email protected]
+			2        2        [email protected]`
 		if d := zdb.Diff(have, want); d != "" {
 			t.Error(d)
 		}
@@ -220,35 +220,12 @@ func TestDBSite(t *testing.T) {
 			zdb.DumpString(ctx, `select user_id, site_id, email from users order by user_id`)
 		want := `
 			site_id  parent  cname               state
-			1        NULL    gctest.localhost    a
-			2        1       update.example.com  a
-			3        1       stats2.stats        d
+			1        NULL    NULL                d
+			2        NULL    update.example.com  a
+			3        2       NULL                d
 			user_id  site_id  email
-			1        1        [email protected]
-			2        1        [email protected]`
-		if d := zdb.Diff(have, want); d != "" {
-			t.Error(d)
-		}
-	}
-
-	{ // Force delete
-		runCmd(t, exit, "db", "delete", "site",
-			"-db="+dbc,
-			"-find=1",
-			"-force",
-		)
-		wantExit(t, exit, out, 0)
-
-		have := zdb.DumpString(ctx, `select site_id, parent, cname, state from sites order by site_id`) +
-			zdb.DumpString(ctx, `select user_id, site_id, email from users order by user_id`)
-		want := `
-			site_id  parent  cname               state
-			1        NULL    gctest.localhost    d
-			2        1       update.example.com  d
-			3        1       stats2.stats        d
-			user_id  site_id  email
-			1        1        [email protected]
-			2        1        [email protected]`
+			1        2        [email protected]
+			2        2        [email protected]`
 		if d := zdb.Diff(have, want); d != "" {
 			t.Error(d)
 		}

cmd/goatcounter/help.go

@@ -237,6 +237,7 @@ commas.
     all            Show debug logs for all of the below.
 
     acme           ACME certificate creation.
+    cli-trace      Show stack traces in errors on the CLI.
     cron           Background "cron" jobs.
     cron-acme      Cron jobs for ACME certificate creations.
     dashboard      Dashboard view.
@@ -245,7 +246,7 @@ commas.
     import-api     Imports from the API.
     memstore       Storing of pageviews in the database.
     monitor        Additional logs in "goatcounter monitor" .
-    cli-trace      Show stack traces in errors on the CLI.
+    session        Internal "session" generation to track visitors .
     startup        Some additional logs during startup.
     vacuum         Deletion of old deleted sites and old pageviews.
 `

cmd/goatcounter/import.go

@@ -42,7 +42,8 @@ Overview:
         $ goatcounter import -site=https://stats.example.com export.csv.gz
 
     You can create an API key with "goatcounter db create apikey -count", or
-    from the web interface in "User → API" from the top-right menu.
+    from the web interface in "[Username in top menu] → API" from the top-right
+    menu.
 
     You must give one filename to import; use - to read from stdin:
 
@@ -309,7 +310,7 @@ func importCSV(fp io.ReadCloser, url, key string, silent bool) error {
 
 			n += len(hits)
 			if !silent {
-				zli.ReplaceLinef("Imported %d rows", n)
+				zli.Replacef("Imported %d rows", n)
 			}
 
 			hits = make([]handlers.APICountRequestHit, 0, 500)
@@ -404,7 +405,7 @@ func importLog(
 			t.Reset(d)
 			persistLog(hits, url, key, silent, follow)
 			if !silent && !follow {
-				zli.ReplaceLinef("Imported %d rows", n)
+				zli.Replacef("Imported %d rows", n)
 			}
 		}
 	}
@@ -429,10 +430,7 @@ func persistLog(hits <-chan handlers.APICountRequestHit, url, key string, silent
 	}
 }
 
-var (
-	importClient = http.Client{Timeout: 5 * time.Second}
-	nSent        int64
-)
+var importClient = http.Client{Timeout: 5 * time.Second}
 
 func importSend(url, key string, silent, follow bool, hits []handlers.APICountRequestHit) error {
 	body, err := json.Marshal(handlers.APICountRequest{NoSessions: true, Hits: hits})

cmd/goatcounter/pg_test.go

@@ -3,7 +3,6 @@
 // in the LICENSE file or at https://license.goatcounter.com
 
 //go:build testpg
-// +build testpg
 
 package main
 

cmd/goatcounter/saas.go

@@ -80,10 +80,10 @@ func cmdSaas(f zli.Flags, ready chan<- struct{}, stop chan struct{}) error {
 		hosts := map[string]http.Handler{
 			d:          zhttp.RedirectHost("//www." + domain),
 			"www." + d: handlers.NewWebsite(db, dev),
-			"*":        handlers.NewBackend(db, acmeh, dev, c.GoatcounterCom, websocket, c.DomainStatic, 15, apiMax),
+			"*":        handlers.NewBackend(db, acmeh, dev, c.GoatcounterCom, websocket, c.DomainStatic, c.BasePath, 15, apiMax),
 		}
 		if dev {
-			hosts[znet.RemovePort(domainStatic)] = handlers.NewStatic(chi.NewRouter(), dev, true)
+			hosts[znet.RemovePort(domainStatic)] = handlers.NewStatic(chi.NewRouter(), dev, true, c.BasePath)
 		}
 
 		return doServe(ctx, db, listen, listenTLS, tlsc, hosts, stop, func() {

cmd/goatcounter/serve.go

@@ -21,6 +21,8 @@ import (
 
 	"github.com/go-chi/chi/v5"
 	"github.com/teamwork/reload"
+	"golang.org/x/net/http2"
+	"golang.org/x/net/http2/h2c"
 	"golang.org/x/text/language"
 	"zgo.at/bgrun"
 	"zgo.at/blackmail"
@@ -79,6 +81,12 @@ Flags:
   -public-port Port your site is publicly accessible on. Only needed if it's
                not 80 or 443.
 
+  -base-path   Path under which GoatCounter is available. Usually GoatCounter
+               runs on its own domain or subdomain ("stats.example.com"), but in
+               some cases it's useful to run GoatCounter under a path
+               ("example.com/stats"), in which case you'll need to set this to
+               "/stats".
+
   -automigrate Automatically run all pending migrations on startup.
 
   -smtp        SMTP relay server, as URL (e.g. "smtp://user:pass@server").
@@ -162,18 +170,25 @@ func cmdServe(f zli.Flags, ready chan<- struct{}, stop chan struct{}) error {
 	var (
 		// TODO(depr): -port is for compat with <2.0
 		port         = f.Int(0, "public-port", "port").Pointer()
+		basePath     = f.String("/", "base-path").Pointer()
 		domainStatic = f.String("", "static").Pointer()
 	)
 	dbConnect, dbConn, dev, automigrate, listen, flagTLS, from, websocket, apiMax, err := flagsServe(f, &v)
 	if err != nil {
 		return err
 	}
 
-	return func(port int, domainStatic string) error {
+	return func(port int, basePath, domainStatic string) error {
 		if flagTLS == "" {
 			flagTLS = map[bool]string{true: "http", false: "acme,rdr"}[dev]
 		}
 
+		basePath = strings.Trim(basePath, "/")
+		if basePath != "" {
+			basePath = "/" + basePath
+		}
+		zhttp.BasePath = basePath
+
 		var domainCount, urlStatic string
 		if domainStatic != "" {
 			if p := strings.Index(domainStatic, ":"); p > -1 {
@@ -183,6 +198,8 @@ func cmdServe(f zli.Flags, ready chan<- struct{}, stop chan struct{}) error {
 			}
 			urlStatic = "//" + domainStatic
 			domainCount = domainStatic
+		} else {
+			urlStatic = basePath
 		}
 
 		//from := flagFrom(from, "cfg.Domain", &v)
@@ -204,17 +221,18 @@ func cmdServe(f zli.Flags, ready chan<- struct{}, stop chan struct{}) error {
 		c.DomainStatic = domainStatic
 		c.Dev = dev
 		c.URLStatic = urlStatic
+		c.BasePath = basePath
 		c.DomainCount = domainCount
 		c.Websocket = websocket
 
 		// Set up HTTP handler and servers.
 		hosts := map[string]http.Handler{
-			"*": handlers.NewBackend(db, acmeh, dev, c.GoatcounterCom, websocket, c.DomainStatic, 60, apiMax),
+			"*": handlers.NewBackend(db, acmeh, dev, c.GoatcounterCom, websocket, c.DomainStatic, c.BasePath, 60, apiMax),
 		}
 		if domainStatic != "" {
 			// May not be needed, but just in case the DomainStatic isn't an
 			// external CDN.
-			hosts[znet.RemovePort(domainStatic)] = handlers.NewStatic(chi.NewRouter(), dev, false)
+			hosts[znet.RemovePort(domainStatic)] = handlers.NewStatic(chi.NewRouter(), dev, false, c.BasePath)
 		}
 
 		cnames, err := lsSites(ctx)
@@ -236,7 +254,7 @@ func cmdServe(f zli.Flags, ready chan<- struct{}, stop chan struct{}) error {
 			}
 			ready <- struct{}{}
 		})
-	}(*port, *domainStatic)
+	}(*port, *basePath, *domainStatic)
 }
 
 func doServe(ctx context.Context, db zdb.DB,
@@ -248,7 +266,7 @@ func doServe(ctx context.Context, db zdb.DB,
 	zlog.Module("startup").Debug(getVersion())
 	ch, err := zhttp.Serve(listenTLS, stop, &http.Server{
 		Addr:        listen,
-		Handler:     zhttp.HostRoute(hosts),
+		Handler:     h2c.NewHandler(zhttp.HostRoute(hosts), &http2.Server{}),
 		TLSConfig:   tlsc,
 		BaseContext: func(net.Listener) context.Context { return ctx },
 	})
@@ -287,16 +305,16 @@ func doServe(ctx context.Context, db zdb.DB,
 		}
 		time.Sleep(100 * time.Millisecond)
 
-		zli.EraseLine()
-		fmt.Fprintf(zli.Stdout, "%d tasks: ", len(r))
+		zli.Erase()
+		fmt.Fprintf(zli.Stdout, "\r%d tasks: ", len(r))
 		for i, t := range r {
 			if i > 0 {
 				fmt.Fprint(zli.Stdout, ", ")
 			}
 			fmt.Fprintf(zli.Stdout, "%s (%s)", t.Task, time.Since(t.Started).Round(time.Second))
 		}
 	}
-
+	fmt.Fprintln(zli.Stdout)
 	db.Close()
 	return nil
 }
@@ -498,7 +516,7 @@ func flagErrors(errors string, v *zvalidate.Validator) {
 					subject = subject[i+7:]
 				}
 
-				err := blackmail.Send(fmt.Sprintf(subject),
+				err := blackmail.Send(subject,
 					blackmail.From("", from),
 					blackmail.To(to),
 					blackmail.BodyText([]byte(msg)))