From c5f251b83f98b75614b34208e19fb46c5539335e Mon Sep 17 00:00:00 2001 From: junoberryferry Date: Sun, 31 Aug 2025 21:32:19 +0000 Subject: [PATCH 01/24] use experimental go json v2 library --- Makefile | 3 ++ go.mod | 2 +- modules/json/json.go | 54 ++++++++++++++++++++++++- modules/json/jsonv2.go | 72 +++++++++++++++++++++++++++++++++ modules/json/jsonv2_fallback.go | 33 +++++++++++++++ 5 files changed, 161 insertions(+), 3 deletions(-) create mode 100644 modules/json/jsonv2.go create mode 100644 modules/json/jsonv2_fallback.go diff --git a/Makefile b/Makefile index 9cd32e4c33f9d..9815bfefed1f2 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,9 @@ DIST := dist DIST_DIRS := $(DIST)/binaries $(DIST)/release IMPORT := code.gitea.io/gitea +# No experiment set by default, but you can set jsonv2 to use go 1.25.0 json v2 experimental changes +export GOEXPERIMENT ?= + GO ?= go SHASUM ?= shasum -a 256 HAS_GO := $(shell hash $(GO) > /dev/null 2>&1 && echo yes) diff --git a/go.mod b/go.mod index 164702c90e5f5..17e6d86eb3cf8 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module code.gitea.io/gitea -go 1.24.6 +go 1.25 // rfc5280 said: "The serial number is an integer assigned by the CA to each certificate." // But some CAs use negative serial number, just relax the check. related: diff --git a/modules/json/json.go b/modules/json/json.go index 444dc8526aeab..185897648ebb7 100644 --- a/modules/json/json.go +++ b/modules/json/json.go @@ -32,13 +32,22 @@ type Interface interface { } var ( - // DefaultJSONHandler default json handler - DefaultJSONHandler Interface = JSONiter{jsoniter.ConfigCompatibleWithStandardLibrary} + // DefaultJSONHandler default json handler - uses JSON v2 if available, otherwise JSONiter + DefaultJSONHandler = getDefaultHandler() _ Interface = StdJSON{} _ Interface = JSONiter{} + _ Interface = JSONv2{} ) +// getDefaultHandler returns the expected JSON implementation +func getDefaultHandler() Interface { + if isJSONv2Available() { + return JSONv2{} + } + return JSONiter{jsoniter.ConfigCompatibleWithStandardLibrary} +} + // StdJSON implements Interface via encoding/json type StdJSON struct{} @@ -97,6 +106,47 @@ func (j JSONiter) Indent(dst *bytes.Buffer, src []byte, prefix, indent string) e return json.Indent(dst, src, prefix, indent) } +// JSONv2 implements Interface via encoding/json/v2 +// Requires GOEXPERIMENT=jsonv2 to be set at build time +type JSONv2 struct{} + +// Marshal implements Interface using JSON v2 - fallback if v2 is not available +func (JSONv2) Marshal(v any) ([]byte, error) { + if !isJSONv2Available() { + return json.Marshal(v) + } + return marshalV2(v) +} + +// Unmarshal implements Interface using JSON v2 - fallback if v2 is not available +func (JSONv2) Unmarshal(data []byte, v any) error { + if !isJSONv2Available() { + return json.Unmarshal(data, v) + } + return unmarshalV2(data, v) +} + +// NewEncoder implements Interface using JSON v2 - fallback if v2 is not available +func (JSONv2) NewEncoder(writer io.Writer) Encoder { + if !isJSONv2Available() { + return json.NewEncoder(writer) + } + return newEncoderV2(writer) +} + +// NewDecoder implements Interface using JSON v2 - fallback if v2 is not available +func (JSONv2) NewDecoder(reader io.Reader) Decoder { + if !isJSONv2Available() { + return json.NewDecoder(reader) + } + return newDecoderV2(reader) +} + +// Indent implements Interface using standard library (JSON v2 doesn't have Indent yet) +func (JSONv2) Indent(dst *bytes.Buffer, src []byte, prefix, indent string) error { + return json.Indent(dst, src, prefix, indent) +} + // Marshal converts object as bytes func Marshal(v any) ([]byte, error) { return DefaultJSONHandler.Marshal(v) diff --git a/modules/json/jsonv2.go b/modules/json/jsonv2.go new file mode 100644 index 0000000000000..93b618a26e425 --- /dev/null +++ b/modules/json/jsonv2.go @@ -0,0 +1,72 @@ +//go:build goexperiment.jsonv2 + +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package json + +import ( + jsonv2 "encoding/json/v2" + "io" +) + +// isJSONv2Available returns true when JSON v2 is available (compiled with GOEXPERIMENT=jsonv2) +func isJSONv2Available() bool { + return true +} + +// marshalV2 uses JSON v2 marshal with v1 compatibility options +func marshalV2(v any) ([]byte, error) { + opts := jsonv2.JoinOptions( + jsonv2.MatchCaseInsensitiveNames(true), + jsonv2.FormatNilSliceAsNull(true), + jsonv2.FormatNilMapAsNull(true), + ) + return jsonv2.Marshal(v, opts) +} + +// unmarshalV2 uses JSON v2 unmarshal with v1 compatibility options +func unmarshalV2(data []byte, v any) error { + opts := jsonv2.JoinOptions( + jsonv2.MatchCaseInsensitiveNames(true), + ) + return jsonv2.Unmarshal(data, v, opts) +} + +// encoderV2 wraps JSON v2 streaming encoder +type encoderV2 struct { + writer io.Writer + opts jsonv2.Options +} + +func (e *encoderV2) Encode(v any) error { + return jsonv2.MarshalWrite(e.writer, v, e.opts) +} + +// newEncoderV2 creates a new JSON v2 streaming encoder +func newEncoderV2(writer io.Writer) Encoder { + opts := jsonv2.JoinOptions( + jsonv2.MatchCaseInsensitiveNames(true), + jsonv2.FormatNilSliceAsNull(true), + jsonv2.FormatNilMapAsNull(true), + ) + return &encoderV2{writer: writer, opts: opts} +} + +// decoderV2 wraps JSON v2 streaming decoder +type decoderV2 struct { + reader io.Reader + opts jsonv2.Options +} + +func (d *decoderV2) Decode(v any) error { + return jsonv2.UnmarshalRead(d.reader, v, d.opts) +} + +// newDecoderV2 creates a new JSON v2 streaming decoder +func newDecoderV2(reader io.Reader) Decoder { + opts := jsonv2.JoinOptions( + jsonv2.MatchCaseInsensitiveNames(true), + ) + return &decoderV2{reader: reader, opts: opts} +} diff --git a/modules/json/jsonv2_fallback.go b/modules/json/jsonv2_fallback.go new file mode 100644 index 0000000000000..326d2f8b4652c --- /dev/null +++ b/modules/json/jsonv2_fallback.go @@ -0,0 +1,33 @@ +//go:build !goexperiment.jsonv2 + +// Copyright 2025 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package json + +import "io" + +// isJSONv2Available returns false when JSON v2 is not available (not compiled with GOEXPERIMENT=jsonv2) +func isJSONv2Available() bool { + return false +} + +// marshalV2 fallback - should not be called when JSON v2 is not available +func marshalV2(v any) ([]byte, error) { + panic("JSON v2 not available - build with GOEXPERIMENT=jsonv2") +} + +// unmarshalV2 fallback - should not be called when JSON v2 is not available +func unmarshalV2(data []byte, v any) error { + panic("JSON v2 not available - build with GOEXPERIMENT=jsonv2") +} + +// newEncoderV2 fallback - should not be called when JSON v2 is not available +func newEncoderV2(writer io.Writer) Encoder { + panic("JSON v2 not available - build with GOEXPERIMENT=jsonv2") +} + +// newDecoderV2 fallback - should not be called when JSON v2 is not available +func newDecoderV2(reader io.Reader) Decoder { + panic("JSON v2 not available - build with GOEXPERIMENT=jsonv2") +} From abc734cb13572d4ae287a81a623715c043401499 Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Tue, 2 Sep 2025 19:50:37 -0400 Subject: [PATCH 02/24] Update go.mod Signed-off-by: techknowlogick --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 4cc2e8a925acf..38bd1e9b722b2 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module code.gitea.io/gitea -go 1.25 +go 1.24.6 // rfc5280 said: "The serial number is an integer assigned by the CA to each certificate." // But some CAs use negative serial number, just relax the check. related: From f9c880570816d5549cd86925626665a6742f163b Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Tue, 2 Sep 2025 19:53:20 -0400 Subject: [PATCH 03/24] Update jsonv2.go Signed-off-by: techknowlogick --- modules/json/jsonv2.go | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/json/jsonv2.go b/modules/json/jsonv2.go index 93b618a26e425..5ad7f8db1c936 100644 --- a/modules/json/jsonv2.go +++ b/modules/json/jsonv2.go @@ -1,4 +1,3 @@ -//go:build goexperiment.jsonv2 // Copyright 2025 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT From 9d83440d597ed498d3cbc72e9a4df5424ba60b07 Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Tue, 2 Sep 2025 19:54:22 -0400 Subject: [PATCH 04/24] Update jsonv2.go Signed-off-by: techknowlogick --- modules/json/jsonv2.go | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/json/jsonv2.go b/modules/json/jsonv2.go index 5ad7f8db1c936..6178ba79ff56b 100644 --- a/modules/json/jsonv2.go +++ b/modules/json/jsonv2.go @@ -1,4 +1,3 @@ - // Copyright 2025 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT From 31492c12fd310c8bf33825310bac83d87939efea Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Tue, 2 Sep 2025 19:55:20 -0400 Subject: [PATCH 05/24] Update jsonv2.go Signed-off-by: techknowlogick --- modules/json/jsonv2.go | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/json/jsonv2.go b/modules/json/jsonv2.go index 6178ba79ff56b..99d8e7f67438d 100644 --- a/modules/json/jsonv2.go +++ b/modules/json/jsonv2.go @@ -1,6 +1,7 @@ // Copyright 2025 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT +//go:build goexperiment.jsonv2 package json import ( From 7b0832cd2f39d06a6f53779b688f40bd6df5e7a4 Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Tue, 2 Sep 2025 19:56:15 -0400 Subject: [PATCH 06/24] Update jsonv2.go Signed-off-by: techknowlogick --- modules/json/jsonv2.go | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/json/jsonv2.go b/modules/json/jsonv2.go index 99d8e7f67438d..e3faedda83a6a 100644 --- a/modules/json/jsonv2.go +++ b/modules/json/jsonv2.go @@ -2,6 +2,7 @@ // SPDX-License-Identifier: MIT //go:build goexperiment.jsonv2 + package json import ( From fe51938ef7293cad6fdfcdc181ad634fbfb63fcd Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Tue, 2 Sep 2025 19:57:43 -0400 Subject: [PATCH 07/24] Update jsonv2_fallback.go Signed-off-by: techknowlogick --- modules/json/jsonv2_fallback.go | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/json/jsonv2_fallback.go b/modules/json/jsonv2_fallback.go index 326d2f8b4652c..d46f49bf4fbab 100644 --- a/modules/json/jsonv2_fallback.go +++ b/modules/json/jsonv2_fallback.go @@ -3,6 +3,7 @@ // Copyright 2025 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT +//go:build goexperiment.jsonv2 package json import "io" From 29d17705fb595ca6061084499d6ba63285e63f92 Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Tue, 2 Sep 2025 19:57:50 -0400 Subject: [PATCH 08/24] Update jsonv2_fallback.go Signed-off-by: techknowlogick --- modules/json/jsonv2_fallback.go | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/json/jsonv2_fallback.go b/modules/json/jsonv2_fallback.go index d46f49bf4fbab..b6ca8af1a662f 100644 --- a/modules/json/jsonv2_fallback.go +++ b/modules/json/jsonv2_fallback.go @@ -1,5 +1,4 @@ //go:build !goexperiment.jsonv2 - // Copyright 2025 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT From 3c3f77fe8e568f41b582f846a494d2b2b5301504 Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Tue, 2 Sep 2025 19:57:57 -0400 Subject: [PATCH 09/24] Update jsonv2_fallback.go Signed-off-by: techknowlogick --- modules/json/jsonv2_fallback.go | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/json/jsonv2_fallback.go b/modules/json/jsonv2_fallback.go index b6ca8af1a662f..73bbb957ab073 100644 --- a/modules/json/jsonv2_fallback.go +++ b/modules/json/jsonv2_fallback.go @@ -1,4 +1,3 @@ -//go:build !goexperiment.jsonv2 // Copyright 2025 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT From b8fb94decae58c2340851d2ddbc6483805bd84ae Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Tue, 2 Sep 2025 20:01:20 -0400 Subject: [PATCH 10/24] Update jsonv2_fallback.go Signed-off-by: techknowlogick --- modules/json/jsonv2_fallback.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/json/jsonv2_fallback.go b/modules/json/jsonv2_fallback.go index 73bbb957ab073..985300dbd73b9 100644 --- a/modules/json/jsonv2_fallback.go +++ b/modules/json/jsonv2_fallback.go @@ -1,7 +1,7 @@ // Copyright 2025 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -//go:build goexperiment.jsonv2 +//go:build !goexperiment.jsonv2 package json import "io" From c748ca8cda652b043860cb0a304b4e50a7b54690 Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Tue, 2 Sep 2025 20:12:09 -0400 Subject: [PATCH 11/24] Update jsonv2_fallback.go Signed-off-by: techknowlogick --- modules/json/jsonv2_fallback.go | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/json/jsonv2_fallback.go b/modules/json/jsonv2_fallback.go index 985300dbd73b9..e78893ed45b88 100644 --- a/modules/json/jsonv2_fallback.go +++ b/modules/json/jsonv2_fallback.go @@ -2,6 +2,7 @@ // SPDX-License-Identifier: MIT //go:build !goexperiment.jsonv2 + package json import "io" From 7fec8381119f364ad9792b7284ae05c0585cfd17 Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Thu, 4 Sep 2025 11:08:25 -0400 Subject: [PATCH 12/24] Update Go version from 1.24.6 to 1.25.0 Signed-off-by: techknowlogick --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index cadb23f069644..09b150e3e5dde 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module code.gitea.io/gitea -go 1.24.6 +go 1.25.0 // rfc5280 said: "The serial number is an integer assigned by the CA to each certificate." // But some CAs use negative serial number, just relax the check. related: From 3c6d690dadb8667b900a437dbef82133ac4b7c88 Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Thu, 4 Sep 2025 11:56:18 -0400 Subject: [PATCH 13/24] bump go.mod --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 09b150e3e5dde..3661d051a9b29 100644 --- a/go.mod +++ b/go.mod @@ -278,7 +278,7 @@ require ( go.uber.org/zap v1.27.0 // indirect go.uber.org/zap/exp v0.3.0 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect - golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 // indirect + golang.org/x/exp v0.0.0-20250819193227-8b4c13bb791b // indirect golang.org/x/mod v0.27.0 // indirect golang.org/x/time v0.12.0 // indirect golang.org/x/tools v0.36.0 // indirect diff --git a/go.sum b/go.sum index b69b380cfe2ab..3e59115f04de2 100644 --- a/go.sum +++ b/go.sum @@ -850,8 +850,8 @@ golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 h1:nDVHiLt8aIbd/VzvPWN6kSOPE7+F/fNFDSXLVYkE/Iw= -golang.org/x/exp v0.0.0-20250305212735-054e65f0b394/go.mod h1:sIifuuw/Yco/y6yb6+bDNfyeQ/MdPUy/hKEMYQV17cM= +golang.org/x/exp v0.0.0-20250819193227-8b4c13bb791b h1:DXr+pvt3nC887026GRP39Ej11UATqWDmWuS99x26cD0= +golang.org/x/exp v0.0.0-20250819193227-8b4c13bb791b/go.mod h1:4QTo5u+SEIbbKW1RacMZq1YEfOBqeXa19JeshGi+zc4= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.30.0 h1:jD5RhkmVAnjqaCUXfbGBrn3lpxbknfN9w2UhHHU+5B4= From b3969becbf2c2a78a89b903dc8bb49ea67d06323 Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Thu, 4 Sep 2025 11:57:39 -0400 Subject: [PATCH 14/24] use fixed go-swagger version --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 4b089821e1377..5651a01af470e 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,7 @@ GOFUMPT_PACKAGE ?= mvdan.cc/gofumpt@v0.8.0 GOLANGCI_LINT_PACKAGE ?= github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.4.0 GXZ_PACKAGE ?= github.com/ulikunitz/xz/cmd/gxz@v0.5.15 MISSPELL_PACKAGE ?= github.com/golangci/misspell/cmd/misspell@v0.7.0 -SWAGGER_PACKAGE ?= github.com/go-swagger/go-swagger/cmd/swagger@v0.32.3 +SWAGGER_PACKAGE ?= github.com/go-swagger/go-swagger/cmd/swagger@717e3cb29becaaf00e56953556c6d80f8a01b286 XGO_PACKAGE ?= src.techknowlogick.com/xgo@latest GO_LICENSES_PACKAGE ?= github.com/google/go-licenses@v1 GOVULNCHECK_PACKAGE ?= golang.org/x/vuln/cmd/govulncheck@v1 From 94d537c8450392bc6a77ce0c0d2627f6908d69ae Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Thu, 4 Sep 2025 12:06:06 -0400 Subject: [PATCH 15/24] Update GOEXPERIMENT to use jsonv2 by default Signed-off-by: techknowlogick --- Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 5651a01af470e..d5136db68fafb 100644 --- a/Makefile +++ b/Makefile @@ -18,8 +18,9 @@ DIST := dist DIST_DIRS := $(DIST)/binaries $(DIST)/release IMPORT := code.gitea.io/gitea -# No experiment set by default, but you can set jsonv2 to use go 1.25.0 json v2 experimental changes -export GOEXPERIMENT ?= +# By default use go's 1.25 experimental json v2 library when building +# TODO: remove when no longer experimental +export GOEXPERIMENT ?= jsonv2 GO ?= go SHASUM ?= shasum -a 256 From 82b2d977e97456d1285b911d79f4364743ddafc8 Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Thu, 4 Sep 2025 12:26:35 -0400 Subject: [PATCH 16/24] fix lint --- modules/globallock/locker_test.go | 6 ++---- modules/json/jsonv2.go | 2 +- modules/log/event_writer_conn_test.go | 6 ++---- modules/queue/workergroup.go | 6 ++---- 4 files changed, 7 insertions(+), 13 deletions(-) diff --git a/modules/globallock/locker_test.go b/modules/globallock/locker_test.go index c9e73c25d2e5b..14cb0ec388898 100644 --- a/modules/globallock/locker_test.go +++ b/modules/globallock/locker_test.go @@ -105,15 +105,13 @@ func testLocker(t *testing.T, locker Locker) { require.NoError(t, err) wg := &sync.WaitGroup{} - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { started := time.Now() release, err := locker.Lock(t.Context(), "test") // should be blocked for seconds defer release() assert.Greater(t, time.Since(started), time.Second) assert.NoError(t, err) - }() + }) time.Sleep(2 * time.Second) release() diff --git a/modules/json/jsonv2.go b/modules/json/jsonv2.go index e3faedda83a6a..73ec03523b16c 100644 --- a/modules/json/jsonv2.go +++ b/modules/json/jsonv2.go @@ -6,7 +6,7 @@ package json import ( - jsonv2 "encoding/json/v2" + jsonv2 "encoding/json/v2" //nolint:depguard // this package wraps it "io" ) diff --git a/modules/log/event_writer_conn_test.go b/modules/log/event_writer_conn_test.go index 2aff37812d639..e7011da79cb8b 100644 --- a/modules/log/event_writer_conn_test.go +++ b/modules/log/event_writer_conn_test.go @@ -62,11 +62,9 @@ func TestConnLogger(t *testing.T) { } expected := fmt.Sprintf("%s%s %s:%d:%s [%c] %s\n", prefix, dateString, event.Filename, event.Line, event.Caller, strings.ToUpper(event.Level.String())[0], event.MsgSimpleText) var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { listenReadAndClose(t, l, expected) - }() + }) logger.SendLogEvent(&event) wg.Wait() diff --git a/modules/queue/workergroup.go b/modules/queue/workergroup.go index 82b0790d5a9c2..3910179a6bb42 100644 --- a/modules/queue/workergroup.go +++ b/modules/queue/workergroup.go @@ -153,10 +153,8 @@ func resetIdleTicker(t *time.Ticker, dur time.Duration) { // doStartNewWorker starts a new worker for the queue, the worker reads from worker's channel and handles the items. func (q *WorkerPoolQueue[T]) doStartNewWorker(wp *workerGroup[T]) { - wp.wg.Add(1) - go func() { - defer wp.wg.Done() + &{wp wg}.Go(func() { log.Debug("Queue %q starts new worker", q.GetName()) defer log.Debug("Queue %q stops idle worker", q.GetName()) @@ -192,7 +190,7 @@ func (q *WorkerPoolQueue[T]) doStartNewWorker(wp *workerGroup[T]) { q.workerNumMu.Unlock() } } - }() + }) } // doFlush flushes the queue: it tries to read all items from the queue and handles them. From 502b4fb4809f0affc899e1aab6d726b87bea280f Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Thu, 4 Sep 2025 12:39:28 -0400 Subject: [PATCH 17/24] clean up modernizer fixes --- modules/queue/workergroup.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/modules/queue/workergroup.go b/modules/queue/workergroup.go index 3910179a6bb42..e76c1ebc1e2a9 100644 --- a/modules/queue/workergroup.go +++ b/modules/queue/workergroup.go @@ -153,9 +153,7 @@ func resetIdleTicker(t *time.Ticker, dur time.Duration) { // doStartNewWorker starts a new worker for the queue, the worker reads from worker's channel and handles the items. func (q *WorkerPoolQueue[T]) doStartNewWorker(wp *workerGroup[T]) { - - &{wp wg}.Go(func() { - + &wp.wg.Go(func() { log.Debug("Queue %q starts new worker", q.GetName()) defer log.Debug("Queue %q stops idle worker", q.GetName()) From d4b1e10f3195dde2b39cfd584865cd83757c3561 Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Thu, 4 Sep 2025 12:50:11 -0400 Subject: [PATCH 18/24] try to fix lint --- modules/queue/workergroup.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/queue/workergroup.go b/modules/queue/workergroup.go index e76c1ebc1e2a9..76435ba32ad42 100644 --- a/modules/queue/workergroup.go +++ b/modules/queue/workergroup.go @@ -153,7 +153,9 @@ func resetIdleTicker(t *time.Ticker, dur time.Duration) { // doStartNewWorker starts a new worker for the queue, the worker reads from worker's channel and handles the items. func (q *WorkerPoolQueue[T]) doStartNewWorker(wp *workerGroup[T]) { - &wp.wg.Go(func() { + wp.wg.Add(1) + go func() { + defer wp.wg.Done() log.Debug("Queue %q starts new worker", q.GetName()) defer log.Debug("Queue %q stops idle worker", q.GetName()) @@ -188,7 +190,7 @@ func (q *WorkerPoolQueue[T]) doStartNewWorker(wp *workerGroup[T]) { q.workerNumMu.Unlock() } } - }) + }() } // doFlush flushes the queue: it tries to read all items from the queue and handles them. From 8f3b21892e24e01c62101b95b5c1e2a8b79124f5 Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Thu, 4 Sep 2025 13:23:51 -0400 Subject: [PATCH 19/24] try to use go1.25 waitgroup logic --- modules/queue/workergroup.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/modules/queue/workergroup.go b/modules/queue/workergroup.go index 76435ba32ad42..c7e33497c6416 100644 --- a/modules/queue/workergroup.go +++ b/modules/queue/workergroup.go @@ -153,9 +153,7 @@ func resetIdleTicker(t *time.Ticker, dur time.Duration) { // doStartNewWorker starts a new worker for the queue, the worker reads from worker's channel and handles the items. func (q *WorkerPoolQueue[T]) doStartNewWorker(wp *workerGroup[T]) { - wp.wg.Add(1) - go func() { - defer wp.wg.Done() + wp.wg.Go(func() { log.Debug("Queue %q starts new worker", q.GetName()) defer log.Debug("Queue %q stops idle worker", q.GetName()) @@ -190,7 +188,7 @@ func (q *WorkerPoolQueue[T]) doStartNewWorker(wp *workerGroup[T]) { q.workerNumMu.Unlock() } } - }() + }) } // doFlush flushes the queue: it tries to read all items from the queue and handles them. From 34af20c731d6a06fec8c85037d97e94e781da673 Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Thu, 4 Sep 2025 13:50:35 -0400 Subject: [PATCH 20/24] fixup test fails --- modules/json/json.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/json/json.go b/modules/json/json.go index 185897648ebb7..656a1060e0eaf 100644 --- a/modules/json/json.go +++ b/modules/json/json.go @@ -200,7 +200,7 @@ func UnmarshalHandleDoubleEncode(bs []byte, v any) error { // To be consistent, we should treat all empty inputs as success return nil } - err := json.Unmarshal(bs, v) + err := DefaultJSONHandler.Unmarshal(bs, v) if err != nil { ok := true rs := []byte{} @@ -217,11 +217,11 @@ func UnmarshalHandleDoubleEncode(bs []byte, v any) error { if len(rs) > 1 && rs[0] == 0xff && rs[1] == 0xfe { rs = rs[2:] } - err = json.Unmarshal(rs, v) + err = DefaultJSONHandler.Unmarshal(rs, v) } } if err != nil && len(bs) > 2 && bs[0] == 0xff && bs[1] == 0xfe { - err = json.Unmarshal(bs[2:], v) + err = DefaultJSONHandler.Unmarshal(bs[2:], v) } return err } From cdcb9bda2efb2d6261ed41adcecf8209187a1eba Mon Sep 17 00:00:00 2001 From: junoberryferry Date: Wed, 10 Sep 2025 23:37:27 +0000 Subject: [PATCH 21/24] adjust jsonv2 output to become similar to v1 --- models/webhook/webhook.go | 9 ++++++- modules/json/json.go | 8 ++++++- modules/json/jsonv2.go | 41 ++++++++++++++++++++++++++++---- services/webhook/deliver_test.go | 4 ++-- services/webhook/matrix_test.go | 2 +- 5 files changed, 55 insertions(+), 9 deletions(-) diff --git a/models/webhook/webhook.go b/models/webhook/webhook.go index 7d4b2e2237db0..794608e970b39 100644 --- a/models/webhook/webhook.go +++ b/models/webhook/webhook.go @@ -150,7 +150,14 @@ func init() { // AfterLoad updates the webhook object upon setting a column func (w *Webhook) AfterLoad() { w.HookEvent = &webhook_module.HookEvent{} - if err := json.Unmarshal([]byte(w.Events), w.HookEvent); err != nil { + + events := w.Events + if events == "" { + // jsonv2 is unable to unmarshal an empty string + return + } + + if err := json.Unmarshal([]byte(events), w.HookEvent); err != nil { log.Error("Unmarshal[%d]: %v", w.ID, err) } } diff --git a/modules/json/json.go b/modules/json/json.go index 656a1060e0eaf..c736e5b7232bf 100644 --- a/modules/json/json.go +++ b/modules/json/json.go @@ -174,7 +174,7 @@ func Indent(dst *bytes.Buffer, src []byte, prefix, indent string) error { // MarshalIndent copied from encoding/json func MarshalIndent(v any, prefix, indent string) ([]byte, error) { - b, err := Marshal(v) + b, err := DefaultJSONHandler.Marshal(v) if err != nil { return nil, err } @@ -200,6 +200,12 @@ func UnmarshalHandleDoubleEncode(bs []byte, v any) error { // To be consistent, we should treat all empty inputs as success return nil } + + trimmed := bytes.TrimSpace(bs) + if len(trimmed) == 0 { + return nil + } + err := DefaultJSONHandler.Unmarshal(bs, v) if err != nil { ok := true diff --git a/modules/json/jsonv2.go b/modules/json/jsonv2.go index 73ec03523b16c..9b41af2e34cb6 100644 --- a/modules/json/jsonv2.go +++ b/modules/json/jsonv2.go @@ -6,6 +6,7 @@ package json import ( + "bytes" jsonv2 "encoding/json/v2" //nolint:depguard // this package wraps it "io" ) @@ -15,18 +16,38 @@ func isJSONv2Available() bool { return true } -// marshalV2 uses JSON v2 marshal with v1 compatibility options -func marshalV2(v any) ([]byte, error) { +// marshalV2Internal uses JSON v2 marshal with v1 compatibility options (no trailing newline) +func marshalV2Internal(v any) ([]byte, error) { opts := jsonv2.JoinOptions( jsonv2.MatchCaseInsensitiveNames(true), jsonv2.FormatNilSliceAsNull(true), jsonv2.FormatNilMapAsNull(true), + jsonv2.Deterministic(true), ) return jsonv2.Marshal(v, opts) } +// marshalV2 uses JSON v2 marshal with v1 compatibility options (with trailing newline for compatibility with standard library) +func marshalV2(v any) ([]byte, error) { + result, err := marshalV2Internal(v) + if err != nil { + return nil, err + } + + return append(result, '\n'), nil +} + // unmarshalV2 uses JSON v2 unmarshal with v1 compatibility options func unmarshalV2(data []byte, v any) error { + if len(data) == 0 { + return nil + } + + data = bytes.TrimSpace(data) + if len(data) == 0 { + return nil + } + opts := jsonv2.JoinOptions( jsonv2.MatchCaseInsensitiveNames(true), ) @@ -40,7 +61,13 @@ type encoderV2 struct { } func (e *encoderV2) Encode(v any) error { - return jsonv2.MarshalWrite(e.writer, v, e.opts) + err := jsonv2.MarshalWrite(e.writer, v, e.opts) + if err != nil { + return err + } + + _, err = e.writer.Write([]byte{'\n'}) + return err } // newEncoderV2 creates a new JSON v2 streaming encoder @@ -49,6 +76,7 @@ func newEncoderV2(writer io.Writer) Encoder { jsonv2.MatchCaseInsensitiveNames(true), jsonv2.FormatNilSliceAsNull(true), jsonv2.FormatNilMapAsNull(true), + jsonv2.Deterministic(true), ) return &encoderV2{writer: writer, opts: opts} } @@ -60,7 +88,12 @@ type decoderV2 struct { } func (d *decoderV2) Decode(v any) error { - return jsonv2.UnmarshalRead(d.reader, v, d.opts) + err := jsonv2.UnmarshalRead(d.reader, v, d.opts) + // Handle EOF more gracefully to match standard library behavior + if err != nil && err.Error() == "unexpected EOF" { + return io.EOF + } + return err } // newDecoderV2 creates a new JSON v2 streaming decoder diff --git a/services/webhook/deliver_test.go b/services/webhook/deliver_test.go index efbef1fc89444..b0e83175747fb 100644 --- a/services/webhook/deliver_test.go +++ b/services/webhook/deliver_test.go @@ -142,13 +142,13 @@ func TestWebhookDeliverHookTask(t *testing.T) { assert.NoError(t, err) assert.Equal(t, `{"data": 42}`, string(body)) - case "/webhook/6db5dc1e282529a8c162c7fe93dd2667494eeb51": + case "/webhook/4ddf3b1533e54f082ae6eadfc1b5530be36c8893": // Version 2 assert.Equal(t, "push", r.Header.Get("X-GitHub-Event")) assert.Equal(t, "application/json", r.Header.Get("Content-Type")) body, err := io.ReadAll(r.Body) assert.NoError(t, err) - assert.Len(t, body, 2147) + assert.Len(t, body, 2047) default: w.WriteHeader(http.StatusNotFound) diff --git a/services/webhook/matrix_test.go b/services/webhook/matrix_test.go index d36d93c5a7385..c39867b4816f0 100644 --- a/services/webhook/matrix_test.go +++ b/services/webhook/matrix_test.go @@ -216,7 +216,7 @@ func TestMatrixJSONPayload(t *testing.T) { require.NoError(t, err) assert.Equal(t, "PUT", req.Method) - assert.Equal(t, "/_matrix/client/r0/rooms/ROOM_ID/send/m.room.message/6db5dc1e282529a8c162c7fe93dd2667494eeb51", req.URL.Path) + assert.Equal(t, "/_matrix/client/r0/rooms/ROOM_ID/send/m.room.message/4ddf3b1533e54f082ae6eadfc1b5530be36c8893", req.URL.Path) assert.Equal(t, "sha256=", req.Header.Get("X-Hub-Signature-256")) assert.Equal(t, "application/json", req.Header.Get("Content-Type")) var body MatrixPayload From ecc304aa53293749f6beb156c32ac593a8f9af63 Mon Sep 17 00:00:00 2001 From: junoberryferry Date: Thu, 11 Sep 2025 00:08:26 +0000 Subject: [PATCH 22/24] resolve vagrant test failure --- tests/integration/api_packages_vagrant_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/api_packages_vagrant_test.go b/tests/integration/api_packages_vagrant_test.go index 22412a8558e62..83e1431dd82c9 100644 --- a/tests/integration/api_packages_vagrant_test.go +++ b/tests/integration/api_packages_vagrant_test.go @@ -165,6 +165,6 @@ func TestPackageVagrant(t *testing.T) { provider := version.Providers[0] assert.Equal(t, packageProvider, provider.Name) assert.Equal(t, "sha512", provider.ChecksumType) - assert.Equal(t, "259bebd6160acad695016d22a45812e26f187aaf78e71a4c23ee3201528346293f991af3468a8c6c5d2a21d7d9e1bdc1bf79b87110b2fddfcc5a0d45963c7c30", provider.Checksum) + assert.Equal(t, "c9967d88db2888a74778b5c62dbc2508921c8b54aca0e2ba34ab3e95e655cdb182bb2989b28e7ab4cab696f2ac7193d7ba9f57dea5191aad0c6a1082991c1ab8", provider.Checksum) }) } From 748b590c89595e80547953b4bcb1d12bd38541c3 Mon Sep 17 00:00:00 2001 From: junoberryferry Date: Thu, 11 Sep 2025 14:23:40 +0000 Subject: [PATCH 23/24] the security check has a panic when using the experimental go library --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0132becff1991..9a274f97e82a8 100644 --- a/Makefile +++ b/Makefile @@ -770,7 +770,7 @@ generate-go: $(TAGS_PREREQ) .PHONY: security-check security-check: - go run $(GOVULNCHECK_PACKAGE) -show color ./... + GOEXPERIMENT= go run $(GOVULNCHECK_PACKAGE) -show color ./... $(EXECUTABLE): $(GO_SOURCES) $(TAGS_PREREQ) ifneq ($(and $(STATIC),$(findstring pam,$(TAGS))),) From 60b26b0368dba35e54537eb4d103f9b038324967 Mon Sep 17 00:00:00 2001 From: junoberryferry Date: Thu, 11 Sep 2025 16:19:22 +0000 Subject: [PATCH 24/24] resolve panic in assetfs parsing of embeded data --- modules/assetfs/embed.go | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/modules/assetfs/embed.go b/modules/assetfs/embed.go index 95176372d10d1..6f2aeb9218e3c 100644 --- a/modules/assetfs/embed.go +++ b/modules/assetfs/embed.go @@ -102,12 +102,29 @@ func NewEmbeddedFS(data []byte) fs.ReadDirFS { efs := &embeddedFS{data: data, files: make(map[string]*embeddedFileInfo)} efs.meta = sync.OnceValue(func() *EmbeddedMeta { var meta EmbeddedMeta + + // look for the separator newline between binary data and JSON metadata + // jsonv2 may end with an extra newline p := bytes.LastIndexByte(data, '\n') if p < 0 { return &meta } - if err := json.Unmarshal(data[p+1:], &meta); err != nil { - panic("embedded file is not valid") + + // if the data ends with a newline, look for the previous newline + // to find the real separator + if p == len(data)-1 { + p = bytes.LastIndexByte(data[:p], '\n') + if p < 0 { + return &meta + } + } + + jsonData := data[p+1:] + if err := json.Unmarshal(jsonData, &meta); err != nil { + panic("embedded file is not valid: " + err.Error()) + } + if meta.Root == nil { + panic("embedded file metadata has nil root") } return &meta }) @@ -150,9 +167,15 @@ func (e *embeddedFS) getFileInfo(fullName string) (*embeddedFileInfo, error) { fields := strings.Split(fullName, "/") fi = e.meta().Root + if fi == nil { + return nil, fs.ErrNotExist + } if fullName != "." { found := true for _, field := range fields { + if fi.Children == nil { + return nil, fs.ErrNotExist + } for _, child := range fi.Children { if found = child.BaseName == field; found { fi = child