diff --git a/.github/workflows/tour_of_beam_backend_integration.yml b/.github/workflows/tour_of_beam_backend_integration.yml index 6c4a7db9db39..90be7e653ff1 100644 --- a/.github/workflows/tour_of_beam_backend_integration.yml +++ b/.github/workflows/tour_of_beam_backend_integration.yml @@ -85,7 +85,7 @@ jobs: uses: ./.github/actions/setup-environment-action with: # pin to the biggest Go version supported by Cloud Functions runtime - go-version: '1.16' + go-version: '1.25' - name: Build Playground router image run: ./gradlew -i playground:backend:containers:router:docker diff --git a/CHANGES.md b/CHANGES.md index b857e09ac298..365605c7065b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -95,6 +95,7 @@ * (Java) KafkaIO dynamic reads no longer require the obsolete `beam_fn_api` experiment ([#29998](https://github.com/apache/beam/issues/29998)). * (Prism) Self-checkpointing splittable DoFns now resume after their requested delay instead of immediately, so polling SDFs no longer busy-spin ([#39848](https://github.com/apache/beam/issues/39848)). * (Java) MongoDbIO read splitting now preserves non-ObjectId `_id` types (e.g. string ids) instead of failing to parse the generated range filters ([#39900](https://github.com/apache/beam/issues/39900)). +* (Go) Fixed GCS glob matching silently dropping objects when the glob pattern contains multi-byte characters ([#39969](https://github.com/apache/beam/issues/39969)). ## Security Fixes @@ -123,6 +124,7 @@ ## New Features / Improvements +* [Flink Runner] Added opt-in static round-robin split assignment for small bounded sources via the new `sourceStaticSplitThresholdMb` pipeline option. The default of 0 keeps the existing lazy pull-based assignment ([#39873](https://github.com/apache/beam/issues/39873)). * Added automatic caching of bounded, single-pane side-input views for classic Java Flink DataStream execution ([#39866](https://github.com/apache/beam/issues/39866)). * Added `GroupIntoBatches` transform and the standard `beam:coder:sharded_key:v1` coder to the Go SDK, along with diff --git a/learning/tour-of-beam/backend/function.go b/learning/tour-of-beam/backend/function.go index eaa1dcad076a..2458b532c38f 100644 --- a/learning/tour-of-beam/backend/function.go +++ b/learning/tour-of-beam/backend/function.go @@ -87,7 +87,7 @@ func MakePlaygroundClient(ctx context.Context) pb.PlaygroundServiceClient { return service.GetMockClient() } else { host := os.Getenv("PLAYGROUND_ROUTER_HOST") - cc, err := grpc.Dial(host, grpc.WithTransportCredentials(insecure.NewCredentials())) + cc, err := grpc.NewClient(host, grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { log.Fatalf("fail to dial playground: %v", err) } diff --git a/learning/tour-of-beam/backend/go.mod b/learning/tour-of-beam/backend/go.mod index d796f60bc9d4..b1dfa3685fc3 100644 --- a/learning/tour-of-beam/backend/go.mod +++ b/learning/tour-of-beam/backend/go.mod @@ -15,7 +15,7 @@ module beam.apache.org/learning/tour-of-beam/backend -go 1.16 +go 1.25.0 require ( github.com/GoogleCloudPlatform/functions-framework-go v1.5.3 @@ -23,10 +23,74 @@ require ( ) require ( - cloud.google.com/go/datastore v1.8.0 - cloud.google.com/go/firestore v1.7.0 // indirect + cel.dev/expr v0.25.1 // indirect + cloud.google.com/go v0.120.0 // indirect + cloud.google.com/go/auth v0.15.0 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/compute/metadata v0.9.0 // indirect + cloud.google.com/go/functions v1.19.3 // indirect + cloud.google.com/go/iam v1.4.0 // indirect + cloud.google.com/go/longrunning v0.6.6 // indirect + cloud.google.com/go/monitoring v1.24.1 // indirect + cloud.google.com/go/storage v1.50.0 // indirect + cloud.google.com/go/trace v1.11.5 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.50.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.50.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cloudevents/sdk-go/v2 v2.15.2 // indirect + github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect + github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-jose/go-jose/v4 v4.1.4 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect + github.com/googleapis/gax-go/v2 v2.14.1 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.43.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect + go.opentelemetry.io/otel v1.43.0 // indirect + go.opentelemetry.io/otel/metric v1.43.0 // indirect + go.opentelemetry.io/otel/sdk v1.43.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect + go.opentelemetry.io/otel/trace v1.43.0 // indirect + go.uber.org/atomic v1.7.0 // indirect + go.uber.org/multierr v1.6.0 // indirect + go.uber.org/zap v1.16.0 // indirect + golang.org/x/crypto v0.52.0 // indirect + golang.org/x/net v0.55.0 // indirect + golang.org/x/oauth2 v0.36.0 // indirect + golang.org/x/sync v0.20.0 // indirect + golang.org/x/sys v0.45.0 // indirect + golang.org/x/text v0.37.0 // indirect + golang.org/x/time v0.11.0 // indirect + google.golang.org/api v0.228.0 // indirect + google.golang.org/appengine/v2 v2.0.2 // indirect + google.golang.org/genproto v0.0.0-20250324211829-b45e905df463 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + honnef.co/go/tools v0.1.3 // indirect +) + +require ( + cloud.google.com/go/datastore v1.20.0 + cloud.google.com/go/firestore v1.18.0 // indirect firebase.google.com/go/v4 v4.9.0 - github.com/stretchr/testify v1.8.0 - google.golang.org/grpc v1.49.0 - google.golang.org/protobuf v1.28.1 + github.com/stretchr/testify v1.11.1 + google.golang.org/grpc v1.82.1 + google.golang.org/protobuf v1.36.11 ) diff --git a/learning/tour-of-beam/backend/go.sum b/learning/tour-of-beam/backend/go.sum index ba9fb37df5d1..1aff793160ad 100644 --- a/learning/tour-of-beam/backend/go.sum +++ b/learning/tour-of-beam/backend/go.sum @@ -1,3 +1,5 @@ +cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= +cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= @@ -24,135 +26,90 @@ cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWc cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= -cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= -cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= -cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= -cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= -cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= -cloud.google.com/go v0.104.0 h1:gSmWO7DY1vOm0MVU6DNXM11BWHHsTUmsC5cv1fuW5X8= -cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRYtA= -cloud.google.com/go/analytics v0.11.0/go.mod h1:DjEWCu41bVbYcKyvlws9Er60YE4a//bK6mnhWvQeFNI= -cloud.google.com/go/area120 v0.5.0/go.mod h1:DE/n4mp+iqVyvxHN41Vf1CR602GiHQjFPusMFW6bGR4= -cloud.google.com/go/artifactregistry v1.6.0/go.mod h1:IYt0oBPSAGYj/kprzsBjZ/4LnG/zOcHyFHjWPCi6SAQ= -cloud.google.com/go/asset v1.5.0/go.mod h1:5mfs8UvcM5wHhqtSv8J1CtxxaQq3AdBxxQi2jGW/K4o= -cloud.google.com/go/assuredworkloads v1.5.0/go.mod h1:n8HOZ6pff6re5KYfBXcFvSViQjDwxFkAkmUFffJRbbY= -cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0= +cloud.google.com/go v0.120.0 h1:wc6bgG9DHyKqF5/vQvX1CiZrtHnxJjBlKUyF9nP6meA= +cloud.google.com/go v0.120.0/go.mod h1:/beW32s8/pGRuj4IILWQNd4uuebeT4dkOhKmkfit64Q= +cloud.google.com/go/auth v0.15.0 h1:Ly0u4aA5vG/fsSsxu98qCQBemXtAtJf+95z9HK+cxps= +cloud.google.com/go/auth v0.15.0/go.mod h1:WJDGqZ1o9E9wKIL+IwStfyn/+s59zl4Bi+1KQNVXLZ8= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/billing v1.4.0/go.mod h1:g9IdKBEFlItS8bTtlrZdVLWSSdSyFUZKXNS02zKMOZY= -cloud.google.com/go/binaryauthorization v1.1.0/go.mod h1:xwnoWu3Y84jbuHa0zd526MJYmtnVXn0syOjaJgy4+dM= -cloud.google.com/go/cloudtasks v1.5.0/go.mod h1:fD92REy1x5woxkKEkLdvavGnPJGEn8Uic9nWuLzqCpY= -cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= -cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= -cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= -cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= -cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= -cloud.google.com/go/compute v1.7.0 h1:v/k9Eueb8aAJ0vZuxKMrgm6kPhCLZU9HxFU+AFDs9Uk= -cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= -cloud.google.com/go/datacatalog v1.5.0/go.mod h1:M7GPLNQeLfWqeIm3iuiruhPzkt65+Bx8dAKvScX8jvs= -cloud.google.com/go/dataflow v0.6.0/go.mod h1:9QwV89cGoxjjSR9/r7eFDqqjtvbKxAK2BaYU6PVk9UM= -cloud.google.com/go/dataform v0.3.0/go.mod h1:cj8uNliRlHpa6L3yVhDOBrUXH+BPAO1+KFMQQNSThKo= -cloud.google.com/go/datalabeling v0.5.0/go.mod h1:TGcJ0G2NzcsXSE/97yWjIZO0bXj0KbVlINXMG9ud42I= -cloud.google.com/go/dataqna v0.5.0/go.mod h1:90Hyk596ft3zUQ8NkFfvICSIfHFh1Bc7C4cK3vbhkeo= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/datastore v1.8.0 h1:2qo2G7hABSeqswa+5Ga3+QB8/ZwKOJmDsCISM9scmsU= -cloud.google.com/go/datastore v1.8.0/go.mod h1:q1CpHVByTlXppdqTcu4LIhCsTn3fhtZ5R7+TajciO+M= -cloud.google.com/go/datastream v1.2.0/go.mod h1:i/uTP8/fZwgATHS/XFu0TcNUhuA0twZxxQ3EyCUQMwo= -cloud.google.com/go/dialogflow v1.15.0/go.mod h1:HbHDWs33WOGJgn6rfzBW1Kv807BE3O1+xGbn59zZWI4= -cloud.google.com/go/documentai v1.7.0/go.mod h1:lJvftZB5NRiFSX4moiye1SMxHx0Bc3x1+p9e/RfXYiU= -cloud.google.com/go/domains v0.6.0/go.mod h1:T9Rz3GasrpYk6mEGHh4rymIhjlnIuB4ofT1wTxDeT4Y= -cloud.google.com/go/edgecontainer v0.1.0/go.mod h1:WgkZ9tp10bFxqO8BLPqv2LlfmQF1X8lZqwW4r1BTajk= -cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= -cloud.google.com/go/firestore v1.7.0 h1:cNkQyruzd5v7FjmL6eeDqwqgX+FbPCjbHxz7vsMhGoo= -cloud.google.com/go/firestore v1.7.0/go.mod h1:0b8DxQkXhbg/PmsjhCUAg4EExIuifAvbHj5Z/iX3BYI= +cloud.google.com/go/datastore v1.20.0 h1:NNpXoyEqIJmZFc0ACcwBEaXnmscUpcG4NkKnbCePmiM= +cloud.google.com/go/datastore v1.20.0/go.mod h1:uFo3e+aEpRfHgtp5pp0+6M0o147KoPaYNaPAKpfh8Ew= +cloud.google.com/go/firestore v1.18.0 h1:cuydCaLS7Vl2SatAeivXyhbhDEIR8BDmtn4egDhIn2s= +cloud.google.com/go/firestore v1.18.0/go.mod h1:5ye0v48PhseZBdcl0qbl3uttu7FIEwEYVaWm0UIEOEU= cloud.google.com/go/functions v1.0.0/go.mod h1:O9KS8UweFVo6GbbbCBKh5yEzbW08PVkg2spe3RfPMd4= -cloud.google.com/go/functions v1.6.0 h1:Oveqoadoi2f+yMpJtf1/OrwhTIzaR38l+6Q8/RPyM18= -cloud.google.com/go/functions v1.6.0/go.mod h1:3H1UA3qiIPRWD7PeZKLvHZ9SaQhR26XIJcC0A5GbvAk= -cloud.google.com/go/gaming v1.5.0/go.mod h1:ol7rGcxP/qHTRQE/RO4bxkXq+Fix0j6D4LFPzYTIrDM= -cloud.google.com/go/gkeconnect v0.5.0/go.mod h1:c5lsNAg5EwAy7fkqX/+goqFsU1Da/jQFqArp+wGNr/o= -cloud.google.com/go/gkehub v0.9.0/go.mod h1:WYHN6WG8w9bXU0hqNxt8rm5uxnk8IH+lPY9J2TV7BK0= -cloud.google.com/go/iam v0.3.0 h1:exkAomrVUuzx9kWFI1wm3KI0uoDeUFPB4kKGzx6x+Gc= -cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= -cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= -cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= -cloud.google.com/go/mediatranslation v0.5.0/go.mod h1:jGPUhGTybqsPQn91pNXw0xVHfuJ3leR1wj37oU3y1f4= -cloud.google.com/go/memcache v1.4.0/go.mod h1:rTOfiGZtJX1AaFUrOgsMHX5kAzaTQ8azHiuDoTPzNsE= -cloud.google.com/go/metastore v1.5.0/go.mod h1:2ZNrDcQwghfdtCwJ33nM0+GrBGlVuh8rakL3vdPY3XY= -cloud.google.com/go/networkconnectivity v1.4.0/go.mod h1:nOl7YL8odKyAOtzNX73/M5/mGZgqqMeryi6UPZTk/rA= -cloud.google.com/go/networksecurity v0.5.0/go.mod h1:xS6fOCoqpVC5zx15Z/MqkfDwH4+m/61A3ODiDV1xmiQ= -cloud.google.com/go/notebooks v1.2.0/go.mod h1:9+wtppMfVPUeJ8fIWPOq1UnATHISkGXGqTkxeieQ6UY= -cloud.google.com/go/osconfig v1.7.0/go.mod h1:oVHeCeZELfJP7XLxcBGTMBvRO+1nQ5tFG9VQTmYS2Fs= -cloud.google.com/go/oslogin v1.4.0/go.mod h1:YdgMXWRaElXz/lDk1Na6Fh5orF7gvmJ0FGLIs9LId4E= -cloud.google.com/go/phishingprotection v0.5.0/go.mod h1:Y3HZknsK9bc9dMi+oE8Bim0lczMU6hrX0UpADuMefr0= -cloud.google.com/go/privatecatalog v0.5.0/go.mod h1:XgosMUvvPyxDjAVNDYxJ7wBW8//hLDDYmnsNcMGq1K0= +cloud.google.com/go/functions v1.19.3 h1:V0vCHSgFTUqKn57+PUXp1UfQY0/aMkveAw7wXeM3Lq0= +cloud.google.com/go/functions v1.19.3/go.mod h1:nOZ34tGWMmwfiSJjoH/16+Ko5106x+1Iji29wzrBeOo= +cloud.google.com/go/iam v1.4.0 h1:ZNfy/TYfn2uh/ukvhp783WhnbVluqf/tzOaqVUPlIPA= +cloud.google.com/go/iam v1.4.0/go.mod h1:gMBgqPaERlriaOV0CUl//XUzDhSfXevn4OEUbg6VRs4= +cloud.google.com/go/logging v1.13.0 h1:7j0HgAp0B94o1YRDqiqm26w4q1rDMH7XNRU34lJXHYc= +cloud.google.com/go/logging v1.13.0/go.mod h1:36CoKh6KA/M0PbhPKMq6/qety2DCAErbhXT62TuXALA= +cloud.google.com/go/longrunning v0.6.6 h1:XJNDo5MUfMM05xK3ewpbSdmt7R2Zw+aQEMbdQR65Rbw= +cloud.google.com/go/longrunning v0.6.6/go.mod h1:hyeGJUrPHcx0u2Uu1UFSoYZLn4lkMrccJig0t4FI7yw= +cloud.google.com/go/monitoring v1.24.1 h1:vKiypZVFD/5a3BbQMvI4gZdl8445ITzXFh257XBgrS0= +cloud.google.com/go/monitoring v1.24.1/go.mod h1:Z05d1/vn9NaujqY2voG6pVQXoJGbp+r3laV+LySt9K0= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/recaptchaenterprise v1.3.1/go.mod h1:OdD+q+y4XGeAlxRaMn1Y7/GveP6zmq76byL6tjPE7d4= -cloud.google.com/go/recaptchaenterprise/v2 v2.1.0/go.mod h1:w9yVqajwroDNTfGuhmOjPDN//rZGySaf6PtFVcSCa7o= -cloud.google.com/go/recommendationengine v0.5.0/go.mod h1:E5756pJcVFeVgaQv3WNpImkFP8a+RptV6dDLGPILjvg= -cloud.google.com/go/recommender v1.5.0/go.mod h1:jdoeiBIVrJe9gQjwd759ecLJbxCDED4A6p+mqoqDvTg= -cloud.google.com/go/redis v1.7.0/go.mod h1:V3x5Jq1jzUcg+UNsRvdmsfuFnit1cfe3Z/PGyq/lm4Y= -cloud.google.com/go/retail v1.8.0/go.mod h1:QblKS8waDmNUhghY2TI9O3JLlFk8jybHeV4BF19FrE4= -cloud.google.com/go/scheduler v1.4.0/go.mod h1:drcJBmxF3aqZJRhmkHQ9b3uSSpQoltBPGPxGAWROx6s= -cloud.google.com/go/secretmanager v1.6.0/go.mod h1:awVa/OXF6IiyaU1wQ34inzQNc4ISIDIrId8qE5QGgKA= -cloud.google.com/go/security v1.5.0/go.mod h1:lgxGdyOKKjHL4YG3/YwIL2zLqMFCKs0UbQwgyZmfJl4= -cloud.google.com/go/security v1.7.0/go.mod h1:mZklORHl6Bg7CNnnjLH//0UlAlaXqiG7Lb9PsPXLfD0= -cloud.google.com/go/securitycenter v1.13.0/go.mod h1:cv5qNAqjY84FCN6Y9z28WlkKXyWsgLO832YiWwkCWcU= -cloud.google.com/go/servicedirectory v1.4.0/go.mod h1:gH1MUaZCgtP7qQiI+F+A+OpeKF/HQWgtAddhTbhL2bs= -cloud.google.com/go/speech v1.6.0/go.mod h1:79tcr4FHCimOp56lwC01xnt/WPJZc4v3gzyT7FoBkCM= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= -cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= -cloud.google.com/go/storage v1.26.0 h1:lYAGjknyDJirSzfwUlkv4Nsnj7od7foxQNH/fqZqles= -cloud.google.com/go/storage v1.26.0/go.mod h1:mk/N7YwIKEWyTvXAWQCIeiCTdLoRH6Pd5xmSnolQLTI= -cloud.google.com/go/talent v1.1.0/go.mod h1:Vl4pt9jiHKvOgF9KoZo6Kob9oV4lwd/ZD5Cto54zDRw= -cloud.google.com/go/videointelligence v1.6.0/go.mod h1:w0DIDlVRKtwPCn/C4iwZIJdvC69yInhW0cfi+p546uU= -cloud.google.com/go/vision v1.2.0/go.mod h1:SmNwgObm5DpFBme2xpyOyasvBc1aPdjvMk2bBk0tKD0= -cloud.google.com/go/vision/v2 v2.2.0/go.mod h1:uCdV4PpN1S0jyCyq8sIM42v2Y6zOLkZs+4R9LrGYwFo= -cloud.google.com/go/webrisk v1.4.0/go.mod h1:Hn8X6Zr+ziE2aNd8SliSDWpEnSS1u4R9+xXZmFiHmGE= -cloud.google.com/go/workflows v1.6.0/go.mod h1:6t9F5h/unJz41YqfBmqSASJSXccBLtD1Vwf+KmJENM0= +cloud.google.com/go/storage v1.50.0 h1:3TbVkzTooBvnZsk7WaAQfOsNrdoM8QHusXA1cpk6QJs= +cloud.google.com/go/storage v1.50.0/go.mod h1:l7XeiD//vx5lfqE3RavfmU9yvk5Pp0Zhcv482poyafY= +cloud.google.com/go/trace v1.11.5 h1:CALS1loyxJMnRiCwZSpdf8ac7iCsjreMxFD2WGxzzHU= +cloud.google.com/go/trace v1.11.5/go.mod h1:TwblCcqNInriu5/qzaeYEIH7wzUcchSdeY2l5wL3Eec= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= firebase.google.com/go/v4 v4.9.0 h1:VCagv+hYOxUGeuyu7J+o2rKJkDp5JQBbA3Bzlof+LMk= firebase.google.com/go/v4 v4.9.0/go.mod h1:bHhRkM3VtGJx19rQdW7GDNLdnA8/T6SsnN5nXk/xdw8= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/GoogleCloudPlatform/functions-framework-go v1.5.3 h1:Xx8uWT4hjgbjuXexbpU6V0yawWOdrbcAzZVyMYJvX8Q= github.com/GoogleCloudPlatform/functions-framework-go v1.5.3/go.mod h1:pq+lZy4vONJ5fjd3q/B6QzWhfHPAbuVweLpxZzMOb9Y= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0 h1:rIkQfkCOVKc1OiRCNcSDD8ml5RJlZbH/Xsq7lbpynwc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0/go.mod h1:RD2SsorTmYhF6HkTmDw7KmPYQk8OBYwTkuasChwv7R4= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.50.0 h1:5IT7xOdq17MtcdtL/vtl6mGfzhaq4m4vpollPRmlsBQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.50.0/go.mod h1:ZV4VOm0/eHR06JLrXWe09068dHpr3TRpY9Uo7T+anuA= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.50.0 h1:nNMpRpnkWDAaqcpxMJvxa/Ud98gjbYwayJY4/9bdjiU= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.50.0/go.mod h1:SZiPHWGOOk3bl8tkevxkoiwPgsIl6CwrWcbwjfHZpdM= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.50.0 h1:ig/FpDD2JofP/NExKQUbn7uOSZzJAQqogfqluZK4ed4= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.50.0/go.mod h1:otE2jQekW/PqXk1Awf5lmfokJx4uwuqcj1ab5SpGeW0= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudevents/sdk-go/v2 v2.6.1 h1:yHtzgmeBvc0TZx1nrnvYXov1CSvkQyvhEhNMs8Z5Mmk= github.com/cloudevents/sdk-go/v2 v2.6.1/go.mod h1:nlXhgFkf0uTopxmRXalyMwS2LG70cRGPrxzmjJgSG0U= +github.com/cloudevents/sdk-go/v2 v2.15.2 h1:54+I5xQEnI73RBhWHxbI1XJcqOFOVJN85vb41+8mHUc= +github.com/cloudevents/sdk-go/v2 v2.15.2/go.mod h1:lL7kSWAE/V8VI4Wh0jbL2v/jvqsm6tjmaQBSvxcv4uE= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -160,17 +117,31 @@ github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5y github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= -github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= -github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= +github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= +github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU= +github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= +github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= +github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= +github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -197,8 +168,9 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -214,16 +186,16 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -239,62 +211,67 @@ github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= -github.com/googleapis/enterprise-certificate-proxy v0.1.0 h1:zO8WHNx/MYiAKJ3d5spxZXZE6KHmIQGQcAzwUzV7qQw= -github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4= +github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= -github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= -github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= -github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= -github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= -github.com/googleapis/gax-go/v2 v2.5.1 h1:kBRZU0PSuI7PspsSb/ChWoVResUcwNVIdpB049pKTiw= -github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo= -github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= +github.com/googleapis/gax-go/v2 v2.14.1 h1:hb0FFeiPaQskmvakKu5EbCbpntQn48jyHuvrkurSS/Q= +github.com/googleapis/gax-go/v2 v2.14.1/go.mod h1:Hb/NubMaVM88SrNkvl8X/o8XWwDJEPqouaLeN2IUxoA= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo= +github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -308,20 +285,47 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/detectors/gcp v1.43.0 h1:62yY3dT7/ShwOxzA0RsKRgshBmfElKI4d/Myu2OxDFU= +go.opentelemetry.io/contrib/detectors/gcp v1.43.0/go.mod h1:RyaZMFY7yi1kAs45S6mbFGz8O8rqB0dTY14uzvG4LCs= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 h1:x7wzEgXfnzJcHDwStJT+mxOz4etr2EcexjqhBvmoakw= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0/go.mod h1:rg+RlpR5dKwaS95IyyZqj5Wd4E13lk/msnTS0Xl9lJM= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.29.0 h1:WDdP9acbMYjbKIyJUhTvtzj601sVJOqgWdUxSdR/Ysc= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.29.0/go.mod h1:BLbf7zbNIONBLPwvFnwNHGj4zge8uTCM/UPIVW1Mq2I= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= +go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= +golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -345,6 +349,7 @@ golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRu golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -357,6 +362,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= +golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -392,17 +399,9 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220708220712-1185a9018129/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20220909164309-bea034e7d591 h1:D0B/7al0LLrVC8aWF4+oxpv/m8bc7ViFfVS8/gXGdqI= -golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -418,16 +417,8 @@ golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= -golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= -golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= -golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1 h1:lxqLZaMad/dJHMFZH0NiNpiEZI/nhgWhe4wgzpE+MuA= -golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -439,8 +430,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f h1:Ax0t5p6N38Ga0dThY21weqDEyz2oklo4IvDkpigvkD8= -golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -487,23 +478,9 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg= -golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -514,13 +491,14 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220609170525-579cf78fd858 h1:Dpdu/EMxGMFgq0CeYMh4fazTD2vtlZRYE7wyynxJb9U= -golang.org/x/time v0.0.0-20220609170525-579cf78fd858/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= +golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -536,6 +514,8 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -572,14 +552,14 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= +golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0= -golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -607,34 +587,15 @@ google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNe google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= -google.golang.org/api v0.59.0/go.mod h1:sT2boj7M9YJxZzgeZqXogmhfmRWDtPzT31xkieUbuZU= -google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= -google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= -google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= -google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= -google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= -google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= -google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= -google.golang.org/api v0.77.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= -google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= -google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg= -google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o= -google.golang.org/api v0.85.0/go.mod h1:AqZf8Ep9uZ2pyTvgL+x0D3Zt0eoT9b5E8fmzfu6FO2g= -google.golang.org/api v0.90.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= -google.golang.org/api v0.93.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= -google.golang.org/api v0.94.0/go.mod h1:eADj+UBuxkh5zlrSntJghuNeg8HwQ1w5lTKkuqaETEI= -google.golang.org/api v0.95.0/go.mod h1:eADj+UBuxkh5zlrSntJghuNeg8HwQ1w5lTKkuqaETEI= -google.golang.org/api v0.96.0 h1:F60cuQPJq7K7FzsxMYHAUJSiXh2oKctHxBMbDygxhfM= -google.golang.org/api v0.96.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= +google.golang.org/api v0.228.0 h1:X2DJ/uoWGnY5obVjewbp8icSL5U4FzuCfy9OjbLSnLs= +google.golang.org/api v0.228.0/go.mod h1:wNvRS1Pbe8r4+IfBIniV8fwCpGwTrYa+kMUDiC5z5a4= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine/v2 v2.0.2 h1:MSqyWy2shDLwG7chbwBJ5uMyw6SNqJzhJHNDwYB0Akk= google.golang.org/appengine/v2 v2.0.2/go.mod h1:PkgRUWz4o1XOvbqtWTkBtCitEJ5Tp4HoVEdMMYQR/8E= @@ -677,7 +638,6 @@ google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210329143202-679c6ae281ee/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= @@ -693,51 +653,13 @@ google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEc google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20210921142501-181ce0d877f6/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211008145708-270636b82663/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211028162531-8db9c33dc351/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= -google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220628213854-d9e0b6570c03/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220722212130-b98a9ff5e252/go.mod h1:GkXuJDJ6aQ7lnJcRF+SJVgFdQhypqgl3LB1C9vabdRE= -google.golang.org/genproto v0.0.0-20220801145646-83ce21fca29f/go.mod h1:iHe1svFLAZg9VWz891+QbRMwUv9O/1Ww+/mngYeThbc= -google.golang.org/genproto v0.0.0-20220810155839-1856144b1d9c/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= -google.golang.org/genproto v0.0.0-20220815135757-37a418bb8959/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= -google.golang.org/genproto v0.0.0-20220817144833-d7fd3f11b9b1/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= -google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= -google.golang.org/genproto v0.0.0-20220829144015-23454907ede3/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= -google.golang.org/genproto v0.0.0-20220829175752-36a9c930ecbf/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk= -google.golang.org/genproto v0.0.0-20220913154956-18f8339a66a5/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= -google.golang.org/genproto v0.0.0-20220914142337-ca0e39ece12f/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= -google.golang.org/genproto v0.0.0-20220915135415-7fd63a7952de/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= -google.golang.org/genproto v0.0.0-20220916172020-2692e8806bfa/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= -google.golang.org/genproto v0.0.0-20220919141832-68c03719ef51/go.mod h1:0Nb8Qy+Sk5eDzHnzlStwW3itdNaWoZA5XeSG+R3JHSo= -google.golang.org/genproto v0.0.0-20220920201722-2b89144ce006 h1:mmbq5q8M1t7dhkLw320YK4PsOXm6jdnUAkErImaIqOg= -google.golang.org/genproto v0.0.0-20220920201722-2b89144ce006/go.mod h1:ht8XFiar2npT/g4vkk7O0WYS1sHOHbdujxbEp7CJWbw= +google.golang.org/genproto v0.0.0-20250324211829-b45e905df463 h1:qEFnJI6AnfZk0NNe8YTyXQh5i//Zxi4gBHwRgp76qpw= +google.golang.org/genproto v0.0.0-20250324211829-b45e905df463/go.mod h1:SqIx1NV9hcvqdLHo7uNZDS5lrUJybQ3evo3+z/WBfA0= +google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 h1:yQugLulqltosq0B/f8l4w9VryjV+N/5gcW0jQ3N8Qec= +google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478/go.mod h1:C6ADNqOxbgdUUeRTU+LCHDPB9ttAMCTff6auwCVa4uc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -763,15 +685,8 @@ google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQ google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= -google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.49.0 h1:WTLtQzmQori5FUH25Pq4WT22oCsv8USpQ+F6rqtsmxw= -google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= +google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -786,17 +701,19 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -807,6 +724,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.1.3 h1:qTakTkI6ni6LFD5sBwwsdSO+AQqbSIxOauHTTQKZ/7o= +honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/learning/tour-of-beam/backend/integration_tests/client_pg.go b/learning/tour-of-beam/backend/integration_tests/client_pg.go index 80fc4e7d4294..fcd815e9247b 100644 --- a/learning/tour-of-beam/backend/integration_tests/client_pg.go +++ b/learning/tour-of-beam/backend/integration_tests/client_pg.go @@ -24,10 +24,14 @@ import ( func GetSnippet(ctx context.Context, snippetId string) (*pb.GetSnippetResponse, error) { routerHost := os.Getenv("PLAYGROUND_ROUTER_HOST") - conn, err := grpc.DialContext(ctx, routerHost, grpc.WithTransportCredentials(insecure.NewCredentials())) + conn, err := grpc.NewClient(routerHost, grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { return nil, fmt.Errorf("dial grpc: %w", err) } + defer func() { + _ = conn.Close() + }() + client := pb.NewPlaygroundServiceClient(conn) req := &pb.GetSnippetRequest{Id: snippetId} diff --git a/playground/backend/go.mod b/playground/backend/go.mod index bad7c9c6a524..8c53478dd333 100644 --- a/playground/backend/go.mod +++ b/playground/backend/go.mod @@ -83,12 +83,12 @@ require ( go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.8.0 // indirect go.uber.org/zap v1.21.0 // indirect - golang.org/x/crypto v0.50.0 // indirect - golang.org/x/net v0.53.0 // indirect + golang.org/x/crypto v0.55.0 // indirect + golang.org/x/net v0.58.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/sync v0.20.0 // indirect - golang.org/x/sys v0.43.0 // indirect - golang.org/x/text v0.36.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.41.0 // indirect golang.org/x/time v0.11.0 // indirect google.golang.org/api v0.228.0 // indirect google.golang.org/genproto v0.0.0-20241118233622-e639e219e697 // indirect diff --git a/playground/backend/go.sum b/playground/backend/go.sum index 61c4b431d32c..3536fdd24689 100644 --- a/playground/backend/go.sum +++ b/playground/backend/go.sum @@ -1857,8 +1857,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= -golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= +golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= +golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1997,8 +1997,8 @@ golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= -golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -2045,8 +2045,8 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -2183,8 +2183,8 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= -golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -2210,8 +2210,8 @@ golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= -golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -2305,8 +2305,8 @@ golang.org/x/tools v0.1.11/go.mod h1:SgwaegtQh8clINPpECJMqnxLv9I09HLqnW3RMqW0CA4 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s= -golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0= +golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= +golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/runners/flink/2.0/src/main/java/org/apache/beam/runners/flink/FlinkPipelineOptions.java b/runners/flink/2.0/src/main/java/org/apache/beam/runners/flink/FlinkPipelineOptions.java index 3fee130d58ee..d3c5fefd9cf7 100644 --- a/runners/flink/2.0/src/main/java/org/apache/beam/runners/flink/FlinkPipelineOptions.java +++ b/runners/flink/2.0/src/main/java/org/apache/beam/runners/flink/FlinkPipelineOptions.java @@ -365,6 +365,17 @@ public Long create(PipelineOptions options) { void setFileInputSplitMaxSizeMB(Long fileInputSplitMaxSizeMB); + @Description( + "Static split assignment threshold in MiB per source reader for bounded sources in Flink " + + "DataStream mode. The default of 0 always uses lazy assignment. A positive value " + + "selects static round-robin assignment for sources with a known, positive estimate " + + "below the threshold and lazy assignment otherwise. Any negative value always uses " + + "static assignment.") + @Default.Long(0) + Long getSourceStaticSplitThresholdMb(); + + void setSourceStaticSplitThresholdMb(Long thresholdMb); + @Description( "Allow drain operation for flink pipelines that contain RequiresStableInput operator. Note that at time of draining," + "the RequiresStableInput contract might be violated if there any processing related failures in the DoFn operator.") diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkPipelineOptions.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkPipelineOptions.java index 6e4d998ed098..a6729356bdcb 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkPipelineOptions.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkPipelineOptions.java @@ -382,6 +382,17 @@ public Long create(PipelineOptions options) { void setFileInputSplitMaxSizeMB(Long fileInputSplitMaxSizeMB); + @Description( + "Static split assignment threshold in MiB per source reader for bounded sources in Flink " + + "DataStream mode. The default of 0 always uses lazy assignment. A positive value " + + "selects static round-robin assignment for sources with a known, positive estimate " + + "below the threshold and lazy assignment otherwise. Any negative value always uses " + + "static assignment.") + @Default.Long(0) + Long getSourceStaticSplitThresholdMb(); + + void setSourceStaticSplitThresholdMb(Long thresholdMb); + @Description( "Allow drain operation for flink pipelines that contain RequiresStableInput operator. Note that at time of draining," + "the RequiresStableInput contract might be violated if there any processing related failures in the DoFn operator.") diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/FlinkSource.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/FlinkSource.java index 0c1ba73b2a2d..f3bbf14f68fd 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/FlinkSource.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/FlinkSource.java @@ -18,12 +18,10 @@ package org.apache.beam.runners.flink.translation.wrappers.streaming.io.source; import java.io.Serializable; -import java.util.List; -import java.util.Map; import java.util.function.Function; import org.apache.beam.runners.core.construction.SerializablePipelineOptions; import org.apache.beam.runners.flink.FlinkPipelineOptions; -import org.apache.beam.runners.flink.translation.utils.SerdeUtils; +import org.apache.beam.runners.flink.translation.wrappers.streaming.io.source.FlinkSourceEnumeratorState.AssignmentMode; import org.apache.beam.runners.flink.translation.wrappers.streaming.io.source.bounded.FlinkBoundedSource; import org.apache.beam.runners.flink.translation.wrappers.streaming.io.source.impulse.BeamImpulseSource; import org.apache.beam.runners.flink.translation.wrappers.streaming.io.source.unbounded.FlinkUnboundedSource; @@ -43,7 +41,7 @@ * @param The data type of the records emitted by the Flink Source. */ public abstract class FlinkSource - implements Source, Map>>> { + implements Source, FlinkSourceEnumeratorState> { protected final String stepName; protected final org.apache.beam.sdk.io.Source beamSource; @@ -102,36 +100,38 @@ public Boundedness getBoundedness() { } @Override - public SplitEnumerator, Map>>> - createEnumerator(SplitEnumeratorContext> enumContext) throws Exception { - return createEnumerator(enumContext, false); - } - - public SplitEnumerator, Map>>> - createEnumerator( - SplitEnumeratorContext> enumContext, boolean splitInitialized) - throws Exception { - + public SplitEnumerator, FlinkSourceEnumeratorState> createEnumerator( + SplitEnumeratorContext> enumContext) throws Exception { + FlinkPipelineOptions options = serializablePipelineOptions.get().as(FlinkPipelineOptions.class); if (boundedness == Boundedness.BOUNDED) { - return new LazyFlinkSourceSplitEnumerator<>( - enumContext, beamSource, serializablePipelineOptions.get(), numSplits, splitInitialized); - } else { - return new FlinkSourceSplitEnumerator<>( - enumContext, beamSource, serializablePipelineOptions.get(), numSplits, splitInitialized); + long thresholdMb = options.getSourceStaticSplitThresholdMb(); + if (thresholdMb < 0) { + return new FlinkSourceSplitEnumerator<>(enumContext, beamSource, options, numSplits); + } + if (thresholdMb > 0) { + return new SizeBasedFlinkSourceSplitEnumerator<>( + enumContext, (BoundedSource) beamSource, options, numSplits); + } + return new LazyFlinkSourceSplitEnumerator<>(enumContext, beamSource, options, numSplits); } + return new FlinkSourceSplitEnumerator<>(enumContext, beamSource, options, numSplits); } @Override - public SplitEnumerator, Map>>> - restoreEnumerator( - SplitEnumeratorContext> enumContext, - Map>> checkpoint) - throws Exception { - SplitEnumerator, Map>>> enumerator = - createEnumerator(enumContext, true); - checkpoint.forEach( - (subtaskId, splitsForSubtask) -> enumerator.addSplitsBack(splitsForSubtask, subtaskId)); - return enumerator; + public SplitEnumerator, FlinkSourceEnumeratorState> restoreEnumerator( + SplitEnumeratorContext> enumContext, + FlinkSourceEnumeratorState checkpoint) + throws Exception { + FlinkPipelineOptions options = serializablePipelineOptions.get().as(FlinkPipelineOptions.class); + if (checkpoint.getAssignmentMode() == AssignmentMode.LAZY) { + return new LazyFlinkSourceSplitEnumerator<>( + enumContext, beamSource, options, numSplits, checkpoint); + } + if (checkpoint.getAssignmentMode() == AssignmentMode.STATIC) { + return new FlinkSourceSplitEnumerator<>( + enumContext, beamSource, options, numSplits, checkpoint); + } + return createEnumerator(enumContext); } @Override @@ -140,9 +140,11 @@ public SimpleVersionedSerializer> getSplitSerializer() { } @Override - public SimpleVersionedSerializer>>> + public SimpleVersionedSerializer> getEnumeratorCheckpointSerializer() { - return SerdeUtils.getNaiveObjectSerializer(); + AssignmentMode legacyAssignmentMode = + boundedness == Boundedness.BOUNDED ? AssignmentMode.LAZY : AssignmentMode.STATIC; + return new FlinkSourceEnumeratorStateSerializer<>(legacyAssignmentMode); } public int getNumSplits() { diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/FlinkSourceEnumeratorState.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/FlinkSourceEnumeratorState.java new file mode 100644 index 000000000000..8a825eeb991f --- /dev/null +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/FlinkSourceEnumeratorState.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.flink.translation.wrappers.streaming.io.source; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** Checkpoint state shared by the lazy and static source split assignment strategies. */ +public final class FlinkSourceEnumeratorState implements Serializable { + private static final long serialVersionUID = 1L; + + private final AssignmentMode assignmentMode; + private final ArrayList> pendingSplits; + + /** Takes ownership of {@code pendingSplits}; the caller must not mutate it afterwards. */ + FlinkSourceEnumeratorState( + AssignmentMode assignmentMode, ArrayList> pendingSplits) { + this.assignmentMode = assignmentMode; + this.pendingSplits = pendingSplits; + } + + AssignmentMode getAssignmentMode() { + return assignmentMode; + } + + List> getPendingSplits() { + return Collections.unmodifiableList(pendingSplits); + } + + enum AssignmentMode { + UNDECIDED, + LAZY, + STATIC + } +} diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/FlinkSourceEnumeratorStateSerializer.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/FlinkSourceEnumeratorStateSerializer.java new file mode 100644 index 000000000000..3470e60d0bda --- /dev/null +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/FlinkSourceEnumeratorStateSerializer.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.flink.translation.wrappers.streaming.io.source; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import javax.annotation.Nullable; +import org.apache.beam.runners.flink.translation.utils.SerdeUtils; +import org.apache.beam.runners.flink.translation.wrappers.streaming.io.source.FlinkSourceEnumeratorState.AssignmentMode; +import org.apache.flink.core.io.SimpleVersionedSerializer; + +/** Serializes source enumerator state and upgrades the map used by earlier runner versions. */ +final class FlinkSourceEnumeratorStateSerializer + implements SimpleVersionedSerializer> { + // Version written by SerdeUtils.getNaiveObjectSerializer, which serialized the + // Map>> state used by earlier runner versions. + static final int LEGACY_MAP_VERSION = 0; + static final int VERSION = 1; + + private final AssignmentMode legacyAssignmentMode; + + FlinkSourceEnumeratorStateSerializer(AssignmentMode legacyAssignmentMode) { + this.legacyAssignmentMode = legacyAssignmentMode; + } + + @Override + public int getVersion() { + return VERSION; + } + + @Override + public byte[] serialize(FlinkSourceEnumeratorState state) throws IOException { + return SerdeUtils.serializeObject(state); + } + + @Override + @SuppressWarnings("unchecked") + public FlinkSourceEnumeratorState deserialize(int version, byte[] serialized) + throws IOException { + if (version == VERSION) { + Object deserialized = SerdeUtils.deserializeObject(serialized); + if (deserialized instanceof FlinkSourceEnumeratorState) { + return (FlinkSourceEnumeratorState) deserialized; + } + throw new IOException( + "Expected source enumerator state of type FlinkSourceEnumeratorState for version " + + version + + ", but got: " + + describe(deserialized)); + } + if (version == LEGACY_MAP_VERSION) { + Object deserialized = SerdeUtils.deserializeObject(serialized); + if (deserialized instanceof Map) { + return upgradeLegacyState((Map) deserialized); + } + throw new IOException( + "Expected legacy source enumerator state of type Map for version " + + version + + ", but got: " + + describe(deserialized)); + } + throw new IOException( + String.format( + "Received source enumerator state version %d, but the highest supported version " + + "is %d.", + version, VERSION)); + } + + @SuppressWarnings("unchecked") + private FlinkSourceEnumeratorState upgradeLegacyState(Map legacyState) + throws IOException { + ArrayList> pendingSplits = new ArrayList<>(); + for (Object value : legacyState.values()) { + List> legacySplits = (List>) value; + if (legacySplits == null) { + throw new IOException("Legacy source enumerator state contains a null split list."); + } + pendingSplits.addAll(legacySplits); + } + return new FlinkSourceEnumeratorState<>(legacyAssignmentMode, pendingSplits); + } + + private static String describe(@Nullable Object obj) { + return obj == null ? "null" : obj.getClass().getName(); + } +} diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/FlinkSourceSplitEnumerator.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/FlinkSourceSplitEnumerator.java index 38d2f3639394..bb8956f5daea 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/FlinkSourceSplitEnumerator.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/FlinkSourceSplitEnumerator.java @@ -21,10 +21,13 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import javax.annotation.Nullable; +import org.apache.beam.runners.flink.translation.wrappers.streaming.io.source.FlinkSourceEnumeratorState.AssignmentMode; import org.apache.beam.sdk.io.BoundedSource; import org.apache.beam.sdk.io.Source; import org.apache.beam.sdk.io.UnboundedSource; @@ -35,27 +38,17 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** - * A Flink {@link org.apache.flink.api.connector.source.SplitEnumerator SplitEnumerator} - * implementation that holds a Beam {@link Source} and does the following: - * - *
    - *
  • Split the Beam {@link Source} to desired number of splits. - *
  • Assign the splits to the Flink Source Reader. - *
- * - *

Note that at this point, this class has a static round-robin split assignment strategy. - * - * @param The output type of the encapsulated Beam {@link Source}. - */ +/** Splits a Beam source and assigns its splits to Flink source readers round-robin. */ public class FlinkSourceSplitEnumerator - implements SplitEnumerator, Map>>> { + implements SplitEnumerator, FlinkSourceEnumeratorState> { private static final Logger LOG = LoggerFactory.getLogger(FlinkSourceSplitEnumerator.class); + private final SplitEnumeratorContext> context; private final Source beamSource; private final PipelineOptions pipelineOptions; private final int numSplits; private final Map>> pendingSplits; + private boolean splitsInitialized; public FlinkSourceSplitEnumerator( @@ -63,8 +56,7 @@ public FlinkSourceSplitEnumerator( Source beamSource, PipelineOptions pipelineOptions, int numSplits) { - - this(context, beamSource, pipelineOptions, numSplits, false); + this(context, beamSource, pipelineOptions, numSplits, null); } public FlinkSourceSplitEnumerator( @@ -72,17 +64,31 @@ public FlinkSourceSplitEnumerator( Source beamSource, PipelineOptions pipelineOptions, int numSplits, - boolean splitsInitialized) { - + @Nullable FlinkSourceEnumeratorState restoredState) { this.context = context; this.beamSource = beamSource; this.pipelineOptions = pipelineOptions; this.numSplits = numSplits; this.pendingSplits = new HashMap<>(numSplits); - this.splitsInitialized = splitsInitialized; + this.splitsInitialized = restoredState != null; + + if (restoredState != null) { + if (restoredState.getAssignmentMode() != AssignmentMode.STATIC) { + throw new IllegalArgumentException( + "Cannot restore the static source enumerator from " + + restoredState.getAssignmentMode() + + " state."); + } + int parallelism = context.currentParallelism(); + for (FlinkSourceSplit split : restoredState.getPendingSplits()) { + int targetSubtask = split.splitIndex() % parallelism; + pendingSplits.computeIfAbsent(targetSubtask, ignored -> new ArrayList<>()).add(split); + } + } LOG.info( - "Created new enumerator with parallelism {}, source {}, numSplits {}, initialized {}", + "Created static source enumerator with parallelism {}, source {}, numSplits {}, " + + "initialized {}", context.currentParallelism(), beamSource, numSplits, @@ -93,52 +99,33 @@ public FlinkSourceSplitEnumerator( public void start() { if (!splitsInitialized) { initializeSplits(); + } else { + sendPendingSplitsToSourceReaders(); } } private void initializeSplits() { context.callAsync( - () -> { - try { - LOG.info("Starting source {}", beamSource); - List> beamSplitSourceList = splitBeamSource(); - Map>> flinkSourceSplitsList = new HashMap<>(); - int i = 0; - for (Source beamSplitSource : beamSplitSourceList) { - int targetSubtask = i % context.currentParallelism(); - List> splitsForTask = - flinkSourceSplitsList.computeIfAbsent( - targetSubtask, ignored -> new ArrayList<>()); - splitsForTask.add(new FlinkSourceSplit<>(i, beamSplitSource)); - i++; - } - return flinkSourceSplitsList; - } catch (Exception e) { - throw new RuntimeException(e); - } - }, + this::splitBeamSource, (sourceSplits, error) -> { if (error != null) { throw new RuntimeException("Failed to start source enumerator.", error); - } else { - pendingSplits.putAll(sourceSplits); - splitsInitialized = true; - sendPendingSplitsToSourceReaders(); } + prepareAssignments(sourceSplits); + splitsInitialized = true; + sendPendingSplitsToSourceReaders(); }); } @Override public void handleSplitRequest(int subtaskId, @Nullable String requesterHostname) { - // Not used. + // Static assignment happens when readers register. } @Override public void addSplitsBack(List> splits, int subtaskId) { LOG.info("Adding splits {} back from subtask {}", splits, subtaskId); - List> splitsForSubtask = - pendingSplits.computeIfAbsent(subtaskId, ignored -> new ArrayList<>()); - splitsForSubtask.addAll(splits); + pendingSplits.computeIfAbsent(subtaskId, ignored -> new ArrayList<>()).addAll(splits); } @Override @@ -146,18 +133,19 @@ public void addReader(int subtaskId) { List> splitsForSubtask = pendingSplits.remove(subtaskId); if (splitsForSubtask != null) { assignSplitsAndLog(splitsForSubtask, subtaskId); - } else { - if (splitsInitialized) { - LOG.info("There is no split for subtask {}. Signaling no more splits.", subtaskId); - context.signalNoMoreSplits(subtaskId); - } + } else if (splitsInitialized) { + LOG.info("There is no split for subtask {}. Signaling no more splits.", subtaskId); + context.signalNoMoreSplits(subtaskId); } } @Override - public Map>> snapshotState(long checkpointId) throws Exception { + public FlinkSourceEnumeratorState snapshotState(long checkpointId) { LOG.info("Taking snapshot for checkpoint {}", checkpointId); - return pendingSplits; + ArrayList> checkpointSplits = new ArrayList<>(); + pendingSplits.values().forEach(checkpointSplits::addAll); + AssignmentMode mode = splitsInitialized ? AssignmentMode.STATIC : AssignmentMode.UNDECIDED; + return new FlinkSourceEnumeratorState<>(mode, checkpointSplits); } @Override @@ -165,34 +153,49 @@ public void close() throws IOException { // NoOp } - // -------------- Private helper methods ---------------------- - private List> splitBeamSource() throws Exception { + private ArrayList> splitBeamSource() throws Exception { + LOG.info("Starting source {}", beamSource); if (beamSource instanceof BoundedSource) { BoundedSource boundedSource = (BoundedSource) beamSource; - long desiredSizeBytes = boundedSource.getEstimatedSizeBytes(pipelineOptions) / numSplits; - return boundedSource.split(desiredSizeBytes, pipelineOptions); - } else if (beamSource instanceof UnboundedSource) { - List> splits = - ((UnboundedSource) beamSource).split(numSplits, pipelineOptions); - LOG.info("Split source {} to {} splits", beamSource, splits); - return splits; - } else { - throw new IllegalStateException("Unknown source type " + beamSource.getClass()); + long estimatedSizeBytes = + FlinkSourceSplitUtils.estimateBoundedSourceSize(boundedSource, pipelineOptions); + return FlinkSourceSplitUtils.splitBoundedSource( + boundedSource, pipelineOptions, numSplits, estimatedSizeBytes); + } + if (beamSource instanceof UnboundedSource) { + return FlinkSourceSplitUtils.splitUnboundedSource( + (UnboundedSource) beamSource, pipelineOptions, numSplits); + } + throw new IllegalStateException("Unknown source type " + beamSource.getClass()); + } + + private void prepareAssignments(List> sourceSplits) { + int parallelism = context.currentParallelism(); + for (FlinkSourceSplit split : sourceSplits) { + int targetSubtask = split.splitIndex() % parallelism; + pendingSplits.computeIfAbsent(targetSubtask, ignored -> new ArrayList<>()).add(split); } } private void sendPendingSplitsToSourceReaders() { + Set assignedReaders = new HashSet<>(); Iterator>>> splitIter = pendingSplits.entrySet().iterator(); while (splitIter.hasNext()) { Map.Entry>> entry = splitIter.next(); - int readerIndex = entry.getKey(); - int targetSubtask = readerIndex % context.currentParallelism(); - if (context.registeredReaders().containsKey(targetSubtask)) { - assignSplitsAndLog(entry.getValue(), targetSubtask); + int subtaskId = entry.getKey(); + if (context.registeredReaders().containsKey(subtaskId)) { + assignSplitsAndLog(entry.getValue(), subtaskId); + assignedReaders.add(subtaskId); splitIter.remove(); } } + + for (int subtaskId : context.registeredReaders().keySet()) { + if (!assignedReaders.contains(subtaskId) && !pendingSplits.containsKey(subtaskId)) { + context.signalNoMoreSplits(subtaskId); + } + } } private void assignSplitsAndLog(List> splits, int subtaskId) { diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/FlinkSourceSplitUtils.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/FlinkSourceSplitUtils.java new file mode 100644 index 000000000000..dda99bf53bef --- /dev/null +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/FlinkSourceSplitUtils.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.flink.translation.wrappers.streaming.io.source; + +import java.util.ArrayList; +import java.util.List; +import org.apache.beam.runners.flink.FlinkPipelineOptions; +import org.apache.beam.sdk.io.BoundedSource; +import org.apache.beam.sdk.io.FileBasedSource; +import org.apache.beam.sdk.io.Source; +import org.apache.beam.sdk.io.UnboundedSource; +import org.apache.beam.sdk.options.PipelineOptions; + +/** Shared Beam source sizing and splitting helpers. */ +final class FlinkSourceSplitUtils { + static final long MEBIBYTE = 1024L * 1024L; + + private FlinkSourceSplitUtils() {} + + static long estimateBoundedSourceSize( + BoundedSource boundedSource, PipelineOptions pipelineOptions) throws Exception { + return boundedSource.getEstimatedSizeBytes(pipelineOptions); + } + + static ArrayList> splitBoundedSource( + BoundedSource boundedSource, + PipelineOptions pipelineOptions, + int numSplits, + long estimatedSizeBytes) + throws Exception { + long desiredSizeBytes = + getDesiredSizeBytes(boundedSource, pipelineOptions, numSplits, estimatedSizeBytes); + return toFlinkSplits(boundedSource.split(desiredSizeBytes, pipelineOptions)); + } + + static ArrayList> splitUnboundedSource( + UnboundedSource unboundedSource, PipelineOptions pipelineOptions, int numSplits) + throws Exception { + return toFlinkSplits(unboundedSource.split(numSplits, pipelineOptions)); + } + + static long getDesiredSizeBytes( + Source beamSource, + PipelineOptions pipelineOptions, + int numSplits, + long estimatedSizeBytes) { + long desiredSizeBytes = estimatedSizeBytes / numSplits; + + long maxSplitSizeMb = + pipelineOptions.as(FlinkPipelineOptions.class).getFileInputSplitMaxSizeMB(); + if (beamSource instanceof FileBasedSource && maxSplitSizeMb > 0) { + return Math.min(desiredSizeBytes, mebibytesToBytes(maxSplitSizeMb)); + } + return desiredSizeBytes; + } + + static long mebibytesToBytes(long mebibytes) { + return mebibytes > Long.MAX_VALUE / MEBIBYTE ? Long.MAX_VALUE : mebibytes * MEBIBYTE; + } + + private static ArrayList> toFlinkSplits( + List> beamSplits) { + ArrayList> flinkSplits = new ArrayList<>(beamSplits.size()); + for (int i = 0; i < beamSplits.size(); i++) { + flinkSplits.add(new FlinkSourceSplit<>(i, beamSplits.get(i))); + } + return flinkSplits; + } +} diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/LazyFlinkSourceSplitEnumerator.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/LazyFlinkSourceSplitEnumerator.java index 94c14b2999b9..bd07d00cd049 100644 --- a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/LazyFlinkSourceSplitEnumerator.java +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/LazyFlinkSourceSplitEnumerator.java @@ -19,123 +19,109 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.CountDownLatch; +import java.util.Optional; import javax.annotation.Nullable; -import org.apache.beam.runners.flink.FlinkPipelineOptions; +import org.apache.beam.runners.flink.translation.wrappers.streaming.io.source.FlinkSourceEnumeratorState.AssignmentMode; import org.apache.beam.sdk.io.BoundedSource; -import org.apache.beam.sdk.io.FileBasedSource; import org.apache.beam.sdk.io.Source; -import org.apache.beam.sdk.io.UnboundedSource; import org.apache.beam.sdk.options.PipelineOptions; import org.apache.flink.api.connector.source.SplitEnumerator; import org.apache.flink.api.connector.source.SplitEnumeratorContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** - * A Flink {@link org.apache.flink.api.connector.source.SplitEnumerator SplitEnumerator} - * implementation that holds a Beam {@link Source} and does the following: - * - *

    - *
  • Split the Beam {@link Source} to desired number of splits. - *
  • Lazily assign the splits to the Flink Source Reader. - *
- * - * @param The output type of the encapsulated Beam {@link Source}. - */ +/** Splits a bounded Beam source and assigns one split for each reader request. */ public class LazyFlinkSourceSplitEnumerator - implements SplitEnumerator, Map>>> { + implements SplitEnumerator, FlinkSourceEnumeratorState> { private static final Logger LOG = LoggerFactory.getLogger(LazyFlinkSourceSplitEnumerator.class); + private final SplitEnumeratorContext> context; private final Source beamSource; private final PipelineOptions pipelineOptions; private final int numSplits; private final List> pendingSplits; - private volatile boolean splitsInitialized; - private final CountDownLatch initializationLatch = new CountDownLatch(1); + private final Map> pendingSplitRequests; + + private boolean splitsInitialized; + + public LazyFlinkSourceSplitEnumerator( + SplitEnumeratorContext> context, + Source beamSource, + PipelineOptions pipelineOptions, + int numSplits) { + this(context, beamSource, pipelineOptions, numSplits, null); + } public LazyFlinkSourceSplitEnumerator( SplitEnumeratorContext> context, Source beamSource, PipelineOptions pipelineOptions, int numSplits, - boolean splitInitialized) { + @Nullable FlinkSourceEnumeratorState restoredState) { this.context = context; this.beamSource = beamSource; this.pipelineOptions = pipelineOptions; this.numSplits = numSplits; this.pendingSplits = new ArrayList<>(numSplits); - this.splitsInitialized = splitInitialized; + this.pendingSplitRequests = new LinkedHashMap<>(); + this.splitsInitialized = restoredState != null; + + if (restoredState != null) { + if (restoredState.getAssignmentMode() != AssignmentMode.LAZY) { + throw new IllegalArgumentException( + "Cannot restore the lazy source enumerator from " + + restoredState.getAssignmentMode() + + " state."); + } + pendingSplits.addAll(restoredState.getPendingSplits()); + } + + LOG.info( + "Created lazy source enumerator with parallelism {}, source {}, numSplits {}, " + + "initialized {}", + context.currentParallelism(), + beamSource, + numSplits, + splitsInitialized); } @Override public void start() { if (!splitsInitialized) { initializeSplits(); + } else { + sendPendingSplitRequests(); } } - public void initializeSplits() { + private void initializeSplits() { context.callAsync( - () -> { - try { - LOG.info("Starting source {}", beamSource); - List> beamSplitSourceList = splitBeamSource(); - int i = 0; - for (Source beamSplitSource : beamSplitSourceList) { - pendingSplits.add(new FlinkSourceSplit<>(i, beamSplitSource)); - i++; - } - return pendingSplits; - } catch (Exception e) { - throw new RuntimeException(e); - } finally { - initializationLatch.countDown(); - } - }, + this::splitBeamSource, (sourceSplits, error) -> { if (error != null) { - pendingSplits.addAll(sourceSplits); throw new RuntimeException("Failed to start source enumerator.", error); } + pendingSplits.addAll(sourceSplits); splitsInitialized = true; + sendPendingSplitRequests(); }); } @Override - public void handleSplitRequest(int subtask, @Nullable String hostname) { - if (!context.registeredReaders().containsKey(subtask)) { - // reader failed between sending the request and now. skip this request. + public void handleSplitRequest(int subtaskId, @Nullable String requesterHostname) { + if (!context.registeredReaders().containsKey(subtaskId)) { return; } - if (LOG.isInfoEnabled()) { - final String hostInfo = - hostname == null ? "(no host locality info)" : "(on host '" + hostname + "')"; - LOG.info("Subtask {} {} is requesting a file source split", subtask, hostInfo); - } - if (!splitsInitialized) { - try { - initializationLatch.await(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - LOG.warn("Interrupted while waiting for splits initialization", e); - return; - } + pendingSplitRequests.put(subtaskId, Optional.ofNullable(requesterHostname)); + return; } - if (!pendingSplits.isEmpty()) { - final FlinkSourceSplit split = pendingSplits.remove(pendingSplits.size() - 1); - context.assignSplit(split, subtask); - LOG.info("Assigned split to subtask {} : {}", subtask, split); - } else { - context.signalNoMoreSplits(subtask); - LOG.info("No more splits available for subtask {}", subtask); - } + assignNextSplit(subtaskId, requesterHostname); } @Override @@ -146,20 +132,14 @@ public void addSplitsBack(List> splits, int subtaskId) { @Override public void addReader(int subtaskId) { - // this source is purely lazy-pull-based, nothing to do upon registration + // Readers request lazy splits when they are ready for work. } @Override - public Map>> snapshotState(long checkpointId) throws Exception { + public FlinkSourceEnumeratorState snapshotState(long checkpointId) { LOG.info("Taking snapshot for checkpoint {}", checkpointId); - return snapshotState(); - } - - public Map>> snapshotState() throws Exception { - // For type compatibility reasons, we return a Map but we do not actually care about the key - Map>> state = new HashMap<>(1); - state.put(1, pendingSplits); - return state; + AssignmentMode mode = splitsInitialized ? AssignmentMode.LAZY : AssignmentMode.UNDECIDED; + return new FlinkSourceEnumeratorState<>(mode, new ArrayList<>(pendingSplits)); } @Override @@ -167,39 +147,37 @@ public void close() throws IOException { // NoOp } - private long getDesiredSizeBytes(int numSplits, BoundedSource boundedSource) throws Exception { - long totalSize = boundedSource.getEstimatedSizeBytes(pipelineOptions); - long defaultSplitSize = totalSize / numSplits; - long maxSplitSize = 0; - if (pipelineOptions != null) { - maxSplitSize = pipelineOptions.as(FlinkPipelineOptions.class).getFileInputSplitMaxSizeMB(); - } - if (beamSource instanceof FileBasedSource && maxSplitSize > 0) { - // Most of the time parallelism is < number of files in source. - // Each file becomes a unique split which commonly create skew. - // This limits the size of splits to reduce skew. - return Math.min(defaultSplitSize, maxSplitSize * 1024 * 1024); - } else { - return defaultSplitSize; + private ArrayList> splitBeamSource() throws Exception { + if (!(beamSource instanceof BoundedSource)) { + throw new IllegalStateException("Lazy assignment requires a bounded source."); } + LOG.info("Starting source {}", beamSource); + BoundedSource boundedSource = (BoundedSource) beamSource; + long estimatedSizeBytes = + FlinkSourceSplitUtils.estimateBoundedSourceSize(boundedSource, pipelineOptions); + return FlinkSourceSplitUtils.splitBoundedSource( + boundedSource, pipelineOptions, numSplits, estimatedSizeBytes); } - // -------------- Private helper methods ---------------------- - private List> splitBeamSource() throws Exception { - if (beamSource instanceof BoundedSource) { - BoundedSource boundedSource = (BoundedSource) beamSource; - long desiredSizeBytes = getDesiredSizeBytes(numSplits, boundedSource); - List> splits = - ((BoundedSource) beamSource).split(desiredSizeBytes, pipelineOptions); - LOG.info("Split bounded source {} in {} splits", beamSource, splits.size()); - return splits; - } else if (beamSource instanceof UnboundedSource) { - List> splits = - ((UnboundedSource) beamSource).split(numSplits, pipelineOptions); - LOG.info("Split source {} to {} splits", beamSource, splits); - return splits; - } else { - throw new IllegalStateException("Unknown source type " + beamSource.getClass()); + private void sendPendingSplitRequests() { + Map> splitRequests = new LinkedHashMap<>(pendingSplitRequests); + pendingSplitRequests.clear(); + splitRequests.forEach( + (subtaskId, hostname) -> assignNextSplit(subtaskId, hostname.orElse(null))); + } + + private void assignNextSplit(int subtaskId, @Nullable String requesterHostname) { + if (!context.registeredReaders().containsKey(subtaskId)) { + return; + } + if (pendingSplits.isEmpty()) { + context.signalNoMoreSplits(subtaskId); + LOG.info("No more splits available for subtask {}", subtaskId); + return; } + + FlinkSourceSplit split = pendingSplits.remove(pendingSplits.size() - 1); + context.assignSplit(split, subtaskId); + LOG.info("Assigned split to subtask {} on host {}: {}", subtaskId, requesterHostname, split); } } diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/SizeBasedFlinkSourceSplitEnumerator.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/SizeBasedFlinkSourceSplitEnumerator.java new file mode 100644 index 000000000000..78b0fb59f2c4 --- /dev/null +++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/SizeBasedFlinkSourceSplitEnumerator.java @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.runners.flink.translation.wrappers.streaming.io.source; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import javax.annotation.Nullable; +import org.apache.beam.runners.flink.FlinkPipelineOptions; +import org.apache.beam.runners.flink.translation.wrappers.streaming.io.source.FlinkSourceEnumeratorState.AssignmentMode; +import org.apache.beam.sdk.io.BoundedSource; +import org.apache.beam.sdk.options.PipelineOptions; +import org.apache.flink.api.connector.source.SplitEnumerator; +import org.apache.flink.api.connector.source.SplitEnumeratorContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Selects static or lazy assignment by estimated bounded-source size. */ +final class SizeBasedFlinkSourceSplitEnumerator + implements SplitEnumerator, FlinkSourceEnumeratorState> { + private static final Logger LOG = + LoggerFactory.getLogger(SizeBasedFlinkSourceSplitEnumerator.class); + + private final SplitEnumeratorContext> context; + private final BoundedSource boundedSource; + private final PipelineOptions pipelineOptions; + private final int numSplits; + private final Map> pendingSplitRequests; + private final List> returnedSplits; + + private @Nullable SplitEnumerator, FlinkSourceEnumeratorState> delegate; + + SizeBasedFlinkSourceSplitEnumerator( + SplitEnumeratorContext> context, + BoundedSource boundedSource, + PipelineOptions pipelineOptions, + int numSplits) { + this.context = context; + this.boundedSource = boundedSource; + this.pipelineOptions = pipelineOptions; + this.numSplits = numSplits; + this.pendingSplitRequests = new LinkedHashMap<>(); + this.returnedSplits = new ArrayList<>(); + } + + @Override + public void start() { + context.callAsync( + this::selectAndSplit, + (initialState, error) -> { + if (error != null) { + throw new RuntimeException("Failed to select a source split assignment mode.", error); + } + + SplitEnumerator, FlinkSourceEnumeratorState> selectedDelegate = + createDelegate(initialState); + delegate = selectedDelegate; + returnedSplits.forEach( + returned -> selectedDelegate.addSplitsBack(returned.splits, returned.subtaskId)); + returnedSplits.clear(); + selectedDelegate.start(); + pendingSplitRequests.forEach( + (subtaskId, hostname) -> + selectedDelegate.handleSplitRequest(subtaskId, hostname.orElse(null))); + pendingSplitRequests.clear(); + }); + } + + @Override + public void handleSplitRequest(int subtaskId, @Nullable String requesterHostname) { + if (delegate == null) { + pendingSplitRequests.put(subtaskId, Optional.ofNullable(requesterHostname)); + } else { + delegate.handleSplitRequest(subtaskId, requesterHostname); + } + } + + @Override + public void addSplitsBack(List> splits, int subtaskId) { + if (delegate == null) { + returnedSplits.add(new ReturnedSplits<>(new ArrayList<>(splits), subtaskId)); + } else { + delegate.addSplitsBack(splits, subtaskId); + } + } + + @Override + public void addReader(int subtaskId) { + if (delegate != null) { + delegate.addReader(subtaskId); + } + } + + @Override + public FlinkSourceEnumeratorState snapshotState(long checkpointId) throws Exception { + if (delegate == null) { + return new FlinkSourceEnumeratorState<>(AssignmentMode.UNDECIDED, new ArrayList<>()); + } + return delegate.snapshotState(checkpointId); + } + + @Override + public void close() throws IOException { + if (delegate != null) { + delegate.close(); + } + } + + private FlinkSourceEnumeratorState selectAndSplit() throws Exception { + long estimatedSizeBytes = + FlinkSourceSplitUtils.estimateBoundedSourceSize(boundedSource, pipelineOptions); + AssignmentMode selectedMode = selectAssignmentMode(estimatedSizeBytes); + ArrayList> splits = + FlinkSourceSplitUtils.splitBoundedSource( + boundedSource, pipelineOptions, numSplits, estimatedSizeBytes); + LOG.info( + "Split bounded source {} into {} splits using {} assignment", + boundedSource, + splits.size(), + selectedMode); + return new FlinkSourceEnumeratorState<>(selectedMode, splits); + } + + private AssignmentMode selectAssignmentMode(long estimatedSizeBytes) { + long thresholdMb = + pipelineOptions.as(FlinkPipelineOptions.class).getSourceStaticSplitThresholdMb(); + if (thresholdMb <= 0) { + throw new IllegalArgumentException( + "Size-based source assignment requires a positive threshold, but received " + + thresholdMb + + "."); + } + if (estimatedSizeBytes <= 0 || estimatedSizeBytes == Long.MAX_VALUE) { + LOG.info( + "Estimated size of bounded source {} is zero or unknown. Using lazy split assignment.", + boundedSource); + return AssignmentMode.LAZY; + } + + int sourceParallelism = context.currentParallelism(); + if (sourceParallelism <= 0) { + throw new IllegalStateException( + "Source parallelism must be positive, but was " + sourceParallelism + "."); + } + long estimatedBytesPerReader = estimatedSizeBytes / sourceParallelism; + long thresholdBytes = FlinkSourceSplitUtils.mebibytesToBytes(thresholdMb); + AssignmentMode selectedMode = + estimatedBytesPerReader >= thresholdBytes ? AssignmentMode.LAZY : AssignmentMode.STATIC; + LOG.info( + "Using {} split assignment for bounded source {}: estimated size {} bytes, source " + + "parallelism {}, estimated bytes per reader {}, static assignment threshold {} " + + "bytes", + selectedMode, + boundedSource, + estimatedSizeBytes, + sourceParallelism, + estimatedBytesPerReader, + thresholdBytes); + return selectedMode; + } + + private SplitEnumerator, FlinkSourceEnumeratorState> createDelegate( + FlinkSourceEnumeratorState initialState) { + if (initialState.getAssignmentMode() == AssignmentMode.LAZY) { + return new LazyFlinkSourceSplitEnumerator<>( + context, boundedSource, pipelineOptions, numSplits, initialState); + } + if (initialState.getAssignmentMode() == AssignmentMode.STATIC) { + return new FlinkSourceSplitEnumerator<>( + context, boundedSource, pipelineOptions, numSplits, initialState); + } + throw new IllegalArgumentException( + "Cannot create a source enumerator for " + + initialState.getAssignmentMode() + + " assignment."); + } + + private static final class ReturnedSplits { + private final List> splits; + private final int subtaskId; + + private ReturnedSplits(List> splits, int subtaskId) { + this.splits = splits; + this.subtaskId = subtaskId; + } + } +} diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/FlinkSourceSplitEnumeratorTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/FlinkSourceSplitEnumeratorTest.java index e911ee72db5e..c4d220d22138 100644 --- a/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/FlinkSourceSplitEnumeratorTest.java +++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/translation/wrappers/streaming/io/source/FlinkSourceSplitEnumeratorTest.java @@ -18,23 +18,303 @@ package org.apache.beam.runners.flink.translation.wrappers.streaming.io.source; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.beam.runners.core.construction.SerializablePipelineOptions; import org.apache.beam.runners.flink.FlinkPipelineOptions; +import org.apache.beam.runners.flink.translation.utils.SerdeUtils; import org.apache.beam.runners.flink.translation.wrappers.streaming.io.TestBoundedCountingSource; import org.apache.beam.runners.flink.translation.wrappers.streaming.io.TestCountingSource; +import org.apache.beam.runners.flink.translation.wrappers.streaming.io.source.FlinkSourceEnumeratorState.AssignmentMode; +import org.apache.beam.sdk.coders.Coder; +import org.apache.beam.sdk.coders.StringUtf8Coder; +import org.apache.beam.sdk.io.BoundedSource; import org.apache.beam.sdk.io.Source; +import org.apache.beam.sdk.options.PipelineOptions; import org.apache.beam.sdk.values.KV; +import org.apache.flink.api.connector.source.SplitEnumerator; import org.apache.flink.connector.testutils.source.reader.TestingSplitEnumeratorContext; +import org.apache.flink.core.io.SimpleVersionedSerializer; import org.junit.Test; -/** Unit tests for {@link FlinkSourceSplitEnumerator}. */ +/** Unit tests for the Flink source split enumerators. */ public class FlinkSourceSplitEnumeratorTest { + private static final long MEBIBYTE = 1024L * 1024L; + private static final long STATIC_SPLIT_THRESHOLD_MB = 6144L; + private static final int SOURCE_PARALLELISM = 2; + private static final int REQUESTED_SPLITS = 4; + + @Test + public void testSelectsAssignmentModeFromEstimatedSizeAndConfiguration() throws Exception { + assertEquals(0L, (long) FlinkPipelineOptions.defaults().getSourceStaticSplitThresholdMb()); + + long thresholdBytes = STATIC_SPLIT_THRESHOLD_MB * MEBIBYTE; + long largeEstimate = 1024L * MEBIBYTE; + long[] estimatedSizes = { + SOURCE_PARALLELISM * thresholdBytes - 1L, + SOURCE_PARALLELISM * thresholdBytes, + -1L, + 0L, + Long.MAX_VALUE, + largeEstimate, + largeEstimate, + largeEstimate, + largeEstimate + }; + long[] configuredThresholds = { + STATIC_SPLIT_THRESHOLD_MB, + STATIC_SPLIT_THRESHOLD_MB, + STATIC_SPLIT_THRESHOLD_MB, + STATIC_SPLIT_THRESHOLD_MB, + STATIC_SPLIT_THRESHOLD_MB, + -1L, + -100L, + 0L, + Long.MAX_VALUE + }; + AssignmentMode[] expectedModes = { + AssignmentMode.STATIC, + AssignmentMode.LAZY, + AssignmentMode.LAZY, + AssignmentMode.LAZY, + AssignmentMode.LAZY, + AssignmentMode.STATIC, + AssignmentMode.STATIC, + AssignmentMode.LAZY, + AssignmentMode.STATIC + }; + + for (int i = 0; i < estimatedSizes.length; i++) { + assertAssignmentMode( + estimatedSizes[i], configuredThresholds[i], expectedModes[i], REQUESTED_SPLITS); + } + } + + @Test + public void testSizeBasedSelectionReplaysLazyRequestAfterInitialization() throws Exception { + FlinkPipelineOptions options = thresholdOptions(); + long estimatedSizeBytes = SOURCE_PARALLELISM * STATIC_SPLIT_THRESHOLD_MB * MEBIBYTE; + TestEstimatedSizeBoundedSource testSource = + TestEstimatedSizeBoundedSource.create(estimatedSizeBytes, REQUESTED_SPLITS); + TestingSplitEnumeratorContext> context = + new TestingSplitEnumeratorContext<>(SOURCE_PARALLELISM); + FlinkSource flinkSource = createBoundedSource(testSource, options, REQUESTED_SPLITS); + + try (SplitEnumerator, FlinkSourceEnumeratorState> enumerator = + flinkSource.createEnumerator(context)) { + enumerator.start(); + context.registerReader(0, "reader-0"); + enumerator.addReader(0); + enumerator.handleSplitRequest(0, "reader-0"); + + context.getExecutorService().triggerAll(); + + assertEquals(1, context.getSplitAssignments().get(0).getAssignedSplits().size()); + assertEquals(AssignmentMode.LAZY, enumerator.snapshotState(1L).getAssignmentMode()); + } + } + + @Test + public void testSizeBasedSelectionAssignsStaticSplitsToEarlyReaders() throws Exception { + FlinkPipelineOptions options = thresholdOptions(); + long estimatedSizeBytes = SOURCE_PARALLELISM * STATIC_SPLIT_THRESHOLD_MB * MEBIBYTE - 1L; + TestEstimatedSizeBoundedSource testSource = + TestEstimatedSizeBoundedSource.create(estimatedSizeBytes, REQUESTED_SPLITS); + TestingSplitEnumeratorContext> context = + new TestingSplitEnumeratorContext<>(SOURCE_PARALLELISM); + FlinkSource flinkSource = createBoundedSource(testSource, options, REQUESTED_SPLITS); + + try (SplitEnumerator, FlinkSourceEnumeratorState> enumerator = + flinkSource.createEnumerator(context)) { + enumerator.start(); + for (int subtaskId = 0; subtaskId < SOURCE_PARALLELISM; subtaskId++) { + context.registerReader(subtaskId, "reader-" + subtaskId); + enumerator.addReader(subtaskId); + } + + context.getExecutorService().triggerAll(); + + assertEquals(REQUESTED_SPLITS, countAssignedSplits(context)); + context + .getSplitAssignments() + .values() + .forEach(state -> assertTrue(state.hasReceivedNoMoreSplitsSignal())); + assertEquals(AssignmentMode.STATIC, enumerator.snapshotState(1L).getAssignmentMode()); + } + } + + @Test + public void testRestoreKeepsLazyAssignmentAcrossRescaleWithoutEstimating() throws Exception { + final int initialParallelism = 4; + final int restoredParallelism = 1; + final int generatedSplits = 4; + FlinkPipelineOptions options = thresholdOptions(); + long thresholdBytes = STATIC_SPLIT_THRESHOLD_MB * MEBIBYTE; + long estimatedSizeBytes = initialParallelism * thresholdBytes; + TestEstimatedSizeBoundedSource testSource = + TestEstimatedSizeBoundedSource.create(estimatedSizeBytes, generatedSplits); + FlinkSource flinkSource = createBoundedSource(testSource, options, generatedSplits); + FlinkSourceEnumeratorState checkpoint; + + TestingSplitEnumeratorContext> initialContext = + new TestingSplitEnumeratorContext<>(initialParallelism); + try (SplitEnumerator, FlinkSourceEnumeratorState> enumerator = + flinkSource.createEnumerator(initialContext)) { + enumerator.start(); + initialContext.getExecutorService().triggerAll(); + checkpoint = roundTripState(flinkSource, enumerator.snapshotState(1L)); + assertEquals(AssignmentMode.LAZY, checkpoint.getAssignmentMode()); + assertEquals(generatedSplits, checkpoint.getPendingSplits().size()); + } + + TestingSplitEnumeratorContext> restoredContext = + new TestingSplitEnumeratorContext<>(restoredParallelism); + try (SplitEnumerator, FlinkSourceEnumeratorState> restored = + flinkSource.restoreEnumerator(restoredContext, checkpoint)) { + restored.start(); + restoredContext.registerReader(0, "reader-0"); + restored.addReader(0); + for (int i = 0; i < generatedSplits; i++) { + restored.handleSplitRequest(0, "reader-0"); + } + restored.handleSplitRequest(0, "reader-0"); + + assertEquals( + generatedSplits, restoredContext.getSplitAssignments().get(0).getAssignedSplits().size()); + assertTrue(restoredContext.getSplitAssignments().get(0).hasReceivedNoMoreSplitsSignal()); + assertEquals( + "restoring a decided strategy must not estimate the source again", + 1, + testSource.getEstimationCalls()); + assertEquals(AssignmentMode.LAZY, restored.snapshotState(2L).getAssignmentMode()); + } + } + + @Test + public void testLegacyLazyCheckpointMapUpgradesToStrategyNeutralState() throws Exception { + FlinkPipelineOptions options = FlinkPipelineOptions.defaults(); + TestEstimatedSizeBoundedSource testSource = TestEstimatedSizeBoundedSource.create(1L, 1); + FlinkSource flinkSource = createBoundedSource(testSource, options, 1); + FlinkSourceSplit pendingSplit = new FlinkSourceSplit<>(0, testSource); + Map>> legacyCheckpoint = + Collections.singletonMap(1, Collections.singletonList(pendingSplit)); + byte[] serialized = SerdeUtils.serializeObject(legacyCheckpoint); + + FlinkSourceEnumeratorState upgraded = + flinkSource.getEnumeratorCheckpointSerializer().deserialize(0, serialized); + + assertEquals(AssignmentMode.LAZY, upgraded.getAssignmentMode()); + assertEquals(1, upgraded.getPendingSplits().size()); + assertEquals(0, upgraded.getPendingSplits().get(0).splitIndex()); + } + + @Test + public void testLegacyStaticCheckpointMapKeepsSplitsOnTheirOriginalReaders() throws Exception { + final int parallelism = 2; + TestEstimatedSizeBoundedSource testSource = TestEstimatedSizeBoundedSource.create(1L, 1); + Map>> legacyCheckpoint = new HashMap<>(); + legacyCheckpoint.put( + 0, + Arrays.asList( + new FlinkSourceSplit<>(0, testSource), new FlinkSourceSplit<>(2, testSource))); + legacyCheckpoint.put(1, Collections.singletonList(new FlinkSourceSplit<>(1, testSource))); + byte[] serialized = SerdeUtils.serializeObject(legacyCheckpoint); + + FlinkSourceEnumeratorState upgraded = + new FlinkSourceEnumeratorStateSerializer(AssignmentMode.STATIC) + .deserialize(0, serialized); + assertEquals(AssignmentMode.STATIC, upgraded.getAssignmentMode()); + assertEquals(3, upgraded.getPendingSplits().size()); + + TestingSplitEnumeratorContext> restoredContext = + new TestingSplitEnumeratorContext<>(parallelism); + try (FlinkSourceSplitEnumerator restored = + new FlinkSourceSplitEnumerator<>( + restoredContext, testSource, staticOptions(), 3, upgraded)) { + restored.start(); + for (int subtaskId = 0; subtaskId < parallelism; subtaskId++) { + restoredContext.registerReader(subtaskId, "reader-" + subtaskId); + restored.addReader(subtaskId); + } + + assertEquals(Arrays.asList(0, 2), assignedSplitIndexesForSubtask(restoredContext, 0)); + assertEquals( + Collections.singletonList(1), assignedSplitIndexesForSubtask(restoredContext, 1)); + } + } + + @Test + public void testSerializerRejectsUnknownVersionsAndUnexpectedPayloads() throws Exception { + FlinkSourceEnumeratorStateSerializer serializer = + new FlinkSourceEnumeratorStateSerializer<>(AssignmentMode.LAZY); + byte[] legacyMapBytes = + SerdeUtils.serializeObject(Collections.singletonMap(1, new ArrayList<>())); + byte[] stateBytes = + serializer.serialize( + new FlinkSourceEnumeratorState<>(AssignmentMode.LAZY, new ArrayList<>())); + + // The payload type must match the version it was written with. + assertThrows(IOException.class, () -> serializer.deserialize(1, legacyMapBytes)); + assertThrows(IOException.class, () -> serializer.deserialize(0, stateBytes)); + assertThrows(IOException.class, () -> serializer.deserialize(2, stateBytes)); + } + + @Test + public void testRestoreRepartitionsStaticSplitsForNewParallelismWithoutEstimating() + throws Exception { + final int initialParallelism = 2; + final int restoredParallelism = 3; + final int generatedSplits = 5; + FlinkPipelineOptions options = thresholdOptions(); + long thresholdBytes = STATIC_SPLIT_THRESHOLD_MB * MEBIBYTE; + long estimatedSizeBytes = initialParallelism * thresholdBytes - 1L; + TestEstimatedSizeBoundedSource testSource = + TestEstimatedSizeBoundedSource.create(estimatedSizeBytes, generatedSplits); + FlinkSource flinkSource = createBoundedSource(testSource, options, generatedSplits); + FlinkSourceEnumeratorState checkpoint; + + TestingSplitEnumeratorContext> initialContext = + new TestingSplitEnumeratorContext<>(initialParallelism); + try (SplitEnumerator, FlinkSourceEnumeratorState> enumerator = + flinkSource.createEnumerator(initialContext)) { + enumerator.start(); + initialContext.getExecutorService().triggerAll(); + checkpoint = roundTripState(flinkSource, enumerator.snapshotState(1L)); + assertEquals(AssignmentMode.STATIC, checkpoint.getAssignmentMode()); + assertEquals(generatedSplits, checkpoint.getPendingSplits().size()); + } + + TestingSplitEnumeratorContext> restoredContext = + new TestingSplitEnumeratorContext<>(restoredParallelism); + try (SplitEnumerator, FlinkSourceEnumeratorState> restored = + flinkSource.restoreEnumerator(restoredContext, checkpoint)) { + restored.start(); + for (int subtaskId = 0; subtaskId < restoredParallelism; subtaskId++) { + restoredContext.registerReader(subtaskId, "reader-" + subtaskId); + restored.addReader(subtaskId); + } + + assertEquals(generatedSplits, countAssignedSplits(restoredContext)); + assertEquals(2, restoredContext.getSplitAssignments().get(0).getAssignedSplits().size()); + assertEquals(2, restoredContext.getSplitAssignments().get(1).getAssignedSplits().size()); + assertEquals(1, restoredContext.getSplitAssignments().get(2).getAssignedSplits().size()); + restoredContext + .getSplitAssignments() + .values() + .forEach(state -> assertTrue(state.hasReceivedNoMoreSplitsSignal())); + assertEquals(1, testSource.getEstimationCalls()); + } + } @Test public void testAssignSplitsWithBoundedSource() throws IOException { @@ -54,13 +334,8 @@ public void testAssignSplitsWithBoundedSource() throws IOException { .forEach( (subtaskId, state) -> { int expectedNumSplitsPerSubtask = numSplits / numSubtasks; - assertEquals( - "Each subtask should have " + expectedNumSplitsPerSubtask + " assigned splits", - expectedNumSplitsPerSubtask, - state.getAssignedSplits().size()); - assertTrue( - "Each subtask should have received NoMoreSplits", - state.hasReceivedNoMoreSplitsSignal()); + assertEquals(expectedNumSplitsPerSubtask, state.getAssignedSplits().size()); + assertTrue(state.hasReceivedNoMoreSplitsSignal()); state .getAssignedSplits() .forEach( @@ -72,8 +347,8 @@ public void testAssignSplitsWithBoundedSource() throws IOException { assertEquals( expectedSplitSize, source.getEstimatedSizeBytes(FlinkPipelineOptions.defaults())); - } catch (Exception e) { - fail("Received exception" + e); + } catch (Exception error) { + fail("Received exception " + error); } }); }); @@ -88,90 +363,79 @@ public void testAssignSplitsWithUnboundedSource() throws IOException { new TestingSplitEnumeratorContext<>(numSubtasks); TestCountingSource testSource = new TestCountingSource(numRecordsPerSplit); - assignSplits(testContext, testSource, numSplits); + try (FlinkSourceSplitEnumerator> enumerator = + new FlinkSourceSplitEnumerator<>( + testContext, testSource, FlinkPipelineOptions.defaults(), numSplits)) { + enumerator.start(); + for (int subtaskId = 0; subtaskId < numSubtasks; subtaskId++) { + testContext.registerReader(subtaskId, String.valueOf(subtaskId)); + enumerator.addReader(subtaskId); + } + testContext.getExecutorService().triggerAll(); + } testContext .getSplitAssignments() .forEach( (subtaskId, state) -> { - int expectedNumSplitsPerSubtask = numSplits / numSubtasks; - assertEquals( - "Each subtask should have " + expectedNumSplitsPerSubtask + " assigned splits", - expectedNumSplitsPerSubtask, - state.getAssignedSplits().size()); - assertTrue( - "Each subtask should have received NoMoreSplits", - state.hasReceivedNoMoreSplitsSignal()); + assertEquals(numSplits / numSubtasks, state.getAssignedSplits().size()); + assertTrue(state.hasReceivedNoMoreSplitsSignal()); }); } @Test - public void testAddSplitsBack() throws IOException { + public void testAddSplitsBackToStaticReader() throws IOException { final int numSubtasks = 2; final int numSplits = 10; - final int totalNumRecords = 10; TestingSplitEnumeratorContext>> testContext = new TestingSplitEnumeratorContext<>(numSubtasks); - TestBoundedCountingSource testSource = - new TestBoundedCountingSource(numSplits, totalNumRecords); - try (FlinkSourceSplitEnumerator> splitEnumerator = - new FlinkSourceSplitEnumerator<>( - testContext, testSource, FlinkPipelineOptions.defaults(), numSplits)) { - splitEnumerator.start(); + TestBoundedCountingSource testSource = new TestBoundedCountingSource(numSplits, numSplits); + + try (FlinkSourceSplitEnumerator> enumerator = + new FlinkSourceSplitEnumerator<>(testContext, testSource, staticOptions(), numSplits)) { + enumerator.start(); testContext.registerReader(0, "0"); - splitEnumerator.addReader(0); + enumerator.addReader(0); testContext.getExecutorService().triggerAll(); - List>> splitsForReader = - testContext.getSplitAssignments().get(0).getAssignedSplits(); - assertEquals(numSplits / numSubtasks, splitsForReader.size()); + List>> returnedSplits = + new ArrayList<>(testContext.getSplitAssignments().get(0).getAssignedSplits()); + assertEquals(numSplits / numSubtasks, returnedSplits.size()); - splitEnumerator.addSplitsBack(splitsForReader, 0); - splitEnumerator.addReader(0); - assertEquals(2 * numSplits / numSubtasks, splitsForReader.size()); + enumerator.addSplitsBack(returnedSplits, 0); + enumerator.addReader(0); + assertEquals( + 2 * numSplits / numSubtasks, + testContext.getSplitAssignments().get(0).getAssignedSplits().size()); } } - @Test - public void testAddSplitsBackAfterRescale() throws Exception { - final int numSubtasks = 2; - final int numSplits = 10; - final int totalNumRecords = 10; - TestingSplitEnumeratorContext>> testContext = - new TestingSplitEnumeratorContext<>(numSubtasks); - TestBoundedCountingSource testSource = - new TestBoundedCountingSource(numSplits, totalNumRecords); - final Map>>> assignment; - try (FlinkSourceSplitEnumerator> splitEnumerator = - new FlinkSourceSplitEnumerator<>( - testContext, testSource, FlinkPipelineOptions.defaults(), numSplits)) { - splitEnumerator.start(); - for (int i = 0; i < numSubtasks; i++) { - testContext.registerReader(i, String.valueOf(i)); - splitEnumerator.addReader(i); - } - testContext.getExecutorService().triggerAll(); - assignment = - testContext.getSplitAssignments().entrySet().stream() - .map(e -> KV.of(e.getKey(), e.getValue().getAssignedSplits())) - .collect(Collectors.toMap(KV::getKey, KV::getValue)); - } + private void assertAssignmentMode( + long estimatedSizeBytes, + long configuredThresholdMb, + AssignmentMode expectedMode, + int generatedSplits) + throws Exception { + FlinkPipelineOptions options = FlinkPipelineOptions.defaults(); + options.setSourceStaticSplitThresholdMb(configuredThresholdMb); + TestEstimatedSizeBoundedSource testSource = + TestEstimatedSizeBoundedSource.create(estimatedSizeBytes, generatedSplits); + TestingSplitEnumeratorContext> context = + new TestingSplitEnumeratorContext<>(SOURCE_PARALLELISM); + FlinkSource flinkSource = createBoundedSource(testSource, options, generatedSplits); - // add tasks back - testContext = new TestingSplitEnumeratorContext<>(numSubtasks); - try (FlinkSourceSplitEnumerator> splitEnumerator = - new FlinkSourceSplitEnumerator<>( - testContext, testSource, FlinkPipelineOptions.defaults(), numSplits, true)) { - splitEnumerator.start(); - assignment.forEach( - (splitId, assignedSplits) -> splitEnumerator.addSplitsBack(assignedSplits, splitId)); - testContext.registerReader(0, "0"); - splitEnumerator.addReader(0); - testContext.getExecutorService().triggerAll(); - - List>> splitsForReader = - testContext.getSplitAssignments().get(0).getAssignedSplits(); - assertEquals(numSplits / numSubtasks, splitsForReader.size()); + try (SplitEnumerator, FlinkSourceEnumeratorState> enumerator = + flinkSource.createEnumerator(context)) { + if (configuredThresholdMb < 0) { + assertTrue(enumerator instanceof FlinkSourceSplitEnumerator); + } else if (configuredThresholdMb == 0) { + assertTrue(enumerator instanceof LazyFlinkSourceSplitEnumerator); + } else { + assertTrue(enumerator instanceof SizeBasedFlinkSourceSplitEnumerator); + } + enumerator.start(); + context.getExecutorService().triggerAll(); + assertEquals(expectedMode, enumerator.snapshotState(1L).getAssignmentMode()); } } @@ -180,17 +444,107 @@ private void assignSplits( Source> source, int numSplits) throws IOException { - try (FlinkSourceSplitEnumerator> splitEnumerator = - new FlinkSourceSplitEnumerator<>( - context, source, FlinkPipelineOptions.defaults(), numSplits)) { - splitEnumerator.start(); - // Add a reader before splitting the beam source. - context.registerReader(0, "0"); - splitEnumerator.addReader(0); + try (FlinkSourceSplitEnumerator> enumerator = + new FlinkSourceSplitEnumerator<>(context, source, staticOptions(), numSplits)) { + enumerator.start(); + for (int subtaskId = 0; subtaskId < context.currentParallelism(); subtaskId++) { + context.registerReader(subtaskId, String.valueOf(subtaskId)); + enumerator.addReader(subtaskId); + } context.getExecutorService().triggerAll(); - context.registerReader(1, "1"); - // Add another reader after splitting the beam source. - splitEnumerator.addReader(1); + } + } + + private static FlinkPipelineOptions staticOptions() { + FlinkPipelineOptions options = FlinkPipelineOptions.defaults(); + options.setSourceStaticSplitThresholdMb(-1L); + return options; + } + + private static FlinkPipelineOptions thresholdOptions() { + FlinkPipelineOptions options = FlinkPipelineOptions.defaults(); + options.setSourceStaticSplitThresholdMb(STATIC_SPLIT_THRESHOLD_MB); + return options; + } + + private static FlinkSource createBoundedSource( + BoundedSource source, FlinkPipelineOptions options, int numSplits) { + return FlinkSource.bounded( + "test-bounded-source", source, new SerializablePipelineOptions(options), numSplits); + } + + private static FlinkSourceEnumeratorState roundTripState( + FlinkSource source, FlinkSourceEnumeratorState state) throws IOException { + SimpleVersionedSerializer> serializer = + source.getEnumeratorCheckpointSerializer(); + byte[] serialized = serializer.serialize(state); + return serializer.deserialize(serializer.getVersion(), serialized); + } + + private static int countAssignedSplits( + TestingSplitEnumeratorContext> context) { + return context.getSplitAssignments().values().stream() + .mapToInt(state -> state.getAssignedSplits().size()) + .sum(); + } + + private static List assignedSplitIndexesForSubtask( + TestingSplitEnumeratorContext> context, int subtaskId) { + List splitIndexes = new ArrayList<>(); + for (FlinkSourceSplit split : + context.getSplitAssignments().get(subtaskId).getAssignedSplits()) { + splitIndexes.add(split.splitIndex()); + } + Collections.sort(splitIndexes); + return splitIndexes; + } + + private static final class TestEstimatedSizeBoundedSource extends BoundedSource { + private final long estimatedSizeBytes; + private final int generatedSplits; + private final AtomicInteger estimationCalls; + + private TestEstimatedSizeBoundedSource( + long estimatedSizeBytes, int generatedSplits, AtomicInteger estimationCalls) { + this.estimatedSizeBytes = estimatedSizeBytes; + this.generatedSplits = generatedSplits; + this.estimationCalls = estimationCalls; + } + + private static TestEstimatedSizeBoundedSource create( + long estimatedSizeBytes, int generatedSplits) { + return new TestEstimatedSizeBoundedSource( + estimatedSizeBytes, generatedSplits, new AtomicInteger()); + } + + @Override + public List> split( + long desiredBundleSizeBytes, PipelineOptions options) { + List splits = new ArrayList<>(generatedSplits); + for (int i = 0; i < generatedSplits; i++) { + splits.add(new TestEstimatedSizeBoundedSource(1L, 1, estimationCalls)); + } + return splits; + } + + @Override + public long getEstimatedSizeBytes(PipelineOptions options) { + estimationCalls.incrementAndGet(); + return estimatedSizeBytes; + } + + @Override + public BoundedReader createReader(PipelineOptions options) { + throw new UnsupportedOperationException("This source is only used to test split assignment"); + } + + @Override + public Coder getOutputCoder() { + return StringUtf8Coder.of(); + } + + private int getEstimationCalls() { + return estimationCalls.get(); } } } diff --git a/sdks/go/pkg/beam/io/filesystem/gcs/gcs.go b/sdks/go/pkg/beam/io/filesystem/gcs/gcs.go index 8d6ce28559f7..6a990f9cbe7a 100644 --- a/sdks/go/pkg/beam/io/filesystem/gcs/gcs.go +++ b/sdks/go/pkg/beam/io/filesystem/gcs/gcs.go @@ -53,14 +53,16 @@ func globToRegex(pattern string) (*regexp.Regexp, error) { var result strings.Builder result.WriteString("^") - for i := 0; i < len(pattern); i++ { - c := pattern[i] + // Scan rune by rune so multi-byte characters are handled correctly. + runes := []rune(pattern) + for i := 0; i < len(runes); i++ { + c := runes[i] switch c { case '*': // Check for ** (double asterisk) - if i+1 < len(pattern) && pattern[i+1] == '*' { + if i+1 < len(runes) && runes[i+1] == '*' { // Check if followed by / (e.g., "**/" matches zero or more path segments) - if i+2 < len(pattern) && pattern[i+2] == '/' { + if i+2 < len(runes) && runes[i+2] == '/' { // **/ matches "" or "something/" or "a/b/c/" result.WriteString("(?:.*/)?") i += 2 // Skip the second * and the / @@ -77,26 +79,26 @@ func globToRegex(pattern string) (*regexp.Regexp, error) { case '[': // Character class - find the closing bracket j := i + 1 - if j < len(pattern) && pattern[j] == '!' { + if j < len(runes) && runes[j] == '!' { j++ } - if j < len(pattern) && pattern[j] == ']' { + if j < len(runes) && runes[j] == ']' { j++ } - for j < len(pattern) && pattern[j] != ']' { + for j < len(runes) && runes[j] != ']' { j++ } - if j >= len(pattern) { + if j >= len(runes) { return nil, fmt.Errorf("syntax error: unclosed '[' in pattern %q", pattern) } else { // Copy the character class, converting ! to ^ for negation result.WriteByte('[') - content := pattern[i+1 : j] + content := runes[i+1 : j] if len(content) > 0 && content[0] == '!' { result.WriteByte('^') content = content[1:] } - result.WriteString(content) + result.WriteString(string(content)) result.WriteByte(']') i = j } diff --git a/sdks/go/pkg/beam/io/filesystem/gcs/gcs_test.go b/sdks/go/pkg/beam/io/filesystem/gcs/gcs_test.go index cd6aab2a2364..351b0b58c13b 100644 --- a/sdks/go/pkg/beam/io/filesystem/gcs/gcs_test.go +++ b/sdks/go/pkg/beam/io/filesystem/gcs/gcs_test.go @@ -322,6 +322,14 @@ func TestGlobToRegex(t *testing.T) { {"file.txt", "file.txt", true}, {"file.txt", "fileXtxt", false}, {"file(1).txt", "file(1).txt", true}, + + // Multi-byte characters must be preserved + {"résumé.txt", "résumé.txt", true}, + {"résumé.txt", "resume.txt", false}, + {"résumé/*.txt", "résumé/file.txt", true}, + {"*/résumé.txt", "dir/résumé.txt", true}, + {"*.txt", "résumé.txt", true}, + {"ïé.txt", "ïé.txt", true}, } for _, tt := range tests { diff --git a/sdks/go/pkg/beam/options/jobopts/options.go b/sdks/go/pkg/beam/options/jobopts/options.go index 327f3895b117..8456c8fd0615 100644 --- a/sdks/go/pkg/beam/options/jobopts/options.go +++ b/sdks/go/pkg/beam/options/jobopts/options.go @@ -207,6 +207,8 @@ func GetPipelineResourceHints() resource.Hints { h = resource.ParseMinRAM(val) case "accelerator", "beam:resources:accelerator:v1": h = resource.Accelerator(val) + case "cpu_count", "beam:resources:cpu_count:v1": + h = resource.ParseCPUCount(val) default: if strings.HasPrefix(name, "beam:resources:") { h = stringHint{urn: name, value: val} diff --git a/sdks/go/pkg/beam/options/jobopts/options_test.go b/sdks/go/pkg/beam/options/jobopts/options_test.go index 89e2d5a721dc..5bcdf39ea599 100644 --- a/sdks/go/pkg/beam/options/jobopts/options_test.go +++ b/sdks/go/pkg/beam/options/jobopts/options_test.go @@ -149,9 +149,11 @@ func TestGetPipelineResourceHints(t *testing.T) { hints.Set("accelerator=pedal_to_the_metal") hints.Set("beam:resources:novel_execution:v1=jaguar") hints.Set("min_ram=1GB") + hints.Set("cpu_count=1") + hints.Set("beam:resources:cpu_count:v1=4") ResourceHints = hints - want := resource.NewHints(resource.ParseMinRAM("1GB"), resource.Accelerator("pedal_to_the_metal"), stringHint{ + want := resource.NewHints(resource.ParseMinRAM("1GB"), resource.Accelerator("pedal_to_the_metal"), resource.ParseCPUCount("4"), stringHint{ urn: "beam:resources:novel_execution:v1", value: "jaguar", }) diff --git a/sdks/go/pkg/beam/options/resource/hint.go b/sdks/go/pkg/beam/options/resource/hint.go index 5efaca80ede0..1d3f1cd37ec4 100644 --- a/sdks/go/pkg/beam/options/resource/hint.go +++ b/sdks/go/pkg/beam/options/resource/hint.go @@ -246,6 +246,21 @@ func (h acceleratorHint) String() string { return fmt.Sprintf("accelerator=%v", h.value) } +// ParseCPUCount converts a number in string form into a hint. +// An invalid format will cause ParseCPUCount to panic. +// +// Hints are advisory only and runners may not respect them. +// +// See https://beam.apache.org/documentation/runtime/resource-hints/ for more information about +// resource hints. +func ParseCPUCount(v string) Hint { + b, err := strconv.ParseUint(v, 10, 64) + if err != nil { + panic(fmt.Sprintf("resource.ParseCPUCount: unable to parse %q: %v", v, err)) + } + return CPUCount(uint64(b)) +} + // CPUCount hints that this scope should be put in a machine with at least this many CPUs or vCPUs. // // Hints are advisory only and runners may not respect them. @@ -280,5 +295,5 @@ func (h CPUCountHint) MergeWithOuter(outer Hint) Hint { } func (h CPUCountHint) String() string { - return fmt.Sprintf("cpu_count=%v", humanize.Bytes(uint64(h.value))) + return fmt.Sprintf("cpu_count=%v", h.value) } diff --git a/sdks/go/pkg/beam/options/resource/hint_test.go b/sdks/go/pkg/beam/options/resource/hint_test.go index 2c4c6506e3a1..35dc5fa0dd76 100644 --- a/sdks/go/pkg/beam/options/resource/hint_test.go +++ b/sdks/go/pkg/beam/options/resource/hint_test.go @@ -143,6 +143,69 @@ func TestCPUCountHint_Payload(t *testing.T) { } } +func TestCPUCountHint_String(t *testing.T) { + tests := []struct { + value uint64 + want string + }{ + {0, "cpu_count=0"}, + {1, "cpu_count=1"}, + {4, "cpu_count=4"}, + {128, "cpu_count=128"}, + } + + for _, test := range tests { + h := CPUCountHint{value: test.value} + if got, want := h.String(), test.want; got != want { + t.Errorf("%v.String() = %v, want %v", h, got, want) + } + } +} + +func TestParseCPUCount(t *testing.T) { + tests := []struct { + value string + payload string + }{ + {"0", "0"}, + {"1", "1"}, + {"2", "2"}, + {"4", "4"}, + {"11", "11"}, + {"2003", "2003"}, + {"12000000", "12000000"}, + {"18446744073709551615", "18446744073709551615"}, + } + + for _, test := range tests { + h := ParseCPUCount(test.value) + if got, want := h.Payload(), []byte(test.payload); !bytes.Equal(got, want) { + t.Errorf("%v.Payload() = %v, want %v", h, string(got), string(want)) + } + } +} + +func TestParseCPUCount_panic(t *testing.T) { + tests := []string{ + "a bad cpu string", + "-1", + "1.5", + "", + "18446744073709551616", + } + + for _, test := range tests { + t.Run(test, func(t *testing.T) { + defer func() { + if r := recover(); r == nil { + t.Errorf("want ParseCPUCount(%q) to panic", test) + } + }() + ParseCPUCount(test) + }) + } +} + func TestMaxActiveBundlesPerWorkerHint_MergeWith(t *testing.T) { low := maxActiveBundlesPerWorkerHint{value: 2} high := maxActiveBundlesPerWorkerHint{value: 4} diff --git a/website/www/site/layouts/shortcodes/flink_java_pipeline_options.html b/website/www/site/layouts/shortcodes/flink_java_pipeline_options.html index 34d6c5243776..8e0fc007f13b 100644 --- a/website/www/site/layouts/shortcodes/flink_java_pipeline_options.html +++ b/website/www/site/layouts/shortcodes/flink_java_pipeline_options.html @@ -187,6 +187,11 @@ Shuts down sources which have been idle for the configured time of milliseconds. Once a source has been shut down, checkpointing is not possible anymore. Shutting down the sources eventually leads to pipeline shutdown (=Flink job finishes) once all input has been processed. Unless explicitly set, this will default to Long.MAX_VALUE when checkpointing is enabled and to 0 when checkpointing is disabled. See https://issues.apache.org/jira/browse/FLINK-2491 for progress on this issue. Default: -1 + + sourceStaticSplitThresholdMb + Static split assignment threshold in MiB per source reader for bounded sources in Flink DataStream mode. The default of 0 always uses lazy assignment. A positive value selects static round-robin assignment for sources with a known, positive estimate below the threshold and lazy assignment otherwise. Any negative value always uses static assignment. + Default: 0 + stateBackend State backend to store Beam's state. Use 'rocksdb' or 'hashmap' (same as 'filesystem'). diff --git a/website/www/site/layouts/shortcodes/flink_python_pipeline_options.html b/website/www/site/layouts/shortcodes/flink_python_pipeline_options.html index e3fe24216a54..d2ab687c42be 100644 --- a/website/www/site/layouts/shortcodes/flink_python_pipeline_options.html +++ b/website/www/site/layouts/shortcodes/flink_python_pipeline_options.html @@ -187,6 +187,11 @@ Shuts down sources which have been idle for the configured time of milliseconds. Once a source has been shut down, checkpointing is not possible anymore. Shutting down the sources eventually leads to pipeline shutdown (=Flink job finishes) once all input has been processed. Unless explicitly set, this will default to Long.MAX_VALUE when checkpointing is enabled and to 0 when checkpointing is disabled. See https://issues.apache.org/jira/browse/FLINK-2491 for progress on this issue. Default: -1 + + source_static_split_threshold_mb + Static split assignment threshold in MiB per source reader for bounded sources in Flink DataStream mode. The default of 0 always uses lazy assignment. A positive value selects static round-robin assignment for sources with a known, positive estimate below the threshold and lazy assignment otherwise. Any negative value always uses static assignment. + Default: 0 + state_backend State backend to store Beam's state. Use 'rocksdb' or 'hashmap' (same as 'filesystem').