diff --git a/.github/workflows/prune.yml b/.github/workflows/prune.yml new file mode 100644 index 000000000..17b6a3cdf --- /dev/null +++ b/.github/workflows/prune.yml @@ -0,0 +1,38 @@ +name: prune + +on: + workflow_dispatch: + schedule: + - cron: "0 0 * * *" + push: + branches: + - main + +jobs: + clean: + runs-on: ubuntu-latest + name: Prune images + steps: + - name: Login to GitHub Container Registry + uses: docker/login-action@v3.0.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Fetch multi-platform package version SHAs + id: multi-arch-digests + run: | + erpc=$(docker manifest inspect ghcr.io/erpc/erpc | jq -r '.manifests.[] | .digest' | paste -s -d ' ' -) + echo "multi-arch-digests=$erpc" >> $GITHUB_OUTPUT + + - uses: snok/container-retention-policy@v3.0.0 + with: + account: erpc + token: ${{ secrets.GITHUB_TOKEN }} + image-names: "erpc" + image-tags: "!latest !main !*.*.*" + skip-shas: ${{ steps.multi-arch-digests.outputs.multi-arch-digests }} + tag-selection: both + cut-off: 0ms + dry-run: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f56633288..7bf4146e6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,6 +10,9 @@ on: description: 'Commit SHA (default: last commit of the current branch)' required: false default: '' + push: + branches: + - main permissions: contents: write @@ -20,17 +23,20 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout + if: github.event.inputs.version_tag != '' uses: actions/checkout@v4 with: fetch-depth: 0 ref: ${{ github.event.inputs.commit_sha || github.ref }} - name: Configure Git + if: github.event.inputs.version_tag != '' run: | git config user.name github-actions git config user.email github-actions@github.com - name: Tag commit and push + if: github.event.inputs.version_tag != '' run: | git tag ${{ github.event.inputs.version_tag }} -f git push origin ${{ github.event.inputs.version_tag }} -f @@ -40,17 +46,20 @@ jobs: needs: tag steps: - name: Checkout + if: github.event.inputs.version_tag != '' uses: actions/checkout@v4 with: fetch-depth: 0 ref: ${{ github.event.inputs.commit_sha || github.ref }} - name: Set up Go + if: github.event.inputs.version_tag != '' uses: actions/setup-go@v5 with: go-version: '1.22.x' - name: Run GoReleaser + if: github.event.inputs.version_tag != '' uses: goreleaser/goreleaser-action@v6 with: distribution: goreleaser @@ -66,7 +75,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - ref: ${{ github.event.inputs.version_tag }} + ref: ${{ github.event.inputs.version_tag || 'main' }} - name: Login to GitHub Container Registry uses: docker/login-action@v3 @@ -87,7 +96,25 @@ jobs: REPO="${{ github.repository }}" echo "repo=${REPO@L}" >> "$GITHUB_OUTPUT" - - name: Build and push Docker image + - name: Generate short SHA + id: short_sha + run: echo "SHORT_SHA=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + + - name: Build and push Docker image from main + if: github.event.inputs.version_tag == '' + uses: docker/build-push-action@v5 + with: + context: . + push: true + platforms: linux/amd64,linux/arm64 + build-args: | + VERSION=main + COMMIT_SHA=${{ steps.short_sha.outputs.SHORT_SHA }} + tags: | + ghcr.io/${{ steps.tag_param.outputs.repo }}:main + + - name: Build and push Docker image with tags + if: github.event.inputs.version_tag != '' uses: docker/build-push-action@v5 with: context: . @@ -95,7 +122,7 @@ jobs: platforms: linux/amd64,linux/arm64 build-args: | VERSION=${{ github.event.inputs.version_tag }} - COMMIT_SHA=${{ github.sha }} + COMMIT_SHA=${{ steps.short_sha.outputs.SHORT_SHA }} tags: | - ghcr.io/${{ steps.tag_param.outputs.repo }}:latest ghcr.io/${{ steps.tag_param.outputs.repo }}:${{ github.event.inputs.version_tag }} + ghcr.io/${{ steps.tag_param.outputs.repo }}:latest diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a58994c1f..0e4c375b1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,4 +25,4 @@ jobs: - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v4.0.1 with: - token: ${{ secrets.CODECOV_TOKEN }} \ No newline at end of file + args: -exclude-dir=test -tests=false ./... diff --git a/.gitignore b/.gitignore index 5e54ac953..4f919e7e6 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,5 @@ coverage.txt .DS_Store erpc.yaml .generated-go-semantic-release-changelog.md -.semrel/ \ No newline at end of file +.semrel/ +__debug* \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index e7002ff52..13c0ae30b 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -11,6 +11,11 @@ "mode": "auto", "program": "${workspaceFolder}/cmd/erpc", "args": ["${workspaceFolder}/erpc.yaml"], + "env": { + "CC": "/usr/bin/cc", + "CXX": "/usr/bin/c++", + "CGO_ENABLED": "1" + } } ] } \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 2dc7f1171..3655f9bda 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,10 +17,10 @@ RUN go mod download COPY . . # Build the application without pprof -RUN CGO_ENABLED=0 GOOS=linux LDFLAGS="-w -s -X main.version=${VERSION} -X main.commitSHA=${COMMIT_SHA}" go build -a -installsuffix cgo -o erpc-server ./cmd/erpc/main.go +RUN CGO_ENABLED=0 GOOS=linux LDFLAGS="-w -s -X common.ErpcVersion=${VERSION} -X common.ErpcCommitSha=${COMMIT_SHA}" go build -a -installsuffix cgo -o erpc-server ./cmd/erpc/main.go # Build the application with pprof -RUN CGO_ENABLED=0 GOOS=linux LDFLAGS="-w -s -X main.version=${VERSION} -X main.commitSHA=${COMMIT_SHA}" go build -a -installsuffix cgo -tags pprof -o erpc-server-pprof ./cmd/erpc/*.go +RUN CGO_ENABLED=0 GOOS=linux LDFLAGS="-w -s -X common.ErpcVersion=${VERSION} -X common.ErpcCommitSha=${COMMIT_SHA}" go build -a -installsuffix cgo -tags pprof -o erpc-server-pprof ./cmd/erpc/*.go # Final stage FROM alpine:3.18 diff --git a/Makefile b/Makefile index a4599be5d..8f61e0cc4 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ build: test: @go clean -testcache @go test ./cmd/... -count 1 -parallel 1 - @go test $$(ls -d */ | grep -v "cmd/" | grep -v "test/" | awk '{print "./" $$1 "..."}') -covermode=atomic -race -count 3 -parallel 1 + @go test $$(ls -d */ | grep -v "cmd/" | grep -v "test/" | awk '{print "./" $$1 "..."}') -covermode=atomic -race -count 5 -parallel 1 .PHONY: coverage coverage: diff --git a/README.md b/README.md index c99c0bb0d..3b84a59cc 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,11 @@ ✅ [Authentication](https://docs.erpc.cloud/config/auth) modules such as basic auth, secret-based, JWT and SIWE.
✅ [Smart batching](https://docs.erpc.cloud/operation/batch) to aggregates multiple RPC or contract calls into one.
+# Case Studies + +[🚀 Moonwell: How eRPC slashed RPC calls by 67%](https://erpc.cloud/case-studies/moonwell)
+[🚀 Chronicle: How eRPC reduced RPC cost by 45%](https://erpc.cloud/case-studies/chronicle) + # Usage & Docs - Visit [docs.erpc.cloud](https://docs.erpc.cloud) for documentation and guides. diff --git a/auth/http.go b/auth/http.go index 0d1aeedfd..9b961e498 100644 --- a/auth/http.go +++ b/auth/http.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/erpc/erpc/common" + "github.com/erpc/erpc/util" "github.com/valyala/fasthttp" ) @@ -19,15 +20,15 @@ func NewPayloadFromHttp(projectId string, nq *common.NormalizedRequest, headers if args.Has("token") { ap.Type = common.AuthTypeSecret ap.Secret = &SecretPayload{ - Value: string(args.Peek("token")), + Value: util.Mem2Str(args.Peek("token")), } } else if tkn := headers.Peek("X-ERPC-Secret-Token"); tkn != nil { ap.Type = common.AuthTypeSecret ap.Secret = &SecretPayload{ - Value: string(tkn), + Value: util.Mem2Str(tkn), } } else if ath := headers.Peek("Authorization"); ath != nil { - ath := strings.TrimSpace(string(ath)) + ath := strings.TrimSpace(util.Mem2Str(ath)) label := strings.ToLower(ath[0:6]) if strings.EqualFold(label, "basic") { @@ -36,7 +37,7 @@ func NewPayloadFromHttp(projectId string, nq *common.NormalizedRequest, headers if err != nil { return nil, err } - parts := strings.Split(string(basicAuth), ":") + parts := strings.Split(util.Mem2Str(basicAuth), ":") if len(parts) != 2 { return nil, errors.New("invalid basic auth must be base64 of username:password") } @@ -55,20 +56,20 @@ func NewPayloadFromHttp(projectId string, nq *common.NormalizedRequest, headers } else if args.Has("jwt") { ap.Type = common.AuthTypeJwt ap.Jwt = &JwtPayload{ - Token: string(args.Peek("jwt")), + Token: util.Mem2Str(args.Peek("jwt")), } } else if args.Has("signature") && args.Has("message") { ap.Type = common.AuthTypeSiwe ap.Siwe = &SiwePayload{ - Signature: string(args.Peek("signature")), - Message: normalizeSiweMessage(string(args.Peek("message"))), + Signature: util.Mem2Str(args.Peek("signature")), + Message: normalizeSiweMessage(util.Mem2Str(args.Peek("message"))), } } else if msg := headers.Peek("X-Siwe-Message"); msg != nil { if sig := headers.Peek("X-Siwe-Signature"); sig != nil { ap.Type = common.AuthTypeSiwe ap.Siwe = &SiwePayload{ - Signature: string(sig), - Message: normalizeSiweMessage(string(msg)), + Signature: util.Mem2Str(sig), + Message: normalizeSiweMessage(util.Mem2Str(msg)), } } } @@ -77,8 +78,8 @@ func NewPayloadFromHttp(projectId string, nq *common.NormalizedRequest, headers if ap.Type == "" { ap.Type = common.AuthTypeNetwork ap.Network = &NetworkPayload{ - Address: string(headers.Peek("X-Forwarded-For")), - ForwardProxies: strings.Split(string(headers.Peek("X-Forwarded-For")), ","), + Address: util.Mem2Str(headers.Peek("X-Forwarded-For")), + ForwardProxies: strings.Split(util.Mem2Str(headers.Peek("X-Forwarded-For")), ","), } } @@ -91,5 +92,5 @@ func normalizeSiweMessage(msg string) string { if err != nil { return msg } - return string(decoded) + return util.Mem2Str(decoded) } diff --git a/cmd/erpc/init_test.go b/cmd/erpc/init_test.go new file mode 100644 index 000000000..685447d56 --- /dev/null +++ b/cmd/erpc/init_test.go @@ -0,0 +1,9 @@ +package main + +import ( + "github.com/rs/zerolog" +) + +func init() { + zerolog.SetGlobalLevel(zerolog.Disabled) +} diff --git a/cmd/erpc/main.go b/cmd/erpc/main.go index b8cc41e63..abc6858e1 100644 --- a/cmd/erpc/main.go +++ b/cmd/erpc/main.go @@ -6,21 +6,17 @@ import ( "os/signal" "syscall" + "github.com/erpc/erpc/common" "github.com/erpc/erpc/erpc" "github.com/erpc/erpc/util" "github.com/rs/zerolog/log" "github.com/spf13/afero" ) -var ( - version = "dev" - commitSHA = "none" -) - func main() { logger := log.With().Logger() - logger.Info().Msgf("starting eRPC version: %s, commit: %s", version, commitSHA) + logger.Info().Msgf("starting eRPC version: %s, commit: %s", common.ErpcVersion, common.ErpcCommitSha) err := erpc.Init( context.Background(), diff --git a/cmd/erpc/main_test.go b/cmd/erpc/main_test.go index 934045d1a..4a836b6cf 100644 --- a/cmd/erpc/main_test.go +++ b/cmd/erpc/main_test.go @@ -3,7 +3,6 @@ package main import ( "bytes" "context" - "encoding/json" "fmt" "io" "math/rand" @@ -158,7 +157,7 @@ func TestInit_HappyPath(t *testing.T) { Times(5). Post(""). Reply(200). - JSON(json.RawMessage(`{"result":{"hash":"0x64d340d2470d2ed0ec979b72d79af9cd09fc4eb2b89ae98728d5fb07fd89baf9"}}`)) + JSON([]byte(`{"result":{"hash":"0x64d340d2470d2ed0ec979b72d79af9cd09fc4eb2b89ae98728d5fb07fd89baf9"}}`)) // // 2) Initialize the eRPC server with a mock configuration diff --git a/common/config.go b/common/config.go index 008d021d1..24f47a1d4 100644 --- a/common/config.go +++ b/common/config.go @@ -10,8 +10,12 @@ import ( "gopkg.in/yaml.v3" ) -var TRUE = true -var FALSE = false +var ( + ErpcVersion = "dev" + ErpcCommitSha = "none" + TRUE = true + FALSE = false +) // Config represents the configuration of the application. type Config struct { @@ -373,6 +377,10 @@ func (c *RateLimitRuleConfig) MarshalZerologObject(e *zerolog.Event) { } func (c *NetworkConfig) NetworkId() string { + if c.Architecture == "" || c.Evm == nil { + return "" + } + switch c.Architecture { case "evm": return util.EvmNetworkId(c.Evm.ChainId) @@ -437,11 +445,11 @@ func (s *UpstreamConfig) UnmarshalYAML(unmarshal func(interface{}) error) error BackoffFactor: 1.5, }, CircuitBreaker: &CircuitBreakerPolicyConfig{ - FailureThresholdCount: 80, - FailureThresholdCapacity: 100, + FailureThresholdCount: 800, + FailureThresholdCapacity: 1000, HalfOpenAfter: "5m", - SuccessThresholdCount: 5, - SuccessThresholdCapacity: 5, + SuccessThresholdCount: 3, + SuccessThresholdCapacity: 3, }, }, RateLimitAutoTune: &RateLimitAutoTuneConfig{ @@ -468,3 +476,41 @@ func (s *UpstreamConfig) UnmarshalYAML(unmarshal func(interface{}) error) error *s = UpstreamConfig(raw) return nil } + +var NetworkDefaultFailsafeConfig = &FailsafeConfig{ + Hedge: &HedgePolicyConfig{ + Delay: "200ms", + MaxCount: 3, + }, + Retry: &RetryPolicyConfig{ + MaxAttempts: 3, + Delay: "1s", + Jitter: "500ms", + BackoffMaxDelay: "10s", + BackoffFactor: 2, + }, + Timeout: &TimeoutPolicyConfig{ + Duration: "30s", + }, +} + +func (c *NetworkConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + type rawNetworkConfig NetworkConfig + raw := rawNetworkConfig{ + Failsafe: NetworkDefaultFailsafeConfig, + } + if err := unmarshal(&raw); err != nil { + return err + } + + if raw.Architecture == "" { + return NewErrInvalidConfig("network.*.architecture is required") + } + + if raw.Architecture == "evm" && raw.Evm == nil { + return NewErrInvalidConfig("network.*.evm is required for evm networks") + } + + *c = NetworkConfig(raw) + return nil +} diff --git a/common/errors.go b/common/errors.go index 9756a1f6b..2e8a9067e 100644 --- a/common/errors.go +++ b/common/errors.go @@ -8,8 +8,6 @@ import ( "regexp" "strings" "time" - - "github.com/bytedance/sonic" ) func IsNull(err interface{}) bool { @@ -97,7 +95,7 @@ func (e *BaseError) Error() string { var detailsStr string if e.Details != nil && len(e.Details) > 0 { - s, er := sonic.Marshal(e.Details) + s, er := SonicCfg.Marshal(e.Details) if er == nil { detailsStr = fmt.Sprintf("(%s)", s) } else { @@ -158,7 +156,7 @@ func (e BaseError) MarshalJSON() ([]byte, error) { causes = append(causes, err.Error()) } } - return sonic.Marshal(&struct { + return SonicCfg.Marshal(&struct { Alias Cause []interface{} `json:"cause"` }{ @@ -166,7 +164,7 @@ func (e BaseError) MarshalJSON() ([]byte, error) { Cause: causes, }) } else if cs, ok := cause.(StandardError); ok { - return sonic.Marshal(&struct { + return SonicCfg.Marshal(&struct { Alias Cause StandardError `json:"cause"` }{ @@ -174,7 +172,7 @@ func (e BaseError) MarshalJSON() ([]byte, error) { Cause: cs, }) } else if cause != nil { - return sonic.Marshal(&struct { + return SonicCfg.Marshal(&struct { Alias Cause BaseError `json:"cause"` }{ @@ -186,7 +184,7 @@ func (e BaseError) MarshalJSON() ([]byte, error) { }) } - return sonic.Marshal(&struct { + return SonicCfg.Marshal(&struct { Alias Cause interface{} `json:"cause"` }{ @@ -700,6 +698,8 @@ func (e *ErrUpstreamsExhausted) SummarizeCauses() string { cbOpen := 0 billing := 0 other := 0 + client := 0 + transport := 0 cancelled := 0 for _, e := range joinedErr.Unwrap() { @@ -728,6 +728,12 @@ func (e *ErrUpstreamsExhausted) SummarizeCauses() string { } else if HasErrorCode(e, ErrCodeUpstreamHedgeCancelled) { cancelled++ continue + } else if HasErrorCode(e, ErrCodeEndpointClientSideException) || HasErrorCode(e, ErrCodeJsonRpcRequestUnmarshal) { + client++ + continue + } else if HasErrorCode(e, ErrCodeEndpointTransportFailure) { + transport++ + continue } else if !HasErrorCode(e, ErrCodeUpstreamMethodIgnored) { other++ } @@ -758,6 +764,9 @@ func (e *ErrUpstreamsExhausted) SummarizeCauses() string { if cancelled > 0 { reasons = append(reasons, fmt.Sprintf("%d hedges cancelled", cancelled)) } + if client > 0 { + reasons = append(reasons, fmt.Sprintf("%d user errors", client)) + } if other > 0 { reasons = append(reasons, fmt.Sprintf("%d other errors", other)) } @@ -927,6 +936,10 @@ var NewErrUpstreamMethodIgnored = func(method string, upstreamId string) error { } } +func (e *ErrUpstreamMethodIgnored) ErrorStatusCode() int { + return http.StatusUnsupportedMediaType +} + type ErrUpstreamSyncing struct{ BaseError } const ErrCodeUpstreamSyncing ErrorCode = "ErrUpstreamSyncing" @@ -961,14 +974,15 @@ type ErrUpstreamHedgeCancelled struct{ BaseError } const ErrCodeUpstreamHedgeCancelled ErrorCode = "ErrUpstreamHedgeCancelled" -var NewErrUpstreamHedgeCancelled = func(upstreamId string) error { +var NewErrUpstreamHedgeCancelled = func(upstreamId string, cause error) error { return &ErrUpstreamHedgeCancelled{ BaseError{ Code: ErrCodeUpstreamHedgeCancelled, - Message: "hedged request cancelled in favor another response", + Message: "hedged request cancelled in favor of another response", Details: map[string]interface{}{ "upstreamId": upstreamId, }, + Cause: cause, }, } } @@ -995,14 +1009,29 @@ type ErrJsonRpcRequestUnmarshal struct { BaseError } -const ErrCodeJsonRpcRequestUnmarshal = "ErrJsonRpcRequestUnmarshal" +const ErrCodeJsonRpcRequestUnmarshal ErrorCode = "ErrJsonRpcRequestUnmarshal" var NewErrJsonRpcRequestUnmarshal = func(cause error) error { + if _, ok := cause.(*BaseError); ok { + return &ErrJsonRpcRequestUnmarshal{ + BaseError{ + Code: ErrCodeJsonRpcRequestUnmarshal, + Message: "failed to unmarshal json-rpc request", + Cause: cause, + }, + } + } else if cause != nil { + return &ErrJsonRpcRequestUnmarshal{ + BaseError{ + Code: ErrCodeJsonRpcRequestUnmarshal, + Message: fmt.Sprintf("%s", cause), + }, + } + } return &ErrJsonRpcRequestUnmarshal{ BaseError{ Code: ErrCodeJsonRpcRequestUnmarshal, Message: "failed to unmarshal json-rpc request", - Cause: cause, }, } } @@ -1318,6 +1347,20 @@ func (e *ErrEndpointClientSideException) ErrorStatusCode() int { return http.StatusBadRequest } +type ErrEndpointTransportFailure struct{ BaseError } + +const ErrCodeEndpointTransportFailure = "ErrEndpointTransportFailure" + +var NewErrEndpointTransportFailure = func(cause error) error { + return &ErrEndpointTransportFailure{ + BaseError{ + Code: ErrCodeEndpointTransportFailure, + Message: "failure when sending request to remote endpoint", + Cause: cause, + }, + } +} + type ErrEndpointServerSideException struct{ BaseError } const ErrCodeEndpointServerSideException = "ErrEndpointServerSideException" @@ -1407,21 +1450,29 @@ var NewErrEndpointMissingData = func(cause error) error { } } -type ErrEndpointEvmLargeRange struct{ BaseError } +type ErrEndpointRequestTooLarge struct{ BaseError } + +const ErrCodeEndpointRequestTooLarge = "ErrEndpointRequestTooLarge" -const ErrCodeEndpointEvmLargeRange = "ErrEndpointEvmLargeRange" +type TooLargeComplaint string -var NewErrEndpointEvmLargeRange = func(cause error) error { - return &ErrEndpointEvmLargeRange{ +const EvmBlockRangeTooLarge TooLargeComplaint = "evm_block_range" +const EvmAddressesTooLarge TooLargeComplaint = "evm_addresses" + +var NewErrEndpointRequestTooLarge = func(cause error, complaint TooLargeComplaint) error { + return &ErrEndpointRequestTooLarge{ BaseError{ - Code: ErrCodeEndpointEvmLargeRange, - Message: "evm remote endpoint complained about large logs range", + Code: ErrCodeEndpointRequestTooLarge, + Message: "remote endpoint complained about large request (e.g. block range, number of addresses, etc)", Cause: cause, + Details: map[string]interface{}{ + "complaint": complaint, + }, }, } } -func (e *ErrEndpointEvmLargeRange) ErrorStatusCode() int { +func (e *ErrEndpointRequestTooLarge) ErrorStatusCode() int { return http.StatusRequestEntityTooLarge } @@ -1441,6 +1492,7 @@ const ( JsonRpcErrorUnknown JsonRpcErrorNumber = -99999 // Standard JSON-RPC codes + JsonRpcErrorCallException JsonRpcErrorNumber = -32000 JsonRpcErrorClientSideException JsonRpcErrorNumber = -32600 JsonRpcErrorUnsupportedException JsonRpcErrorNumber = -32601 JsonRpcErrorInvalidArgument JsonRpcErrorNumber = -32602 @@ -1456,7 +1508,6 @@ const ( JsonRpcErrorMissingData JsonRpcErrorNumber = -32014 JsonRpcErrorNodeTimeout JsonRpcErrorNumber = -32015 JsonRpcErrorUnauthorized JsonRpcErrorNumber = -32016 - JsonRpcErrorCallException JsonRpcErrorNumber = -32017 ) // This struct represents an json-rpc error with erpc structure (i.e. code is string) @@ -1518,7 +1569,7 @@ type ErrJsonRpcExceptionExternal struct { Message string `json:"message,omitempty"` // Some errors such as execution reverted carry "data" field which has additional information - Data string `json:"data,omitempty"` + Data interface{} `json:"data,omitempty"` } func NewErrJsonRpcExceptionExternal(code int, message string, data string) *ErrJsonRpcExceptionExternal { @@ -1633,3 +1684,8 @@ func IsCapacityIssue(err error) bool { HasErrorCode(err, ErrCodeAuthRateLimitRuleExceeded) || HasErrorCode(err, ErrCodeEndpointCapacityExceeded) } + +func IsClientError(err error) bool { + return err != nil && (HasErrorCode(err, ErrCodeEndpointClientSideException) || + HasErrorCode(err, ErrCodeJsonRpcRequestUnmarshal)) +} diff --git a/common/evm.go b/common/evm.go index 588d04bf7..8e1ae6d6d 100644 --- a/common/evm.go +++ b/common/evm.go @@ -8,3 +8,14 @@ const ( EvmNodeTypeSequencer EvmNodeType = "sequencer" EvmNodeTypeExecution EvmNodeType = "execution" ) + +func IsEvmWriteMethod(method string) bool { + return method == "eth_sendRawTransaction" || + method == "eth_sendTransaction" || + method == "eth_createAccessList" || + method == "eth_submitTransaction" || + method == "eth_submitWork" || + method == "eth_newFilter" || + method == "eth_newBlockFilter" || + method == "eth_newPendingTransactionFilter" +} diff --git a/common/evm_block_ref.go b/common/evm_block_ref.go index 1e21efe79..834cedb02 100644 --- a/common/evm_block_ref.go +++ b/common/evm_block_ref.go @@ -47,6 +47,9 @@ func ExtractEvmBlockReferenceFromRequest(r *JsonRpcRequest) (string, int64, erro if len(r.Params) > 0 { if bns, ok := r.Params[0].(string); ok { if strings.HasPrefix(bns, "0x") { + if len(bns) == 66 { + return bns, 0, nil + } bni, err := HexToInt64(bns) if err != nil { return "", 0, err @@ -79,15 +82,14 @@ func ExtractEvmBlockReferenceFromRequest(r *JsonRpcRequest) (string, int64, erro return "", 0, nil - case "eth_getBalance", - "eth_getCode", - "eth_getTransactionCount", - "eth_call", - "eth_feeHistory", + case "eth_feeHistory", "eth_getAccount": if len(r.Params) > 1 { if bns, ok := r.Params[1].(string); ok { if strings.HasPrefix(bns, "0x") { + if len(bns) == 66 { + return bns, 0, nil + } bni, err := HexToInt64(bns) if err != nil { return bns, 0, err @@ -101,6 +103,58 @@ func ExtractEvmBlockReferenceFromRequest(r *JsonRpcRequest) (string, int64, erro return "", 0, fmt.Errorf("unexpected missing 2nd parameter for method %s: %+v", r.Method, r.Params) } + case "eth_getBalance", + "eth_getTransactionCount", + "eth_getCode", + "eth_call": + if len(r.Params) > 1 { + switch secondParam := r.Params[1].(type) { + case string: + if strings.HasPrefix(secondParam, "0x") { + // Handle the 2nd parameter as a blockHash HEX String + if len(secondParam) == 66 { // 32 bytes * 2 + "0x" + return secondParam, 0, nil + } + // Handle the 2nd parameter as a blockNumber HEX String + bni, err := HexToInt64(secondParam) + if err != nil { + return secondParam, 0, err + } + return strconv.FormatInt(bni, 10), bni, nil + } + return "", 0, nil + + case map[string]interface{}: + // Handle the 2nd parameter as an object + if blockNumber, exists := secondParam["blockNumber"]; exists { + if bns, ok := blockNumber.(string); ok && strings.HasPrefix(bns, "0x") { + bni, err := HexToInt64(bns) + if err != nil { + return bns, 0, err + } + return strconv.FormatInt(bni, 10), bni, nil + } + return "", 0, nil + } + + if blockHash, exists := secondParam["blockHash"]; exists { + if bh, ok := blockHash.(string); ok && strings.HasPrefix(bh, "0x") { + return bh, 0, nil + } + return "", 0, nil + } + + // If neither blockNumber nor blockHash is provided + return "", 0, nil + + default: + // If the 2nd parameter is neither string nor map + return "", 0, nil + } + } else { + return "", 0, fmt.Errorf("unexpected missing 2nd parameter for method %s: %+v", r.Method, r.Params) + } + case "eth_chainId", "eth_getTransactionReceipt", "eth_getTransactionByHash", @@ -130,16 +184,48 @@ func ExtractEvmBlockReferenceFromRequest(r *JsonRpcRequest) (string, int64, erro case "eth_getProof", "eth_getStorageAt": if len(r.Params) > 2 { - if bns, ok := r.Params[2].(string); ok { - if strings.HasPrefix(bns, "0x") { - bni, err := HexToInt64(bns) + switch thirdParam := r.Params[2].(type) { + case string: + if strings.HasPrefix(thirdParam, "0x") { + // Handle the 3rd parameter as a blockHash HEX String + if len(thirdParam) == 66 { // 32 bytes * 2 + "0x" + return thirdParam, 0, nil + } + // Handle the 3rd parameter as a HEX String + bni, err := HexToInt64(thirdParam) if err != nil { - return bns, 0, err + return thirdParam, 0, err } return strconv.FormatInt(bni, 10), bni, nil - } else { + } + return "", 0, nil + + case map[string]interface{}: + // Handle the 3rd parameter as an object + if blockNumber, exists := thirdParam["blockNumber"]; exists { + if bns, ok := blockNumber.(string); ok && strings.HasPrefix(bns, "0x") { + bni, err := HexToInt64(bns) + if err != nil { + return bns, 0, err + } + return strconv.FormatInt(bni, 10), bni, nil + } return "", 0, nil } + + if blockHash, exists := thirdParam["blockHash"]; exists { + if bh, ok := blockHash.(string); ok && strings.HasPrefix(bh, "0x") { + return bh, 0, nil + } + return "", 0, nil + } + + // If neither blockNumber nor blockHash is provided + return "", 0, nil + + default: + // If the 3rd parameter is neither string nor map + return "", 0, nil } } else { return "", 0, fmt.Errorf("unexpected missing 3rd parameter for method %s: %+v", r.Method, r.Params) @@ -164,56 +250,45 @@ func ExtractEvmBlockReferenceFromResponse(rpcReq *JsonRpcRequest, rpcResp *JsonR switch rpcReq.Method { case "eth_getTransactionReceipt", "eth_getTransactionByHash": - if rpcResp.Result != nil { - result, err := rpcResp.ParsedResult() - if err != nil { - return "", 0, err - } - rpcResp.RLock() - defer rpcResp.RUnlock() - if tx, ok := result.(map[string]interface{}); ok { - var blockRef string - var blockNumber int64 - blockRef, _ = tx["blockHash"].(string) - if bns, ok := tx["blockNumber"].(string); ok && bns != "" { - bn, err := HexToInt64(bns) - if err != nil { - return "", 0, err - } - blockNumber = bn - } - if blockRef == "" && blockNumber > 0 { - blockRef = strconv.FormatInt(blockNumber, 10) + if len(rpcResp.Result) > 0 { + blockRef, _ := rpcResp.PeekStringByPath("blockHash") + blockNumberStr, _ := rpcResp.PeekStringByPath("blockNumber") + + var blockNumber int64 + if blockNumberStr != "" { + bn, err := HexToInt64(blockNumberStr) + if err != nil { + return "", 0, err } - return blockRef, blockNumber, nil + blockNumber = bn + } + + if blockRef == "" && blockNumber > 0 { + blockRef = strconv.FormatInt(blockNumber, 10) } + + return blockRef, blockNumber, nil } case "eth_getBlockByNumber": - if rpcResp.Result != nil { - result, err := rpcResp.ParsedResult() - if err != nil { - return "", 0, err - } - rpcResp.RLock() - defer rpcResp.RUnlock() - if blk, ok := result.(map[string]interface{}); ok { - var blockRef string - var blockNumber int64 - blockRef, _ = blk["hash"].(string) - if bns, ok := blk["number"].(string); ok && bns != "" { - bn, err := HexToInt64(bns) - if err != nil { - return "", 0, err - } - blockNumber = bn - } - if blockRef == "" && blockNumber > 0 { - blockRef = strconv.FormatInt(blockNumber, 10) + if len(rpcResp.Result) > 0 { + blockRef, _ := rpcResp.PeekStringByPath("hash") + blockNumberStr, _ := rpcResp.PeekStringByPath("number") + + var blockNumber int64 + if blockNumberStr != "" { + bn, err := HexToInt64(blockNumberStr) + if err != nil { + return "", 0, err } - return blockRef, blockNumber, nil + blockNumber = bn + } + + if blockRef == "" && blockNumber > 0 { + blockRef = strconv.FormatInt(blockNumber, 10) } - } + return blockRef, blockNumber, nil + } default: return "", 0, nil } diff --git a/common/evm_block_ref_test.go b/common/evm_block_ref_test.go index 2c69f0559..2cd563c25 100644 --- a/common/evm_block_ref_test.go +++ b/common/evm_block_ref_test.go @@ -1,7 +1,6 @@ package common import ( - "encoding/json" "testing" "github.com/stretchr/testify/assert" @@ -180,6 +179,44 @@ func TestExtractBlockReference(t *testing.T) { expectedNum: 436, expectedErr: false, }, + { + name: "eth_call with blockHash object", + request: &JsonRpcRequest{ + Method: "eth_call", + Params: []interface{}{ + map[string]interface{}{ + "from": nil, + "to": "0x6b175474e89094c44da98b954eedeac495271d0f", + "data": "0x70a082310000000000000000000000006E0d01A76C3Cf4288372a29124A26D4353EE51BE", + }, + map[string]interface{}{ + "blockHash": "0x3f07a9c83155594c000642e7d60e8a8a00038d03e9849171a05ed0e2d47acbb3", + }, + }, + }, + expectedRef: "0x3f07a9c83155594c000642e7d60e8a8a00038d03e9849171a05ed0e2d47acbb3", + expectedNum: 0, + expectedErr: false, + }, + { + name: "eth_call with blockNumber object", + request: &JsonRpcRequest{ + Method: "eth_call", + Params: []interface{}{ + map[string]interface{}{ + "from": nil, + "to": "0x6b175474e89094c44da98b954eedeac495271d0f", + "data": "0x70a082310000000000000000000000006E0d01A76C3Cf4288372a29124A26D4353EE51BE", + }, + map[string]interface{}{ + "blockNumber": "0x1b4", + }, + }, + }, + expectedRef: "436", + expectedNum: 436, + expectedErr: false, + }, { name: "eth_feeHistory", request: &JsonRpcRequest{ @@ -254,6 +291,38 @@ func TestExtractBlockReference(t *testing.T) { expectedNum: 436, expectedErr: false, }, + { + name: "eth_getProof with blockHash object", + request: &JsonRpcRequest{ + Method: "eth_getProof", + Params: []interface{}{ + "0x7F0d15C7FAae65896648C8273B6d7E43f58Fa842", + []interface{}{"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"}, + map[string]interface{}{ + "blockHash": "0x3f07a9c83155594c000642e7d60e8a8a00038d03e9849171a05ed0e2d47acbb3", + }, + }, + }, + expectedRef: "0x3f07a9c83155594c000642e7d60e8a8a00038d03e9849171a05ed0e2d47acbb3", + expectedNum: 0, + expectedErr: false, + }, + { + name: "eth_getProof with blockNumber object", + request: &JsonRpcRequest{ + Method: "eth_getProof", + Params: []interface{}{ + "0x7F0d15C7FAae65896648C8273B6d7E43f58Fa842", + []interface{}{"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"}, + map[string]interface{}{ + "blockNumber": "0x1b4", + }, + }, + }, + expectedRef: "436", + expectedNum: 436, + expectedErr: false, + }, { name: "eth_getStorageAt", request: &JsonRpcRequest{ @@ -274,7 +343,7 @@ func TestExtractBlockReference(t *testing.T) { Method: "eth_getTransactionReceipt", }, response: &JsonRpcResponse{ - Result: json.RawMessage(`{"blockNumber":"0x1b4","blockHash":"0xaaaaaabbbbccccc"}`), + Result: []byte(`{"blockNumber":"0x1b4","blockHash":"0xaaaaaabbbbccccc"}`), }, expectedRef: "*", expectedNum: 436, diff --git a/common/evm_json_rpc.go b/common/evm_json_rpc.go index a3159266d..85a71ff9d 100644 --- a/common/evm_json_rpc.go +++ b/common/evm_json_rpc.go @@ -5,33 +5,33 @@ import ( "strings" ) -func NormalizeEvmHttpJsonRpc(nrq *NormalizedRequest, r *JsonRpcRequest) error { - r.Lock() - defer r.Unlock() +func NormalizeEvmHttpJsonRpc(nrq *NormalizedRequest, jrq *JsonRpcRequest) { + jrq.Lock() + defer jrq.Unlock() - switch r.Method { + // Intentionally ignore the error to be more robust against params we don't understand but upstreams would be fine + switch jrq.Method { case "eth_getBlockByNumber", "eth_getUncleByBlockNumberAndIndex", "eth_getTransactionByBlockNumberAndIndex", "eth_getUncleCountByBlockNumber", "eth_getBlockTransactionCountByNumber": - if len(r.Params) > 0 { - bks, ok := r.Params[0].(string) + if len(jrq.Params) > 0 { + bks, ok := jrq.Params[0].(string) if !ok { - bkf, ok := r.Params[0].(float64) - if !ok { - return fmt.Errorf("invalid block number, must be 0x hex string, or number or latest/finalized") - } - bks = fmt.Sprintf("%f", bkf) - b, err := NormalizeHex(bks) - if err == nil { - r.Params[0] = b + bkf, ok := jrq.Params[0].(float64) + if ok { + bks = fmt.Sprintf("%f", bkf) + b, err := NormalizeHex(bks) + if err == nil { + jrq.Params[0] = b + } } } if strings.HasPrefix(bks, "0x") { b, err := NormalizeHex(bks) if err == nil { - r.Params[0] = b + jrq.Params[0] = b } } } @@ -41,41 +41,57 @@ func NormalizeEvmHttpJsonRpc(nrq *NormalizedRequest, r *JsonRpcRequest) error { "eth_getTransactionCount", "eth_call", "eth_estimateGas": - if len(r.Params) > 1 { - b, err := NormalizeHex(r.Params[1]) - if err != nil { - return err + if len(jrq.Params) > 1 { + if strValue, ok := jrq.Params[1].(string); ok { + if strings.HasPrefix(strValue, "0x") { + b, err := NormalizeHex(strValue) + if err == nil { + jrq.Params[1] = b + } + } + } else if mapValue, ok := jrq.Params[1].(map[string]interface{}); ok { + if blockNumber, ok := mapValue["blockNumber"]; ok { + b, err := NormalizeHex(blockNumber) + if err == nil { + mapValue["blockNumber"] = b + } + } } - r.Params[1] = b } case "eth_getStorageAt": - if len(r.Params) > 2 { - b, err := NormalizeHex(r.Params[2]) - if err != nil { - return err + if len(jrq.Params) > 2 { + if strValue, ok := jrq.Params[2].(string); ok { + if strings.HasPrefix(strValue, "0x") { + b, err := NormalizeHex(strValue) + if err == nil { + jrq.Params[2] = b + } + } + } else if mapValue, ok := jrq.Params[2].(map[string]interface{}); ok { + if blockNumber, ok := mapValue["blockNumber"]; ok { + b, err := NormalizeHex(blockNumber) + if err == nil { + mapValue["blockNumber"] = b + } + } } - r.Params[2] = b } case "eth_getLogs": - if len(r.Params) > 0 { - if paramsMap, ok := r.Params[0].(map[string]interface{}); ok { + if len(jrq.Params) > 0 { + if paramsMap, ok := jrq.Params[0].(map[string]interface{}); ok { if fromBlock, ok := paramsMap["fromBlock"]; ok { b, err := NormalizeHex(fromBlock) - if err != nil { - return err + if err == nil { + paramsMap["fromBlock"] = b } - paramsMap["fromBlock"] = b } if toBlock, ok := paramsMap["toBlock"]; ok { b, err := NormalizeHex(toBlock) - if err != nil { - return err + if err == nil { + paramsMap["toBlock"] = b } - paramsMap["toBlock"] = b } } } } - - return nil } diff --git a/common/init_test.go b/common/init_test.go new file mode 100644 index 000000000..1bb373eb2 --- /dev/null +++ b/common/init_test.go @@ -0,0 +1,9 @@ +package common + +import ( + "github.com/rs/zerolog" +) + +func init() { + zerolog.SetGlobalLevel(zerolog.Disabled) +} diff --git a/common/json_rpc.go b/common/json_rpc.go index 93d169660..d95f55ce4 100644 --- a/common/json_rpc.go +++ b/common/json_rpc.go @@ -1,220 +1,486 @@ package common import ( + "bytes" "crypto/sha256" "encoding/json" "fmt" "io" - "log" "sort" + "strconv" "strings" "sync" - "github.com/bytedance/sonic" + "github.com/bytedance/sonic/ast" + "github.com/erpc/erpc/util" "github.com/rs/zerolog" + "github.com/rs/zerolog/log" ) type JsonRpcResponse struct { - sync.RWMutex - - JSONRPC string `json:"jsonrpc,omitempty"` - ID interface{} `json:"id,omitempty"` - Result json.RawMessage `json:"result,omitempty"` - Error *ErrJsonRpcExceptionExternal `json:"error,omitempty"` - - parsedResult interface{} + // ID is mainly set based on incoming request. + // During parsing of response from upstream we'll still parse it + // so that batch requests are correctly identified. + // The "id" field is already parsed-value used within erpc, + // and idBytes is a lazy-loaded byte representation of the ID used when writing responses. + id int64 + idBytes []byte + idMu sync.RWMutex + + // Error is the parsed error from the response bytes, and potentially normalized based on vendor-specific drivers. + // errBytes is a lazy-loaded byte representation of the Error used when writing responses. + // When upstream response is received we're going to set the errBytes to incoming bytes and parse the error. + // Internal components (any error not initiated by upstream) will set the Error field directly and then we'll + // lazy-load the errBytes for writing responses by marshalling the Error field. + Error *ErrJsonRpcExceptionExternal + errBytes []byte + errMu sync.RWMutex + + // Result is the raw bytes of the result from the response, and is used when writing responses. + // Ideally we don't need to parse these bytes. In cases where we need a specific field (e.g. blockNumber) + // we use Sonic library to traverse directly to target such field (vs marshalling the whole result in memory). + Result []byte + resultMu sync.RWMutex + cachedNode *ast.Node } -func NewJsonRpcResponse(id interface{}, result interface{}, rpcError *ErrJsonRpcExceptionExternal) (*JsonRpcResponse, error) { - resultRaw, err := sonic.Marshal(result) +func NewJsonRpcResponse(id int64, result interface{}, rpcError *ErrJsonRpcExceptionExternal) (*JsonRpcResponse, error) { + resultRaw, err := SonicCfg.Marshal(result) if err != nil { return nil, err } return &JsonRpcResponse{ - JSONRPC: "2.0", - ID: id, - Result: resultRaw, - Error: rpcError, + id: id, + Result: resultRaw, + Error: rpcError, }, nil } -func (r *JsonRpcResponse) ParsedResult() (interface{}, error) { - r.RLock() - if r.parsedResult != nil { - defer r.RUnlock() - return r.parsedResult, nil +func NewJsonRpcResponseFromBytes(id []byte, resultRaw []byte, errBytes []byte) (*JsonRpcResponse, error) { + jr := &JsonRpcResponse{ + idBytes: id, + Result: resultRaw, } - r.RUnlock() - r.Lock() - defer r.Unlock() + if len(errBytes) > 0 { + err := jr.ParseError(util.Mem2Str(errBytes)) + if err != nil { + return nil, err + } + } - // Double-check in case another goroutine initialized it - if r.parsedResult != nil { - return r.parsedResult, nil + if len(id) > 0 { + err := jr.parseID() + if err != nil { + return nil, err + } } - if r.Result == nil { - return nil, nil + return jr, nil +} + +func (r *JsonRpcResponse) parseID() error { + r.idMu.Lock() + defer r.idMu.Unlock() + + var rawID interface{} + err := SonicCfg.Unmarshal(r.idBytes, &rawID) + if err != nil { + return err } - defer func() { - if rec := recover(); rec != nil { - // Catch the panic and log the raw JSON - log.Printf("Panic occurred: %v\n", rec) - log.Printf("Raw JSON response: %s\n", r.Result) + switch v := rawID.(type) { + case float64: + r.id = int64(v) + case string: + if v == "" { + return nil + } else { + parsedID, err := strconv.ParseInt(v, 0, 64) + if err != nil { + // Try parsing as float if integer parsing fails + parsedFloat, err := strconv.ParseFloat(v, 64) + if err != nil { + return err + } + r.id = int64(parsedFloat) + } else { + r.id = parsedID + } } - }() - err := sonic.Unmarshal(r.Result, &r.parsedResult) - if err != nil { - return nil, err + case nil: + return nil + default: + return fmt.Errorf("unsupported ID type: %T", v) } - return r.parsedResult, nil + // Update idBytes with the parsed int64 value + r.idBytes, err = SonicCfg.Marshal(r.id) + return err } -func (r *JsonRpcResponse) MarshalZerologObject(e *zerolog.Event) { - if r == nil { - return - } +func (r *JsonRpcResponse) SetID(id int64) error { + r.idMu.Lock() + defer r.idMu.Unlock() - r.Lock() - defer r.Unlock() + r.id = id + var err error + r.idBytes, err = SonicCfg.Marshal(id) + if err != nil { + return err + } - e.Interface("id", r.ID). - Interface("result", r.Result). - Interface("error", r.Error) + return nil } -// Custom unmarshal method for JsonRpcResponse -func (r *JsonRpcResponse) UnmarshalJSON(data []byte) error { - if r == nil { - return nil +func (r *JsonRpcResponse) ID() int64 { + r.idMu.RLock() + + if r.id != 0 { + r.idMu.RUnlock() + return r.id } + r.idMu.RUnlock() - r.Lock() - defer r.Unlock() + r.idMu.Lock() + defer r.idMu.Unlock() - type Alias JsonRpcResponse - aux := &struct { - Error json.RawMessage `json:"error,omitempty"` - *Alias - }{ - Alias: (*Alias)(r), + err := SonicCfg.Unmarshal(r.idBytes, &r.id) + if err != nil { + log.Error().Err(err).Interface("response", r).Bytes("idBytes", r.idBytes).Msg("failed to unmarshal id") } - if err := sonic.Unmarshal(data, &aux); err != nil { + return r.id +} + +func (r *JsonRpcResponse) SetIDBytes(idBytes []byte) error { + r.idMu.Lock() + defer r.idMu.Unlock() + + r.idBytes = idBytes + return r.parseID() +} + +func (r *JsonRpcResponse) ParseFromStream(reader io.Reader, expectedSize int) error { + data, err := util.ReadAll(reader, 16*1024, expectedSize) // 16KB + if err != nil { return err } - // Special case upstream does not return proper json-rpc response - if aux.Error == nil && aux.Result == nil && aux.ID == nil { - // Special case #1: there is numeric "code" and "message" in the "data" - sp1 := &struct { - Code int `json:"code,omitempty"` - Message string `json:"message,omitempty"` - Data string `json:"data,omitempty"` - }{} - if err := sonic.Unmarshal(data, &sp1); err == nil { - if sp1.Code != 0 || sp1.Message != "" || sp1.Data != "" { - r.Error = NewErrJsonRpcExceptionExternal( - sp1.Code, - sp1.Message, - sp1.Data, - ) - return nil + // Parse the JSON data into an ast.Node + searcher := ast.NewSearcher(util.Mem2Str(data)) + searcher.CopyReturn = false + searcher.ConcurrentRead = false + searcher.ValidateJSON = false + + // Extract the "id" field + if idNode, err := searcher.GetByPath("id"); err == nil { + if rawID, err := idNode.Raw(); err == nil { + r.idMu.Lock() + defer r.idMu.Unlock() + r.idBytes = util.Str2Mem(rawID) + } + } + + if resultNode, err := searcher.GetByPath("result"); err == nil { + if rawResult, err := resultNode.Raw(); err == nil { + r.resultMu.Lock() + defer r.resultMu.Unlock() + r.Result = util.Str2Mem(rawResult) + r.cachedNode = &resultNode + } else { + return err + } + } else if errorNode, err := searcher.GetByPath("error"); err == nil { + if rawError, err := errorNode.Raw(); err == nil { + if err := r.ParseError(rawError); err != nil { + return err } + } else { + return err } - // Special case #2: there is only "error" field with string in the body - sp2 := &struct { - Error string `json:"error"` - }{} - if err := sonic.Unmarshal(data, &sp2); err == nil && sp2.Error != "" { + } else if err := r.ParseError(util.Mem2Str(data)); err != nil { + return err + } + + return nil +} + +func (r *JsonRpcResponse) ParseError(raw string) error { + r.errMu.Lock() + defer r.errMu.Unlock() + + r.errBytes = nil + + // First attempt to unmarshal the error as a typical JSON-RPC error + var rpcErr ErrJsonRpcExceptionExternal + if err := SonicCfg.UnmarshalFromString(raw, &rpcErr); err != nil { + // Special case: check for non-standard error structures in the raw data + if raw == "" || raw == "null" { r.Error = NewErrJsonRpcExceptionExternal( int(JsonRpcErrorServerSideException), - sp2.Error, + "unexpected empty response from upstream endpoint", "", ) return nil } + } - if len(data) == 0 { - r.Error = NewErrJsonRpcExceptionExternal( - int(JsonRpcErrorServerSideException), - "unexpected empty response from upstream endpoint", - "", - ) - } else if data[0] == '{' || data[0] == '[' { - r.Error = NewErrJsonRpcExceptionExternal( - int(JsonRpcErrorServerSideException), - fmt.Sprintf("unexpected response json structure from upstream: %s", string(data)), - "", - ) - } else { + // Check if the error is well-formed and has necessary fields + if rpcErr.Code != 0 || rpcErr.Message != "" { + r.Error = &rpcErr + r.errBytes = util.Str2Mem(raw) + return nil + } + + // Handle further special cases + // Special case #1: numeric "code", "message", and "data" + sp1 := &struct { + Code int `json:"code,omitempty"` + Message string `json:"message,omitempty"` + Data string `json:"data,omitempty"` + }{} + if err := SonicCfg.UnmarshalFromString(raw, sp1); err == nil { + if sp1.Code != 0 || sp1.Message != "" || sp1.Data != "" { r.Error = NewErrJsonRpcExceptionExternal( - int(JsonRpcErrorServerSideException), - string(data), - "", + sp1.Code, + sp1.Message, + sp1.Data, ) + return nil } + } + + // Special case #2: only "error" field as a string + sp2 := &struct { + Error string `json:"error"` + }{} + if err := SonicCfg.UnmarshalFromString(raw, sp2); err == nil && sp2.Error != "" { + r.Error = NewErrJsonRpcExceptionExternal( + int(JsonRpcErrorServerSideException), + sp2.Error, + "", + ) return nil } - if aux.Error != nil { - var code int - var msg string - var data string + // If no match, treat the raw data as message string + r.Error = NewErrJsonRpcExceptionExternal( + int(JsonRpcErrorServerSideException), + raw, + "", + ) + return nil +} - var customObjectError map[string]interface{} - if err := sonic.Unmarshal(aux.Error, &customObjectError); err == nil { - if c, ok := customObjectError["code"]; ok { - if cf, ok := c.(float64); ok { - code = int(cf) - } - } - if m, ok := customObjectError["message"]; ok { - if tm, ok := m.(string); ok { - msg = tm - } - } - if d, ok := customObjectError["data"]; ok { - if dt, ok := d.(string); ok { - data = dt - } else { - data = fmt.Sprintf("%v", d) - } - } - } else { - var customStringError string - if err := sonic.Unmarshal(aux.Error, &customStringError); err == nil { - code = int(JsonRpcErrorServerSideException) - msg = customStringError - } - } +func (r *JsonRpcResponse) PeekStringByPath(path ...interface{}) (string, error) { + err := r.ensureCachedNode() + if err != nil { + return "", err + } - r.Error = NewErrJsonRpcExceptionExternal( - code, - msg, - data, - ) + n := r.cachedNode.GetByPath(path...) + if n == nil { + return "", fmt.Errorf("could not get '%s' from json-rpc response", path) + } + if n.Error() != "" { + return "", fmt.Errorf("error getting '%s' from json-rpc response: %s", path, n.Error()) + } + + return n.String() +} + +func (r *JsonRpcResponse) MarshalZerologObject(e *zerolog.Event) { + if r == nil { + return + } + + r.resultMu.RLock() + defer r.resultMu.RUnlock() + r.errMu.RLock() + defer r.errMu.RUnlock() + + e.Int64("id", r.ID()). + Int("resultSize", len(r.Result)). + Interface("error", r.Error) +} + +func (r *JsonRpcResponse) ensureCachedNode() error { + r.resultMu.RLock() + defer r.resultMu.RUnlock() + + if r.cachedNode == nil { + srchr := ast.NewSearcher(util.Mem2Str(r.Result)) + srchr.ValidateJSON = false + srchr.ConcurrentRead = false + srchr.CopyReturn = false + n, err := srchr.GetByPath() + if err != nil { + return err + } + r.cachedNode = &n } return nil } +// MarshalJSON must not be used for majority of use-cases, +// as it requires marshalling the whole response into a buffer in memory. +// GetReader is a lighter approach to be used when needed. +// Unfortunately, when requests are batched, MarshalJSON is called for each response, +// causing unnecessary memory allocations. +// To avoid this, send single requests as batching feature is generally an anti-pattern. +func (r *JsonRpcResponse) MarshalJSON() ([]byte, error) { + rdr, err := r.GetReader() + if err != nil { + return nil, err + } + return util.ReadAll(rdr, 16*1024, 0) +} + +// GetReader is a custom implementation of marshalling json-rpc response, +// this approach uses minimum memory allocations and is faster than a generic JSON marshaller. +func (r *JsonRpcResponse) GetReader() (io.Reader, error) { + r.idMu.RLock() + defer r.idMu.RUnlock() + + r.errMu.RLock() + defer r.errMu.RUnlock() + + r.resultMu.RLock() + defer r.resultMu.RUnlock() + + var err error + totalReaders := 3 + + if r.Error != nil || len(r.errBytes) > 0 { + totalReaders += 2 + } + if r.Result != nil || len(r.Result) > 0 { + totalReaders += 2 + } + + readers := make([]io.Reader, totalReaders) + idx := 0 + readers[idx] = bytes.NewReader([]byte(`{"jsonrpc":"2.0","id":`)) + idx++ + readers[idx] = bytes.NewReader(r.idBytes) + idx++ + + if len(r.errBytes) > 0 { + readers[idx] = bytes.NewReader([]byte(`,"error":`)) + idx++ + readers[idx] = bytes.NewReader(r.errBytes) + idx++ + } else if r.Error != nil { + r.errBytes, err = SonicCfg.Marshal(r.Error) + if err != nil { + return nil, err + } + readers[idx] = bytes.NewReader([]byte(`,"error":`)) + idx++ + readers[idx] = bytes.NewReader(r.errBytes) + idx++ + } + + if len(r.Result) > 0 { + readers[idx] = bytes.NewReader([]byte(`,"result":`)) + idx++ + readers[idx] = bytes.NewReader(r.Result) + idx++ + } + + readers[idx] = bytes.NewReader([]byte{'}'}) + + return io.MultiReader(readers...), nil +} + +func (r *JsonRpcResponse) Clone() (*JsonRpcResponse, error) { + if r == nil { + return nil, nil + } + + return &JsonRpcResponse{ + id: r.id, + idBytes: r.idBytes, + Error: r.Error, + errBytes: r.errBytes, + Result: r.Result, + cachedNode: r.cachedNode, + }, nil +} + +// +// +// JSON-RPC Request +// + type JsonRpcRequest struct { sync.RWMutex JSONRPC string `json:"jsonrpc,omitempty"` - ID interface{} `json:"id,omitempty"` + ID int64 `json:"id,omitempty"` Method string `json:"method"` Params []interface{} `json:"params"` } +func (r *JsonRpcRequest) UnmarshalJSON(data []byte) error { + type Alias JsonRpcRequest + aux := &struct { + *Alias + ID json.RawMessage `json:"id,omitempty"` + }{ + Alias: (*Alias)(r), + } + aux.JSONRPC = "2.0" + + if err := SonicCfg.Unmarshal(data, &aux); err != nil { + return err + } + + if aux.ID == nil { + r.ID = util.RandomID() + } else { + var id interface{} + if err := SonicCfg.Unmarshal(aux.ID, &id); err != nil { + return err + } + + switch v := id.(type) { + case float64: + r.ID = int64(v) + case string: + if v == "" { + r.ID = util.RandomID() + } else { + var parsedID float64 + if err := SonicCfg.Unmarshal([]byte(v), &parsedID); err != nil { + return err + } + r.ID = int64(parsedID) + } + case nil: + r.ID = util.RandomID() + default: + r.ID = util.RandomID() + } + } + + if r.ID == 0 { + r.ID = util.RandomID() + } + + return nil +} + func (r *JsonRpcRequest) MarshalZerologObject(e *zerolog.Event) { if r == nil { return } + + r.RLock() + defer r.RUnlock() + e.Str("method", r.Method). Interface("params", r.Params). Interface("id", r.ID) @@ -244,16 +510,16 @@ func (r *JsonRpcRequest) CacheHash() (string, error) { func hashValue(h io.Writer, v interface{}) error { switch t := v.(type) { case bool: - _, err := h.Write([]byte(fmt.Sprintf("%t", t))) + _, err := h.Write(util.Str2Mem(fmt.Sprintf("%t", t))) return err case int: - _, err := h.Write([]byte(fmt.Sprintf("%d", t))) + _, err := h.Write(util.Str2Mem(fmt.Sprintf("%d", t))) return err case float64: - _, err := h.Write([]byte(fmt.Sprintf("%f", t))) + _, err := h.Write(util.Str2Mem(fmt.Sprintf("%f", t))) return err case string: - _, err := h.Write([]byte(strings.ToLower(t))) + _, err := h.Write(util.Str2Mem(strings.ToLower(t))) return err case []interface{}: for _, i := range t { @@ -270,7 +536,7 @@ func hashValue(h io.Writer, v interface{}) error { } sort.Strings(keys) for _, k := range keys { - if _, err := h.Write([]byte(k)); err != nil { + if _, err := h.Write(util.Str2Mem(k)); err != nil { return err } err := hashValue(h, t[k]) @@ -320,6 +586,26 @@ func TranslateToJsonRpcException(err error) error { ) } + if HasErrorCode(err, ErrCodeUpstreamMethodIgnored) { + return NewErrJsonRpcExceptionInternal( + 0, + JsonRpcErrorUnsupportedException, + "method ignored by upstream", + err, + nil, + ) + } + + if HasErrorCode(err, ErrCodeJsonRpcRequestUnmarshal) { + return NewErrJsonRpcExceptionInternal( + 0, + JsonRpcErrorParseException, + "failed to parse json-rpc request", + err, + nil, + ) + } + var msg = "internal server error" if se, ok := err.(StandardError); ok { msg = se.DeepestMessage() diff --git a/common/request.go b/common/request.go index 8194e61f2..790090581 100644 --- a/common/request.go +++ b/common/request.go @@ -2,15 +2,18 @@ package common import ( "fmt" - "math" - "math/rand" + "strconv" "sync" + "sync/atomic" "github.com/bytedance/sonic" + "github.com/erpc/erpc/util" "github.com/rs/zerolog" "github.com/valyala/fasthttp" ) +const RequestContextKey ContextKey = "request" + type RequestDirectives struct { // Instruct the proxy to retry if response from the upstream appears to be empty // indicating a missing data or non-synced data (empty array for logs, null for block, null for tx receipt, etc). @@ -38,18 +41,16 @@ type RequestDirectives struct { type NormalizedRequest struct { sync.RWMutex - Attempt int - network Network body []byte - uid string + uid atomic.Value method string directives *RequestDirectives jsonRpcRequest *JsonRpcRequest - lastValidResponse *NormalizedResponse - lastUpstream Upstream + lastValidResponse atomic.Pointer[NormalizedResponse] + lastUpstream atomic.Value } type UniqueRequestKey struct { @@ -70,9 +71,7 @@ func (r *NormalizedRequest) SetLastUpstream(upstream Upstream) *NormalizedReques if r == nil { return r } - r.Lock() - defer r.Unlock() - r.lastUpstream = upstream + r.lastUpstream.Store(upstream) return r } @@ -80,27 +79,24 @@ func (r *NormalizedRequest) LastUpstream() Upstream { if r == nil { return nil } - r.Lock() - defer r.Unlock() - return r.lastUpstream + if lu := r.lastUpstream.Load(); lu != nil { + return lu.(Upstream) + } + return nil } func (r *NormalizedRequest) SetLastValidResponse(response *NormalizedResponse) { if r == nil { return } - r.Lock() - defer r.Unlock() - r.lastValidResponse = response + r.lastValidResponse.Store(response) } func (r *NormalizedRequest) LastValidResponse() *NormalizedResponse { if r == nil { return nil } - r.RLock() - defer r.RUnlock() - return r.lastValidResponse + return r.lastValidResponse.Load() } func (r *NormalizedRequest) Network() Network { @@ -110,62 +106,44 @@ func (r *NormalizedRequest) Network() Network { return r.network } -func (r *NormalizedRequest) Id() string { +func (r *NormalizedRequest) Id() int64 { if r == nil { - return "" + return 0 } - if r.uid != "" { - return r.uid - } - - r.RLock() if r.jsonRpcRequest != nil { - defer r.RUnlock() - if id, ok := r.jsonRpcRequest.ID.(string); ok { - r.uid = id - return id - } else if id, ok := r.jsonRpcRequest.ID.(float64); ok { - r.uid = fmt.Sprintf("%d", int64(id)) - return r.uid - } else { - r.uid = fmt.Sprintf("%v", r.jsonRpcRequest.ID) - return r.uid - } - } - r.RUnlock() - - r.Lock() - defer r.Unlock() - - if r.uid != "" { - return r.uid + return r.jsonRpcRequest.ID } if len(r.body) > 0 { idnode, err := sonic.Get(r.body, "id") if err == nil { ids, err := idnode.String() - if err == nil { + if err != nil { idf, err := idnode.Float64() if err != nil { idn, err := idnode.Int64() if err != nil { - r.uid = fmt.Sprintf("%d", idn) - return r.uid + r.uid.Store(idn) + return idn } } else { - r.uid = fmt.Sprintf("%d", int64(idf)) - return r.uid + uid := int64(idf) + r.uid.Store(uid) + return uid } } else { - r.uid = ids + uid, err := strconv.ParseInt(ids, 0, 64) + if err != nil { + uid = 0 + } + r.uid.Store(uid) + return uid } - return r.uid } } - return "" + return 0 } func (r *NormalizedRequest) NetworkId() string { @@ -185,25 +163,25 @@ func (r *NormalizedRequest) ApplyDirectivesFromHttp( queryArgs *fasthttp.Args, ) { drc := &RequestDirectives{ - RetryEmpty: string(headers.Peek("X-ERPC-Retry-Empty")) != "false", - RetryPending: string(headers.Peek("X-ERPC-Retry-Pending")) != "false", - SkipCacheRead: string(headers.Peek("X-ERPC-Skip-Cache-Read")) == "true", - UseUpstream: string(headers.Peek("X-ERPC-Use-Upstream")), + RetryEmpty: util.Mem2Str(headers.Peek("X-ERPC-Retry-Empty")) != "false", + RetryPending: util.Mem2Str(headers.Peek("X-ERPC-Retry-Pending")) != "false", + SkipCacheRead: util.Mem2Str(headers.Peek("X-ERPC-Skip-Cache-Read")) == "true", + UseUpstream: util.Mem2Str(headers.Peek("X-ERPC-Use-Upstream")), } - if useUpstream := string(queryArgs.Peek("use-upstream")); useUpstream != "" { + if useUpstream := util.Mem2Str(queryArgs.Peek("use-upstream")); useUpstream != "" { drc.UseUpstream = useUpstream } - if retryEmpty := string(queryArgs.Peek("retry-empty")); retryEmpty != "" { + if retryEmpty := util.Mem2Str(queryArgs.Peek("retry-empty")); retryEmpty != "" { drc.RetryEmpty = retryEmpty != "false" } - if retryPending := string(queryArgs.Peek("retry-pending")); retryPending != "" { + if retryPending := util.Mem2Str(queryArgs.Peek("retry-pending")); retryPending != "" { drc.RetryPending = retryPending != "false" } - if skipCacheRead := string(queryArgs.Peek("skip-cache-read")); skipCacheRead != "" { + if skipCacheRead := util.Mem2Str(queryArgs.Peek("skip-cache-read")); skipCacheRead != "" { drc.SkipCacheRead = skipCacheRead != "false" } @@ -249,7 +227,7 @@ func (r *NormalizedRequest) JsonRpcRequest() (*JsonRpcRequest, error) { } rpcReq := new(JsonRpcRequest) - if err := sonic.Unmarshal(r.body, rpcReq); err != nil { + if err := SonicCfg.Unmarshal(r.body, rpcReq); err != nil { return nil, NewErrJsonRpcRequestUnmarshal(err) } @@ -258,14 +236,6 @@ func (r *NormalizedRequest) JsonRpcRequest() (*JsonRpcRequest, error) { return nil, NewErrJsonRpcRequestUnresolvableMethod(rpcReq) } - if rpcReq.JSONRPC == "" { - rpcReq.JSONRPC = "2.0" - } - - if rpcReq.ID == nil { - rpcReq.ID = rand.Intn(math.MaxInt32) // #nosec G404 - } - r.jsonRpcRequest = rpcReq return rpcReq, nil @@ -284,8 +254,7 @@ func (r *NormalizedRequest) Method() (string, error) { if len(r.body) > 0 { method, err := sonic.Get(r.body, "method") if err != nil { - r.method = "n/a" - return r.method, err + return "", NewErrJsonRpcRequestUnmarshal(err) } m, err := method.String() r.method = m @@ -304,7 +273,7 @@ func (r *NormalizedRequest) MarshalZerologObject(e *zerolog.Event) { if r.jsonRpcRequest != nil { e.Object("jsonRpc", r.jsonRpcRequest) } else if r.body != nil { - e.Str("body", string(r.body)) + e.Str("body", util.Mem2Str(r.body)) } } } @@ -326,11 +295,12 @@ func (r *NormalizedRequest) EvmBlockNumber() (int64, error) { return bn, nil } - if r.lastValidResponse == nil { + lvr := r.lastValidResponse.Load() + if lvr == nil { return 0, nil } - bn, err = r.lastValidResponse.EvmBlockNumber() + bn, err = lvr.EvmBlockNumber() if err != nil { return 0, err } @@ -344,11 +314,13 @@ func (r *NormalizedRequest) MarshalJSON() ([]byte, error) { } if r.jsonRpcRequest != nil { - return sonic.Marshal(r.jsonRpcRequest) + return SonicCfg.Marshal(map[string]interface{}{ + "method": r.jsonRpcRequest.Method, + }) } if m, _ := r.Method(); m != "" { - return sonic.Marshal(map[string]interface{}{ + return SonicCfg.Marshal(map[string]interface{}{ "method": m, }) } diff --git a/common/response.go b/common/response.go index 03635deeb..93714e010 100644 --- a/common/response.go +++ b/common/response.go @@ -1,17 +1,20 @@ package common import ( + "io" "sync" + "sync/atomic" - "github.com/bytedance/sonic" + "github.com/rs/zerolog/log" ) type NormalizedResponse struct { sync.RWMutex - request *NormalizedRequest - body []byte - err error + request *NormalizedRequest + body io.ReadCloser + expectedSize int + err error fromCache bool attempts int @@ -19,8 +22,8 @@ type NormalizedResponse struct { hedges int upstream Upstream - jsonRpcResponse *JsonRpcResponse - evmBlockNumber int64 + jsonRpcResponse atomic.Pointer[JsonRpcResponse] + evmBlockNumber atomic.Int64 } type ResponseMetadata interface { @@ -102,18 +105,45 @@ func (r *NormalizedResponse) WithFromCache(fromCache bool) *NormalizedResponse { return r } -func (r *NormalizedResponse) WithBody(body []byte) *NormalizedResponse { +func (r *NormalizedResponse) JsonRpcResponse() (*JsonRpcResponse, error) { + if r == nil { + return nil, nil + } + + if jrr := r.jsonRpcResponse.Load(); jrr != nil { + return jrr, nil + } + + jrr := &JsonRpcResponse{} + + if r.body != nil { + err := jrr.ParseFromStream(r.body, r.expectedSize) + if err != nil { + return nil, err + } + } + + r.jsonRpcResponse.Store(jrr) + return jrr, nil +} + +func (r *NormalizedResponse) WithBody(body io.ReadCloser) *NormalizedResponse { r.body = body return r } +func (r *NormalizedResponse) WithExpectedSize(expectedSize int) *NormalizedResponse { + r.expectedSize = expectedSize + return r +} + func (r *NormalizedResponse) WithError(err error) *NormalizedResponse { r.err = err return r } func (r *NormalizedResponse) WithJsonRpcResponse(jrr *JsonRpcResponse) *NormalizedResponse { - r.jsonRpcResponse = jrr + r.jsonRpcResponse.Store(jrr) return r } @@ -124,27 +154,6 @@ func (r *NormalizedResponse) Request() *NormalizedRequest { return r.request } -func (r *NormalizedResponse) Body() []byte { - if r == nil { - return nil - } - if r.body != nil { - return r.body - } - - jrr, err := r.JsonRpcResponse() - if err != nil { - return nil - } - - r.body, err = sonic.Marshal(jrr) - if err != nil { - return nil - } - - return r.body -} - func (r *NormalizedResponse) Error() error { if r.err != nil { return r.err @@ -155,19 +164,22 @@ func (r *NormalizedResponse) Error() error { func (r *NormalizedResponse) IsResultEmptyish() bool { jrr, err := r.JsonRpcResponse() + + jrr.resultMu.RLock() + defer jrr.resultMu.RUnlock() + if err == nil { if jrr == nil { return true } - // Use raw result to avoid json unmarshalling for performance reasons - if jrr.Result == nil || - len(jrr.Result) == 0 || - (jrr.Result[0] == '"' && jrr.Result[1] == '0' && jrr.Result[2] == 'x' && jrr.Result[3] == '"' && len(jrr.Result) == 4) || - (jrr.Result[0] == 'n' && jrr.Result[1] == 'u' && jrr.Result[2] == 'l' && jrr.Result[3] == 'l' && len(jrr.Result) == 4) || - (jrr.Result[0] == '"' && jrr.Result[1] == '"' && len(jrr.Result) == 2) || - (jrr.Result[0] == '[' && jrr.Result[1] == ']' && len(jrr.Result) == 2) || - (jrr.Result[0] == '{' && jrr.Result[1] == '}' && len(jrr.Result) == 2) { + lnr := len(jrr.Result) + if lnr == 0 || + (lnr == 4 && jrr.Result[0] == '"' && jrr.Result[1] == '0' && jrr.Result[2] == 'x' && jrr.Result[3] == '"') || + (lnr == 4 && jrr.Result[0] == 'n' && jrr.Result[1] == 'u' && jrr.Result[2] == 'l' && jrr.Result[3] == 'l') || + (lnr == 2 && jrr.Result[0] == '"' && jrr.Result[1] == '"') || + (lnr == 2 && jrr.Result[0] == '[' && jrr.Result[1] == ']') || + (lnr == 2 && jrr.Result[0] == '{' && jrr.Result[1] == '}') { return true } } @@ -175,45 +187,23 @@ func (r *NormalizedResponse) IsResultEmptyish() bool { return false } -func (r *NormalizedResponse) JsonRpcResponse() (*JsonRpcResponse, error) { +func (r *NormalizedResponse) IsObjectNull() bool { if r == nil { - return nil, nil - } - - if r.jsonRpcResponse != nil { - return r.jsonRpcResponse, nil + return true } - jrr := &JsonRpcResponse{} - err := sonic.Unmarshal(r.body, jrr) - if err != nil { - if len(r.body) == 0 { - jrr.Error = NewErrJsonRpcExceptionExternal( - int(JsonRpcErrorServerSideException), - "unexpected empty response from upstream endpoint", - "", - ) - } else { - jrr.Error = NewErrJsonRpcExceptionExternal( - int(JsonRpcErrorServerSideException), - string(r.body), - "", - ) - } + jrr, _ := r.JsonRpcResponse() + if jrr == nil { + return true } - r.jsonRpcResponse = jrr + jrr.resultMu.RLock() + defer jrr.resultMu.RUnlock() - return jrr, nil -} - -func (r *NormalizedResponse) IsObjectNull() bool { - if r == nil { - return true - } + jrr.errMu.RLock() + defer jrr.errMu.RUnlock() - jrr, _ := r.JsonRpcResponse() - if jrr == nil && r.body == nil { + if len(jrr.Result) == 0 && jrr.Error == nil && jrr.ID() == 0 { return true } @@ -225,10 +215,8 @@ func (r *NormalizedResponse) EvmBlockNumber() (int64, error) { return 0, nil } - r.RLock() - if r.evmBlockNumber != 0 { - defer r.RUnlock() - return r.evmBlockNumber, nil + if n := r.evmBlockNumber.Load(); n != 0 { + return n, nil } r.RUnlock() @@ -241,47 +229,61 @@ func (r *NormalizedResponse) EvmBlockNumber() (int64, error) { return 0, err } - _, bn, err := ExtractEvmBlockReferenceFromResponse(rq, r.jsonRpcResponse) + jrr := r.jsonRpcResponse.Load() + if jrr == nil { + return 0, nil + } + + _, bn, err := ExtractEvmBlockReferenceFromResponse(rq, jrr) if err != nil { return 0, err } - r.Lock() - r.evmBlockNumber = bn - r.Unlock() + r.evmBlockNumber.Store(bn) return bn, nil } -func (r *NormalizedResponse) String() string { - if r == nil { - return "" - } - if r.body != nil && len(r.body) > 0 { - return string(r.body) - } - if r.err != nil { - return r.err.Error() - } - if r.jsonRpcResponse != nil { - b, _ := sonic.Marshal(r.jsonRpcResponse) - return string(b) +func (r *NormalizedResponse) MarshalJSON() ([]byte, error) { + r.RLock() + defer r.RUnlock() + + if jrr := r.jsonRpcResponse.Load(); jrr != nil { + return SonicCfg.Marshal(jrr) } - return "" + + return nil, nil } -func (r *NormalizedResponse) MarshalJSON() ([]byte, error) { - if r.body != nil { - return r.body, nil - } +func (r *NormalizedResponse) GetReader() (io.Reader, error) { + r.RLock() + defer r.RUnlock() - if r.jsonRpcResponse != nil { - return sonic.Marshal(r.jsonRpcResponse) + if jrr := r.jsonRpcResponse.Load(); jrr != nil { + return jrr.GetReader() } return nil, nil } +func (r *NormalizedResponse) Release() { + r.Lock() + defer r.Unlock() + + // If body is not closed yet, close it + if r.body != nil { + err := r.body.Close() + if err != nil { + log.Error().Err(err).Interface("response", r).Msg("failed to close response body") + } + r.body = nil + } + + r.jsonRpcResponse.Store(nil) +} + +// CopyResponseForRequest creates a copy of the response for another request +// We use references for underlying Result and Error fields to save memory. func CopyResponseForRequest(resp *NormalizedResponse, req *NormalizedRequest) (*NormalizedResponse, error) { req.RLock() defer req.RUnlock() @@ -293,27 +295,18 @@ func CopyResponseForRequest(resp *NormalizedResponse, req *NormalizedRequest) (* r := NewNormalizedResponse() r.WithRequest(req) - // We need to use request ID because response ID can be different - // in case of multiplexed requests, where we only sent 1 actual request to the upstream. - if resp.jsonRpcResponse != nil { - jrr, err := req.JsonRpcRequest() + if ejrr := resp.jsonRpcResponse.Load(); ejrr != nil { + // We need to use request ID because response ID can be different for multiplexed requests + // where we only sent 1 actual request to the upstream. + jrr, err := ejrr.Clone() if err != nil { return nil, err } - r.WithJsonRpcResponse(&JsonRpcResponse{ - JSONRPC: resp.jsonRpcResponse.JSONRPC, - ID: jrr.ID, - Result: resp.jsonRpcResponse.Result, - Error: resp.jsonRpcResponse.Error, - }) - } else if resp.body != nil { - r.WithBody(resp.body) - jrr, err := r.JsonRpcResponse() + err = jrr.SetID(req.jsonRpcRequest.ID) if err != nil { return nil, err } - r.body = nil // This enforces re-marshalling the response body if anyone needs it - jrr.ID = req.jsonRpcRequest.ID + r.WithJsonRpcResponse(jrr) } return r, nil diff --git a/common/sonic.go b/common/sonic.go new file mode 100644 index 000000000..e8102d76e --- /dev/null +++ b/common/sonic.go @@ -0,0 +1,58 @@ +package common + +import ( + "reflect" + + "github.com/bytedance/sonic" + "github.com/bytedance/sonic/option" +) + +var SonicCfg sonic.API + +func init() { + err := sonic.Pretouch( + reflect.TypeOf(NormalizedResponse{}), + option.WithCompileMaxInlineDepth(1), + ) + if err != nil { + panic(err) + } + err = sonic.Pretouch( + reflect.TypeOf(JsonRpcResponse{}), + option.WithCompileMaxInlineDepth(1), + ) + if err != nil { + panic(err) + } + err = sonic.Pretouch( + reflect.TypeOf(JsonRpcRequest{}), + option.WithCompileMaxInlineDepth(1), + ) + if err != nil { + panic(err) + } + err = sonic.Pretouch( + reflect.TypeOf(NormalizedRequest{}), + option.WithCompileMaxInlineDepth(1), + ) + if err != nil { + panic(err) + } + err = sonic.Pretouch( + reflect.TypeOf(BaseError{}), + option.WithCompileMaxInlineDepth(1), + ) + if err != nil { + panic(err) + } + SonicCfg = sonic.Config{ + CopyString: false, + NoQuoteTextMarshaler: true, + NoValidateJSONMarshaler: true, + NoValidateJSONSkip: true, + EscapeHTML: false, + SortMapKeys: false, + CompactMarshaler: true, + ValidateString: false, + }.Froze() +} diff --git a/common/utils.go b/common/utils.go index e27f1c3f5..c2a3006eb 100644 --- a/common/utils.go +++ b/common/utils.go @@ -10,6 +10,8 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" ) +type ContextKey string + // HexToUint64 converts a hexadecimal string to its decimal representation as a string. func HexToUint64(hexValue string) (uint64, error) { // Create a new big.Int @@ -31,12 +33,13 @@ func HexToUint64(hexValue string) (uint64, error) { } func HexToInt64(hexValue string) (int64, error) { - hexValue = strings.TrimPrefix(hexValue, "0x") - return strconv.ParseInt(hexValue, 16, 64) + normalizedHex, err := NormalizeHex(hexValue) + if err != nil { + return 0, err + } + return strconv.ParseInt(normalizedHex, 0, 64) } -type ContextKey string - func NormalizeHex(value interface{}) (string, error) { if bn, ok := value.(string); ok { // Check if blockNumber is already in hex format @@ -49,8 +52,7 @@ func NormalizeHex(value interface{}) (string, error) { return fmt.Sprintf("0x%x", value), nil // Convert back to hex without leading 0s } - // If blockNumber is base 10 digits, convert to hex without 0 padding - value, err := strconv.ParseUint(bn, 10, 64) + value, err := strconv.ParseUint(bn, 0, 64) if err == nil && value > 0 { return fmt.Sprintf("0x%x", value), nil } diff --git a/data/memory.go b/data/memory.go index 1f1f6bf70..ca1faa068 100644 --- a/data/memory.go +++ b/data/memory.go @@ -81,23 +81,6 @@ func (m *MemoryConnector) getWithWildcard(_ context.Context, _, partitionKey, ra return "", common.NewErrRecordNotFound(fmt.Sprintf("PK: %s RK: %s", partitionKey, rangeKey), MemoryDriverName) } -func (m *MemoryConnector) Query(ctx context.Context, index, partitionKey, rangeKey string) ([]*DataRow, error) { - prefix := strings.TrimSuffix(partitionKey, "*") - var results []*DataRow - - for _, key := range m.cache.Keys() { - parts := strings.Split(key, ":") - if len(parts) == 2 && strings.HasPrefix(parts[0], prefix) { - if rangeKey == "" || (strings.HasSuffix(rangeKey, "*") && strings.HasPrefix(parts[1], strings.TrimSuffix(rangeKey, "*"))) || parts[1] == rangeKey { - value, _ := m.cache.Get(key) - results = append(results, &DataRow{Value: value}) - } - } - } - - return results, nil -} - func (m *MemoryConnector) Delete(ctx context.Context, index, partitionKey, rangeKey string) error { if strings.HasSuffix(partitionKey, "*") || strings.HasSuffix(rangeKey, "*") { return m.deleteWithWildcard(ctx, index, partitionKey, rangeKey) diff --git a/data/mock_connector.go b/data/mock_connector.go index a21daf390..dc7fa2410 100644 --- a/data/mock_connector.go +++ b/data/mock_connector.go @@ -2,7 +2,10 @@ package data import ( "context" + "time" + "github.com/erpc/erpc/common" + "github.com/rs/zerolog" "github.com/stretchr/testify/mock" ) @@ -45,3 +48,39 @@ func (m *MockConnector) HasTTL(method string) bool { func NewMockConnector() *MockConnector { return &MockConnector{} } + +// MockMemoryConnector extends MemoryConnector with a fake delay feature +type MockMemoryConnector struct { + MemoryConnector + fakeDelay time.Duration +} + +// NewMockMemoryConnector creates a new MockMemoryConnector +func NewMockMemoryConnector(ctx context.Context, logger *zerolog.Logger, cfg *common.MemoryConnectorConfig, fakeDelay time.Duration) (*MockMemoryConnector, error) { + baseConnector, err := NewMemoryConnector(ctx, logger, cfg) + if err != nil { + return nil, err + } + + return &MockMemoryConnector{ + MemoryConnector: *baseConnector, + fakeDelay: fakeDelay, + }, nil +} + +// Set overrides the base Set method to include a fake delay +func (m *MockMemoryConnector) Set(ctx context.Context, partitionKey, rangeKey, value string) error { + time.Sleep(m.fakeDelay) + return m.MemoryConnector.Set(ctx, partitionKey, rangeKey, value) +} + +// Get overrides the base Get method to include a fake delay +func (m *MockMemoryConnector) Get(ctx context.Context, index, partitionKey, rangeKey string) (string, error) { + time.Sleep(m.fakeDelay) + return m.MemoryConnector.Get(ctx, index, partitionKey, rangeKey) +} + +// SetFakeDelay allows changing the fake delay dynamically +func (m *MockMemoryConnector) SetFakeDelay(delay time.Duration) { + m.fakeDelay = delay +} diff --git a/data/value.go b/data/value.go deleted file mode 100644 index feece6ffe..000000000 --- a/data/value.go +++ /dev/null @@ -1,39 +0,0 @@ -package data - -import ( - "sync" - - "github.com/bytedance/sonic" - "github.com/erpc/erpc/common" -) - -type DataRow struct { - PK string - RK string - Value string - - mu sync.Mutex - parsedJsonRpcRes *common.JsonRpcResponse -} - -func NewDataValue(v string) *DataRow { - return &DataRow{Value: v} -} - -func (d *DataRow) AsJsonRpcResponse() (*common.JsonRpcResponse, error) { - d.mu.Lock() - defer d.mu.Unlock() - - if d.parsedJsonRpcRes != nil { - return d.parsedJsonRpcRes, nil - } - - var result common.JsonRpcResponse - err := sonic.Unmarshal([]byte(d.Value), &result) - if err != nil { - return nil, err - } - - d.parsedJsonRpcRes = &result - return d.parsedJsonRpcRes, nil -} diff --git a/docs/pages/config/database.mdx b/docs/pages/config/database.mdx index ddea2abec..1b81137a2 100644 --- a/docs/pages/config/database.mdx +++ b/docs/pages/config/database.mdx @@ -79,7 +79,7 @@ database: ### Redis -Redis is useful when you need to store cached data temporarily with eviction policy (e.g. certain amount of memory). +Redis is useful when you need to store cached data temporarily with **eviction policy** (e.g. certain amount of memory). ```yaml filename="erpc.yaml" # ... @@ -93,6 +93,12 @@ database: # ... ``` +Example of Redis config with eviction policy: +```conf +maxmemory 2000mb +maxmemory-policy allkeys-lru +``` + ### PostgreSQL Useful when you need to store cached data permanently without TTL i.e. forever. diff --git a/docs/pages/config/example.mdx b/docs/pages/config/example.mdx index 336973df2..8893a9ca3 100644 --- a/docs/pages/config/example.mdx +++ b/docs/pages/config/example.mdx @@ -74,6 +74,7 @@ server: listenV6: false httpHostV6: "[::]" httpPort: 4000 + maxTimeout: 30s # Optional Prometheus metrics server. metrics: diff --git a/docs/pages/config/rate-limiters.mdx b/docs/pages/config/rate-limiters.mdx index 501b5eff6..5891ee27e 100644 --- a/docs/pages/config/rate-limiters.mdx +++ b/docs/pages/config/rate-limiters.mdx @@ -60,3 +60,47 @@ rateLimiters: maxCount: 300 period: 1s ``` + +## Auto-tuner + +The auto-tuner feature allows dynamic adjustment of rate limits based on the upstream's performance. It's particularly useful in the following scenarios: + +1. When you're unsure about the actual RPS limit imposed by the provider. +2. When you need to update the limits dynamically based on the provider's current capacity. + +The auto-tuner is enabled by default when an upstream has any rate limit budget defined. Here's an example configuration with explanations: + +```yaml +upstreams: + - id: example-upstream + type: evm + endpoint: https://example-endpoint.com + rateLimitBudget: example-budget + rateLimitAutoTune: + enabled: true # Enable auto-tuning (default: true) + adjustmentPeriod: "1m" # How often to adjust the rate limit (default: "1m") + errorRateThreshold: 0.1 # Maximum acceptable error rate (default: 0.1) + increaseFactor: 1.05 # Factor to increase the limit by (default: 1.05) + decreaseFactor: 0.9 # Factor to decrease the limit by (default: 0.9) + minBudget: 1 # Minimum rate limit (default: 0) + maxBudget: 10000 # Maximum rate limit (default: 10000) +``` + +It's recommended to set `minBudget` to at least 1. This ensures that some requests are always routed to the upstream, allowing the auto-tuner to re-adjust if the provider can handle more requests. + +The auto-tuner works by monitoring the "rate limited" (e.g. 429 status code) error rate of requests to the upstream. If the 'rate-limited' error rate is below the `errorRateThreshold`, it gradually increases the rate limit by the `increaseFactor`. If the 'rate-limited' error rate exceeds the threshold, it quickly decreases the rate limit by the `decreaseFactor`. + +By default, the auto-tuner is enabled with the following configuration: + +```yaml +rateLimitAutoTune: + enabled: true + adjustmentPeriod: "1m" + errorRateThreshold: 0.1 + increaseFactor: 1.05 + decreaseFactor: 0.9 + minBudget: 0 + maxBudget: 10000 +``` + +You can override these defaults by specifying the desired values in your configuration. \ No newline at end of file diff --git a/docs/pages/deployment/railway.mdx b/docs/pages/deployment/railway.mdx index 195cbaec2..3e4f259e9 100644 --- a/docs/pages/deployment/railway.mdx +++ b/docs/pages/deployment/railway.mdx @@ -30,12 +30,29 @@ The eRPC template includes pre-configured environment variables to simplify the - `BLASTAPI_API_KEY`: *API key from balastapi.io* - `DRPC_API_KEY`: *API key from drpc.org* +### Usage in your services + +To reduce cost and overhead use private network (`.railway.internal`) to connect to eRPC, from your backend services (such as indexers or mev bots): + +```js +const result = await fetch("https://my-erpc.railway.internal/main/evm/1", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + method: "eth_getBlockByNumber", + params: ["0x1203319"], + }), +}); +``` + ### Send your first request Within your `erpc` service, find the proxy URL under `Settings > Networking > Public Networking`. ```bash -curl --location 'https://erpc-production-6dd3.up.railway.app/main/evm/1' \ +curl --location 'https://erpc-xxxxxxx.up.railway.app/main/evm/1' \ --header 'Content-Type: application/json' \ --data '{ "method": "eth_getBlockByNumber", diff --git a/docs/pages/operation/batch.mdx b/docs/pages/operation/batch.mdx index 490203cdf..520aae452 100644 --- a/docs/pages/operation/batch.mdx +++ b/docs/pages/operation/batch.mdx @@ -4,6 +4,15 @@ import { Callout } from "nextra/components"; eRPC automatically batches requests towards upstreams which support it. Additionally you can send batched requests (an array of multiple requests) to eRPC itself. + + Most often json-rpc batching for EVM is [anti-pattern](https://www.quicknode.com/guides/quicknode-products/apis/guide-to-efficient-rpc-requests#avoid-batching-multiple-rpc-requests), as it increases resource consumption without significant benefits: + - All requests will be as slow as the slowest request inside the batch. + - JSON handling will be more expensive causing memory spikes and OOM errors. + - Handling partial failures will be burdensome for the client (status code is always 200 OK). + - Many 3rd-party providers (Alchemy, Infura, etc) charge based on number of method calls, not actual requests. + - When running eRPC in private network locally close to your services, overhead of many single requests is negligible. + + ### How it works? * When an upstream is configured to support batching, eRPC will accumulate as many requests as possible for that upstream, even if you send many single requests. diff --git a/docs/pages/operation/production.mdx b/docs/pages/operation/production.mdx index 2f81281e5..cbfc14118 100644 --- a/docs/pages/operation/production.mdx +++ b/docs/pages/operation/production.mdx @@ -2,20 +2,38 @@ Here are some recommendations for running eRPC in production. +## Memory usage + +Biggest memory usage contributor in eRPC is size of responses of your requests. For example, for common requests such as `eth_getBlockByNumber` or `eth_getTransactionReceipt` the size (<1MB) will be relatively smaller than `debug_traceTransaction` (which could potentially be up to 50MB). When using eRPC in Kubernetes for example your might see occesional `OOMKilled` errors which is most often because of high RPS of large request/responses. + +In majority of use-cases eRPC uses around 256MB of memory (and 1vCPU). To find the ideal memory limit based on your use-case start with a high limit first (e.g. 16GB) and route your production traffic (either shadow or real) to see what is the usage based on your request patterns. + +For more control you can configure Go's garbage collection with the following env variables (e.g. when facing OOM Killed errors on Kubernetes): + +```bash +# This flag controls when GC kicks in, for example when memory is increased by 30% try to run GC: +export GOGC=30 + +# This flag instructs Go to do a GC when memory goes over the 2GiB limit. +# IMPORTANT: if this value is too low, it might cause high GC frequency, +# which in turn might impact the performance without giving much memory benefits. +export GOMEMLIMIT=2GiB +``` + ## Failsafe policies Make sure to configure [retry policy](/config/failsafe#retry-policy) on both network-level and upstream-level. -- Network-level retry configuration is useful to try other upstreams if one has an issue. Even when only 1 upstream network-level retry is useful. Recommendation is to configure `maxCount` to be equal to the number of upstreams. +- Network-level retry configuration is useful to try other upstreams if one has an issue. Even when you only have 1 upstream, network-level retry is still useful. Recommendation is to configure `maxCount` to be equal to the number of upstreams. - Upstream-level retry configuration covers intermittent issues with a specific upstream. It is recommended to set at least 2 and at most 5 as `maxCount`. [Timeout policy](/config/failsafe#timeout-policy) depends on the expected response time for your use-case, for example when using "trace" methods on EVM chains, providers might take up to 10 seconds to respond. Therefore a low timeout might ultimately always fail. If you are not using heavy methods such as trace or large getLogs, you can use `3s` as a default timeout. -[Hedge policy](/config/failsafe#hedge-policy) is **highly-recommended** if you prefer "fast response as soon as possible". For example setting `500ms` as "delay" will make sure if upstream A did not respond under 500 milliseconds, simultaneously another request to upstream B will be fired, and eRPC will respond back as soon as any of them comes back with result faster. Note that since more requests are sent it might incur higher costs to achieve the "fast response" goal. +[Hedge policy](/config/failsafe#hedge-policy) is **highly-recommended** if you prefer "fast response as soon as possible". For example setting `500ms` as "delay" will make sure if upstream A did not respond under 500 milliseconds, simultaneously another request to upstream B will be fired, and eRPC will respond back as soon as any of them comes back with result faster. Note: since more requests are sent, it might incur higher costs to achieve the "fast response" goal. ## Caching database -Storing cached RPC responses requires high storage for read-heavy use-cases such as indexing 100m blocks on Arbitrum. eRPC is designed to be robust towards cache database issues, so even if database is compeltely down it will not impact the RPC availability. +Storing cached RPC responses requires high storage for read-heavy use-cases such as indexing 100m blocks on Arbitrum. eRPC is designed to be robust towards cache database issues, so even if database is completely down it will not impact the RPC availability. As described in [Database](/config/database) section depending on your requirements choose the right type. You can start with Redis which is easiest to setup, and if amount of cached data is larger than available memory you can switch to PostgreSQL. diff --git a/docs/pages/operation/url.mdx b/docs/pages/operation/url.mdx index 87f22682d..7d3e432e6 100644 --- a/docs/pages/operation/url.mdx +++ b/docs/pages/operation/url.mdx @@ -67,6 +67,20 @@ curl --location 'http://localhost:4000/main' \ }' ``` -# Batch requests +## Batch requests You can batch multiple calls across any number of networks, in a single request. Read more about it in [Batch requests](/operation/batch) page. + +## Healthcheck + +eRPC has a built-in `/healthcheck` endpoint that can be used to check the health of the service within Kubernetes, Railway, etc. + +```bash +curl http://localhost:4000/healthcheck -v +# < HTTP/1.1 200 OK +# OK +``` + + +Currently this endpoint checks active projects and their upstreams (i.e. those which received at least 1 request) for total error rate, and it will return a non-200 response if all endpoints have a +99% error rate. + diff --git a/erpc/erpc.go b/erpc/erpc.go index c4f9a1f82..984bf5722 100644 --- a/erpc/erpc.go +++ b/erpc/erpc.go @@ -56,3 +56,7 @@ func (e *ERPC) GetNetwork(projectId string, networkId string) (*Network, error) func (e *ERPC) GetProject(projectId string) (*PreparedProject, error) { return e.projectsRegistry.GetProject(projectId) } + +func (e *ERPC) GetProjects() []*PreparedProject { + return e.projectsRegistry.GetAll() +} diff --git a/erpc/erpc_test.go b/erpc/erpc_test.go index 6bca40830..393a43310 100644 --- a/erpc/erpc_test.go +++ b/erpc/erpc_test.go @@ -2,10 +2,7 @@ package erpc import ( "context" - "encoding/json" - "fmt" "math/rand" - "os" "sync" "testing" "time" @@ -18,7 +15,7 @@ import ( ) func init() { - log.Logger = log.Level(zerolog.ErrorLevel).Output(zerolog.ConsoleWriter{Out: os.Stderr}) + zerolog.SetGlobalLevel(zerolog.Disabled) } var erpcMu sync.Mutex @@ -84,10 +81,10 @@ func TestErpc_UpstreamsRegistryCorrectPriorityChange(t *testing.T) { // 30% chance of failure if rand.Intn(100) < 30 { r.Status(500) - r.JSON(json.RawMessage(`{"error":{"code":-32000,"message":"internal server error"}}`)) + r.JSON([]byte(`{"error":{"code":-32000,"message":"internal server error"}}`)) } else { r.Status(200) - r.JSON(json.RawMessage(`{"result":{"hash":"0x123456789","fromHost":"rpc1"}}`)) + r.JSON([]byte(`{"result":{"hash":"0x123456789","fromHost":"rpc1"}}`)) } }) } @@ -96,7 +93,7 @@ func TestErpc_UpstreamsRegistryCorrectPriorityChange(t *testing.T) { Persist(). Post(""). Reply(200). - JSON(json.RawMessage(`{"result":{"hash":"0x123456789","fromHost":"rpc2"}}`)) + JSON([]byte(`{"result":{"hash":"0x123456789","fromHost":"rpc2"}}`)) lg := log.With().Logger() ctx1, cancel1 := context.WithCancel(context.Background()) @@ -134,7 +131,6 @@ func TestErpc_UpstreamsRegistryCorrectPriorityChange(t *testing.T) { cancel2() sortedUpstreams, err := nw.upstreamsRegistry.GetSortedUpstreams("evm:123", "eth_getTransactionReceipt") - fmt.Printf("Checking upstream order: %v\n", sortedUpstreams) expectedOrder := []string{"rpc2", "rpc1"} assert.NoError(t, err) diff --git a/erpc/evm_json_rpc_cache.go b/erpc/evm_json_rpc_cache.go index 60a2efdc4..948e5795e 100644 --- a/erpc/evm_json_rpc_cache.go +++ b/erpc/evm_json_rpc_cache.go @@ -2,14 +2,13 @@ package erpc import ( "context" - "encoding/json" "errors" "fmt" "time" - "github.com/bytedance/sonic" "github.com/erpc/erpc/common" "github.com/erpc/erpc/data" + "github.com/erpc/erpc/util" "github.com/rs/zerolog" ) @@ -63,6 +62,9 @@ func (c *EvmJsonRpcCache) Get(ctx context.Context, req *common.NormalizedRequest return nil, err } + rpcReq.RLock() + defer rpcReq.RUnlock() + hasTTL := c.conn.HasTTL(rpcReq.Method) blockRef, blockNumber, err := common.ExtractEvmBlockReferenceFromRequest(rpcReq) @@ -99,10 +101,11 @@ func (c *EvmJsonRpcCache) Get(ctx context.Context, req *common.NormalizedRequest } jrr := &common.JsonRpcResponse{ - JSONRPC: rpcReq.JSONRPC, - ID: rpcReq.ID, - Error: nil, - Result: json.RawMessage(resultString), + Result: util.Str2Mem(resultString), + } + err = jrr.SetID(rpcReq.ID) + if err != nil { + return nil, err } return common.NewNormalizedResponse(). @@ -124,7 +127,7 @@ func (c *EvmJsonRpcCache) Set(ctx context.Context, req *common.NormalizedRequest lg := c.logger.With().Str("networkId", req.NetworkId()).Str("method", rpcReq.Method).Logger() - shouldCache, err := shouldCache(lg, req, resp, rpcReq, rpcResp) + shouldCache, err := shouldCacheResponse(lg, req, resp, rpcReq, rpcResp) if !shouldCache || err != nil { return err } @@ -145,27 +148,18 @@ func (c *EvmJsonRpcCache) Set(ctx context.Context, req *common.NormalizedRequest return nil } - if !hasTTL { - if blockRef == "" && blockNumber == 0 { - // Do not cache if we can't resolve a block reference (e.g. latest block requests) - lg.Debug(). - Str("blockRef", blockRef). - Int64("blockNumber", blockNumber). - Msg("will not cache the response because it has no block reference or block number") - return nil - } - - if blockNumber > 0 { - s, e := c.shouldCacheForBlock(blockNumber) - if !s || e != nil { + if blockNumber > 0 { + s, e := c.shouldCacheForBlock(blockNumber) + if !s || e != nil { + if lg.GetLevel() <= zerolog.DebugLevel { lg.Debug(). Err(e). Str("blockRef", blockRef). Int64("blockNumber", blockNumber). - Interface("result", rpcResp.Result). + Str("result", util.Mem2Str(rpcResp.Result)). Msg("will not cache the response because block is not finalized") - return e } + return e } } @@ -174,25 +168,22 @@ func (c *EvmJsonRpcCache) Set(ctx context.Context, req *common.NormalizedRequest return err } - lg.Debug(). - Str("blockRef", blockRef). - Str("primaryKey", pk). - Str("rangeKey", rk). - Int64("blockNumber", blockNumber). - Interface("result", rpcResp.Result). - Msg("caching the response") - - resultBytes, err := sonic.Marshal(rpcResp.Result) - if err != nil { - return err + if lg.GetLevel() <= zerolog.DebugLevel { + lg.Debug(). + Str("blockRef", blockRef). + Str("primaryKey", pk). + Str("rangeKey", rk). + Int64("blockNumber", blockNumber). + Str("result", util.Mem2Str(rpcResp.Result)). + Msg("caching the response") } ctx, cancel := context.WithTimeoutCause(ctx, 5*time.Second, errors.New("evm json-rpc cache driver timeout during set")) defer cancel() - return c.conn.Set(ctx, pk, rk, string(resultBytes)) + return c.conn.Set(ctx, pk, rk, util.Mem2Str(rpcResp.Result)) } -func shouldCache( +func shouldCacheResponse( lg zerolog.Logger, req *common.NormalizedRequest, resp *common.NormalizedResponse, @@ -265,8 +256,7 @@ func (c *EvmJsonRpcCache) DeleteByGroupKey(ctx context.Context, groupKeys ...str } func (c *EvmJsonRpcCache) shouldCacheForBlock(blockNumber int64) (bool, error) { - b, e := c.network.EvmIsBlockFinalized(blockNumber) - return b, e + return c.network.EvmIsBlockFinalized(blockNumber) } func generateKeysForJsonRpcRequest(req *common.NormalizedRequest, blockRef string) (string, string, error) { diff --git a/erpc/evm_json_rpc_cache_test.go b/erpc/evm_json_rpc_cache_test.go index 2d7ef5699..fb1d45af3 100644 --- a/erpc/evm_json_rpc_cache_test.go +++ b/erpc/evm_json_rpc_cache_test.go @@ -9,6 +9,7 @@ import ( "github.com/erpc/erpc/data" "github.com/erpc/erpc/health" "github.com/erpc/erpc/upstream" + "github.com/erpc/erpc/util" "github.com/erpc/erpc/vendors" "github.com/rs/zerolog" "github.com/stretchr/testify/assert" @@ -64,12 +65,13 @@ func TestEvmJsonRpcCache_Set(t *testing.T) { mockConnector, _, cache := createCacheTestFixtures(10, 15, nil) req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","method":"eth_getTransactionByHash","params":["0x123"],"id":1}`)) - resp := common.NewNormalizedResponse().WithBody([]byte(`{"hash":"0x123","blockNumber":null}`)) + resp := common.NewNormalizedResponse().WithBody(util.StringToReaderCloser(`{"result":{"hash":"0x123","blockNumber":null}}`)) err := cache.Set(context.Background(), req, resp) assert.NoError(t, err) mockConnector.AssertNotCalled(t, "Set") + mockConnector.On("HasTTL", mock.AnythingOfType("string")).Return(false) }) t.Run("CacheIfBlockNumberIsFinalizedWhenBlockIsIrrelevantForPrimaryKey", func(t *testing.T) { @@ -77,7 +79,7 @@ func TestEvmJsonRpcCache_Set(t *testing.T) { req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","method":"eth_getTransactionReceipt","params":["0xabc",false],"id":1}`)) req.SetNetwork(mockNetwork) - resp := common.NewNormalizedResponse().WithBody([]byte(`{"result":{"hash":"0xabc","blockNumber":"0x2"}}`)) + resp := common.NewNormalizedResponse().WithBody(util.StringToReaderCloser(`{"result":{"hash":"0xabc","blockNumber":"0x2"}}`)) mockConnector.On("Set", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) mockConnector.On("HasTTL", mock.AnythingOfType("string")).Return(false) @@ -93,7 +95,7 @@ func TestEvmJsonRpcCache_Set(t *testing.T) { req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","method":"eth_getBlockByNumber","params":["0x2",false],"id":1}`)) req.SetNetwork(mockNetwork) - resp := common.NewNormalizedResponse().WithBody([]byte(`{"result":{"hash":"0xabc","number":"0x2"}}`)) + resp := common.NewNormalizedResponse().WithBody(util.StringToReaderCloser(`{"result":{"hash":"0xabc","number":"0x2"}}`)) mockConnector.On("Set", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) mockConnector.On("HasTTL", mock.AnythingOfType("string")).Return(false) @@ -106,9 +108,10 @@ func TestEvmJsonRpcCache_Set(t *testing.T) { t.Run("SkipWhenNoRefAndNoBlockNumberFound", func(t *testing.T) { mockConnector, _, cache := createCacheTestFixtures(10, 15, nil) + mockConnector.On("HasTTL", mock.AnythingOfType("string")).Return(false) req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","method":"eth_getBalance","params":["0x123","latest"],"id":1}`)) - resp := common.NewNormalizedResponse().WithBody([]byte(`"0x1234"`)) + resp := common.NewNormalizedResponse().WithBody(util.StringToReaderCloser(`{"result":"0x1234"}`)) err := cache.Set(context.Background(), req, resp) @@ -118,6 +121,7 @@ func TestEvmJsonRpcCache_Set(t *testing.T) { t.Run("CacheIfBlockRefFoundWhetherBlockNumberExistsOrNot", func(t *testing.T) { mockConnector, mockNetwork, cache := createCacheTestFixtures(10, 15, nil) + mockConnector.On("HasTTL", mock.AnythingOfType("string")).Return(false) testCases := []struct { name string @@ -146,7 +150,7 @@ func TestEvmJsonRpcCache_Set(t *testing.T) { t.Run(tc.name, func(t *testing.T) { req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","method":"` + tc.method + `","params":` + tc.params + `,"id":1}`)) req.SetNetwork(mockNetwork) - resp := common.NewNormalizedResponse().WithBody([]byte(tc.result)) + resp := common.NewNormalizedResponse().WithBody(util.StringToReaderCloser(tc.result)) mockConnector.On("Set", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil).Once() mockConnector.On("HasTTL", mock.AnythingOfType("string")).Return(false) @@ -166,7 +170,7 @@ func TestEvmJsonRpcCache_Set(t *testing.T) { req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","method":"eth_getBlockByNumber","params":["0x1",false],"id":1}`)) req.SetNetwork(mockNetwork) - resp := common.NewNormalizedResponse().WithBody([]byte(`{"result":{"number":"0x1","hash":"0xabc"}}`)) + resp := common.NewNormalizedResponse().WithBody(util.StringToReaderCloser(`{"result":{"number":"0x1","hash":"0xabc"}}`)) mockConnector.On("Set", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) mockConnector.On("HasTTL", mock.AnythingOfType("string")).Return(false) @@ -181,7 +185,7 @@ func TestEvmJsonRpcCache_Set(t *testing.T) { mockConnector, _, cache := createCacheTestFixtures(10, 15, nil) req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","method":"eth_getBlockByNumber","params":["0x399",false],"id":1}`)) - resp := common.NewNormalizedResponse().WithBody([]byte(`{"result":{"number":"0x399","hash":"0xdef"}}`)) + resp := common.NewNormalizedResponse().WithBody(util.StringToReaderCloser(`{"result":{"number":"0x399","hash":"0xdef"}}`)) mockConnector.On("HasTTL", mock.AnythingOfType("string")).Return(false) err := cache.Set(context.Background(), req, resp) @@ -195,7 +199,7 @@ func TestEvmJsonRpcCache_Set(t *testing.T) { req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","method":"eth_getBalance","params":["0x123","latest"],"id":1}`)) req.SetNetwork(mockNetwork) - resp := common.NewNormalizedResponse().WithBody([]byte(`{"result":"0x0"}`)) + resp := common.NewNormalizedResponse().WithBody(util.StringToReaderCloser(`{"result":"0x0"}`)) mockConnector.On("HasTTL", mock.AnythingOfType("string")).Return(false) err := cache.Set(context.Background(), req, resp) @@ -209,7 +213,7 @@ func TestEvmJsonRpcCache_Set(t *testing.T) { req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","method":"eth_getBalance","params":["0x123","latest"],"id":1}`)) req.SetNetwork(mockNetwork) - resp := common.NewNormalizedResponse().WithBody([]byte(`{"result":"0x0"}`)) + resp := common.NewNormalizedResponse().WithBody(util.StringToReaderCloser(`{"result":"0x0"}`)) mockConnector.On("HasTTL", mock.AnythingOfType("string")).Return(false) err := cache.Set(context.Background(), req, resp) @@ -223,7 +227,7 @@ func TestEvmJsonRpcCache_Set(t *testing.T) { req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","method":"eth_getBalance","params":["0x123","0x14"],"id":1}`)) req.SetNetwork(mockNetwork) - resp := common.NewNormalizedResponse().WithBody([]byte(`{"result":"0x0"}`)) + resp := common.NewNormalizedResponse().WithBody(util.StringToReaderCloser(`{"result":"0x0"}`)) mockConnector.On("HasTTL", mock.AnythingOfType("string")).Return(false) err := cache.Set(context.Background(), req, resp) @@ -237,7 +241,7 @@ func TestEvmJsonRpcCache_Set(t *testing.T) { req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","method":"eth_getBalance","params":["0x123","latest"],"id":1}`)) req.SetNetwork(mockNetwork) - resp := common.NewNormalizedResponse().WithBody([]byte(`{"result":"0x0"}`)) + resp := common.NewNormalizedResponse().WithBody(util.StringToReaderCloser(`{"result":"0x0"}`)) mockConnector.On("HasTTL", mock.AnythingOfType("string")).Return(false) err := cache.Set(context.Background(), req, resp) @@ -251,7 +255,7 @@ func TestEvmJsonRpcCache_Set(t *testing.T) { req := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","method":"eth_getBalance","params":["0x123","0x5"],"id":1}`)) req.SetNetwork(mockNetwork) - resp := common.NewNormalizedResponse().WithBody([]byte(`{"result":"0x0"}`)) + resp := common.NewNormalizedResponse().WithBody(util.StringToReaderCloser(`{"result":"0x0"}`)) mockConnector.On("Set", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) mockConnector.On("HasTTL", mock.AnythingOfType("string")).Return(false) diff --git a/erpc/healthcheck.go b/erpc/healthcheck.go new file mode 100644 index 000000000..9602de15a --- /dev/null +++ b/erpc/healthcheck.go @@ -0,0 +1,53 @@ +package erpc + +import ( + "errors" + "fmt" + "sort" + + "github.com/bytedance/sonic" + "github.com/valyala/fasthttp" +) + +func (s *HttpServer) handleHealthCheck(ctx *fasthttp.RequestCtx, encoder sonic.Encoder) { + logger := s.logger.With().Str("handler", "healthcheck").Logger() + + if s.erpc == nil { + handleErrorResponse(&logger, nil, errors.New("eRPC is not initialized"), ctx, encoder) + return + } + + projects := s.erpc.GetProjects() + + for _, project := range projects { + h, err := project.gatherHealthInfo() + if err != nil { + handleErrorResponse(&logger, nil, err, ctx, encoder) + return + } + + if h.Upstreams != nil && len(h.Upstreams) > 0 { + metricsTracker := project.upstreamsRegistry.GetMetricsTracker() + allErrorRates := []float64{} + for _, ups := range h.Upstreams { + cfg := ups.Config() + mts := metricsTracker.GetUpstreamMethodMetrics(cfg.Id, "*", "*") + if mts != nil && mts.RequestsTotal > 0 { + allErrorRates = append(allErrorRates, float64(mts.ErrorsTotal)/float64(mts.RequestsTotal)) + } + } + + if len(allErrorRates) > 0 { + sort.Float64s(allErrorRates) + if allErrorRates[0] > 0.99 { + handleErrorResponse(&logger, nil, fmt.Errorf("all upstreams are down: %+v", allErrorRates), ctx, encoder) + return + } + } + } + } + + logger.Info().Msg("Healthcheck passed") + ctx.SetStatusCode(fasthttp.StatusOK) + ctx.SetBodyString("OK") +} diff --git a/erpc/http_server.go b/erpc/http_server.go index 0ed314bbc..28008ca14 100644 --- a/erpc/http_server.go +++ b/erpc/http_server.go @@ -1,12 +1,13 @@ package erpc import ( - "bytes" "context" "encoding/json" "errors" "fmt" "net" + "path" + "runtime/debug" "strings" "sync" @@ -16,6 +17,7 @@ import ( "github.com/erpc/erpc/auth" "github.com/erpc/erpc/common" "github.com/erpc/erpc/health" + "github.com/erpc/erpc/util" "github.com/rs/zerolog" "github.com/valyala/fasthttp" ) @@ -27,12 +29,6 @@ type HttpServer struct { logger *zerolog.Logger } -var bufPool = sync.Pool{ - New: func() interface{} { - return new(bytes.Buffer) - }, -} - func NewHttpServer(ctx context.Context, logger *zerolog.Logger, cfg *common.ServerConfig, erpc *ERPC) *HttpServer { reqMaxTimeout, err := time.ParseDuration(cfg.MaxTimeout) if err != nil { @@ -57,6 +53,7 @@ func NewHttpServer(ctx context.Context, logger *zerolog.Logger, cfg *common.Serv ), ReadTimeout: 5 * time.Second, WriteTimeout: 10 * time.Second, + Name: fmt.Sprintf("erpc (%s/%s)", common.ErpcVersion, common.ErpcCommitSha), } go func() { @@ -76,7 +73,7 @@ func (s *HttpServer) createRequestHandler(mainCtx context.Context, reqMaxTimeout defer func() { defer func() { recover() }() if r := recover(); r != nil { - msg := fmt.Sprintf("unexpected server panic on top-level handler: %v -> %s", r, string(debug.Stack())) + msg := fmt.Sprintf("unexpected server panic on top-level handler: %v -> %s", r, util.Mem2Str(debug.Stack())) s.logger.Error().Msgf(msg) fastCtx.SetStatusCode(fasthttp.StatusInternalServerError) fastCtx.Response.Header.Set("Content-Type", "application/json") @@ -84,38 +81,25 @@ func (s *HttpServer) createRequestHandler(mainCtx context.Context, reqMaxTimeout } }() - buf := bufPool.Get().(*bytes.Buffer) - defer bufPool.Put(buf) - buf.Reset() - encoder := json.NewEncoder(buf) + encoder := common.SonicCfg.NewEncoder(fastCtx.Response.BodyWriter()) + encoder.SetEscapeHTML(false) - segments := strings.Split(string(fastCtx.Path()), "/") - if len(segments) != 2 && len(segments) != 3 && len(segments) != 4 { - handleErrorResponse(s.logger, nil, common.NewErrInvalidUrlPath(string(fastCtx.Path())), fastCtx, encoder, buf) + projectId, architecture, chainId, isAdmin, isHealthCheck, err := s.parseUrlPath(fastCtx) + if err != nil { + handleErrorResponse(s.logger, nil, err, fastCtx, encoder) return } - projectId := segments[1] - architecture, chainId := "", "" - isAdmin := false - - if len(segments) == 4 { - architecture = segments[2] - chainId = segments[3] - } else if len(segments) == 3 { - if segments[2] == "admin" { - isAdmin = true - } else { - handleErrorResponse(s.logger, nil, common.NewErrInvalidUrlPath(string(fastCtx.Path())), fastCtx, encoder, buf) - return - } + if isHealthCheck { + s.handleHealthCheck(fastCtx, encoder) + return } lg := s.logger.With().Str("projectId", projectId).Str("architecture", architecture).Str("chainId", chainId).Logger() project, err := s.erpc.GetProject(projectId) if err != nil { - handleErrorResponse(&lg, nil, err, fastCtx, encoder, buf) + handleErrorResponse(&lg, nil, err, fastCtx, encoder) return } @@ -134,11 +118,21 @@ func (s *HttpServer) createRequestHandler(mainCtx context.Context, reqMaxTimeout lg.Debug().Msgf("received request with body: %s", body) var requests []json.RawMessage - err = sonic.Unmarshal(body, &requests) - isBatch := err == nil - + isBatch := len(body) > 0 && body[0] == '[' if !isBatch { requests = []json.RawMessage{body} + } else { + err = common.SonicCfg.Unmarshal(body, &requests) + if err != nil { + handleErrorResponse( + &lg, + nil, + common.NewErrJsonRpcRequestUnmarshal(err), + fastCtx, + encoder, + ) + return + } } responses := make([]interface{}, len(requests)) @@ -155,7 +149,7 @@ func (s *HttpServer) createRequestHandler(mainCtx context.Context, reqMaxTimeout defer func() { defer func() { recover() }() if r := recover(); r != nil { - msg := fmt.Sprintf("unexpected server panic on per-request handler: %v -> %s", r, string(debug.Stack())) + msg := fmt.Sprintf("unexpected server panic on per-request handler: %v -> %s", r, util.Mem2Str(debug.Stack())) lg.Error().Msgf(msg) fastCtx.SetStatusCode(fasthttp.StatusInternalServerError) fastCtx.Response.Header.Set("Content-Type", "application/json") @@ -218,7 +212,7 @@ func (s *HttpServer) createRequestHandler(mainCtx context.Context, reqMaxTimeout if architecture == "" || chainId == "" { var req map[string]interface{} - if err := sonic.Unmarshal(rawReq, &req); err != nil { + if err := common.SonicCfg.Unmarshal(rawReq, &req); err != nil { responses[index] = processErrorBody(&rlg, nq, common.NewErrInvalidRequest(err)) return } @@ -267,6 +261,11 @@ func (s *HttpServer) createRequestHandler(mainCtx context.Context, reqMaxTimeout if isBatch { fastCtx.SetStatusCode(fasthttp.StatusOK) err = encoder.Encode(responses) + for _, resp := range responses { + if r, ok := resp.(*common.NormalizedResponse); ok { + r.Release() + } + } if err != nil { fastCtx.SetStatusCode(fasthttp.StatusInternalServerError) fastCtx.Response.Header.Set("Content-Type", "application/json") @@ -277,25 +276,64 @@ func (s *HttpServer) createRequestHandler(mainCtx context.Context, reqMaxTimeout res := responses[0] setResponseHeaders(res, fastCtx) setResponseStatusCode(res, fastCtx) - err = encoder.Encode(res) + if r, ok := res.(*common.NormalizedResponse); ok { + rdr, err := r.GetReader() + if err != nil { + fastCtx.SetStatusCode(fasthttp.StatusInternalServerError) + fastCtx.SetBodyString(fmt.Sprintf(`{"jsonrpc":"2.0","error":{"code":-32603,"message":"%s"}}`, err.Error())) + return + } + fastCtx.Response.SetBodyStream(rdr, -1) + r.Release() + } else { + err = encoder.Encode(res) + } if err != nil { fastCtx.SetStatusCode(fasthttp.StatusInternalServerError) fastCtx.SetBodyString(fmt.Sprintf(`{"jsonrpc":"2.0","error":{"code":-32603,"message":"%s"}}`, err.Error())) return } } + } +} + +func (s *HttpServer) parseUrlPath(fctx *fasthttp.RequestCtx) ( + projectId, architecture, chainId string, + isAdmin bool, + isHealthCheck bool, + err error, +) { + isPost := fctx.IsPost() + ps := path.Clean(util.Mem2Str(fctx.Path())) + segments := strings.Split(ps, "/") + + if len(segments) != 2 && len(segments) != 3 && len(segments) != 4 { + return "", "", "", false, false, common.NewErrInvalidUrlPath(ps) + } - fastCtx.SetBody(buf.Bytes()) + if isPost && len(segments) == 4 { + projectId = segments[1] + architecture = segments[2] + chainId = segments[3] + } else if isPost && len(segments) == 3 && segments[2] == "admin" { + projectId = segments[1] + isAdmin = true + } else if len(segments) == 2 && segments[1] == "healthcheck" { + isHealthCheck = true + } else { + return "", "", "", false, false, common.NewErrInvalidUrlPath(ps) } + + return projectId, architecture, chainId, isAdmin, isHealthCheck, nil } func (s *HttpServer) handleCORS(ctx *fasthttp.RequestCtx, corsConfig *common.CORSConfig) bool { - origin := string(ctx.Request.Header.Peek("Origin")) + origin := util.Mem2Str(ctx.Request.Header.Peek("Origin")) if origin == "" { return true } - health.MetricCORSRequestsTotal.WithLabelValues(string(ctx.Path()), origin).Inc() + health.MetricCORSRequestsTotal.WithLabelValues(util.Mem2Str(ctx.Path()), origin).Inc() allowed := false for _, allowedOrigin := range corsConfig.AllowedOrigins { @@ -307,7 +345,7 @@ func (s *HttpServer) handleCORS(ctx *fasthttp.RequestCtx, corsConfig *common.COR if !allowed { s.logger.Debug().Str("origin", origin).Msg("CORS request from disallowed origin") - health.MetricCORSDisallowedOriginTotal.WithLabelValues(string(ctx.Path()), origin).Inc() + health.MetricCORSDisallowedOriginTotal.WithLabelValues(util.Mem2Str(ctx.Path()), origin).Inc() if ctx.IsOptions() { ctx.SetStatusCode(fasthttp.StatusNoContent) @@ -331,7 +369,7 @@ func (s *HttpServer) handleCORS(ctx *fasthttp.RequestCtx, corsConfig *common.COR } if ctx.IsOptions() { - health.MetricCORSPreflightRequestsTotal.WithLabelValues(string(ctx.Path()), origin).Inc() + health.MetricCORSPreflightRequestsTotal.WithLabelValues(util.Mem2Str(ctx.Path()), origin).Inc() ctx.SetStatusCode(fasthttp.StatusNoContent) return false } @@ -344,21 +382,28 @@ func setResponseHeaders(res interface{}, fastCtx *fasthttp.RequestCtx) { var ok bool rm, ok = res.(common.ResponseMetadata) if !ok { - var jrsp, errObj map[string]interface{} + var jrsp map[string]interface{} + var hjrsp *HttpJsonRpcErrorResponse if jrsp, ok = res.(map[string]interface{}); ok { - if errObj, ok = jrsp["error"].(map[string]interface{}); ok { - if err, ok := errObj["cause"].(error); ok { - uer := &common.ErrUpstreamsExhausted{} + var err error + if err, ok = jrsp["cause"].(error); ok { + uer := &common.ErrUpstreamsExhausted{} + if ok = errors.As(err, &uer); ok { + rm = uer + } else { + uer := &common.ErrUpstreamRequest{} if ok = errors.As(err, &uer); ok { rm = uer - } else { - uer := &common.ErrUpstreamRequest{} - if ok = errors.As(err, &uer); ok { - rm = uer - } } } } + } else if hjrsp, ok = res.(*HttpJsonRpcErrorResponse); ok { + if err := hjrsp.Cause; err != nil { + uer := &common.ErrUpstreamRequest{} + if ok = errors.As(err, &uer); ok { + rm = uer + } + } } } if ok && rm != nil { @@ -380,20 +425,25 @@ func setResponseStatusCode(respOrErr interface{}, fastCtx *fasthttp.RequestCtx) if err, ok := respOrErr.(error); ok { fastCtx.SetStatusCode(decideErrorStatusCode(err)) } else if resp, ok := respOrErr.(map[string]interface{}); ok { - if errObj, ok := resp["error"].(map[string]interface{}); ok { - if cause, ok := errObj["cause"].(error); ok { - fastCtx.SetStatusCode(decideErrorStatusCode(cause)) - } else { - fastCtx.SetStatusCode(fasthttp.StatusOK) - } + if cause, ok := resp["cause"].(error); ok { + fastCtx.SetStatusCode(decideErrorStatusCode(cause)) } else { fastCtx.SetStatusCode(fasthttp.StatusOK) } + } else if hjrsp, ok := respOrErr.(*HttpJsonRpcErrorResponse); ok { + fastCtx.SetStatusCode(decideErrorStatusCode(hjrsp.Cause)) } else { fastCtx.SetStatusCode(fasthttp.StatusOK) } } +type HttpJsonRpcErrorResponse struct { + Jsonrpc string `json:"jsonrpc"` + Id interface{} `json:"id"` + Error interface{} `json:"error"` + Cause error `json:"-"` +} + func processErrorBody(logger *zerolog.Logger, nq *common.NormalizedRequest, err error) interface{} { if !common.IsNull(err) { if nq != nil { @@ -426,15 +476,18 @@ func processErrorBody(logger *zerolog.Logger, nq *common.NormalizedRequest, err } jre := &common.ErrJsonRpcExceptionInternal{} if errors.As(err, &jre) { - return map[string]interface{}{ - "jsonrpc": jsonrpcVersion, - "id": reqId, - "error": map[string]interface{}{ - "code": jre.NormalizedCode(), - "message": jre.Message, - "data": jre.Details["data"], - "cause": err, - }, + errObj := map[string]interface{}{ + "code": jre.NormalizedCode(), + "message": jre.Message, + } + if jre.Details["data"] != nil { + errObj["data"] = jre.Details["data"] + } + return &HttpJsonRpcErrorResponse{ + Jsonrpc: jsonrpcVersion, + Id: reqId, + Error: errObj, + Cause: err, } } @@ -458,7 +511,7 @@ func decideErrorStatusCode(err error) int { return fasthttp.StatusInternalServerError } -func handleErrorResponse(logger *zerolog.Logger, nq *common.NormalizedRequest, err error, ctx *fasthttp.RequestCtx, encoder sonic.Encoder, buf *bytes.Buffer) { +func handleErrorResponse(logger *zerolog.Logger, nq *common.NormalizedRequest, err error, ctx *fasthttp.RequestCtx, encoder sonic.Encoder) { resp := processErrorBody(logger, nq, err) setResponseStatusCode(err, ctx) err = encoder.Encode(resp) @@ -469,7 +522,6 @@ func handleErrorResponse(logger *zerolog.Logger, nq *common.NormalizedRequest, e ctx.SetBodyString(fmt.Sprintf(`{"jsonrpc":"2.0","error":{"code":-32603,"message":"%s"}}`, err.Error())) } else { ctx.Response.Header.Set("Content-Type", "application/json") - ctx.SetBody(buf.Bytes()) } } diff --git a/erpc/http_server_test.go b/erpc/http_server_test.go index 0994e96a4..db7f0d295 100644 --- a/erpc/http_server_test.go +++ b/erpc/http_server_test.go @@ -35,7 +35,7 @@ func TestHttpServer_RaceTimeouts(t *testing.T) { cfg := &common.Config{ Server: &common.ServerConfig{ - MaxTimeout: "200ms", // Set a very short timeout for testing + MaxTimeout: "500ms", // Set a very short timeout for testing }, Projects: []*common.ProjectConfig{ { @@ -142,7 +142,7 @@ func TestHttpServer_RaceTimeouts(t *testing.T) { if result.statusCode != http.StatusGatewayTimeout { t.Errorf("unexpected status code: %d", result.statusCode) } - assert.Contains(t, result.body, "Timeout") + assert.Contains(t, result.body, "timeout") } }) @@ -151,7 +151,7 @@ func TestHttpServer_RaceTimeouts(t *testing.T) { Post("/"). Times(10). Reply(200). - Delay(300 * time.Millisecond). // Delay longer than the server timeout + Delay(700 * time.Millisecond). // Delay longer than the server timeout JSON(map[string]interface{}{ "jsonrpc": "2.0", "id": 1, @@ -161,7 +161,7 @@ func TestHttpServer_RaceTimeouts(t *testing.T) { for i := 0; i < 10; i++ { statusCode, body := sendRequest() assert.Equal(t, http.StatusGatewayTimeout, statusCode) - assert.Contains(t, body, "Timeout") + assert.Contains(t, body, "timeout") } }) @@ -173,7 +173,7 @@ func TestHttpServer_RaceTimeouts(t *testing.T) { if i%2 == 0 { delay = 1 * time.Millisecond // shorter than the server timeout } else { - delay = 200 * time.Millisecond // longer than the server timeout + delay = 700 * time.Millisecond // longer than the server timeout } gock.New("http://rpc1.localhost"). Post("/"). @@ -212,7 +212,7 @@ func TestHttpServer_RaceTimeouts(t *testing.T) { for _, result := range results { if result.statusCode == http.StatusGatewayTimeout { timeouts++ - assert.Contains(t, result.body, "Timeout") + assert.Contains(t, result.body, "timeout") } else { successes++ assert.Contains(t, result.body, "blockNumber") @@ -249,6 +249,9 @@ func TestHttpServer_SingleUpstream(t *testing.T) { ChainId: 1, }, VendorName: "llama", + JsonRpc: &common.JsonRpcUpstreamConfig{ + SupportsBatch: &common.FALSE, + }, }, }, }, @@ -256,252 +259,359 @@ func TestHttpServer_SingleUpstream(t *testing.T) { RateLimiters: &common.RateLimiterConfig{}, } - sendRequest, baseURL := createServerTestFixtures(cfg, t) - - t.Run("ConcurrentEthGetBlockNumber", func(t *testing.T) { - defer gock.Off() - const concurrentRequests = 10 - - gock.New("http://rpc1.localhost"). - Post("/"). - Times(concurrentRequests). - Reply(200). - JSON(map[string]interface{}{ - "jsonrpc": "2.0", - "id": 1, - "result": "0x444444", - }) - - var wg sync.WaitGroup - results := make([]struct { - statusCode int - body string - }, concurrentRequests) - - for i := 0; i < concurrentRequests; i++ { - wg.Add(1) - go func(index int) { - defer wg.Done() - body := fmt.Sprintf(`{"jsonrpc":"2.0","method":"eth_getBlockNumber","params":[%d],"id":1}`, index) - results[index].statusCode, results[index].body = sendRequest(body, nil, nil) - }(i) - } - - wg.Wait() - - for i, result := range results { - assert.Equal(t, http.StatusOK, result.statusCode, "Status code should be 200 for request %d", i) + cfgCases := []func(*common.Config){ + // Case 1: Upstream supports batch requests + func(cfg *common.Config) { + cfg.Projects[0].Upstreams[0].JsonRpc.SupportsBatch = &common.TRUE + }, - var response map[string]interface{} - err := sonic.Unmarshal([]byte(result.body), &response) - assert.NoError(t, err, "Should be able to decode response for request %d", i) - assert.Equal(t, "0x444444", response["result"], "Unexpected result for request %d", i) - } + // Case 2: Upstream does not support batch requests + func(cfg *common.Config) { + cfg.Projects[0].Upstreams[0].JsonRpc.SupportsBatch = &common.FALSE + }, - assert.True(t, gock.IsDone(), "All mocks should have been called") - }) + // Case 3: Caching is enabled + func(cfg *common.Config) { + cfg.Database = &common.DatabaseConfig{ + EvmJsonRpcCache: &common.ConnectorConfig{ + Driver: "memory", + Memory: &common.MemoryConnectorConfig{ + MaxItems: 100, + }, + }, + } + }, - t.Run("InvalidJSON", func(t *testing.T) { - statusCode, body := sendRequest(`{"invalid json`, nil, nil) + // Case 4: Caching is disabled + func(cfg *common.Config) { + cfg.Database.EvmJsonRpcCache = nil + }, + } - fmt.Println(body) + for _, applyCfgOverride := range cfgCases { + applyCfgOverride(cfg) + sendRequest, baseURL := createServerTestFixtures(cfg, t) - assert.Equal(t, http.StatusBadRequest, statusCode) + t.Run("ConcurrentEthGetBlockNumber", func(t *testing.T) { + defer gock.Off() + const concurrentRequests = 100 - var errorResponse map[string]interface{} - err := sonic.Unmarshal([]byte(body), &errorResponse) - require.NoError(t, err) + gock.New("http://rpc1.localhost"). + Post("/"). + Times(concurrentRequests). + Reply(200). + JSON(map[string]interface{}{ + "jsonrpc": "2.0", + "id": 1, + "result": "0x444444", + }) + + var wg sync.WaitGroup + results := make([]struct { + statusCode int + body string + }, concurrentRequests) + + for i := 0; i < concurrentRequests; i++ { + wg.Add(1) + go func(index int) { + defer wg.Done() + body := fmt.Sprintf(`{"jsonrpc":"2.0","method":"eth_getBlockNumber","params":[%d],"id":1}`, index) + results[index].statusCode, results[index].body = sendRequest(body, nil, nil) + }(i) + } - assert.Contains(t, errorResponse, "error") - errorObj := errorResponse["error"].(map[string]interface{}) - errStr, _ := sonic.Marshal(errorObj) - assert.Contains(t, string(errStr), "ErrJsonRpcRequestUnmarshal") - }) + wg.Wait() - t.Run("UnsupportedMethod", func(t *testing.T) { - defer gock.Off() + for i, result := range results { + assert.Equal(t, http.StatusOK, result.statusCode, "Status code should be 200 for request %d", i) - gock.New("http://rpc1.localhost"). - Post("/"). - Reply(200). - JSON(map[string]interface{}{ - "jsonrpc": "2.0", - "id": 1, - "error": map[string]interface{}{ - "code": -32601, - "message": "Method not found", - }, - }) + var response map[string]interface{} + err := sonic.Unmarshal([]byte(result.body), &response) + assert.NoError(t, err, "Should be able to decode response for request %d", i) + assert.Equal(t, "0x444444", response["result"], "Unexpected result for request %d", i) + } - statusCode, body := sendRequest(`{"jsonrpc":"2.0","method":"unsupported_method","params":[],"id":1}`, nil, nil) + assert.True(t, gock.IsDone(), "All mocks should have been called") + }) - assert.Equal(t, http.StatusUnsupportedMediaType, statusCode) + t.Run("InvalidJSON", func(t *testing.T) { + statusCode, body := sendRequest(`{"invalid json`, nil, nil) - var errorResponse map[string]interface{} - err := sonic.Unmarshal([]byte(body), &errorResponse) - require.NoError(t, err) + fmt.Println(body) - assert.Contains(t, errorResponse, "error") - errorObj := errorResponse["error"].(map[string]interface{}) - assert.Equal(t, float64(-32601), errorObj["code"]) - assert.Contains(t, errorObj["message"], "Method not found") + assert.Equal(t, http.StatusBadRequest, statusCode) - assert.True(t, gock.IsDone(), "All mocks should have been called") - }) + var errorResponse map[string]interface{} + err := sonic.Unmarshal([]byte(body), &errorResponse) + require.NoError(t, err) - // Test case: Request with invalid project ID - t.Run("InvalidProjectID", func(t *testing.T) { - req, err := http.NewRequest("POST", baseURL+"/invalid_project/evm/1", strings.NewReader(`{"jsonrpc":"2.0","method":"eth_getBlockNumber","params":[],"id":1}`)) - require.NoError(t, err) - req.Header.Set("Content-Type", "application/json") + assert.Contains(t, errorResponse, "error") + errorObj := errorResponse["error"].(map[string]interface{}) + errStr, _ := sonic.Marshal(errorObj) + assert.Contains(t, string(errStr), "failed to parse") + }) - client := &http.Client{ - Timeout: 10 * time.Second, - } - resp, err := client.Do(req) - require.NoError(t, err) - defer resp.Body.Close() + t.Run("UnsupportedMethod", func(t *testing.T) { + defer gock.Off() + cfg.Projects[0].Upstreams[0].IgnoreMethods = []string{} - assert.Equal(t, http.StatusNotFound, resp.StatusCode) + gock.New("http://rpc1.localhost"). + Post("/"). + Reply(200). + Map(func(res *http.Response) *http.Response { + sg := `{"jsonrpc":"2.0","id":1,"error":{"code":-32601,"message":"Method not found"}}` + if *cfg.Projects[0].Upstreams[0].JsonRpc.SupportsBatch { + sg = "[" + sg + "]" + } + res.Body = io.NopCloser(strings.NewReader(sg)) + return res + }) + + statusCode, body := sendRequest(`{"jsonrpc":"2.0","method":"unsupported_method","params":[],"id":1}`, nil, nil) + assert.Equal(t, http.StatusUnsupportedMediaType, statusCode) + + var errorResponse map[string]interface{} + err := sonic.Unmarshal([]byte(body), &errorResponse) + require.NoError(t, err) + + assert.Contains(t, errorResponse, "error") + errorObj := errorResponse["error"].(map[string]interface{}) + assert.Equal(t, float64(-32601), errorObj["code"]) + }) - body, err := io.ReadAll(resp.Body) - require.NoError(t, err) + t.Run("IgnoredMethod", func(t *testing.T) { + defer gock.Off() + cfg.Projects[0].Upstreams[0].IgnoreMethods = []string{"ignored_method"} - var errorResponse map[string]interface{} - err = sonic.Unmarshal(body, &errorResponse) - require.NoError(t, err) + gock.New("http://rpc1.localhost"). + Post("/"). + Reply(200). + Map(func(res *http.Response) *http.Response { + sg := `{"jsonrpc":"2.0","id":1,"error":{"code":-32601,"message":"Method not found"}}` + if *cfg.Projects[0].Upstreams[0].JsonRpc.SupportsBatch { + sg = "[" + sg + "]" + } + res.Body = io.NopCloser(strings.NewReader(sg)) + return res + }) + + statusCode, body := sendRequest(`{"jsonrpc":"2.0","method":"ignored_method","params":[],"id":1}`, nil, nil) + assert.Equal(t, http.StatusUnsupportedMediaType, statusCode) + + var errorResponse map[string]interface{} + err := sonic.Unmarshal([]byte(body), &errorResponse) + require.NoError(t, err) + + assert.Contains(t, errorResponse, "error") + errorObj := errorResponse["error"].(map[string]interface{}) + assert.Equal(t, float64(-32601), errorObj["code"]) + }) - assert.Contains(t, errorResponse, "error") - errorObj := errorResponse["error"].(map[string]interface{}) - assert.Contains(t, errorObj["message"], "project not configured") - }) + // Test case: Request with invalid project ID + t.Run("InvalidProjectID", func(t *testing.T) { + req, err := http.NewRequest("POST", baseURL+"/invalid_project/evm/1", strings.NewReader(`{"jsonrpc":"2.0","method":"eth_getBlockNumber","params":[],"id":1}`)) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/json") - t.Run("UpstreamLatencyAndTimeout", func(t *testing.T) { - gock.New("http://rpc1.localhost"). - Post("/"). - Reply(200). - Delay(6 * time.Second). // Delay longer than the server timeout - JSON(map[string]interface{}{ - "jsonrpc": "2.0", - "id": 1, - "result": "0x1111111", - }) + client := &http.Client{ + Timeout: 10 * time.Second, + } + resp, err := client.Do(req) + require.NoError(t, err) + defer resp.Body.Close() - statusCode, body := sendRequest(`{"jsonrpc":"2.0","method":"eth_getBlockNumber","params":[],"id":1}`, nil, nil) + assert.Equal(t, http.StatusNotFound, resp.StatusCode) - assert.Equal(t, http.StatusGatewayTimeout, statusCode) + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) - var errorResponse map[string]interface{} - err := sonic.Unmarshal([]byte(body), &errorResponse) - require.NoError(t, err) + var errorResponse map[string]interface{} + err = sonic.Unmarshal(body, &errorResponse) + require.NoError(t, err) - assert.Contains(t, errorResponse, "error") - errorObj := errorResponse["error"].(map[string]interface{}) - errStr, _ := sonic.Marshal(errorObj) - assert.Contains(t, string(errStr), "ErrEndpointRequestTimeout") + assert.Contains(t, errorResponse, "error") + errorObj := errorResponse["error"].(map[string]interface{}) + assert.Contains(t, errorObj["message"], "project not configured") + }) - assert.True(t, gock.IsDone(), "All mocks should have been called") - }) + t.Run("UpstreamLatencyAndTimeout", func(t *testing.T) { + gock.New("http://rpc1.localhost"). + Post("/"). + Reply(200). + Delay(6 * time.Second). // Delay longer than the server timeout + JSON(map[string]interface{}{ + "jsonrpc": "2.0", + "id": 1, + "result": "0x1111111", + }) - t.Run("UnexpectedPlainErrorResponseFromUpstream", func(t *testing.T) { - gock.New("http://rpc1.localhost"). - Post("/"). - Times(1). - Reply(200). - BodyString("error code: 1015") + statusCode, body := sendRequest(`{"jsonrpc":"2.0","method":"eth_getBlockNumber","params":[],"id":1}`, nil, nil) - statusCode, body := sendRequest(`{"jsonrpc":"2.0","method":"eth_getBlockNumber","params":[],"id":1}`, nil, nil) + assert.Equal(t, http.StatusGatewayTimeout, statusCode) + var errorResponse map[string]interface{} + err := sonic.Unmarshal([]byte(body), &errorResponse) + require.NoError(t, err) - assert.Equal(t, http.StatusTooManyRequests, statusCode) - assert.Contains(t, body, "error code: 1015") + assert.Contains(t, errorResponse, "error") + errorObj := errorResponse["error"].(map[string]interface{}) + errStr, _ := sonic.Marshal(errorObj) + assert.Contains(t, string(errStr), "timeout") - assert.True(t, gock.IsDone(), "All mocks should have been called") - }) + assert.True(t, gock.IsDone(), "All mocks should have been called") + }) - t.Run("UnexpectedServerErrorResponseFromUpstream", func(t *testing.T) { - gock.New("http://rpc1.localhost"). - Post("/"). - Times(1). - Reply(500). - BodyString(`{"error":{"code":-39999,"message":"my funky error"}}`) + t.Run("UnexpectedPlainErrorResponseFromUpstream", func(t *testing.T) { + gock.New("http://rpc1.localhost"). + Post("/"). + Times(1). + Reply(200). + BodyString("error code: 1015") - statusCode, body := sendRequest(`{"jsonrpc":"2.0","method":"eth_getBlockNumber","params":[],"id":1}`, nil, nil) + statusCode, body := sendRequest(`{"jsonrpc":"2.0","method":"eth_getBlockNumber","params":[],"id":1}`, nil, nil) - assert.Equal(t, http.StatusInternalServerError, statusCode) - assert.Contains(t, body, "-32603") - assert.Contains(t, body, "my funky error") + assert.Equal(t, http.StatusTooManyRequests, statusCode) + assert.Contains(t, body, "error code: 1015") - assert.True(t, gock.IsDone(), "All mocks should have been called") - }) -} + assert.True(t, gock.IsDone(), "All mocks should have been called") + }) -func createServerTestFixtures(cfg *common.Config, t *testing.T) ( - func(body string, headers map[string]string, queryParams map[string]string) (int, string), - string, -) { - gock.EnableNetworking() - gock.NetworkingFilter(func(req *http.Request) bool { - shouldMakeRealCall := strings.Split(req.URL.Host, ":")[0] == "localhost" - return shouldMakeRealCall - }) - defer gock.Off() + t.Run("UnexpectedServerErrorResponseFromUpstream", func(t *testing.T) { + gock.New("http://rpc1.localhost"). + Post("/"). + Times(1). + Reply(500). + BodyString(`{"error":{"code":-39999,"message":"my funky error"}}`) - logger := zerolog.New(zerolog.NewConsoleWriter()) - ctx := context.Background() - // ctx, cancel := context.WithCancel(context.Background()) - // defer cancel() + statusCode, body := sendRequest(`{"jsonrpc":"2.0","method":"eth_getBlockNumber","params":[],"id":1}`, nil, nil) - erpcInstance, err := NewERPC(ctx, &logger, nil, cfg) - require.NoError(t, err) + assert.Equal(t, http.StatusInternalServerError, statusCode) + assert.Contains(t, body, "-32603") + assert.Contains(t, body, "my funky error") - httpServer := NewHttpServer(ctx, &logger, cfg.Server, erpcInstance) + assert.True(t, gock.IsDone(), "All mocks should have been called") + }) - listener, err := net.Listen("tcp", "127.0.0.1:0") - require.NoError(t, err) - port := listener.Addr().(*net.TCPAddr).Port + t.Run("MissingIDInJsonRpcRequest", func(t *testing.T) { + var id interface{} + gock.New("http://rpc1.localhost"). + Post("/"). + Times(1). + SetMatcher(gock.NewEmptyMatcher()). + AddMatcher(func(req *http.Request, ereq *gock.Request) (bool, error) { + if !strings.Contains(req.URL.Host, "rpc1") { + return false, nil + } + bodyBytes, err := io.ReadAll(req.Body) + if err != nil { + return false, err + } + idNode, _ := sonic.Get(bodyBytes, "id") + id, _ = idNode.Interface() + if id == nil { + idNode, _ = sonic.Get(bodyBytes, 0, "id") + id, _ = idNode.Interface() + } + return true, nil + }). + Reply(200). + Map(func(res *http.Response) *http.Response { + var respTxt string + if *cfg.Projects[0].Upstreams[0].JsonRpc.SupportsBatch { + respTxt = `[{"jsonrpc":"2.0","id":THIS_WILL_BE_REPLACED,"result":"0x123456"}]` + } else { + respTxt = `{"jsonrpc":"2.0","id":THIS_WILL_BE_REPLACED,"result":"0x123456"}` + } + idp, err := sonic.Marshal(id) + require.NoError(t, err) + res.Body = io.NopCloser(strings.NewReader(strings.Replace(respTxt, "THIS_WILL_BE_REPLACED", string(idp), 1))) + return res + }) + + statusCode, body := sendRequest(`{"jsonrpc":"2.0","method":"eth_traceDebug","params":[]}`, nil, nil) + + assert.Equal(t, http.StatusOK, statusCode) + assert.Contains(t, body, "0x123456") + + assert.True(t, gock.IsDone(), "All mocks should have been called") + }) - go func() { - err := httpServer.server.Serve(listener) - if err != nil && err != http.ErrServerClosed { - t.Errorf("Server error: %v", err) - } - }() - // defer httpServer.server.Shutdown() + t.Run("AutoAddIDandJSONRPCFieldstoRequest", func(t *testing.T) { + gock.New("http://rpc1.localhost"). + Post("/"). + Times(1). + SetMatcher(gock.NewEmptyMatcher()). + AddMatcher(func(req *http.Request, ereq *gock.Request) (bool, error) { + if !strings.Contains(req.URL.Host, "rpc1") { + return false, nil + } + bodyBytes, err := io.ReadAll(req.Body) + if err != nil { + return false, err + } + bodyStr := string(bodyBytes) + if !strings.Contains(bodyStr, "\"id\"") { + t.Fatalf("No id found in request") + } + if !strings.Contains(bodyStr, "\"jsonrpc\"") { + t.Fatalf("No jsonrpc found in request") + } + if !strings.Contains(bodyStr, "\"method\"") { + t.Fatalf("No method found in request") + } + if !strings.Contains(bodyStr, "\"params\"") { + t.Fatalf("No params found in request") + } + return true, nil + }). + Reply(200). + BodyString(`{"jsonrpc":"2.0","id":1,"result":"0x123456"}`) - time.Sleep(1000 * time.Millisecond) + sendRequest(`{"method":"eth_traceDebug","params":[]}`, nil, nil) + }) - baseURL := fmt.Sprintf("http://localhost:%d", port) + t.Run("AutoAddIDWhen0IsProvided", func(t *testing.T) { + gock.New("http://rpc1.localhost"). + Post("/"). + SetMatcher(gock.NewEmptyMatcher()). + AddMatcher(func(req *http.Request, ereq *gock.Request) (bool, error) { + if !strings.Contains(req.URL.Host, "rpc1") { + return false, nil + } + bodyBytes, err := io.ReadAll(req.Body) + if err != nil { + return false, err + } + fmt.Println(string(bodyBytes)) + bodyStr := string(bodyBytes) + if !strings.Contains(bodyStr, "\"id\"") { + t.Fatalf("No id found in request") + } + idNode, err := sonic.Get(bodyBytes, 0, "id") + require.NoError(t, err) + id, err := idNode.Int64() + require.NoError(t, err) + if id == 0 { + t.Fatalf("Expected id to be 0, got %d from body: %s", id, bodyStr) + } + return true, nil + }). + Reply(200). + BodyString(`{"jsonrpc":"2.0","id":1,"result":"0x123456"}`) - sendRequest := func(body string, headers map[string]string, queryParams map[string]string) (int, string) { - req, err := http.NewRequest("POST", baseURL+"/test_project/evm/1", strings.NewReader(body)) - require.NoError(t, err) - req.Header.Set("Content-Type", "application/json") - for k, v := range headers { - req.Header.Set(k, v) - } - q := req.URL.Query() - for k, v := range queryParams { - q.Add(k, v) - } - req.URL.RawQuery = q.Encode() + sendRequest(`{"jsonrpc":"2.0","method":"eth_traceDebug","params":[],"id":0}`, nil, nil) + }) - client := &http.Client{ - Timeout: 10 * time.Second, - } - resp, err := client.Do(req) - if err != nil { - return 0, err.Error() - } - defer resp.Body.Close() + t.Run("AlwaysPropagateUpstreamErrorDataField", func(t *testing.T) { + gock.New("http://rpc1.localhost"). + Post("/"). + Reply(400). + BodyString(`{"jsonrpc":"2.0","id":1,"error":{"code":-32602,"message":"Invalid params","data":{"range":"the range 55074203 - 55124202 exceeds the range allowed for your plan (49999 > 2000)."}}}`) - respBody, err := io.ReadAll(resp.Body) - if err != nil { - return resp.StatusCode, err.Error() - } - return resp.StatusCode, string(respBody) + _, body := sendRequest(`{"jsonrpc":"2.0","method":"eth_getLogs","params":[],"id":1}`, nil, nil) + assert.Contains(t, body, "the range 55074203 - 55124202") + }) } - - return sendRequest, baseURL } func TestHttpServer_MultipleUpstreams(t *testing.T) { @@ -613,3 +723,122 @@ func TestHttpServer_MultipleUpstreams(t *testing.T) { assert.True(t, gock.IsDone(), "All mocks should have been called") }) } + +func TestHttpServer_IntegrationTests(t *testing.T) { + cfg := &common.Config{ + Server: &common.ServerConfig{ + MaxTimeout: "5s", + }, + Projects: []*common.ProjectConfig{ + { + Id: "test_project", + Networks: []*common.NetworkConfig{ + { + Architecture: common.ArchitectureEvm, + Evm: &common.EvmNetworkConfig{ + ChainId: 1, + }, + }, + }, + Upstreams: []*common.UpstreamConfig{ + { + Id: "drpc1", + Type: common.UpstreamTypeEvm, + Endpoint: "https://lb.drpc.org/ogrpc?network=ethereum", + Evm: &common.EvmUpstreamConfig{ + ChainId: 1, + }, + }, + }, + }, + }, + RateLimiters: &common.RateLimiterConfig{}, + } + + sendRequest, _ := createServerTestFixtures(cfg, t) + + t.Run("DrpcUnsupportedMethod", func(t *testing.T) { + gock.New("https://lb.drpc.org"). + Post("/"). + Reply(200). + JSON(map[string]interface{}{ + "jsonrpc": "2.0", + "id": 111, + "error": map[string]interface{}{ + "code": -32601, + "message": "the method trace_transaction does not exist/is not available", + }, + }) + statusCode, _ := sendRequest(`{"jsonrpc":"2.0","method":"trace_transaction","params":[],"id":111}`, nil, nil) + assert.Equal(t, http.StatusBadRequest, statusCode) + }) +} + +func createServerTestFixtures(cfg *common.Config, t *testing.T) ( + func(body string, headers map[string]string, queryParams map[string]string) (int, string), + string, +) { + gock.EnableNetworking() + gock.NetworkingFilter(func(req *http.Request) bool { + shouldMakeRealCall := strings.Split(req.URL.Host, ":")[0] == "localhost" + return shouldMakeRealCall + }) + defer gock.Off() + + logger := zerolog.New(zerolog.NewConsoleWriter()) + ctx := context.Background() + // ctx, cancel := context.WithCancel(context.Background()) + // defer cancel() + + erpcInstance, err := NewERPC(ctx, &logger, nil, cfg) + require.NoError(t, err) + + httpServer := NewHttpServer(ctx, &logger, cfg.Server, erpcInstance) + + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + port := listener.Addr().(*net.TCPAddr).Port + + go func() { + err := httpServer.server.Serve(listener) + if err != nil && err != http.ErrServerClosed { + t.Errorf("Server error: %v", err) + } + }() + // defer httpServer.server.Shutdown() + + time.Sleep(1000 * time.Millisecond) + + baseURL := fmt.Sprintf("http://localhost:%d", port) + + sendRequest := func(body string, headers map[string]string, queryParams map[string]string) (int, string) { + req, err := http.NewRequest("POST", baseURL+"/test_project/evm/1", strings.NewReader(body)) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/json") + for k, v := range headers { + req.Header.Set(k, v) + } + q := req.URL.Query() + for k, v := range queryParams { + q.Add(k, v) + } + req.URL.RawQuery = q.Encode() + + client := &http.Client{ + Timeout: 10 * time.Second, + } + resp, err := client.Do(req) + if err != nil { + return 0, err.Error() + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return resp.StatusCode, err.Error() + } + return resp.StatusCode, string(respBody) + } + + return sendRequest, baseURL +} diff --git a/erpc/init.go b/erpc/init.go index 477ca86a0..e2a8fb691 100644 --- a/erpc/init.go +++ b/erpc/init.go @@ -68,9 +68,8 @@ func Init( // 3) Expose Transports // logger.Info().Msg("initializing transports") - var httpServer *HttpServer if cfg.Server != nil { - httpServer = NewHttpServer(ctx, &logger, cfg.Server, erpcInstance) + httpServer := NewHttpServer(ctx, &logger, cfg.Server, erpcInstance) go func() { if err := httpServer.Start(&logger); err != nil { if err != http.ErrServerClosed { diff --git a/erpc/init_test.go b/erpc/init_test.go new file mode 100644 index 000000000..0b13df5a1 --- /dev/null +++ b/erpc/init_test.go @@ -0,0 +1,9 @@ +package erpc + +import ( + "github.com/rs/zerolog" +) + +func init() { + zerolog.SetGlobalLevel(zerolog.Disabled) +} diff --git a/erpc/multiplexer.go b/erpc/multiplexer.go index 1934a85b7..f435c2b5b 100644 --- a/erpc/multiplexer.go +++ b/erpc/multiplexer.go @@ -11,6 +11,7 @@ type Multiplexer struct { err error done chan struct{} mu *sync.RWMutex + once sync.Once } func NewMultiplexer() *Multiplexer { @@ -21,9 +22,11 @@ func NewMultiplexer() *Multiplexer { } func (inf *Multiplexer) Close(resp *common.NormalizedResponse, err error) { - inf.mu.Lock() - defer inf.mu.Unlock() - inf.resp = resp - inf.err = err - close(inf.done) + inf.once.Do(func() { + inf.mu.Lock() + defer inf.mu.Unlock() + inf.resp = resp + inf.err = err + close(inf.done) + }) } diff --git a/erpc/networks.go b/erpc/networks.go index b119e7dd6..6b5553a0b 100644 --- a/erpc/networks.go +++ b/erpc/networks.go @@ -22,8 +22,8 @@ type Network struct { ProjectId string Logger *zerolog.Logger - inFlightMutex *sync.Mutex - inFlightRequests map[string]*Multiplexer + inFlightRequests *sync.Map + evmStatePollers map[string]*upstream.EvmStatePoller failsafePolicies []failsafe.Policy[*common.NormalizedResponse] failsafeExecutor failsafe.Executor[*common.NormalizedResponse] @@ -31,8 +31,6 @@ type Network struct { cacheDal data.CacheDAL metricsTracker *health.Tracker upstreamsRegistry *upstream.UpstreamsRegistry - - evmStatePollers map[string]*upstream.EvmStatePoller } func (n *Network) Bootstrap(ctx context.Context) error { @@ -78,17 +76,15 @@ func (n *Network) Forward(ctx context.Context, req *common.NormalizedRequest) (* req.SetNetwork(n) method, _ := req.Method() - lg := n.Logger.With().Str("method", method).Str("id", req.Id()).Str("ptr", fmt.Sprintf("%p", req)).Logger() + lg := n.Logger.With().Str("method", method).Int64("id", req.Id()).Str("ptr", fmt.Sprintf("%p", req)).Logger() // 1) In-flight multiplexing var inf *Multiplexer mlxHash, err := req.CacheHash() if err == nil && mlxHash != "" { - n.inFlightMutex.Lock() - var exists bool - if inf, exists = n.inFlightRequests[mlxHash]; exists { - n.inFlightMutex.Unlock() - lg.Debug().Msgf("found similar in-flight request, waiting for result") + if vinf, exists := n.inFlightRequests.Load(mlxHash); exists { + inf = vinf.(*Multiplexer) + lg.Debug().Str("mlx", mlxHash).Msgf("found similar in-flight request, waiting for result") health.MetricNetworkMultiplexedRequests.WithLabelValues(n.ProjectId, n.NetworkId, method).Inc() inf.mu.RLock() @@ -110,6 +106,8 @@ func (n *Network) Forward(ctx context.Context, req *common.NormalizedRequest) (* } return resp, inf.err case <-ctx.Done(): + n.inFlightRequests.Delete(mlxHash) + err := ctx.Err() if errors.Is(err, context.DeadlineExceeded) { return nil, common.NewErrNetworkRequestTimeout(time.Since(startTime)) @@ -119,13 +117,13 @@ func (n *Network) Forward(ctx context.Context, req *common.NormalizedRequest) (* } } inf = NewMultiplexer() - n.inFlightRequests[mlxHash] = inf - n.inFlightMutex.Unlock() defer func() { - n.inFlightMutex.Lock() - defer n.inFlightMutex.Unlock() - delete(n.inFlightRequests, mlxHash) + if r := recover(); r != nil { + lg.Error().Msgf("panic in multiplexer cleanup: %v", r) + } + n.inFlightRequests.Delete(mlxHash) }() + n.inFlightRequests.Store(mlxHash, inf) } // 2) Get from cache if exists @@ -204,9 +202,11 @@ func (n *Network) Forward(ctx context.Context, req *common.NormalizedRequest) (* var execution failsafe.Execution[*common.NormalizedResponse] var errorsByUpstream = map[string]error{} + ectx := context.WithValue(ctx, common.RequestContextKey, req) + i := 0 resp, execErr := n.failsafeExecutor. - WithContext(ctx). + WithContext(ectx). GetWithExecution(func(exec failsafe.Execution[*common.NormalizedResponse]) (*common.NormalizedResponse, error) { req.Lock() execution = exec @@ -218,6 +218,9 @@ func (n *Network) Forward(ctx context.Context, req *common.NormalizedRequest) (* // Upstream-level retry is handled by the upstream itself (and its own failsafe policies). ln := len(upsList) for count := 0; count < ln; count++ { + if err := exec.Context().Err(); err != nil { + return nil, err + } // We need to use write-lock here because "i" is being updated. req.Lock() u := upsList[i] @@ -239,15 +242,18 @@ func (n *Network) Forward(ctx context.Context, req *common.NormalizedRequest) (* ulg := lg.With().Str("upstreamId", u.Config().Id).Logger() - rp, er := tryForward(u, exec.Context(), &ulg) - resp, err := n.normalizeResponse(req, rp, er) + resp, err := tryForward(u, exec.Context(), &ulg) + if e := n.normalizeResponse(req, resp); e != nil { + ulg.Error().Err(e).Msgf("failed to normalize response") + err = e + } - isClientErr := err != nil && common.HasErrorCode(err, common.ErrCodeEndpointClientSideException) + isClientErr := common.IsClientError(err) isHedged := exec.Hedges() > 0 - if isHedged && err != nil && errors.Is(err, context.Canceled) { + if isHedged && (err != nil && errors.Is(err, context.Canceled)) { ulg.Debug().Err(err).Msgf("discarding hedged request to upstream") - return nil, common.NewErrUpstreamHedgeCancelled(u.Config().Id) + return nil, common.NewErrUpstreamHedgeCancelled(u.Config().Id, context.Cause(exec.Context())) } if isHedged { ulg.Debug().Msgf("forwarded hedged request to upstream") @@ -334,7 +340,9 @@ func (n *Network) Forward(ctx context.Context, req *common.NormalizedRequest) (* } if n.cacheDal != nil { + resp.RLock() go (func(resp *common.NormalizedResponse) { + defer resp.RUnlock() c, cancel := context.WithTimeoutCause(context.Background(), 10*time.Second, errors.New("cache driver timeout during set")) defer cancel() err := n.cacheDal.Set(c, req, resp) @@ -355,6 +363,10 @@ func (n *Network) Forward(ctx context.Context, req *common.NormalizedRequest) (* } func (n *Network) EvmIsBlockFinalized(blockNumber int64) (bool, error) { + if n == nil || n.evmStatePollers == nil || len(n.evmStatePollers) == 0 { + return false, nil + } + for _, poller := range n.evmStatePollers { if fin, err := poller.IsBlockFinalized(blockNumber); err != nil { if common.HasErrorCode(err, common.ErrCodeFinalizedBlockUnavailable) { @@ -399,22 +411,16 @@ func (n *Network) enrichStatePoller(method string, req *common.NormalizedRequest if blkTag, ok := jrq.Params[0].(string); ok { if blkTag == "finalized" || blkTag == "latest" { jrs, _ := resp.JsonRpcResponse() - if jrs != nil { - res, err := jrs.ParsedResult() + bnh, err := jrs.PeekStringByPath("number") + if err == nil { + blockNumber, err := common.HexToInt64(bnh) if err == nil { - blk, ok := res.(map[string]interface{}) + poller, ok := n.evmStatePollers[resp.Upstream().Config().Id] if ok { - bnh, ok := blk["number"].(string) - if ok { - blockNumber, err := common.HexToInt64(bnh) - if err == nil { - poller := n.evmStatePollers[resp.Upstream().Config().Id] - if blkTag == "finalized" { - poller.SuggestFinalizedBlock(blockNumber) - } else if blkTag == "latest" { - poller.SuggestLatestBlock(blockNumber) - } - } + if blkTag == "finalized" { + poller.SuggestFinalizedBlock(blockNumber) + } else if blkTag == "latest" { + poller.SuggestLatestBlock(blockNumber) } } } @@ -425,47 +431,26 @@ func (n *Network) enrichStatePoller(method string, req *common.NormalizedRequest } } -func (n *Network) normalizeResponse(req *common.NormalizedRequest, resp *common.NormalizedResponse, err error) (*common.NormalizedResponse, error) { +func (n *Network) normalizeResponse(req *common.NormalizedRequest, resp *common.NormalizedResponse) error { switch n.Architecture() { case common.ArchitectureEvm: if resp != nil { // This ensures that even if upstream gives us wrong/missing ID we'll // use correct one from original incoming request. - jrr, _ := resp.JsonRpcResponse() - if jrr != nil { - jrq, _ := req.JsonRpcRequest() - if jrq != nil { - jrr.ID = jrq.ID + if jrr, err := resp.JsonRpcResponse(); err == nil { + jrq, err := req.JsonRpcRequest() + if err != nil { + return err + } + err = jrr.SetID(jrq.ID) + if err != nil { + return err } } } - - if err == nil { - return resp, nil - } - - if common.HasErrorCode(err, common.ErrCodeJsonRpcExceptionInternal) { - return resp, err - } else if common.HasErrorCode(err, common.ErrCodeJsonRpcRequestUnmarshal) { - return resp, common.NewErrJsonRpcExceptionInternal( - 0, - common.JsonRpcErrorParseException, - "failed to parse json-rpc request", - err, - nil, - ) - } - - return resp, common.NewErrJsonRpcExceptionInternal( - 0, - common.JsonRpcErrorServerSideException, - fmt.Sprintf("failed request on evm network %s", n.NetworkId), - err, - nil, - ) - default: - return resp, err } + + return nil } func (n *Network) acquireRateLimitPermit(req *common.NormalizedRequest) error { diff --git a/erpc/networks_registry.go b/erpc/networks_registry.go index 0bd61dd48..289568ee8 100644 --- a/erpc/networks_registry.go +++ b/erpc/networks_registry.go @@ -67,8 +67,7 @@ func NewNetwork( metricsTracker: metricsTracker, rateLimitersRegistry: rateLimitersRegistry, - inFlightMutex: &sync.Mutex{}, - inFlightRequests: make(map[string]*Multiplexer), + inFlightRequests: &sync.Map{}, failsafePolicies: policies, failsafeExecutor: failsafe.NewExecutor(policies...), } diff --git a/erpc/networks_test.go b/erpc/networks_test.go index d600121b1..84a457760 100644 --- a/erpc/networks_test.go +++ b/erpc/networks_test.go @@ -3,19 +3,26 @@ package erpc import ( "bytes" "context" - "encoding/json" "errors" "fmt" + "runtime" + "sync" + "sync/atomic" + + // "fmt" "io" "net/http" - "os" + + // "os" "strings" - "sync" + + // "sync" "testing" "time" "github.com/bytedance/sonic" "github.com/erpc/erpc/common" + "github.com/erpc/erpc/data" "github.com/erpc/erpc/health" "github.com/erpc/erpc/upstream" "github.com/erpc/erpc/util" @@ -29,10 +36,6 @@ import ( var TRUE = true -func init() { - log.Logger = log.Level(zerolog.ErrorLevel).Output(zerolog.ConsoleWriter{Out: os.Stderr}) -} - func TestNetwork_Forward(t *testing.T) { t.Run("ForwardCorrectlyRateLimitedOnNetworkLevel", func(t *testing.T) { @@ -218,13 +221,13 @@ func TestNetwork_Forward(t *testing.T) { t.Run("ForwardUpstreamRetryIntermittentFailuresWithoutSuccessAndNoErrCode", func(t *testing.T) { defer gock.Off() - var requestBytes = json.RawMessage(`{"jsonrpc":"2.0","id":1,"method":"eth_traceTransaction","params":["0x1273c18",false]}`) + var requestBytes = []byte(`{"jsonrpc":"2.0","id":1,"method":"eth_traceTransaction","params":["0x1273c18",false]}`) gock.New("http://rpc1.localhost"). Times(3). Post(""). Reply(503). - JSON(json.RawMessage(`{"error":{"message":"some random provider issue"}}`)) + JSON([]byte(`{"error":{"message":"some random provider issue"}}`)) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -322,13 +325,13 @@ func TestNetwork_Forward(t *testing.T) { defer gock.Clean() defer gock.CleanUnmatchedRequest() - var requestBytes = json.RawMessage(`{"jsonrpc":"2.0","id":9199,"method":"eth_traceTransaction","params":["0x1273c18",false]}`) + var requestBytes = []byte(`{"jsonrpc":"2.0","id":9199,"method":"eth_traceTransaction","params":["0x1273c18",false]}`) gock.New("http://rpc1.localhost"). Times(3). Post(""). Reply(503). - JSON(json.RawMessage(`{"jsonrpc":"2.0","id":9199,"error":{"code":-32600,"message":"some random provider issue"}}`)) + JSON([]byte(`{"jsonrpc":"2.0","id":9199,"error":{"code":-32603,"message":"some random provider issue"}}`)) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -407,8 +410,8 @@ func TestNetwork_Forward(t *testing.T) { fakeReq := common.NewNormalizedRequest(requestBytes) _, err = ntw.Forward(ctx, fakeReq) - if len(gock.Pending()) > 0 { - t.Errorf("Expected all mocks to be consumed, got %v left", len(gock.Pending())) + if left := anyTestMocksLeft(); left > 0 { + t.Errorf("Expected all mocks to be consumed, got %v left", left) for _, pending := range gock.Pending() { t.Errorf("Pending mock: %v", pending) } @@ -426,25 +429,25 @@ func TestNetwork_Forward(t *testing.T) { t.Run("ForwardSkipsNonRetryableFailuresFromUpstreams", func(t *testing.T) { defer gock.Off() - var requestBytes = json.RawMessage(`{"jsonrpc":"2.0","id":1,"method":"eth_traceTransaction","params":["0x1273c18",false]}`) + var requestBytes = []byte(`{"jsonrpc":"2.0","id":1,"method":"eth_traceTransaction","params":["0x1273c18",false]}`) gock.New("http://rpc1.localhost"). Times(1). Post(""). Reply(401). - JSON(json.RawMessage(`{"error":{"code":-32016,"message":"unauthorized rpc1"}}`)) + JSON([]byte(`{"error":{"code":-32016,"message":"unauthorized rpc1"}}`)) gock.New("http://rpc2.localhost"). Times(2). Post(""). Reply(503). - JSON(json.RawMessage(`{"error":"random rpc2 unavailable"}`)) + JSON([]byte(`{"error":"random rpc2 unavailable"}`)) gock.New("http://rpc2.localhost"). Times(1). Post(""). Reply(200). - JSON(json.RawMessage(`{"result":"0x1234567"}`)) + JSON([]byte(`{"result":"0x1234567"}`)) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -570,25 +573,25 @@ func TestNetwork_Forward(t *testing.T) { t.Run("ForwardNotSkipsRetryableFailuresFromUpstreams", func(t *testing.T) { defer gock.Off() - var requestBytes = json.RawMessage(`{"jsonrpc":"2.0","id":1,"method":"eth_traceTransaction","params":["0x1273c18",false]}`) + var requestBytes = []byte(`{"jsonrpc":"2.0","id":1,"method":"eth_traceTransaction","params":["0x1273c18",false]}`) gock.New("http://rpc1.localhost"). Times(3). Post(""). Reply(503). - JSON(json.RawMessage(`{"error":"random rpc1 unavailable"}`)) + JSON([]byte(`{"error":"random rpc1 unavailable"}`)) gock.New("http://rpc2.localhost"). Times(3). Post(""). Reply(503). - JSON(json.RawMessage(`{"error":"random rpc2 unavailable"}`)) + JSON([]byte(`{"error":"random rpc2 unavailable"}`)) gock.New("http://rpc2.localhost"). Times(1). Post(""). Reply(200). - JSON(json.RawMessage(`{"result":"0x1234567"}`)) + JSON([]byte(`{"result":"0x1234567"}`)) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -718,7 +721,7 @@ func TestNetwork_Forward(t *testing.T) { defer gock.CleanUnmatchedRequest() // Prepare a JSON-RPC request payload as a byte array - var requestBytes = json.RawMessage(`{ + var requestBytes = []byte(`{ "jsonrpc": "2.0", "method": "eth_getLogs", "params": [{ @@ -737,7 +740,7 @@ func TestNetwork_Forward(t *testing.T) { return strings.Contains(safeReadBody(request), "latest") }). Reply(200). - JSON(json.RawMessage(`{"result": {"number":"0x9"}}`)) + JSON([]byte(`{"result": {"number":"0x9"}}`)) // Mock the response for the finalized block number request gock.New("http://rpc1.localhost"). @@ -747,19 +750,19 @@ func TestNetwork_Forward(t *testing.T) { return strings.Contains(safeReadBody(request), "finalized") }). Reply(200). - JSON(json.RawMessage(`{"result": {"number":"0x8"}}`)) + JSON([]byte(`{"result": {"number":"0x8"}}`)) // Mock an empty logs response from the first upstream gock.New("http://rpc1.localhost"). Post(""). Reply(200). - JSON(json.RawMessage(`{"result":[], "fromHost":"rpc1"}`)) + JSON([]byte(`{"result":[]}`)) // Mock a non-empty logs response from the second upstream gock.New("http://rpc2.localhost"). Post(""). Reply(200). - JSON(json.RawMessage(`{"result":[{"logIndex":444}], "fromHost":"rpc2"}`)) + JSON([]byte(`{"result":[{"logIndex":444}]}`)) // Set up a context and a cancellation function ctx, cancel := context.WithCancel(context.Background()) @@ -887,31 +890,14 @@ func TestNetwork_Forward(t *testing.T) { } // Convert the raw response to a map to access custom fields like fromHost - var responseMap map[string]interface{} - err = sonic.Unmarshal(resp.Body(), &responseMap) + jrr, err := resp.JsonRpcResponse() if err != nil { - t.Fatalf("Failed to unmarshal response body: %v", err) - } - - // Check if fromHost exists and is a string - fromHost, ok := responseMap["fromHost"].(string) - if !ok { - t.Fatalf("Expected fromHost to be a string, got %T", responseMap["fromHost"]) - } - - // Assert the value of fromHost - if fromHost != "rpc1" { - t.Errorf("Expected fromHost to be %q, got %q", "rpc1", fromHost) + t.Fatalf("Failed to get JsonRpcResponse: %v", err) } // Check that the result field is an empty array as expected - result, ok := responseMap["result"].([]interface{}) - if !ok { - t.Fatalf("Expected result to be []interface{}, got %T", responseMap["result"]) - } - - if len(result) != 0 { - t.Fatalf("Expected empty result array") + if len(jrr.Result) != 2 || jrr.Result[0] != '[' || jrr.Result[1] != ']' { + t.Fatalf("Expected result to be an empty array, got %T", jrr.Result) } }) @@ -922,7 +908,7 @@ func TestNetwork_Forward(t *testing.T) { defer gock.CleanUnmatchedRequest() // Prepare a JSON-RPC request payload as a byte array - var requestBytes = json.RawMessage(`{ + var requestBytes = []byte(`{ "jsonrpc": "2.0", "method": "eth_getLogs", "params": [{ @@ -934,36 +920,36 @@ func TestNetwork_Forward(t *testing.T) { }`) // Mock the response for the latest block number request - gock.New("http://rpc1.localhost"). + gock.New("http://alchemy.com/rpc1"). Post(""). Persist(). Filter(func(request *http.Request) bool { return strings.Contains(safeReadBody(request), "latest") }). Reply(200). - JSON(json.RawMessage(`{"result": {"number":"0x9"}}`)) + JSON([]byte(`{"result": {"number":"0x9"}}`)) // Mock the response for the finalized block number request - gock.New("http://rpc1.localhost"). + gock.New("http://alchemy.com/rpc1"). Post(""). Persist(). Filter(func(request *http.Request) bool { return strings.Contains(safeReadBody(request), "finalized") }). Reply(200). - JSON(json.RawMessage(`{"result": {"number":"0x8"}}`)) + JSON([]byte(`{"result": {"number":"0x8"}}`)) // Mock an empty logs response from the first upstream - gock.New("http://rpc1.localhost"). + gock.New("http://alchemy.com/rpc1"). Post(""). Reply(200). - JSON(json.RawMessage(`{"result":[], "fromHost":"rpc1"}`)) + JSON([]byte(`{"result":[]}`)) // Mock a non-empty logs response from the second upstream - gock.New("http://rpc2.localhost"). + gock.New("http://alchemy.com/rpc2"). Post(""). Reply(200). - JSON(json.RawMessage(`{"result":[{"logIndex":444}], "fromHost":"rpc2"}`)) + JSON([]byte(`{"result":[{"logIndex":444}]}`)) // Set up a context and a cancellation function ctx, cancel := context.WithCancel(context.Background()) @@ -989,7 +975,7 @@ func TestNetwork_Forward(t *testing.T) { up1 := &common.UpstreamConfig{ Type: common.UpstreamTypeEvm, Id: "rpc1", - Endpoint: "http://rpc1.localhost", + Endpoint: "http://alchemy.com/rpc1", Evm: &common.EvmUpstreamConfig{ ChainId: 123, Syncing: &common.TRUE, @@ -998,7 +984,7 @@ func TestNetwork_Forward(t *testing.T) { up2 := &common.UpstreamConfig{ Type: common.UpstreamTypeEvm, Id: "rpc2", - Endpoint: "http://rpc2.localhost", + Endpoint: "http://alchemy.com/rpc2", Evm: &common.EvmUpstreamConfig{ ChainId: 123, }, @@ -1095,30 +1081,198 @@ func TestNetwork_Forward(t *testing.T) { } // Convert the raw response to a map to access custom fields like fromHost - var responseMap map[string]interface{} - err = sonic.Unmarshal(resp.Body(), &responseMap) + jrr, err := resp.JsonRpcResponse() + if err != nil { + t.Fatalf("Failed to get JsonRpcResponse: %v", err) + } + var result []interface{} + err = sonic.Unmarshal(jrr.Result, &result) if err != nil { t.Fatalf("Failed to unmarshal response body: %v", err) } - // Check if fromHost exists and is a string - fromHost, ok := responseMap["fromHost"].(string) - if !ok { - t.Fatalf("Expected fromHost to be a string, got %T", responseMap["fromHost"]) + if len(result) == 0 { + t.Fatalf("Expected non-empty result array") } + }) - // Assert the value of fromHost - if fromHost != "rpc2" { - t.Errorf("Expected fromHost to be %q, got %q", "rpc2", fromHost) + t.Run("ForwardWithMinimumMemoryAllocation", func(t *testing.T) { + // Clean up any gock mocks after the test runs + defer gock.Off() + defer gock.Clean() + defer gock.CleanUnmatchedRequest() + + // Prepare a JSON-RPC request payload as a byte array + var requestBytes = []byte(`{ + "jsonrpc": "2.0", + "method": "debug_traceTransaction", + "params": [ + "0x1234567890abcdef1234567890abcdef12345678" + ], + "id": 1 + }`) + + // Mock the response for the latest block number request + gock.New("http://rpc1.localhost"). + Post(""). + Persist(). + Filter(func(request *http.Request) bool { + return strings.Contains(safeReadBody(request), "latest") + }). + Reply(200). + JSON([]byte(`{"result":{"number":"0x9"}}`)) + + // Mock the response for the finalized block number request + gock.New("http://rpc1.localhost"). + Post(""). + Persist(). + Filter(func(request *http.Request) bool { + return strings.Contains(safeReadBody(request), "finalized") + }). + Reply(200). + JSON([]byte(`{"result": {"number":"0x8"}}`)) + + sampleSize := 100 * 1024 * 1024 + allowedOverhead := 30 * 1024 * 1024 + largeResult := strings.Repeat("x", sampleSize) + + // Mock the response for the latest block number request + gock.New("http://rpc1.localhost"). + Post(""). + Persist(). + Reply(200). + JSON([]byte(fmt.Sprintf(`{"result":"%s"}`, largeResult))) + + // Set up a context and a cancellation function + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Initialize various components for the test environment + clr := upstream.NewClientRegistry(&log.Logger) + fsCfg := &common.FailsafeConfig{} + rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + Budgets: []*common.RateLimitBudgetConfig{}, + }, &log.Logger) + if err != nil { + t.Fatal(err) } + vndr := vendors.NewVendorsRegistry() + mt := health.NewTracker("prjA", 2*time.Second) - result, ok := responseMap["result"].([]interface{}) - if !ok { - t.Fatalf("Expected result to be []interface{}, got %T", responseMap["result"]) + // Set up upstream configurations + up1 := &common.UpstreamConfig{ + Type: common.UpstreamTypeEvm, + Id: "rpc1", + Endpoint: "http://rpc1.localhost", + Evm: &common.EvmUpstreamConfig{ + ChainId: 123, + }, + } + // Initialize the upstreams registry + upr := upstream.NewUpstreamsRegistry( + &log.Logger, + "prjA", + []*common.UpstreamConfig{up1}, + rlr, + vndr, mt, 1*time.Second, + ) + err = upr.Bootstrap(ctx) + if err != nil { + t.Fatal(err) + } + err = upr.PrepareUpstreamsForNetwork(util.EvmNetworkId(123)) + if err != nil { + t.Fatal(err) } - if len(result) == 0 { - t.Fatalf("Expected non-empty result array") + // Create and register clients for both upstreams + pup1, err := upr.NewUpstream( + "prjA", + up1, + &log.Logger, + mt, + ) + if err != nil { + t.Fatal(err) + } + cl1, err := clr.GetOrCreateClient(pup1) + if err != nil { + t.Fatal(err) + } + pup1.Client = cl1 + + // Set up the network configuration + ntw, err := NewNetwork( + &log.Logger, + "prjA", + &common.NetworkConfig{ + Architecture: common.ArchitectureEvm, + Evm: &common.EvmNetworkConfig{ + ChainId: 123, + BlockTrackerInterval: "10h", + }, + Failsafe: fsCfg, + }, + rlr, + upr, + mt, + ) + if err != nil { + t.Fatal(err) + } + + // Bootstrap the network and make the simulated request + ntw.Bootstrap(ctx) + time.Sleep(100 * time.Millisecond) + + poller1 := ntw.evmStatePollers["rpc1"] + poller1.SuggestLatestBlock(9) + poller1.SuggestFinalizedBlock(8) + + time.Sleep(100 * time.Millisecond) + + // Create a fake request and forward it through the network + fakeReq := common.NewNormalizedRequest(requestBytes) + + // Measure memory usage before request + var mBefore runtime.MemStats + runtime.GC() + runtime.ReadMemStats(&mBefore) + + resp, err := ntw.Forward(ctx, fakeReq) + + // Measure memory usage after parsing + var mAfter runtime.MemStats + runtime.GC() + runtime.ReadMemStats(&mAfter) + + // Calculate the difference in memory usage + memUsed := mAfter.Alloc - mBefore.Alloc + memUsedMB := float64(memUsed) / (1024 * 1024) + + // Log the memory usage + t.Logf("Memory used for request: %.2f MB", memUsedMB) + + // Check that memory used does not exceed sample size + overhead + expectedMemUsage := uint64(sampleSize) + uint64(allowedOverhead) + expectedMemUsageMB := float64(expectedMemUsage) / (1024 * 1024) + if memUsed > expectedMemUsage { + t.Fatalf("Memory usage exceeded expected limit of %.2f MB used %.2f MB", expectedMemUsageMB, memUsedMB) + } + + if err != nil { + t.Fatalf("Expected nil error, got %v", err) + } + + // Convert the raw response to a map to access custom fields like fromHost + jrr, err := resp.JsonRpcResponse() + if err != nil { + t.Fatalf("Failed to get JsonRpcResponse: %v", err) + } + + // add 2 for quote marks + if len(jrr.Result) != sampleSize+2 { + t.Fatalf("Expected result to be %d, got %d", sampleSize+2, len(jrr.Result)) } }) @@ -1129,16 +1283,16 @@ func TestNetwork_Forward(t *testing.T) { defer gock.CleanUnmatchedRequest() // Prepare a JSON-RPC request payload as a byte array - var requestBytes = json.RawMessage(`{ - "jsonrpc": "2.0", - "method": "eth_getLogs", - "params": [{ - "address": "0x1234567890abcdef1234567890abcdef12345678", - "fromBlock": "0x4", - "toBlock": "0x7" - }], - "id": 1 - }`) + var requestBytes = []byte(`{ + "jsonrpc": "2.0", + "method": "eth_getLogs", + "params": [{ + "address": "0x1234567890abcdef1234567890abcdef12345678", + "fromBlock": "0x4", + "toBlock": "0x7" + }], + "id": 1 + }`) // Mock the response for the latest block number request gock.New("http://rpc1.localhost"). @@ -1148,7 +1302,7 @@ func TestNetwork_Forward(t *testing.T) { return strings.Contains(safeReadBody(request), "latest") }). Reply(200). - JSON(json.RawMessage(`{"result": {"number":"0x9"}}`)) + JSON([]byte(`{"result":{"number":"0x9"}}`)) // Mock the response for the finalized block number request gock.New("http://rpc1.localhost"). @@ -1158,19 +1312,19 @@ func TestNetwork_Forward(t *testing.T) { return strings.Contains(safeReadBody(request), "finalized") }). Reply(200). - JSON(json.RawMessage(`{"result": {"number":"0x8"}}`)) + JSON([]byte(`{"result":{"number":"0x8"}}`)) // Mock an empty logs response from the first upstream gock.New("http://rpc1.localhost"). Post(""). Reply(200). - JSON(json.RawMessage(`{"result":[], "fromHost":"rpc1"}`)) + JSON([]byte(`{"result":[]}`)) // Mock a non-empty logs response from the second upstream gock.New("http://rpc2.localhost"). Post(""). Reply(200). - JSON(json.RawMessage(`{"result":[{"logIndex":444}], "fromHost":"rpc2"}`)) + JSON([]byte(`{"result":[{"logIndex":444,"fromHost":"rpc2"}]}`)) // Set up a context and a cancellation function ctx, cancel := context.WithCancel(context.Background()) @@ -1303,31 +1457,22 @@ func TestNetwork_Forward(t *testing.T) { } // Convert the raw response to a map to access custom fields like fromHost - var responseMap map[string]interface{} - err = sonic.Unmarshal(resp.Body(), &responseMap) + jrr, err := resp.JsonRpcResponse() if err != nil { - t.Fatalf("Failed to unmarshal response body: %v", err) + t.Fatalf("Failed to get JSON-RPC response: %v", err) } - // Check if fromHost exists and is a string - fromHost, ok := responseMap["fromHost"].(string) - if !ok { - t.Fatalf("Expected fromHost to be a string, got %T", responseMap["fromHost"]) + if jrr.Result == nil { + t.Fatalf("Expected non-nil result") } - // Assert the value of fromHost + fromHost, err := jrr.PeekStringByPath(0, "fromHost") + if err != nil { + t.Fatalf("Failed to get fromHost from result: %v", err) + } if fromHost != "rpc2" { t.Errorf("Expected fromHost to be %q, got %q", "rpc2", fromHost) } - - result, ok := responseMap["result"].([]interface{}) - if !ok { - t.Fatalf("Expected result to be []interface{}, got %T", responseMap["result"]) - } - - if len(result) == 0 { - t.Fatalf("Expected non-empty result array") - } }) t.Run("RetryWhenBlockIsNotFinalized", func(t *testing.T) { @@ -1337,16 +1482,16 @@ func TestNetwork_Forward(t *testing.T) { defer gock.CleanUnmatchedRequest() // Prepare a JSON-RPC request payload as a byte array - var requestBytes = json.RawMessage(`{ - "jsonrpc": "2.0", - "method": "eth_getLogs", - "params": [{ - "address": "0x1234567890abcdef1234567890abcdef12345678", - "fromBlock": "0x0", - "toBlock": "0x1273c18" - }], - "id": 1 - }`) + var requestBytes = []byte(`{ + "jsonrpc": "2.0", + "method": "eth_getLogs", + "params": [{ + "address": "0x1234567890abcdef1234567890abcdef12345678", + "fromBlock": "0x0", + "toBlock": "0x1273c18" + }], + "id": 1 + }`) // Mock the response for the latest block number request gock.New("http://rpc1.localhost"). @@ -1356,7 +1501,7 @@ func TestNetwork_Forward(t *testing.T) { return strings.Contains(safeReadBody(request), "latest") }). Reply(200). - JSON(json.RawMessage(`{"result": {"number":"0x1273c17"}}`)) + JSON([]byte(`{"result": {"number":"0x1273c17"}}`)) // Mock the response for the finalized block number request gock.New("http://rpc1.localhost"). @@ -1366,19 +1511,19 @@ func TestNetwork_Forward(t *testing.T) { return strings.Contains(safeReadBody(request), "finalized") }). Reply(200). - JSON(json.RawMessage(`{"result": {"number":"0x1273c17"}}`)) + JSON([]byte(`{"result": {"number":"0x1273c17"}}`)) // Mock an empty logs response from the first upstream gock.New("http://rpc1.localhost"). Post(""). Reply(200). - JSON(json.RawMessage(`{"result":[]}`)) + JSON([]byte(`{"result":[]}`)) // Mock a non-empty logs response from the second upstream gock.New("http://rpc2.localhost"). Post(""). Reply(200). - JSON(json.RawMessage(`{"result":[{"logIndex":444, "fromHost":"rpc2"}]}`)) + JSON([]byte(`{"result":[{"logIndex":444, "fromHost":"rpc2"}]}`)) // Set up a context and a cancellation function ctx, cancel := context.WithCancel(context.Background()) @@ -1508,29 +1653,10 @@ func TestNetwork_Forward(t *testing.T) { t.Fatalf("Expected non-nil result") } - res, err := jrr.ParsedResult() + fromHost, err := jrr.PeekStringByPath(0, "fromHost") if err != nil { - t.Fatalf("Failed to get parsed result: %v", err) - } - result, ok := res.([]interface{}) - if !ok { - t.Fatalf("Expected Result to be []interface{}, got %T", jrr.Result) - } - - if len(result) == 0 { - t.Fatalf("Expected non-empty result array") + t.Fatalf("Failed to get fromHost from result: %v", err) } - - firstLog, ok := result[0].(map[string]interface{}) - if !ok { - t.Fatalf("Expected first log to be map[string]interface{}, got %T", result[0]) - } - - fromHost, ok := firstLog["fromHost"].(string) - if !ok { - t.Fatalf("Expected fromHost to be string, got %T", firstLog["fromHost"]) - } - if fromHost != "rpc2" { t.Errorf("Expected fromHost to be %q, got %q", "rpc2", fromHost) } @@ -1543,16 +1669,16 @@ func TestNetwork_Forward(t *testing.T) { defer gock.CleanUnmatchedRequest() // Prepare a JSON-RPC request payload as a byte array - var requestBytes = json.RawMessage(`{ - "jsonrpc": "2.0", - "method": "eth_getLogs", - "params": [{ - "address": "0x1234567890abcdef1234567890abcdef12345678", - "fromBlock": "0x0", - "toBlock": "0x1273c18" - }], - "id": 1 - }`) + var requestBytes = []byte(`{ + "jsonrpc": "2.0", + "method": "eth_getLogs", + "params": [{ + "address": "0x1234567890abcdef1234567890abcdef12345678", + "fromBlock": "0x0", + "toBlock": "0x1273c18" + }], + "id": 1 + }`) // Mock the response for the latest block number request gock.New("http://rpc1.localhost"). @@ -1562,7 +1688,7 @@ func TestNetwork_Forward(t *testing.T) { return strings.Contains(safeReadBody(request), "latest") }). Reply(200). - JSON(json.RawMessage(`{"result": {"number":"0x0"}}`)) // latest block not available + JSON([]byte(`{"result": {"number":"0x0"}}`)) // latest block not available // Mock the response for the finalized block number request gock.New("http://rpc1.localhost"). @@ -1572,19 +1698,19 @@ func TestNetwork_Forward(t *testing.T) { return strings.Contains(safeReadBody(request), "finalized") }). Reply(200). - JSON(json.RawMessage(`{"result": {"number":"0x0"}}`)) // finalzied block not available + JSON([]byte(`{"result": {"number":"0x0"}}`)) // finalzied block not available // Mock an empty logs response from the first upstream gock.New("http://rpc1.localhost"). Post(""). Reply(200). - JSON(json.RawMessage(`{"result":[]}`)) + JSON([]byte(`{"result":[]}`)) // Mock a non-empty logs response from the second upstream gock.New("http://rpc2.localhost"). Post(""). Reply(200). - JSON(json.RawMessage(`{"result":[{"logIndex":444, "fromHost":"rpc2"}]}`)) + JSON([]byte(`{"result":[{"logIndex":444, "fromHost":"rpc2"}]}`)) // Set up a context and a cancellation function ctx, cancel := context.WithCancel(context.Background()) @@ -1714,29 +1840,10 @@ func TestNetwork_Forward(t *testing.T) { t.Fatalf("Expected non-nil result") } - res, err := jrr.ParsedResult() + fromHost, err := jrr.PeekStringByPath(0, "fromHost") if err != nil { - t.Fatalf("Failed to get parsed result: %v", err) - } - result, ok := res.([]interface{}) - if !ok { - t.Fatalf("Expected Result to be []interface{}, got %T", jrr.Result) - } - - if len(result) == 0 { - t.Fatalf("Expected non-empty result array") - } - - firstLog, ok := result[0].(map[string]interface{}) - if !ok { - t.Fatalf("Expected first log to be map[string]interface{}, got %T", result[0]) - } - - fromHost, ok := firstLog["fromHost"].(string) - if !ok { - t.Fatalf("Expected fromHost to be string, got %T", firstLog["fromHost"]) + t.Fatalf("Failed to get fromHost from result: %v", err) } - if fromHost != "rpc2" { t.Errorf("Expected fromHost to be %q, got %q", "rpc2", fromHost) } @@ -1754,7 +1861,7 @@ func TestNetwork_Forward(t *testing.T) { return strings.Contains(b, "latest") }). Reply(200). - JSON(json.RawMessage(`{"result": {"number":"0xA98AC7"}}`)) + JSON([]byte(`{"result": {"number":"0xA98AC7"}}`)) // Mock the response for the finalized block number request gock.New("http://rpc1.localhost"). @@ -1765,7 +1872,7 @@ func TestNetwork_Forward(t *testing.T) { return strings.Contains(b, "finalized") }). Reply(200). - JSON(json.RawMessage(`{"result":{"number":"0x21E88E"}}`)) + JSON([]byte(`{"result":{"number":"0x21E88E"}}`)) gock.New("http://rpc2.localhost"). Post(""). @@ -1775,7 +1882,7 @@ func TestNetwork_Forward(t *testing.T) { return strings.Contains(b, "latest") }). Reply(200). - JSON(json.RawMessage(`{"result": {"number":"0x32DCD5"}}`)) + JSON([]byte(`{"result": {"number":"0x32DCD5"}}`)) // Mock the response for the finalized block number request gock.New("http://rpc2.localhost"). @@ -1786,7 +1893,7 @@ func TestNetwork_Forward(t *testing.T) { return strings.Contains(b, "finalized") }). Reply(200). - JSON(json.RawMessage(`{"result":{"number":"0x2A62B1C"}}`)) + JSON([]byte(`{"result":{"number":"0x2A62B1C"}}`)) // Mock a pending transaction response from the first upstream gock.New("http://rpc1.localhost"). @@ -1796,7 +1903,7 @@ func TestNetwork_Forward(t *testing.T) { return strings.Contains(b, "eth_getTransactionByHash") }). Reply(200). - JSON(json.RawMessage(`{"result":{"blockNumber":null,"hash":"0xabcdef","fromHost":"rpc1"}}`)) + JSON([]byte(`{"result":{"blockNumber":null,"hash":"0xabcdef","fromHost":"rpc1"}}`)) // Mock a non-pending transaction response from the second upstream gock.New("http://rpc2.localhost"). @@ -1806,7 +1913,7 @@ func TestNetwork_Forward(t *testing.T) { return strings.Contains(b, "eth_getTransactionByHash") }). Reply(200). - JSON(json.RawMessage(`{"result":{"blockNumber":"0x54C563","hash":"0xabcdef","fromHost":"rpc2"}}`)) + JSON([]byte(`{"result":{"blockNumber":"0x54C563","hash":"0xabcdef","fromHost":"rpc2"}}`)) // Set up a context and a cancellation function ctx, cancel := context.WithCancel(context.Background()) @@ -1921,12 +2028,12 @@ func TestNetwork_Forward(t *testing.T) { time.Sleep(1000 * time.Millisecond) // Create a fake request and forward it through the network - fakeReq := common.NewNormalizedRequest(json.RawMessage(`{ - "jsonrpc": "2.0", - "method": "eth_getTransactionByHash", - "params": ["0xabcdef"], - "id": 1 - }`)) + fakeReq := common.NewNormalizedRequest([]byte(`{ + "jsonrpc": "2.0", + "method": "eth_getTransactionByHash", + "params": ["0xabcdef"], + "id": 1 + }`)) fakeReq.ApplyDirectivesFromHttp(&fasthttp.RequestHeader{}, &fasthttp.Args{}) resp, err := ntw.Forward(ctx, fakeReq) @@ -1944,28 +2051,18 @@ func TestNetwork_Forward(t *testing.T) { t.Fatalf("Expected non-nil result") } - res, err := jrr.ParsedResult() + blockNumber, err := jrr.PeekStringByPath("blockNumber") if err != nil { - t.Fatalf("Failed to get parsed result: %v", err) - } - result, ok := res.(map[string]interface{}) - if !ok { - t.Fatalf("Expected Result to be map[string]interface{}, got %T", jrr.Result) - } - - blockNumber, ok := result["blockNumber"].(string) - if !ok { - t.Fatalf("Expected blockNumber to be string, got %T", result["blockNumber"]) + t.Fatalf("Failed to get blockNumber from result: %v", err) } if blockNumber != "0x54C563" { t.Errorf("Expected blockNumber to be %q, got %q", "0x54C563", blockNumber) } - fromHost, ok := result["fromHost"].(string) - if !ok { - t.Fatalf("Expected fromHost to be string, got %T", result["fromHost"]) + fromHost, err := jrr.PeekStringByPath("fromHost") + if err != nil { + t.Fatalf("Failed to get fromHost from result: %v", err) } - if fromHost != "rpc2" { t.Errorf("Expected fromHost to be %q, got %q", "rpc2", fromHost) } @@ -1983,7 +2080,7 @@ func TestNetwork_Forward(t *testing.T) { return strings.Contains(b, "latest") }). Reply(200). - JSON(json.RawMessage(`{"result": {"number":"0xA98AC7"}}`)) + JSON([]byte(`{"result":{"number":"0xA98AC7"}}`)) // Mock the response for the finalized block number request gock.New("http://rpc1.localhost"). @@ -1994,7 +2091,7 @@ func TestNetwork_Forward(t *testing.T) { return strings.Contains(b, "finalized") }). Reply(200). - JSON(json.RawMessage(`{"result":{"number":"0x21E88E"}}`)) + JSON([]byte(`{"result":{"number":"0x21E88E"}}`)) gock.New("http://rpc2.localhost"). Post(""). @@ -2004,7 +2101,7 @@ func TestNetwork_Forward(t *testing.T) { return strings.Contains(b, "latest") }). Reply(200). - JSON(json.RawMessage(`{"result": {"number":"0x32DCD5"}}`)) + JSON([]byte(`{"result":{"number":"0x32DCD5"}}`)) // Mock the response for the finalized block number request gock.New("http://rpc2.localhost"). @@ -2015,7 +2112,7 @@ func TestNetwork_Forward(t *testing.T) { return strings.Contains(b, "finalized") }). Reply(200). - JSON(json.RawMessage(`{"result":{"number":"0x2A62B1C"}}`)) + JSON([]byte(`{"result":{"number":"0x2A62B1C"}}`)) // Mock a pending transaction response from the first upstream gock.New("http://rpc1.localhost"). @@ -2025,7 +2122,7 @@ func TestNetwork_Forward(t *testing.T) { return strings.Contains(b, "eth_getTransactionByHash") }). Reply(200). - JSON(json.RawMessage(`{"result":{"blockNumber":null,"hash":"0xabcdef","fromHost":"rpc1"}}`)) + JSON([]byte(`{"result":{"blockNumber":null,"hash":"0xabcdef","fromHost":"rpc1"}}`)) // Set up a context and a cancellation function ctx, cancel := context.WithCancel(context.Background()) @@ -2140,12 +2237,12 @@ func TestNetwork_Forward(t *testing.T) { time.Sleep(1000 * time.Millisecond) // Create a fake request and forward it through the network - fakeReq := common.NewNormalizedRequest(json.RawMessage(`{ - "jsonrpc": "2.0", - "method": "eth_getTransactionByHash", - "params": ["0xabcdef"], - "id": 1 - }`)) + fakeReq := common.NewNormalizedRequest([]byte(`{ + "jsonrpc": "2.0", + "method": "eth_getTransactionByHash", + "params": ["0xabcdef"], + "id": 1 + }`)) hdr := &fasthttp.RequestHeader{} hdr.Set("x-erpc-retry-pending", "false") fakeReq.ApplyDirectivesFromHttp(hdr, &fasthttp.Args{}) @@ -2165,30 +2262,20 @@ func TestNetwork_Forward(t *testing.T) { t.Fatalf("Expected non-nil result") } - res, err := jrr.ParsedResult() + blockNumber, err := jrr.PeekStringByPath("blockNumber") if err != nil { - t.Fatalf("Failed to get parsed result: %v", err) - } - result, ok := res.(map[string]interface{}) - if !ok { - t.Fatalf("Expected Result to be map[string]interface{}, got %T", jrr.Result) - } - - blockNumber, ok := result["blockNumber"].(string) - if ok { - t.Fatalf("Expected blockNumber to be nil, got %v", result["blockNumber"]) + t.Fatalf("Failed to get blockNumber from result: %v", err) } if blockNumber != "" { t.Errorf("Expected blockNumber to be empty, got %q", blockNumber) } - fromHost, ok := result["fromHost"].(string) - if !ok { - t.Fatalf("Expected fromHost to be string, got %T", result["fromHost"]) + fromHost, err := jrr.PeekStringByPath("fromHost") + if err != nil { + t.Fatalf("Failed to get fromHost from result: %v", err) } - if fromHost != "rpc1" { - t.Errorf("Expected fromHost to be %q, got %q", "rpc1", fromHost) + t.Fatalf("Expected fromHost to be string, got %T", fromHost) } }) @@ -2197,14 +2284,14 @@ func TestNetwork_Forward(t *testing.T) { defer gock.Clean() defer gock.CleanUnmatchedRequest() - var requestBytes = json.RawMessage(`{"jsonrpc":"2.0","id":9199,"method":"eth_traceTransaction","params":["0x1273c18",false]}`) + var requestBytes = []byte(`{"jsonrpc":"2.0","id":9199,"method":"eth_traceTransaction","params":["0x1273c18",false]}`) // Mock the upstream response gock.New("http://rpc1.localhost"). Post(""). Times(2). // Expect two calls Reply(200). - JSON(json.RawMessage(`{"result":{"hash":"0x64d340d2470d2ed0ec979b72d79af9cd09fc4eb2b89ae98728d5fb07fd89baf9","fromHost":"rpc1"}}`)) + JSON([]byte(`{"result":{"hash":"0x64d340d2470d2ed0ec979b72d79af9cd09fc4eb2b89ae98728d5fb07fd89baf9","fromHost":"rpc1"}}`)) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -2311,13 +2398,13 @@ func TestNetwork_Forward(t *testing.T) { defer gock.Clean() defer gock.CleanUnmatchedRequest() - var requestBytes = json.RawMessage(`{"jsonrpc":"2.0","id":9199,"method":"eth_traceTransaction","params":["0x1273c18",false]}`) + var requestBytes = []byte(`{"jsonrpc":"2.0","id":9199,"method":"eth_traceTransaction","params":["0x1273c18",false]}`) gock.New("http://rpc1.localhost"). Times(1). Post(""). Reply(404). - JSON(json.RawMessage(`{"jsonrpc":"2.0","id":9199,"error":{"code":-32601,"message":"Method not supported"}}`)) + JSON([]byte(`{"jsonrpc":"2.0","id":9199,"error":{"code":-32601,"message":"Method not supported"}}`)) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -2415,13 +2502,13 @@ func TestNetwork_Forward(t *testing.T) { defer gock.Clean() defer gock.CleanUnmatchedRequest() - var requestBytes = json.RawMessage(`{"jsonrpc":"2.0","id":9199,"method":"eth_call","params":[{"to":"0x362fa9d0bca5d19f743db50738345ce2b40ec99f","data":"0xa4baa10c"}]}`) + var requestBytes = []byte(`{"jsonrpc":"2.0","id":9199,"method":"eth_call","params":[{"to":"0x362fa9d0bca5d19f743db50738345ce2b40ec99f","data":"0xa4baa10c"}]}`) gock.New("http://rpc1.localhost"). Times(1). Post(""). Reply(404). - JSON(json.RawMessage(`{"jsonrpc":"2.0","id":9199,"error":{"code":-32000,"message":"historical backend error: execution reverted: Dai/insufficient-balance"}}`)) + JSON([]byte(`{"jsonrpc":"2.0","id":9199,"error":{"code":-32000,"message":"historical backend error: execution reverted: Dai/insufficient-balance"}}`)) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -2522,13 +2609,13 @@ func TestNetwork_Forward(t *testing.T) { defer gock.Clean() defer gock.CleanUnmatchedRequest() - var requestBytes = json.RawMessage(`{"jsonrpc":"2.0","id":9199,"method":"eth_traceTransaction","params":["0x1273c18",false]}`) + var requestBytes = []byte(`{"jsonrpc":"2.0","id":9199,"method":"eth_traceTransaction","params":["0x1273c18",false]}`) gock.New("http://rpc1.alchemy.com.localhost"). Times(1). Post(""). Reply(503). - JSON(json.RawMessage(`{"jsonrpc":"2.0","id":9179,"error":{"code":-32600,"message":"Monthly capacity limit exceeded."}}`)) + JSON([]byte(`{"jsonrpc":"2.0","id":9179,"error":{"code":-32600,"message":"Monthly capacity limit exceeded."}}`)) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -2725,13 +2812,13 @@ func TestNetwork_Forward(t *testing.T) { defer gock.Clean() defer gock.CleanUnmatchedRequest() - var requestBytes = json.RawMessage(`{"jsonrpc":"2.0","id":1,"method":"eth_traceTransaction","params":["0x1273c18",false]}`) + var requestBytes = []byte(`{"jsonrpc":"2.0","id":1,"method":"eth_traceTransaction","params":["0x1273c18",false]}`) gock.New("http://rpc1.localhost"). Times(3). Post(""). Reply(503). - JSON(json.RawMessage(`{"error":{"message":"some random provider issue"}}`)) + JSON([]byte(`{"error":{"message":"some random provider issue"}}`)) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -2830,18 +2917,18 @@ func TestNetwork_Forward(t *testing.T) { defer gock.Clean() defer gock.CleanUnmatchedRequest() - var requestBytes = json.RawMessage(`{"jsonrpc":"2.0","id":1,"method":"eth_traceTransaction","params":["0x1273c18",false]}`) + var requestBytes = []byte(`{"jsonrpc":"2.0","id":1,"method":"eth_traceTransaction","params":["0x1273c18",false]}`) gock.New("http://rpc1.localhost"). Times(3). Post(""). Reply(503). - JSON(json.RawMessage(`{"error":{"message":"some random provider issue"}}`)) + JSON([]byte(`{"error":{"message":"some random provider issue"}}`)) gock.New("http://rpc1.localhost"). Post(""). Reply(200). - JSON(json.RawMessage(`{"result":{"hash":"0x64d340d2470d2ed0ec979b72d79af9cd09fc4eb2b89ae98728d5fb07fd89baf9"}}`)) + JSON([]byte(`{"result":{"hash":"0x64d340d2470d2ed0ec979b72d79af9cd09fc4eb2b89ae98728d5fb07fd89baf9"}}`)) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -2938,16 +3025,9 @@ func TestNetwork_Forward(t *testing.T) { t.Fatalf("Expected result, got %v", jrr) } - res, err := jrr.ParsedResult() - if err != nil { - t.Fatalf("Failed to get parsed result: %v", err) - } - result, ok := res.(map[string]interface{}) - if !ok { - t.Fatalf("Expected result to be a map, got %T", jrr.Result) - } - if hash, ok := result["hash"]; !ok || hash == "" { - t.Fatalf("Expected hash to exist and be non-empty, got %v", result) + hash, err := jrr.PeekStringByPath("hash") + if err != nil || hash == "" { + t.Fatalf("Expected hash to exist and be non-empty, got %v", hash) } }) @@ -2956,13 +3036,13 @@ func TestNetwork_Forward(t *testing.T) { defer gock.Clean() defer gock.CleanUnmatchedRequest() - var requestBytes = json.RawMessage(`{"jsonrpc":"2.0","id":1,"method":"eth_traceTransaction","params":["0x1273c18",false]}`) + var requestBytes = []byte(`{"jsonrpc":"2.0","id":1,"method":"eth_traceTransaction","params":["0x1273c18",false]}`) gock.New("http://rpc1.localhost"). Post(""). Reply(200). Delay(100 * time.Millisecond). - JSON(json.RawMessage(`{"result":{"hash":"0x64d340d2470d2ed0ec979b72d79af9cd09fc4eb2b89ae98728d5fb07fd89baf9"}}`)) + JSON([]byte(`{"result":{"hash":"0x64d340d2470d2ed0ec979b72d79af9cd09fc4eb2b89ae98728d5fb07fd89baf9"}}`)) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -3065,13 +3145,13 @@ func TestNetwork_Forward(t *testing.T) { defer gock.Clean() defer gock.CleanUnmatchedRequest() - var requestBytes = json.RawMessage(`{"jsonrpc":"2.0","id":1,"method":"eth_traceTransaction","params":["0x1273c18",false]}`) + var requestBytes = []byte(`{"jsonrpc":"2.0","id":1,"method":"eth_traceTransaction","params":["0x1273c18",false]}`) gock.New("http://rpc1.localhost"). Post(""). Reply(200). Delay(100 * time.Millisecond). - JSON(json.RawMessage(`{"result":{"hash":"0x64d340d2470d2ed0ec979b72d79af9cd09fc4eb2b89ae98728d5fb07fd89baf9"}}`)) + JSON([]byte(`{"result":{"hash":"0x64d340d2470d2ed0ec979b72d79af9cd09fc4eb2b89ae98728d5fb07fd89baf9"}}`)) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -3172,18 +3252,18 @@ func TestNetwork_Forward(t *testing.T) { defer gock.Clean() defer gock.CleanUnmatchedRequest() - var requestBytes = json.RawMessage(`{"jsonrpc":"2.0","id":1,"method":"eth_traceTransaction","params":["0x1273c18",false]}`) + var requestBytes = []byte(`{"jsonrpc":"2.0","id":1,"method":"eth_traceTransaction","params":["0x1273c18",false]}`) gock.New("http://rpc1.localhost"). Post(""). Reply(200). - JSON(json.RawMessage(`{"result":{"hash":"0x64d340d2470d2ed0ec979b72d79af9cd09fc4eb2b89ae98728d5fb07fd89baf9","fromHost":"rpc1"}}`)). + JSON([]byte(`{"result":{"hash":"0x64d340d2470d2ed0ec979b72d79af9cd09fc4eb2b89ae98728d5fb07fd89baf9","fromHost":"rpc1"}}`)). Delay(500 * time.Millisecond) gock.New("http://rpc2.localhost"). Post(""). Reply(200). - JSON(json.RawMessage(`{"result":{"hash":"0x64d340d2470d2ed0ec979b72d79af9cd09fc4eb2b89ae98728d5fb07fd89baf9","fromHost":"rpc2"}}`)). + JSON([]byte(`{"result":{"hash":"0x64d340d2470d2ed0ec979b72d79af9cd09fc4eb2b89ae98728d5fb07fd89baf9","fromHost":"rpc2"}}`)). Delay(200 * time.Millisecond) ctx, cancel := context.WithCancel(context.Background()) @@ -3306,44 +3386,167 @@ func TestNetwork_Forward(t *testing.T) { t.Fatalf("Expected result, got nil") } - res, err := jrr.ParsedResult() + fromHost, err := jrr.PeekStringByPath("fromHost") + if err != nil || fromHost != "rpc2" { + t.Errorf("Expected fromHost to be %v, got %v", "rpc2", fromHost) + } + }) + + t.Run("ForwardHedgePolicyNotTriggered", func(t *testing.T) { + defer gock.Off() + defer gock.Clean() + defer gock.CleanUnmatchedRequest() + + var requestBytes = []byte(`{"jsonrpc":"2.0","id":1,"method":"eth_traceTransaction","params":["0x1273c18",false]}`) + + gock.New("http://rpc1.localhost"). + Post(""). + Reply(200). + JSON([]byte(`{"result":{"hash":"0x64d340d2470d2ed0ec979b72d79af9cd09fc4eb2b89ae98728d5fb07fd89baf9","fromHost":"rpc1"}}`)). + Delay(20 * time.Millisecond) + + log.Logger.Info().Msgf("Mocks registered: %d", len(gock.Pending())) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + clr := upstream.NewClientRegistry(&log.Logger) + fsCfg := &common.FailsafeConfig{ + Hedge: &common.HedgePolicyConfig{ + Delay: "100ms", + MaxCount: 5, + }, + } + rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + Budgets: []*common.RateLimitBudgetConfig{}, + }, &log.Logger) if err != nil { - t.Fatalf("Failed to get parsed result: %v", err) + t.Fatal(err) } - result, ok := res.(map[string]interface{}) - if !ok { - t.Fatalf("Expected Result to be map[string]interface{}, got %T", jrr.Result) + vndr := vendors.NewVendorsRegistry() + mt := health.NewTracker("prjA", 2*time.Second) + up1 := &common.UpstreamConfig{ + Type: common.UpstreamTypeEvm, + Id: "rpc1", + Endpoint: "http://rpc1.localhost", + Evm: &common.EvmUpstreamConfig{ + ChainId: 123, + }, + } + up2 := &common.UpstreamConfig{ + Type: common.UpstreamTypeEvm, + Id: "rpc2", + Endpoint: "http://rpc2.localhost", + Evm: &common.EvmUpstreamConfig{ + ChainId: 123, + }, + } + upr := upstream.NewUpstreamsRegistry( + &log.Logger, + "prjA", + []*common.UpstreamConfig{up1, up2}, + rlr, + vndr, mt, 1*time.Second, + ) + err = upr.Bootstrap(ctx) + if err != nil { + t.Fatal(err) + } + err = upr.PrepareUpstreamsForNetwork(util.EvmNetworkId(123)) + if err != nil { + t.Fatal(err) + } + pup1, err := upr.NewUpstream( + "prjA", + up1, + &log.Logger, + mt, + ) + if err != nil { + t.Fatal(err) + } + cl1, err := clr.GetOrCreateClient(pup1) + if err != nil { + t.Fatal(err) } + pup1.Client = cl1 - fromHost, ok := result["fromHost"].(string) - if !ok { - t.Fatalf("Expected fromHost to be string, got %T", result["fromHost"]) + pup2, err := upr.NewUpstream( + "prjA", + up2, + &log.Logger, + mt, + ) + if err != nil { + t.Fatal(err) + } + cl2, err := clr.GetOrCreateClient(pup2) + if err != nil { + t.Fatal(err) } + pup2.Client = cl2 - if fromHost != "rpc2" { - t.Errorf("Expected fromHost to be %v, got %v", "rpc2", fromHost) + ntw, err := NewNetwork( + &log.Logger, + "prjA", + &common.NetworkConfig{ + Architecture: common.ArchitectureEvm, + Evm: &common.EvmNetworkConfig{ + ChainId: 123, + }, + Failsafe: fsCfg, + }, + rlr, + upr, + mt, + ) + if err != nil { + t.Fatal(err) + } + + fakeReq := common.NewNormalizedRequest(requestBytes) + resp, err := ntw.Forward(ctx, fakeReq) + + if len(gock.Pending()) > 0 { + t.Errorf("Expected all mocks to be consumed, got %v left", len(gock.Pending())) + for _, pending := range gock.Pending() { + t.Errorf("Pending mock: %v", pending) + } + } else { + t.Logf("All mocks consumed") + } + + if err != nil { + t.Fatalf("Expected nil error, got %v", err) + } + + jrr, err := resp.JsonRpcResponse() + if err != nil { + t.Fatalf("Expected nil error, got %v", err) + } + if jrr.Result == nil { + t.Fatalf("Expected result, got nil") + } + + fromHost, err := jrr.PeekStringByPath("fromHost") + if err != nil || fromHost != "rpc1" { + t.Errorf("Expected fromHost to be %v, got %v", "rpc1", fromHost) } }) - t.Run("ForwardHedgePolicyNotTriggered", func(t *testing.T) { + t.Run("ForwardHedgePolicySkipsWriteMethods", func(t *testing.T) { defer gock.Off() defer gock.Clean() defer gock.CleanUnmatchedRequest() - var requestBytes = json.RawMessage(`{"jsonrpc":"2.0","id":1,"method":"eth_traceTransaction","params":["0x1273c18",false]}`) + var requestBytes = []byte(`{"jsonrpc":"2.0","id":1,"method":"eth_sendRawTransaction","params":["0x1273c18"]}`) gock.New("http://rpc1.localhost"). Post(""). Reply(200). - JSON(json.RawMessage(`{"result":{"hash":"0x64d340d2470d2ed0ec979b72d79af9cd09fc4eb2b89ae98728d5fb07fd89baf9","fromHost":"rpc1"}}`)). + JSON([]byte(`{"result":{"hash":"0x64d340d2470d2ed0ec979b72d79af9cd09fc4eb2b89ae98728d5fb07fd89baf9","fromHost":"rpc1"}}`)). Delay(2000 * time.Millisecond) - gock.New("http://alchemy.com"). - Post(""). - Reply(200). - JSON(json.RawMessage(`{"result":{"hash":"0x64d340d2470d2ed0ec979b72d79af9cd09fc4eb2b89ae98728d5fb07fd89baf9","fromHost":"rpc2"}}`)). - Delay(10 * time.Millisecond) - log.Logger.Info().Msgf("Mocks registered: %d", len(gock.Pending())) ctx, cancel := context.WithCancel(context.Background()) @@ -3467,176 +3670,30 @@ func TestNetwork_Forward(t *testing.T) { t.Fatalf("Expected result, got nil") } - res, err := jrr.ParsedResult() - if err != nil { - t.Fatalf("Failed to get parsed result: %v", err) - } - result, ok := res.(map[string]interface{}) - if !ok { - t.Fatalf("Expected result to be map[string]interface{}, got %T", jrr.Result) - } - if result["fromHost"] != "rpc2" { - t.Errorf("Expected fromHost to be %v, got %v", "rpc2", result["fromHost"]) + fromHost, err := jrr.PeekStringByPath("fromHost") + if err != nil || fromHost != "rpc1" { + t.Errorf("Expected fromHost to be %v, got %v", "rpc1", fromHost) } }) - // t.Run("ForwardHedgePolicyIgnoresNegativeScoreUpstream", func(t *testing.T) { - // defer gock.Off() - // defer gock.Clean() - // defer gock.CleanUnmatchedRequest() - // var requestBytes = json.RawMessage(`{"jsonrpc":"2.0","id":1,"method":"eth_traceTransaction","params":["0x1273c18",false]}`) - // log.Logger.Info().Msgf("Mocks registered before: %d", len(gock.Pending())) - // gock.New("http://rpc1.localhost"). - // Post(""). - // Times(3). - // Reply(200). - // JSON(json.RawMessage(`{"result":{"hash":"0x64d340d2470d2ed0ec979b72d79af9cd09fc4eb2b89ae98728d5fb07fd89baf9","fromHost":"rpc1"}}`)). - // Delay(100 * time.Millisecond) - // log.Logger.Info().Msgf("Mocks registered after: %d", len(gock.Pending())) - // ctx, cancel := context.WithCancel(context.Background()) - // defer cancel() - // clr := upstream.NewClientRegistry(&log.Logger) - // fsCfg := &common.FailsafeConfig{ - // Hedge: &common.HedgePolicyConfig{ - // Delay: "30ms", - // MaxCount: 2, - // }, - // } - // rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ - // Budgets: []*common.RateLimitBudgetConfig{}, - // }) - // if err != nil { - // t.Fatal(err) - // } - // vndr := vendors.NewVendorsRegistry() - // mt := health.NewTracker("prjA", 2*time.Second) - // up1 := &common.UpstreamConfig{ - // Type: common.UpstreamTypeEvm, - // Id: "rpc1", - // Endpoint: "http://rpc1.localhost", - // Evm: &common.EvmUpstreamConfig{ - // ChainId: 123, - // }, - // } - // up2 := &common.UpstreamConfig{ - // Type: common.UpstreamTypeEvm, - // Id: "rpc2", - // Endpoint: "http://alchemy.com", - // Evm: &common.EvmUpstreamConfig{ - // ChainId: 123, - // }, - // } - // upr := upstream.NewUpstreamsRegistry( - // &log.Logger, - // "prjA", - // []*common.UpstreamConfig{up1, up2}, - // rlr, - // vndr, - // mt, - // ) - // err = upr.Bootstrap(ctx) - // if err != nil { - // t.Fatal(err) - // } - // err = upr.PrepareUpstreamsForNetwork(util.EvmNetworkId(123)) - // if err != nil { - // t.Fatal(err) - // } - // pup1, err := upr.NewUpstream( - // "prjA", - // up1, - // &log.Logger, - // mt, - // ) - // if err != nil { - // t.Fatal(err) - // } - // cl1, err := clr.CreateClient(pup1) - // if err != nil { - // t.Fatal(err) - // } - // pup1.Score = 2 - // pup1.Client = cl1 - // pup2, err := upr.NewUpstream( - // "prjA", - // up2, - // &log.Logger, - // mt, - // ) - // if err != nil { - // t.Fatal(err) - // } - // cl2, err := clr.CreateClient(pup2) - // if err != nil { - // t.Fatal(err) - // } - // pup2.Client = cl2 - // pup2.Score = -2 - // ntw, err := NewNetwork( - // &log.Logger, - // "prjA", - // &common.NetworkConfig{ - // Architecture: common.ArchitectureEvm, - // Evm: &common.EvmNetworkConfig{ - // ChainId: 123, - // }, - // Failsafe: fsCfg, - // }, - // rlr, - // upr, - // mt, - // ) - // if err != nil { - // t.Fatal(err) - // } - // fakeReq := common.NewNormalizedRequest(requestBytes) - // resp, err := ntw.Forward(ctx, fakeReq) - // time.Sleep(50 * time.Millisecond) - // if len(gock.Pending()) > 0 { - // t.Errorf("Expected all mocks to be consumed, got %v left", len(gock.Pending())) - // for _, pending := range gock.Pending() { - // t.Errorf("Pending mock: %v", pending) - // } - // } else { - // t.Logf("All mocks consumed") - // } - // if err != nil { - // t.Fatalf("Expected nil error, got %v", err) - // } - // jrr, err := resp.JsonRpcResponse() - // if err != nil { - // t.Fatalf("Expected nil error, got %v", err) - // } - // if jrr.Result == nil { - // t.Fatalf("Expected result, got nil") - // } - // result, ok := jrr.Result.(map[string]interface{}) - // if !ok { - // t.Fatalf("Expected result to be map[string]interface{}, got %T", jrr.Result) - // } - // if result["fromHost"] != "rpc1" { - // t.Errorf("Expected fromHost to be %v, got %v", "rpc1", result["fromHost"]) - // } - // }) - t.Run("ForwardCBOpensAfterConstantFailure", func(t *testing.T) { defer gock.Off() defer gock.Clean() defer gock.CleanUnmatchedRequest() - var requestBytes = json.RawMessage(`{"jsonrpc":"2.0","id":1,"method":"eth_traceTransaction","params":["0x1273c18",false]}`) + var requestBytes = []byte(`{"jsonrpc":"2.0","id":1,"method":"eth_traceTransaction","params":["0x1273c18",false]}`) gock.New("http://rpc1.localhost"). Post(""). Times(2). Reply(200). - JSON(json.RawMessage(`{"result":{"hash":"0x64d340d2470d2ed0ec979b72d79af9cd09fc4eb2b89ae98728d5fb07fd89baf9"}}`)) + JSON([]byte(`{"result":{"hash":"0x64d340d2470d2ed0ec979b72d79af9cd09fc4eb2b89ae98728d5fb07fd89baf9"}}`)) gock.New("http://rpc1.localhost"). Post(""). Times(2). Reply(503). - JSON(json.RawMessage(`{"error":{"message":"some random provider issue"}}`)) + JSON([]byte(`{"error":{"message":"some random provider issue"}}`)) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -3744,13 +3801,13 @@ func TestNetwork_Forward(t *testing.T) { defer gock.Clean() defer gock.CleanUnmatchedRequest() - var requestBytes = json.RawMessage(`{"jsonrpc":"2.0","id":1,"method":"eth_traceTransaction","params":["0x1273c18",false]}`) + var requestBytes = []byte(`{"jsonrpc":"2.0","id":1,"method":"eth_traceTransaction","params":["0x1273c18",false]}`) gock.New("http://rpc1.localhost"). Post(""). Times(1). Reply(503). - JSON(json.RawMessage(`{"error":{"message":"some random provider issue"}}`)) + JSON([]byte(`{"error":{"message":"some random provider issue"}}`)) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -3865,25 +3922,25 @@ func TestNetwork_Forward(t *testing.T) { defer gock.Clean() defer gock.CleanUnmatchedRequest() - var requestBytes = json.RawMessage(`{"jsonrpc":"2.0","id":1,"method":"eth_traceTransaction","params":["0x1273c18",false]}`) + var requestBytes = []byte(`{"jsonrpc":"2.0","id":1,"method":"eth_traceTransaction","params":["0x1273c18",false]}`) gock.New("http://rpc1.localhost"). Post(""). Times(3). Reply(200). - JSON(json.RawMessage(`{"result":{"hash":"0x64d340d2470d2ed0ec979b72d79af9cd09fc4eb2b89ae98728d5fb07fd89baf9"}}`)) + JSON([]byte(`{"result":{"hash":"0x64d340d2470d2ed0ec979b72d79af9cd09fc4eb2b89ae98728d5fb07fd89baf9"}}`)) gock.New("http://rpc1.localhost"). Post(""). Times(3). Reply(503). - JSON(json.RawMessage(`{"error":{"message":"some random provider issue"}}`)) + JSON([]byte(`{"error":{"message":"some random provider issue"}}`)) gock.New("http://rpc1.localhost"). Post(""). Times(3). Reply(200). - JSON(json.RawMessage(`{"result":{"hash":"0x64d340d2470d2ed0ec979b72d79af9cd09fc4eb2b89ae98728d5fb07fd89baf9"}}`)) + JSON([]byte(`{"result":{"hash":"0x64d340d2470d2ed0ec979b72d79af9cd09fc4eb2b89ae98728d5fb07fd89baf9"}}`)) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -3996,16 +4053,9 @@ func TestNetwork_Forward(t *testing.T) { if err != nil { t.Fatalf("Expected nil error, got %v", err) } - res, err := jrr.ParsedResult() - if err != nil { - t.Fatalf("Failed to get parsed result: %v", err) - } - result, ok := res.(map[string]interface{}) - if !ok { - t.Fatalf("Expected result to be map[string]interface{}, got %T", jrr.Result) - } - if result["hash"] == "" { - t.Errorf("Expected hash to exist, got %v", result) + hash, err := jrr.PeekStringByPath("hash") + if err != nil || hash == "" { + t.Fatalf("Expected hash to exist and be non-empty, got %v", hash) } }) @@ -4014,12 +4064,12 @@ func TestNetwork_Forward(t *testing.T) { defer gock.Clean() defer gock.CleanUnmatchedRequest() - var requestBytes = json.RawMessage(`{"jsonrpc":"2.0","id":1,"method":"eth_traceTransaction","params":["0x1273c18",false]}`) + var requestBytes = []byte(`{"jsonrpc":"2.0","id":1,"method":"eth_traceTransaction","params":["0x1273c18",false]}`) gock.New("http://rpc1.localhost"). Post(""). Reply(500). - JSON(json.RawMessage(`{"error":{"code":-39999,"message":"my funky random error"}}`)) + JSON([]byte(`{"error":{"code":-39999,"message":"my funky random error"}}`)) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -4118,17 +4168,17 @@ func TestNetwork_Forward(t *testing.T) { defer gock.Clean() defer gock.CleanUnmatchedRequest() - var requestBytes = json.RawMessage(`{"jsonrpc":"2.0","id":1,"method":"eth_traceTransaction","params":["0x1273c18",false]}`) + var requestBytes = []byte(`{"jsonrpc":"2.0","id":1,"method":"eth_traceTransaction","params":["0x1273c18",false]}`) gock.New("http://rpc1.localhost"). Post(""). Reply(500). - JSON(json.RawMessage(`{"error":{"message":"Internal error"}}`)) + JSON([]byte(`{"error":{"message":"Internal error"}}`)) gock.New("http://rpc2.localhost"). Post(""). Reply(200). - JSON(json.RawMessage(`{"result":{"hash":"0x64d340d2470d2ed0ec979b72d79af9cd09fc4eb2b89ae98728d5fb07fd89baf9","fromHost":"rpc2"}}`)) + JSON([]byte(`{"result":{"hash":"0x64d340d2470d2ed0ec979b72d79af9cd09fc4eb2b89ae98728d5fb07fd89baf9","fromHost":"rpc2"}}`)) log.Logger.Info().Msgf("Mocks registered: %d", len(gock.Pending())) @@ -4256,22 +4306,99 @@ func TestNetwork_Forward(t *testing.T) { t.Logf("jrr.Result type: %T", jrr.Result) t.Logf("jrr.Result content: %+v", jrr.Result) - res, err := jrr.ParsedResult() + fromHost, err := jrr.PeekStringByPath("fromHost") + if err != nil || fromHost != "rpc2" { + t.Errorf("Expected fromHost to be %v, got %v", "rpc2", fromHost) + } + }) + + t.Run("ForwardIgnoredMethod", func(t *testing.T) { + var requestBytes = []byte(`{"jsonrpc":"2.0","id":1,"method":"ignored_method","params":["0x1273c18",false]}`) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + clr := upstream.NewClientRegistry(&log.Logger) + fsCfg := &common.FailsafeConfig{ + Retry: &common.RetryPolicyConfig{ + MaxAttempts: 2, + }, + } + rlr, err := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{ + Budgets: []*common.RateLimitBudgetConfig{}, + }, &log.Logger) + if err != nil { + t.Fatal(err) + } + + vndr := vendors.NewVendorsRegistry() + mt := health.NewTracker("prjA", 2*time.Second) + up1 := &common.UpstreamConfig{ + Type: common.UpstreamTypeEvm, + Id: "rpc1", + Endpoint: "http://rpc1.localhost", + Evm: &common.EvmUpstreamConfig{ + ChainId: 123, + }, + IgnoreMethods: []string{"ignored_method"}, + } + upr := upstream.NewUpstreamsRegistry( + &log.Logger, + "prjA", + []*common.UpstreamConfig{up1}, + rlr, + vndr, mt, 1*time.Second, + ) + err = upr.Bootstrap(ctx) if err != nil { - t.Fatalf("Failed to get parsed result: %v", err) + t.Fatal(err) } - result, ok := res.(map[string]interface{}) - if !ok { - t.Fatalf("Expected Result to be map[string]interface{}, got %T", jrr.Result) + err = upr.PrepareUpstreamsForNetwork(util.EvmNetworkId(123)) + if err != nil { + t.Fatal(err) + } + pup1, err := upr.NewUpstream( + "prjA", + up1, + &log.Logger, + mt, + ) + if err != nil { + t.Fatal(err) } + cl1, err := clr.GetOrCreateClient(pup1) + if err != nil { + t.Fatal(err) + } + pup1.Client = cl1 - fromHost, ok := result["fromHost"].(string) - if !ok { - t.Fatalf("Expected fromHost to be string, got %T", result["fromHost"]) + ntw, err := NewNetwork( + &log.Logger, + "prjA", + &common.NetworkConfig{ + Architecture: common.ArchitectureEvm, + Evm: &common.EvmNetworkConfig{ + ChainId: 123, + }, + Failsafe: fsCfg, + }, + rlr, + upr, + mt, + ) + if err != nil { + t.Fatal(err) } - if fromHost != "rpc2" { - t.Errorf("Expected fromHost to be %v, got %v", "rpc2", fromHost) + fakeReq := common.NewNormalizedRequest(requestBytes) + _, err = ntw.Forward(ctx, fakeReq) + + if err == nil { + t.Fatalf("Expected non-nil error, got nil") + } + + if !common.HasErrorCode(err, common.ErrCodeUpstreamMethodIgnored) { + t.Fatalf("Expected error code %v, got %v", common.ErrCodeUpstreamMethodIgnored, err) } }) @@ -4280,17 +4407,17 @@ func TestNetwork_Forward(t *testing.T) { defer gock.Clean() defer gock.CleanUnmatchedRequest() - var requestBytes = json.RawMessage(`{"jsonrpc": "2.0","method": "eth_getLogs","params":[{"address":"0x1234567890abcdef1234567890abcdef12345678"}],"id": 1}`) + var requestBytes = []byte(`{"jsonrpc": "2.0","method": "eth_getLogs","params":[{"address":"0x1234567890abcdef1234567890abcdef12345678"}],"id": 1}`) gock.New("http://rpc1.localhost"). Post(""). Reply(200). - JSON(json.RawMessage(`{"result":[]}`)) + JSON([]byte(`{"result":[]}`)) gock.New("http://rpc2.localhost"). Post(""). Reply(200). - JSON(json.RawMessage(`{"result":[{"logIndex":444,"fromHost":"rpc2"}]}`)) + JSON([]byte(`{"result":[{"logIndex":444,"fromHost":"rpc2"}]}`)) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -4411,30 +4538,8 @@ func TestNetwork_Forward(t *testing.T) { t.Fatalf("Expected non-nil result") } - res, err := jrr.ParsedResult() - if err != nil { - t.Fatalf("Failed to get parsed result: %v", err) - } - result, ok := res.([]interface{}) - if !ok { - t.Fatalf("Expected Result to be []interface{}, got %T", jrr.Result) - } - - if len(result) == 0 { - t.Fatalf("Expected non-empty result array") - } - - firstLog, ok := result[0].(map[string]interface{}) - if !ok { - t.Fatalf("Expected first log to be map[string]interface{}, got %T", result[0]) - } - - fromHost, ok := firstLog["fromHost"].(string) - if !ok { - t.Fatalf("Expected fromHost to be string, got %T", firstLog["fromHost"]) - } - - if fromHost != "rpc2" { + fromHost, err := jrr.PeekStringByPath(0, "fromHost") + if err != nil || fromHost != "rpc2" { t.Errorf("Expected fromHost to be %q, got %q", "rpc2", fromHost) } }) @@ -4444,9 +4549,9 @@ func TestNetwork_Forward(t *testing.T) { defer gock.Clean() defer gock.CleanUnmatchedRequest() - var requestBytes = json.RawMessage(`{"jsonrpc": "2.0","method": "eth_getLogs","params":[{"address":"0x1234567890abcdef1234567890abcdef12345678"}],"id": 1}`) + var requestBytes = []byte(`{"jsonrpc": "2.0","method": "eth_getLogs","params":[{"address":"0x1234567890abcdef1234567890abcdef12345678"}],"id": 1}`) - emptyResponse := json.RawMessage(`{"jsonrpc": "2.0","id": 1,"result":[]}`) + emptyResponse := []byte(`{"jsonrpc": "2.0","id": 1,"result":[]}`) gock.New("http://rpc1.localhost"). Post(""). @@ -4547,17 +4652,8 @@ func TestNetwork_Forward(t *testing.T) { t.Fatalf("Expected non-nil result") } - res, err := jrr.ParsedResult() - if err != nil { - t.Fatalf("Failed to get parsed result: %v", err) - } - result, ok := res.([]interface{}) - if !ok { - t.Fatalf("Expected Result to be []interface{}, got %T", jrr.Result) - } - - if len(result) != 0 { - t.Errorf("Expected empty array result, got array of length %d", len(result)) + if len(jrr.Result) != 2 || jrr.Result[0] != '[' || jrr.Result[1] != ']' { + t.Errorf("Expected empty array result, got %s", string(jrr.Result)) } }) @@ -4566,12 +4662,12 @@ func TestNetwork_Forward(t *testing.T) { defer gock.Clean() defer gock.CleanUnmatchedRequest() - var requestBytes = json.RawMessage(`{"jsonrpc":"2.0","id":1,"method":"eth_getLogs","params":["0x1273c18"]}`) + var requestBytes = []byte(`{"jsonrpc":"2.0","id":1,"method":"eth_getLogs","params":["0x1273c18"]}`) gock.New("http://rpc1.localhost"). Post(""). Reply(429). - JSON(json.RawMessage(`{"code":-32007,"message":"300/second request limit reached - reduce calls per second or upgrade your account at quicknode.com"}`)) + JSON([]byte(`{"error":{"code":-32007,"message":"300/second request limit reached - reduce calls per second or upgrade your account at quicknode.com"}}`)) log.Logger.Info().Msgf("Mocks registered: %d", len(gock.Pending())) @@ -4664,7 +4760,7 @@ func TestNetwork_Forward(t *testing.T) { defer gock.Clean() defer gock.CleanUnmatchedRequest() - var requestBytes = json.RawMessage(`{"jsonrpc":"2.0","id":1,"method":"eth_getBlockByNumber","params":["0x1273c18",false]}`) + var requestBytes = []byte(`{"jsonrpc":"2.0","id":1,"method":"eth_getBlockByNumber","params":["0x1273c18",false]}`) gock.New("http://rpc1.localhost"). Post(""). @@ -4761,7 +4857,7 @@ func TestNetwork_Forward(t *testing.T) { defer gock.Clean() defer gock.CleanUnmatchedRequest() - var requestBytes = json.RawMessage(`{"jsonrpc":"2.0","id":1,"method":"eth_getBlockByNumber","params":["0x1273c18",false]}`) + var requestBytes = []byte(`{"jsonrpc":"2.0","id":1,"method":"eth_getBlockByNumber","params":["0x1273c18",false]}`) gock.New("http://rpc1.localhost"). Post(""). @@ -4992,7 +5088,7 @@ func TestNetwork_Forward(t *testing.T) { defer gock.Clean() defer gock.CleanUnmatchedRequest() - var requestBytes = json.RawMessage(`{"jsonrpc": "2.0","method": "eth_getLogs","params":[{"address":"0x1234567890abcdef1234567890abcdef12345678"}],"id": 1}`) + var requestBytes = []byte(`{"jsonrpc":"2.0","method":"eth_getLogs","params":[{"address":"0x1234567890abcdef1234567890abcdef12345678"}],"id": 1}`) gock.New("https://rpc.hypersync.xyz"). Post(""). @@ -5002,7 +5098,7 @@ func TestNetwork_Forward(t *testing.T) { gock.New("http://rpc1.localhost"). Post(""). Reply(200). - JSON(json.RawMessage(`{"result":[{"logIndex":444}], "fromHost":"rpc1"}`)) + JSON([]byte(`{"result":[{"logIndex":444}]}`)) log.Logger.Info().Msgf("Mocks registered: %d", len(gock.Pending())) @@ -5093,34 +5189,266 @@ func TestNetwork_Forward(t *testing.T) { } // Convert the raw response to a map to access custom fields like fromHost - var responseMap map[string]interface{} - err = sonic.Unmarshal(resp.Body(), &responseMap) + jrr, err := resp.JsonRpcResponse() if err != nil { - t.Fatalf("Failed to unmarshal response body: %v", err) - } - - // Check if fromHost exists and is a string - fromHost, ok := responseMap["fromHost"].(string) - if !ok { - t.Fatalf("Expected fromHost to be a string, got %T", responseMap["fromHost"]) - } - - // Assert the value of fromHost - if fromHost != "rpc1" { - t.Errorf("Expected fromHost to be %q, got %q", "rpc1", fromHost) + t.Fatalf("Failed to get JSON-RPC response: %v", err) } // Check that the result field is an empty array as expected - result, ok := responseMap["result"].([]interface{}) - if !ok { - t.Fatalf("Expected result to be []interface{}, got %T", responseMap["result"]) + result := []interface{}{} + err = sonic.Unmarshal(jrr.Result, &result) + if err != nil { + t.Fatalf("Failed to unmarshal result: %v", err) } - if len(result) == 0 { t.Fatalf("Expected non-empty result array") } }) + for i := 0; i < 10; i++ { + t.Run("ResponseReleasedBeforeCacheSet", func(t *testing.T) { + resetGock() + defer resetGock() + + network := setupTestNetwork(t, nil) + gock.New("http://rpc1.localhost"). + Post("/"). + MatchType("json"). + JSON(map[string]interface{}{ + "jsonrpc": "2.0", + "method": "eth_getTransactionReceipt", + "params": []interface{}{"0x1111"}, + "id": 11111, + }). + Reply(200). + JSON(map[string]interface{}{ + "jsonrpc": "2.0", + "id": 11111, + "result": map[string]interface{}{ + "blockNumber": "0x1111", + }, + }) + gock.New("http://rpc1.localhost"). + Post("/"). + MatchType("json"). + JSON(map[string]interface{}{ + "jsonrpc": "2.0", + "method": "eth_getBalance", + "params": []interface{}{"0x2222", "0x2222"}, + "id": 22222, + }). + Reply(200). + JSON(map[string]interface{}{ + "jsonrpc": "2.0", + "id": 22222, + "result": "0x22222222222222", + }) + + // Create a slow cache to increase the chance of a race condition + conn, errc := data.NewMockMemoryConnector(context.Background(), &log.Logger, &common.MemoryConnectorConfig{ + MaxItems: 1000, + }, 100*time.Millisecond) + if errc != nil { + t.Fatalf("Failed to create mock memory connector: %v", errc) + } + slowCache := (&EvmJsonRpcCache{ + conn: conn, + logger: &log.Logger, + }).WithNetwork(network) + network.cacheDal = slowCache + + // Make the request + req1 := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","method":"eth_getTransactionReceipt","params":["0x1111"],"id":11111}`)) + req2 := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","method":"eth_getBalance","params":["0x2222", "0x2222"],"id":22222}`)) + + // Use a WaitGroup to ensure both goroutines complete + var wg sync.WaitGroup + wg.Add(2) + + var jrr1Atomic atomic.Value + var jrr2Atomic atomic.Value + + // Goroutine 1: Make the request and immediately release the response + go func() { + defer wg.Done() + + resp1, err := network.Forward(context.Background(), req1) + if err != nil { + t.Errorf("Unexpected error: %v", err) + return + } + jrr1Value, _ := resp1.JsonRpcResponse() + jrr1Atomic.Store(jrr1Value) + // Simulate immediate release of the response + resp1.Release() + + resp2, err := network.Forward(context.Background(), req2) + if err != nil { + t.Errorf("Unexpected error: %v", err) + return + } + jrr2Value, _ := resp2.JsonRpcResponse() + jrr2Atomic.Store(jrr2Value) + resp2.Release() + }() + + // Goroutine 2: Access the response concurrently + go func() { + defer wg.Done() + time.Sleep(2000 * time.Millisecond) + var res1 string + var res2 string + jrr1 := jrr1Atomic.Load().(*common.JsonRpcResponse) + jrr2 := jrr2Atomic.Load().(*common.JsonRpcResponse) + if jrr1 != nil { + if j, e := jrr1.MarshalJSON(); e != nil { + t.Errorf("Failed to marshal json-rpc response: %v", e) + } else { + var obj map[string]interface{} + common.SonicCfg.Unmarshal(j, &obj) + res1, _ = common.SonicCfg.MarshalToString(obj["result"]) + } + _ = jrr1.ID() + } + if jrr2 != nil { + if j, e := jrr2.MarshalJSON(); e != nil { + t.Errorf("Failed to marshal json-rpc response: %v", e) + } else { + var obj map[string]interface{} + common.SonicCfg.Unmarshal(j, &obj) + res2, _ = common.SonicCfg.MarshalToString(obj["result"]) + } + _ = jrr2.ID() + } + assert.NotEmpty(t, res1) + assert.NotEmpty(t, res2) + assert.NotEqual(t, res1, res2) + cache1, e1 := slowCache.Get(context.Background(), req1) + cache2, e2 := slowCache.Get(context.Background(), req2) + assert.NoError(t, e1) + assert.NoError(t, e2) + cjrr1, _ := cache1.JsonRpcResponse() + cjrr2, _ := cache2.JsonRpcResponse() + assert.NotNil(t, cjrr1) + assert.NotNil(t, cjrr2) + if cjrr1 != nil { + // cjrr1.RLock() + assert.Equal(t, res1, string(cjrr1.Result)) + // cjrr1.RUnlock() + } + if cjrr2 != nil { + // cjrr2.Lock() + assert.Equal(t, res2, string(cjrr2.Result)) + // cjrr2.Unlock() + } + }() + + // Wait for both goroutines to complete + wg.Wait() + }) + } + + t.Run("BatchRequestValidationAndRetry", func(t *testing.T) { + defer gock.Off() + defer gock.Clean() + defer gock.CleanUnmatchedRequest() + + setupMocksForEvmStatePoller() + + // Set up the test environment + network := setupTestNetwork(t, &common.UpstreamConfig{ + Type: common.UpstreamTypeEvm, + Id: "test", + Endpoint: "http://rpc1.localhost", + Evm: &common.EvmUpstreamConfig{ + ChainId: 123, + }, + JsonRpc: &common.JsonRpcUpstreamConfig{ + SupportsBatch: &common.TRUE, + }, + Failsafe: &common.FailsafeConfig{ + Retry: &common.RetryPolicyConfig{ + MaxAttempts: 2, + }, + }, + }) + + // Mock the response for the batch request + gock.New("http://rpc1.localhost"). + Post("/"). + Reply(200). + BodyString(`[ + { + "jsonrpc": "2.0", + "id": 32, + "error": { + "code": -32602, + "message": "Invalid params", + "data": { + "range": "the range 56224203 - 56274202 exceeds the range allowed for your plan (49999 > 2000)." + } + } + }, + { + "jsonrpc": "2.0", + "id": 43, + "error": { + "code": -32600, + "message": "Invalid Request", + "data": { + "message": "Cancelled due to validation errors in batch request" + } + } + } + ]`) + gock.New("http://rpc1.localhost"). + Post("/"). + Reply(200). + BodyString(`[ + { + "jsonrpc": "2.0", + "id": 43, + "result": "0x22222222222222" + } + ]`) + + // Create normalized requests + req1 := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":32,"method":"eth_getLogs","params":[{"fromBlock":"0x35A35CB","toBlock":"0x35AF7CA"}]}`)) + req2 := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":43,"method":"eth_getBalance","params":["0x742d35Cc6634C0532925a3b844Bc454e4438f44e", "latest"]}`)) + + // Process requests + var resp1, resp2 *common.NormalizedResponse + var err1, err2 error + + wg := sync.WaitGroup{} + wg.Add(2) + go func() { + defer wg.Done() + resp1, err1 = network.Forward(context.Background(), req1) + }() + go func() { + defer wg.Done() + resp2, err2 = network.Forward(context.Background(), req2) + }() + wg.Wait() + + // Assertions for the first request (server-side error, should be retried) + assert.Error(t, err1, "Expected an error for the first request") + assert.Nil(t, resp1, "Expected nil response for the first request") + assert.False(t, common.IsRetryableTowardsUpstream(err1), "Expected a retryable error for the first request") + assert.True(t, common.HasErrorCode(err1, common.ErrCodeEndpointClientSideException), "Expected a client-side exception error for the second request") + + // Assertions for the second request (client-side error, should not be retried) + assert.Nil(t, err2, "Expected no error for the second request") + assert.NotNil(t, resp2, "Expected non-nil response for the second request") + + if left := anyTestMocksLeft(); left > 0 { + t.Errorf("Expected all test mocks to be consumed, got %v left", left) + for _, pending := range gock.Pending() { + t.Errorf("Pending mock: %v", pending) + } + } + }) } func TestNetwork_InFlightRequests(t *testing.T) { @@ -5128,17 +5456,17 @@ func TestNetwork_InFlightRequests(t *testing.T) { resetGock() defer resetGock() - network := setupTestNetwork(t) + network := setupTestNetwork(t, nil) requestBytes := []byte(`{"jsonrpc":"2.0","method":"eth_getLogs","params":[]}`) gock.New("http://rpc1.localhost"). Post("/"). - Times(1). + Persist(). Filter(func(request *http.Request) bool { return strings.Contains(safeReadBody(request), "eth_getLogs") }). Reply(200). - Delay(1 * time.Second). // Delay a bit so in-flight multiplexing kicks in + Delay(3 * time.Second). // Delay a bit so in-flight multiplexing kicks in BodyString(`{"jsonrpc":"2.0","id":1,"result":"0x1"}`) var wg sync.WaitGroup @@ -5154,7 +5482,7 @@ func TestNetwork_InFlightRequests(t *testing.T) { } wg.Wait() - if left := anyTestMocksLeft(); left > 0 { + if left := anyTestMocksLeft(); left > 1 { t.Errorf("Expected all test mocks to be consumed, got %v left", left) for _, pending := range gock.Pending() { t.Errorf("Pending mock: %v", pending) @@ -5166,7 +5494,7 @@ func TestNetwork_InFlightRequests(t *testing.T) { resetGock() defer resetGock() - network := setupTestNetwork(t) + network := setupTestNetwork(t, nil) requestBytes := []byte(`{"jsonrpc":"2.0","method":"eth_getLogs","params":[]}`) gock.New("http://rpc1.localhost"). @@ -5204,14 +5532,15 @@ func TestNetwork_InFlightRequests(t *testing.T) { resetGock() defer resetGock() - network := setupTestNetwork(t) + network := setupTestNetwork(t, nil) requestBytes := []byte(`{"jsonrpc":"2.0","method":"eth_getLogs","params":[]}`) gock.New("http://rpc1.localhost"). Post("/"). - Times(1). + Persist(). Filter(func(request *http.Request) bool { - return strings.Contains(safeReadBody(request), "eth_getLogs") + bd := safeReadBody(request) + return strings.Contains(bd, "eth_getLogs") }). Reply(200). Delay(100 * time.Second). @@ -5227,25 +5556,20 @@ func TestNetwork_InFlightRequests(t *testing.T) { req := common.NewNormalizedRequest(requestBytes) resp, err := network.Forward(ctx, req) assert.Error(t, err) - assert.True(t, common.HasErrorCode(err, "ErrNetworkRequestTimeout") || common.HasErrorCode(err, "ErrEndpointRequestTimeout")) + if !common.HasErrorCode(err, "ErrNetworkRequestTimeout") && !common.HasErrorCode(err, "ErrEndpointRequestTimeout") { + t.Errorf("Expected ErrNetworkRequestTimeout or ErrEndpointRequestTimeout, got %v", err) + } assert.Nil(t, resp) }() } wg.Wait() - - if left := anyTestMocksLeft(); left > 0 { - t.Errorf("Expected all test mocks to be consumed, got %v left", left) - for _, pending := range gock.Pending() { - t.Errorf("Pending mock: %v", pending) - } - } }) t.Run("MixedSuccessAndFailureConcurrentRequests", func(t *testing.T) { resetGock() defer resetGock() - network := setupTestNetwork(t) + network := setupTestNetwork(t, nil) successRequestBytes := []byte(`{"jsonrpc":"2.0","method":"eth_getLogs","params":[]}`) failureRequestBytes := []byte(`{"jsonrpc":"2.0","method":"eth_getBalance","params":["0x123"]}`) @@ -5302,7 +5626,7 @@ func TestNetwork_InFlightRequests(t *testing.T) { resetGock() defer resetGock() - network := setupTestNetwork(t) + network := setupTestNetwork(t, nil) requestBytes := []byte(`{"jsonrpc":"2.0","method":"eth_getLogs","params":[]}`) gock.New("http://rpc1.localhost"). @@ -5339,22 +5663,22 @@ func TestNetwork_InFlightRequests(t *testing.T) { resetGock() defer resetGock() - network := setupTestNetwork(t) + network := setupTestNetwork(t, nil) // Mock the response from the upstream gock.New("http://rpc1.localhost"). Post("/"). - Times(1). + Persist(). Reply(200). - Delay(1 * time.Second). + Delay(3 * time.Second). BodyString(`{"jsonrpc":"2.0","id":4,"result":"0x1"}`) - totalRequests := 100 + totalRequests := int64(100) // Prepare requests with different IDs requestTemplate := `{"jsonrpc":"2.0","method":"eth_getBalance","params":["0x742d35Cc6634C0532925a3b844Bc454e4438f44e", "latest"],"id":%d}` requests := make([]*common.NormalizedRequest, totalRequests) - for i := 0; i < totalRequests; i++ { + for i := int64(0); i < totalRequests; i++ { reqBytes := []byte(fmt.Sprintf(requestTemplate, i+1)) requests[i] = common.NewNormalizedRequest(reqBytes) } @@ -5364,9 +5688,9 @@ func TestNetwork_InFlightRequests(t *testing.T) { responses := make([]*common.NormalizedResponse, totalRequests) errors := make([]error, totalRequests) - for i := 0; i < totalRequests; i++ { + for i := int64(0); i < totalRequests; i++ { wg.Add(1) - go func(index int) { + go func(index int64) { defer wg.Done() responses[index], errors[index] = network.Forward(context.Background(), requests[index]) }(i) @@ -5374,16 +5698,123 @@ func TestNetwork_InFlightRequests(t *testing.T) { wg.Wait() // Verify results - for i := 0; i < totalRequests; i++ { + for i := int64(0); i < totalRequests; i++ { assert.NoError(t, errors[i], "Request %d should not return an error", i+1) assert.NotNil(t, responses[i], "Request %d should return a response", i+1) if responses[i] != nil { jrr, err := responses[i].JsonRpcResponse() assert.NoError(t, err, "Response %d should be a valid JSON-RPC response", i+1) - assert.Equal(t, float64(i+1), jrr.ID, "Response ID should match the request ID for request %d", i+1) + assert.Equal(t, i+1, jrr.ID(), "Response ID should match the request ID for request %d", i+1) + } + } + + if left := anyTestMocksLeft(); left > 1 { + t.Errorf("Expected all test mocks to be consumed, got %v left", left) + for _, pending := range gock.Pending() { + t.Errorf("Pending mock: %v", pending) + } + } + }) + + t.Run("ContextCancellationDuringRequest", func(t *testing.T) { + resetGock() + defer resetGock() + + network := setupTestNetwork(t, nil) + requestBytes := []byte(`{"jsonrpc":"2.0","method":"eth_getLogs","params":[]}`) + + gock.New("http://rpc1.localhost"). + Post("/"). + Filter(func(request *http.Request) bool { + return strings.Contains(safeReadBody(request), "eth_getLogs") + }). + Reply(200). + Delay(2 * time.Second). // Delay to ensure context cancellation occurs before response + BodyString(`{"jsonrpc":"2.0","id":1,"result":"0x1"}`) + + ctx, cancel := context.WithCancel(context.Background()) + + var wg sync.WaitGroup + + wg.Add(1) + go func() { + defer wg.Done() + time.Sleep(500 * time.Millisecond) // Wait a bit before cancelling + cancel() + }() + + req := common.NewNormalizedRequest(requestBytes) + resp, err := network.Forward(ctx, req) + + wg.Wait() // Ensure cancellation has occurred + + assert.Error(t, err) + assert.Nil(t, resp) + assert.True(t, strings.Contains(err.Error(), "context canceled")) + + // Verify cleanup + inFlightCount := 0 + network.inFlightRequests.Range(func(key, value interface{}) bool { + inFlightCount++ + return true + }) + assert.Equal(t, 0, inFlightCount, "in-flight requests map should be empty after context cancellation") + + if left := anyTestMocksLeft(); left > 0 { + t.Errorf("Expected all test mocks to be consumed, got %v left", left) + for _, pending := range gock.Pending() { + t.Errorf("Pending mock: %v", pending) } } + }) + + t.Run("LongRunningRequest", func(t *testing.T) { + resetGock() + defer resetGock() + + network := setupTestNetwork(t, nil) + requestBytes := []byte(`{"jsonrpc":"2.0","method":"eth_getLogs","params":[]}`) + + gock.New("http://rpc1.localhost"). + Post("/"). + Filter(func(request *http.Request) bool { + return strings.Contains(safeReadBody(request), "eth_getLogs") + }). + Reply(200). + Delay(5 * time.Second). // Simulate a long-running request + BodyString(`{"jsonrpc":"2.0","id":1,"result":"0x1"}`) + + var wg sync.WaitGroup + wg.Add(1) + + go func() { + defer wg.Done() + req := common.NewNormalizedRequest(requestBytes) + resp, err := network.Forward(context.Background(), req) + assert.NoError(t, err) + assert.NotNil(t, resp) + }() + + // Check in-flight requests during processing + time.Sleep(1 * time.Second) + inFlightCount := 0 + + network.inFlightRequests.Range(func(key, value interface{}) bool { + inFlightCount++ + return true + }) + assert.Equal(t, 1, inFlightCount, "should have one in-flight request during processing") + + wg.Wait() // Wait for the request to complete + + // Verify cleanup after completion + inFlightCount = 0 + network.inFlightRequests.Range(func(key, value interface{}) bool { + inFlightCount++ + return true + }) + assert.Equal(t, 0, inFlightCount, "in-flight requests map should be empty after request completion") if left := anyTestMocksLeft(); left > 0 { t.Errorf("Expected all test mocks to be consumed, got %v left", left) @@ -5394,7 +5825,7 @@ func TestNetwork_InFlightRequests(t *testing.T) { }) } -func setupTestNetwork(t *testing.T) *Network { +func setupTestNetwork(t *testing.T, upstreamConfig *common.UpstreamConfig) *Network { t.Helper() setupMocksForEvmStatePoller() @@ -5402,13 +5833,15 @@ func setupTestNetwork(t *testing.T) *Network { rateLimitersRegistry, _ := upstream.NewRateLimitersRegistry(&common.RateLimiterConfig{}, &log.Logger) metricsTracker := health.NewTracker("test", time.Minute) - upstreamConfig := &common.UpstreamConfig{ - Type: common.UpstreamTypeEvm, - Id: "test", - Endpoint: "http://rpc1.localhost", - Evm: &common.EvmUpstreamConfig{ - ChainId: 123, - }, + if upstreamConfig == nil { + upstreamConfig = &common.UpstreamConfig{ + Type: common.UpstreamTypeEvm, + Id: "test", + Endpoint: "http://rpc1.localhost", + Evm: &common.EvmUpstreamConfig{ + ChainId: 123, + }, + } } upstreamsRegistry := upstream.NewUpstreamsRegistry( &log.Logger, @@ -5437,10 +5870,19 @@ func setupTestNetwork(t *testing.T) *Network { err = upstreamsRegistry.Bootstrap(context.Background()) assert.NoError(t, err) time.Sleep(100 * time.Millisecond) + err = upstreamsRegistry.PrepareUpstreamsForNetwork(util.EvmNetworkId(123)) assert.NoError(t, err) time.Sleep(100 * time.Millisecond) + err = network.Bootstrap(context.Background()) + assert.NoError(t, err) + time.Sleep(100 * time.Millisecond) + + h, _ := common.HexToInt64("0x1273c18") + network.evmStatePollers["test"].SuggestFinalizedBlock(h) + network.evmStatePollers["test"].SuggestLatestBlock(h) + return network } @@ -5455,7 +5897,7 @@ func setupMocksForEvmStatePoller() { return strings.Contains(safeReadBody(request), "eth_getBlockByNumber") }). Reply(200). - JSON(json.RawMessage(`{"result": {"number":"0x1273c18"}}`)) + JSON([]byte(`{"result": {"number":"0x1273c18"}}`)) gock.New("http://rpc2.localhost"). Post(""). Persist(). @@ -5463,7 +5905,7 @@ func setupMocksForEvmStatePoller() { return strings.Contains(safeReadBody(request), "eth_getBlockByNumber") }). Reply(200). - JSON(json.RawMessage(`{"result": {"number":"0x1273c18"}}`)) + JSON([]byte(`{"result": {"number":"0x1273c18"}}`)) } func anyTestMocksLeft() int { diff --git a/erpc/projects.go b/erpc/projects.go index 9d7b86ef8..d20553a2c 100644 --- a/erpc/projects.go +++ b/erpc/projects.go @@ -3,11 +3,9 @@ package erpc import ( "context" "fmt" - "math" "strconv" "strings" "sync" - // "time" "github.com/erpc/erpc/auth" "github.com/erpc/erpc/common" @@ -74,12 +72,12 @@ func (p *PreparedProject) Forward(ctx context.Context, networkId string, nq *com if err != nil { return nil, err } - method, _ := nq.Method() - if err := p.acquireRateLimitPermit(nq); err != nil { return nil, err } + method, _ := nq.Method() + timer := prometheus.NewTimer(prometheus.ObserverFunc(func(v float64) { health.MetricNetworkRequestDuration.WithLabelValues( p.Config.Id, @@ -90,7 +88,7 @@ func (p *PreparedProject) Forward(ctx context.Context, networkId string, nq *com defer timer.ObserveDuration() health.MetricNetworkRequestsReceived.WithLabelValues(p.Config.Id, network.NetworkId, method).Inc() - lg := p.Logger.With().Str("method", method).Str("id", nq.Id()).Str("ptr", fmt.Sprintf("%p", nq)).Logger() + lg := p.Logger.With().Str("method", method).Int64("id", nq.Id()).Str("ptr", fmt.Sprintf("%p", nq)).Logger() lg.Debug().Msgf("forwarding request to network") resp, err := network.Forward(ctx, nq) @@ -127,27 +125,8 @@ func (p *PreparedProject) initializeNetwork(networkId string) (*Network, error) } if nwCfg == nil { - allUps, err := p.upstreamsRegistry.GetSortedUpstreams(networkId, "*") - if err != nil { - return nil, err - } nwCfg = &common.NetworkConfig{ - Failsafe: &common.FailsafeConfig{ - Hedge: &common.HedgePolicyConfig{ - Delay: "200ms", - MaxCount: 3, - }, - Retry: &common.RetryPolicyConfig{ - MaxAttempts: int(math.Min(float64(len(allUps)), 3)), - Delay: "1s", - Jitter: "500ms", - BackoffMaxDelay: "10s", - BackoffFactor: 2, - }, - Timeout: &common.TimeoutPolicyConfig{ - Duration: "30s", - }, - }, + Failsafe: common.NetworkDefaultFailsafeConfig, } s := strings.Split(networkId, ":") diff --git a/erpc/projects_registry.go b/erpc/projects_registry.go index 003a8eec0..1311ca082 100644 --- a/erpc/projects_registry.go +++ b/erpc/projects_registry.go @@ -133,6 +133,14 @@ func (r *ProjectsRegistry) RegisterProject(prjCfg *common.ProjectConfig) (*Prepa return pp, nil } +func (r *ProjectsRegistry) GetAll() []*PreparedProject { + projects := make([]*PreparedProject, 0, len(r.preparedProjects)) + for _, project := range r.preparedProjects { + projects = append(projects, project) + } + return projects +} + func (r *ProjectsRegistry) loadProject(projectId string) (*PreparedProject, error) { for _, prjCfg := range r.staticProjects { if prjCfg.Id == projectId { diff --git a/go.mod b/go.mod index 8cd8a2c24..3516b2e95 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.22.3 require ( github.com/IGLOU-EU/go-wildcard/v2 v2.0.2 github.com/aws/aws-sdk-go v1.55.4 - github.com/bytedance/sonic v1.12.2 + github.com/bytedance/sonic v1.12.3 github.com/ethereum/go-ethereum v1.14.7 github.com/failsafe-go/failsafe-go v0.6.8 github.com/golang-jwt/jwt/v4 v4.5.0 @@ -22,10 +22,12 @@ require ( gopkg.in/yaml.v3 v3.0.1 ) +replace github.com/failsafe-go/failsafe-go v0.6.8 => github.com/aramalipoor/failsafe-go v0.0.0-20241002125322-de01986d3951 + require ( github.com/andybalholm/brotli v1.1.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bits-and-blooms/bitset v1.13.0 // indirect + github.com/bits-and-blooms/bitset v1.14.2 // indirect github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect github.com/bytedance/sonic/loader v0.2.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect diff --git a/go.sum b/go.sum index fb52fd5f6..1818fa083 100644 --- a/go.sum +++ b/go.sum @@ -4,12 +4,14 @@ github.com/IGLOU-EU/go-wildcard/v2 v2.0.2/go.mod h1:/sUMQ5dk2owR0ZcjRI/4AZ+bUFF5 github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= +github.com/aramalipoor/failsafe-go v0.0.0-20241002125322-de01986d3951 h1:oyM25QnrKnHMGHoZo+B8GmIexJ6B3wjaAODDQHEa1g4= +github.com/aramalipoor/failsafe-go v0.0.0-20241002125322-de01986d3951/go.mod h1:wtwrTcEhvCOtq/Te7tPDq5nBQQMWlduXO1pm0PoWn1w= github.com/aws/aws-sdk-go v1.55.4 h1:u7sFWQQs5ivGuYvCxi7gJI8nN/P9Dq04huLaw39a4lg= github.com/aws/aws-sdk-go v1.55.4/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE= -github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/bits-and-blooms/bitset v1.14.2 h1:YXVoyPndbdvcEVcseEovVfp0qjJp7S+i5+xgp/Nfbdc= +github.com/bits-and-blooms/bitset v1.14.2/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= @@ -18,8 +20,8 @@ github.com/btcsuite/btcd/btcec/v2 v2.3.2 h1:5n0X6hX0Zk+6omWcihdYvdAlGf2DfasC0GMf github.com/btcsuite/btcd/btcec/v2 v2.3.2/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY1W/17j2MW85J04= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= -github.com/bytedance/sonic v1.12.2 h1:oaMFuRTpMHYLpCntGca65YWt5ny+wAceDERTkT2L9lg= -github.com/bytedance/sonic v1.12.2/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= +github.com/bytedance/sonic v1.12.3 h1:W2MGa7RCU1QTeYRTPE3+88mVC0yXmsRQRChiyVocVjU= +github.com/bytedance/sonic v1.12.3/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/bytedance/sonic/loader v0.2.0 h1:zNprn+lsIP06C/IqCHs3gPQIvnvpKbbxyXQP1iU4kWM= github.com/bytedance/sonic/loader v0.2.0/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= @@ -48,8 +50,6 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/ethereum/go-ethereum v1.14.7 h1:EHpv3dE8evQmpVEQ/Ne2ahB06n2mQptdwqaMNhAT29g= github.com/ethereum/go-ethereum v1.14.7/go.mod h1:Mq0biU2jbdmKSZoqOj29017ygFrMnB5/Rifwp980W4o= -github.com/failsafe-go/failsafe-go v0.6.8 h1:ERrJMknjXdtDVrx1s05uE5MCDhGTiF7rQ98z6bdVUOw= -github.com/failsafe-go/failsafe-go v0.6.8/go.mod h1:LAo0yJE2PXn1z4T22bkmUxPryrTHUvMhvnwik9x2uq8= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -324,10 +324,10 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 h1:wpZ8pe2x1Q3f2KyT5f8oP/fa9rHAKgFPr/HZdNuS+PQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 h1:Zy9XzmMEflZ/MAaA7vNcoebnRAld7FsPW1EeBB7V0m8= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= -google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= -google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 h1:1GBuWVLM/KMVUv1t1En5Gs+gFZCNd360GGb4sSxtrhU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= +google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/test/cmd/main.go b/test/cmd/main.go index f161145ac..4d63f868e 100644 --- a/test/cmd/main.go +++ b/test/cmd/main.go @@ -15,9 +15,14 @@ import ( func main() { // Define server configurations serverConfigs := []test.ServerConfig{ - {Port: 8081, FailureRate: 0.1, LimitedRate: 0.9, MinDelay: 50 * time.Millisecond, MaxDelay: 200 * time.Millisecond, SampleFile: "test/samples/evm-json-rpc.json"}, - {Port: 8082, FailureRate: 0.3, LimitedRate: 0.8, MinDelay: 100 * time.Millisecond, MaxDelay: 300 * time.Millisecond, SampleFile: "test/samples/evm-json-rpc.json"}, - {Port: 8083, FailureRate: 0.05, LimitedRate: 0.9, MinDelay: 30 * time.Millisecond, MaxDelay: 150 * time.Millisecond, SampleFile: "test/samples/evm-json-rpc.json"}, + {Port: 9081, FailureRate: 0.1, LimitedRate: 0.005, MinDelay: 50 * time.Millisecond, MaxDelay: 200 * time.Millisecond, SampleFile: "test/samples/evm-json-rpc.json"}, + {Port: 9082, FailureRate: 0.3, LimitedRate: 0.2, MinDelay: 100 * time.Millisecond, MaxDelay: 300 * time.Millisecond, SampleFile: "test/samples/evm-json-rpc.json"}, + {Port: 9083, FailureRate: 0.05, LimitedRate: 0.01, MinDelay: 30 * time.Millisecond, MaxDelay: 150 * time.Millisecond, SampleFile: "test/samples/evm-json-rpc.json"}, + } + + // Add servers up to 9185 + for i := 9084; i <= 9185; i++ { + serverConfigs = append(serverConfigs, test.ServerConfig{Port: i, FailureRate: 0.05, LimitedRate: 0.01, MinDelay: 30 * time.Millisecond, MaxDelay: 150 * time.Millisecond, SampleFile: "test/samples/evm-json-rpc.json"}) } // Create fake servers using the existing function diff --git a/test/fake_server.go b/test/fake_server.go index 34c25f914..115f8a4a7 100644 --- a/test/fake_server.go +++ b/test/fake_server.go @@ -2,16 +2,19 @@ package test import ( + "bytes" "encoding/json" "fmt" + "io" "math/rand" "net/http" "os" + "strconv" + "strings" "sync" "time" "github.com/bytedance/sonic" - "github.com/erpc/erpc/common" ) type JSONRPCRequest struct { @@ -101,40 +104,118 @@ func (fs *FakeServer) handleRequest(w http.ResponseWriter, r *http.Request) { fs.requestsHandled++ fs.mu.Unlock() - // Simulate delay - time.Sleep(fs.randomDelay()) - - // Decode JSON-RPC request - var req JSONRPCRequest - err := json.NewDecoder(r.Body).Decode(&req) + // Read the request body + body, err := io.ReadAll(r.Body) if err != nil { + fs.sendErrorResponse(w, nil, -32700, "Parse error") + return + } + + // Try to unmarshal as a batch request + var batchReq []JSONRPCRequest + err = json.Unmarshal(body, &batchReq) + if err == nil { + // Handle batch request + fs.handleBatchRequest(w, batchReq) + return + } + + // Try to unmarshal as a single request + var singleReq JSONRPCRequest + err = json.Unmarshal(body, &singleReq) + if err == nil { + // Handle single request + fs.handleSingleRequest(w, singleReq) + return + } + + // Invalid JSON-RPC request + fs.sendErrorResponse(w, nil, -32600, "Invalid Request") +} + +func (fs *FakeServer) handleBatchRequest(w http.ResponseWriter, batchReq []JSONRPCRequest) { + if len(batchReq) == 0 { + // Empty batch request + fs.sendErrorResponse(w, nil, -32600, "Invalid Request") + return + } + + responses := make([]JSONRPCResponse, 0, len(batchReq)) + for _, req := range batchReq { + response := fs.processSingleRequest(req) + if response != nil { + response.ID = req.ID + responses = append(responses, *response) + } + } + + if len(responses) > 0 { + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(responses) + } +} + +func (fs *FakeServer) handleSingleRequest(w http.ResponseWriter, req JSONRPCRequest) { + response := fs.processSingleRequest(req) + if response != nil { + w.Header().Set("Content-Type", "application/json") + response.ID = req.ID + + buf := &bytes.Buffer{} + if err := json.NewEncoder(buf).Encode(response); err != nil { + fs.sendErrorResponse(w, req.ID, -32000, "Internal error") + return + } + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Length", strconv.Itoa(len(buf.Bytes()))) + w.Write(buf.Bytes()) + } +} + +func (fs *FakeServer) processSingleRequest(req JSONRPCRequest) *JSONRPCResponse { + // Validate request + if req.Jsonrpc != "2.0" || req.Method == "" { fs.mu.Lock() fs.requestsFailed++ fs.mu.Unlock() - http.Error(w, "Invalid JSON-RPC request", http.StatusBadRequest) - return + return &JSONRPCResponse{ + Jsonrpc: "2.0", + Error: map[string]interface{}{"code": -32600, "message": "Invalid Request"}, + ID: req.ID, + } } + // Simulate delay + time.Sleep(fs.randomDelay()) + + // Simulate rate limiting if rand.Float64() < fs.LimitedRate { fs.mu.Lock() fs.requestsLimited++ fs.mu.Unlock() - http.Error(w, "Request limited", http.StatusTooManyRequests) - return + if req.ID != nil { + return &JSONRPCResponse{ + Jsonrpc: "2.0", + Error: map[string]interface{}{"code": -32000, "message": "simulated capacity exceeded"}, + ID: req.ID, + } + } + return nil // Notification, no response } - // Simulate failure based on failure rate + // Simulate failure if rand.Float64() < fs.FailureRate { fs.mu.Lock() fs.requestsFailed++ fs.mu.Unlock() - response := JSONRPCResponse{ - Jsonrpc: "2.0", - Error: map[string]interface{}{"code": common.JsonRpcErrorServerSideException, "message": "simulated internal server failure"}, - ID: req.ID, + if req.ID != nil { + return &JSONRPCResponse{ + Jsonrpc: "2.0", + Error: map[string]interface{}{"code": -32001, "message": "simulated internal server failure"}, + ID: req.ID, + } } - json.NewEncoder(w).Encode(response) - return + return nil // Notification, no response } fs.mu.Lock() @@ -143,7 +224,7 @@ func (fs *FakeServer) handleRequest(w http.ResponseWriter, r *http.Request) { // Find matching sample or use default response response := fs.findMatchingSample(req) - if response == nil { + if response == nil && req.ID != nil { response = &JSONRPCResponse{ Jsonrpc: "2.0", Result: fmt.Sprintf("Default response for method: %s", req.Method), @@ -151,6 +232,20 @@ func (fs *FakeServer) handleRequest(w http.ResponseWriter, r *http.Request) { } } + return response +} + +func (fs *FakeServer) sendErrorResponse(w http.ResponseWriter, id interface{}, code int, message string) { + fs.mu.Lock() + fs.requestsFailed++ + fs.mu.Unlock() + + response := JSONRPCResponse{ + Jsonrpc: "2.0", + Error: map[string]interface{}{"code": code, "message": message}, + ID: id, + } + w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(response) } @@ -166,6 +261,18 @@ func (fs *FakeServer) findMatchingSample(req JSONRPCRequest) *JSONRPCResponse { } } + if txt, ok := sample.Response.Result.(string); ok { + if strings.HasPrefix(txt, "bigfake:") { + size, err := strconv.Atoi(txt[len("bigfake:"):]) + fmt.Printf("Generating big response of size %d\n", size) + if err != nil { + return nil + } + // generate a random string with the size of the number + sample.Response.Result = make([]byte, size) + } + } + return &sample.Response } } diff --git a/test/init_test.go b/test/init_test.go new file mode 100644 index 000000000..ddfb2fef8 --- /dev/null +++ b/test/init_test.go @@ -0,0 +1,9 @@ +package test + +import ( + "github.com/rs/zerolog" +) + +func init() { + zerolog.SetGlobalLevel(zerolog.Disabled) +} diff --git a/test/k6/run.js b/test/k6/run.js index 57decda4e..03a97fe35 100644 --- a/test/k6/run.js +++ b/test/k6/run.js @@ -5,10 +5,16 @@ import { Rate } from 'k6/metrics'; const errorRate = new Rate('errors'); export const options = { - stages: [ - { duration: '20s', target: 10000 }, - { duration: '5m', target: 10000 }, - ], + scenarios: { + constant_request_rate: { + executor: 'constant-arrival-rate', + rate: 500, + timeUnit: '1s', + duration: '30m', + preAllocatedVUs: 1000, + maxVUs: 1000, + }, + }, ext: { loadimpact: { distribution: { @@ -27,6 +33,13 @@ const samplePayload = JSON.stringify({ false ] }); +// const samplePayload = JSON.stringify({ +// "jsonrpc": "2.0", +// "method": "debug_traceTransaction", +// "params": [ +// "0xe6c2decd68012e0245599ddf93c232bf92884758393a502852cbf2f393e3d99c" +// ] +// }); export default function () { const params = { @@ -39,8 +52,13 @@ export default function () { check(res, { 'status is 200': (r) => r.status === 200, 'response has no error': (r) => { - const body = JSON.parse(r.body); - return body && (body.error === undefined || body.error === null); + try { + const body = JSON.parse(r.body); + return body && (body.error === undefined || body.error === null); + } catch (e) { + console.log(`Unmarshal error: ${e} for body: ${r.body}`); + return false; + } }, }); diff --git a/test/samples/evm-json-rpc.json b/test/samples/evm-json-rpc.json index 92b94468c..2686ebab3 100644 --- a/test/samples/evm-json-rpc.json +++ b/test/samples/evm-json-rpc.json @@ -1436,5 +1436,18 @@ "status": "0x1" } } + }, + { + "request": { + "jsonrpc": "2.0", + "method": "debug_traceTransaction", + "params": [ + "0xe6c2decd68012e0245599ddf93c232bf92884758393a502852cbf2f393e3d99c" + ] + }, + "response": { + "jsonrpc": "2.0", + "result": "bigfake:50000000" + } } ] \ No newline at end of file diff --git a/upstream/alchemy_http_json_rpc_client.go b/upstream/alchemy_http_json_rpc_client.go index 8f82d55fc..61397b6f4 100644 --- a/upstream/alchemy_http_json_rpc_client.go +++ b/upstream/alchemy_http_json_rpc_client.go @@ -50,6 +50,8 @@ var alchemyNetworkSubdomains = map[int64]string{ 84532: "base-sepolia", 97: "bnb-testnet", 999999999: "zora-sepolia", + 534351: "scroll-sepolia", + 534352: "scroll-mainnet", } type AlchemyHttpJsonRpcClient struct { diff --git a/upstream/envio_http_json_rpc_client.go b/upstream/envio_http_json_rpc_client.go index 96d2276e8..e4d37c2bd 100644 --- a/upstream/envio_http_json_rpc_client.go +++ b/upstream/envio_http_json_rpc_client.go @@ -129,7 +129,12 @@ func (c *EnvioHttpJsonRpcClient) SupportsNetwork(networkId string) (bool, error) return false, err } - cidh, err := common.NormalizeHex(jrr.Result) + cids, err := jrr.PeekStringByPath() + if err != nil { + return false, err + } + + cidh, err := common.NormalizeHex(cids) if err != nil { return false, err } diff --git a/upstream/evm_state_poller.go b/upstream/evm_state_poller.go index 15da1c13f..4495a9c95 100644 --- a/upstream/evm_state_poller.go +++ b/upstream/evm_state_poller.go @@ -29,7 +29,7 @@ type EvmStatePoller struct { // - When self-hosted nodes have issues flipping back and forth between syncing state. // - When a third-party provider is using a degraded network of upstream nodes that are in different states. // - // To save memory, 0 means we haven't checked, 1 means we have checked but it's still syncing OR + // To save memory, 0 means we haven't checked, 1+ means we have checked but it's still syncing OR // we haven't received enough "synced" responses to assume it's fully synced. synced int8 @@ -315,28 +315,11 @@ func (e *EvmStatePoller) fetchBlock(ctx context.Context, blockTag string) (int64 return 0, jrr.Error } - // If result is nil or has an invalid structure, return an error - result, err := jrr.ParsedResult() + numberStr, err := jrr.PeekStringByPath("number") if err != nil { - return 0, err - } - resultMap, ok := result.(map[string]interface{}) - if !ok || resultMap == nil || resultMap["number"] == nil { return 0, &common.BaseError{ Code: "ErrEvmStatePoller", - Message: "block not found", - Details: map[string]interface{}{ - "blockTag": blockTag, - "result": jrr.Result, - }, - } - } - - numberStr, ok := resultMap["number"].(string) - if !ok { - return 0, &common.BaseError{ - Code: "ErrEvmStatePoller", - Message: "block number is not a string", + Message: "cannot get block number from block data", Details: map[string]interface{}{ "blockTag": blockTag, "result": jrr.Result, @@ -370,20 +353,17 @@ func (e *EvmStatePoller) fetchSyncingState(ctx context.Context) (bool, error) { return false, jrr.Error } - res, err := jrr.ParsedResult() + var syncing bool + err = common.SonicCfg.Unmarshal(jrr.Result, &syncing) if err != nil { - return false, err - } - - if syncing, ok := res.(bool); ok { - return syncing, nil + return false, &common.BaseError{ + Code: "ErrEvmStatePoller", + Message: "cannot get syncing state result type (must be boolean)", + Details: map[string]interface{}{ + "result": util.Mem2Str(jrr.Result), + }, + } } - return false, &common.BaseError{ - Code: "ErrEvmStatePoller", - Message: "invalid syncing state result type (must be boolean)", - Details: map[string]interface{}{ - "result": res, - }, - } + return syncing, nil } diff --git a/upstream/failsafe.go b/upstream/failsafe.go index a7df7c3d9..9c143b269 100644 --- a/upstream/failsafe.go +++ b/upstream/failsafe.go @@ -66,7 +66,7 @@ func CreateFailSafePolicies(logger *zerolog.Logger, scope Scope, component strin } if fsCfg.Hedge != nil { - p, err := createHedgePolicy(component, fsCfg.Hedge) + p, err := createHedgePolicy(logger, component, fsCfg.Hedge) if err != nil { return nil, err } @@ -147,6 +147,7 @@ func createCircuitBreakerPolicy(logger *zerolog.Logger, component string, cfg *c } } lg.Msg("failure caught that will be considered for circuit breaker") + // TODO emit a custom prometheus metric to track CB root causes? }) builder.HandleIf(func(result *common.NormalizedResponse, err error) bool { @@ -186,7 +187,7 @@ func createCircuitBreakerPolicy(logger *zerolog.Logger, component string, cfg *c return builder.Build(), nil } -func createHedgePolicy(component string, cfg *common.HedgePolicyConfig) (failsafe.Policy[*common.NormalizedResponse], error) { +func createHedgePolicy(logger *zerolog.Logger, component string, cfg *common.HedgePolicyConfig) (failsafe.Policy[*common.NormalizedResponse], error) { delay, err := time.ParseDuration(cfg.Delay) if err != nil { return nil, common.NewErrFailsafeConfiguration(fmt.Errorf("failed to parse hedge.delay: %v", err), map[string]interface{}{ @@ -200,6 +201,20 @@ func createHedgePolicy(component string, cfg *common.HedgePolicyConfig) (failsaf builder = builder.WithMaxHedges(cfg.MaxCount) } + builder.OnHedge(func(event failsafe.ExecutionEvent[*common.NormalizedResponse]) bool { + req := event.Context().Value(common.RequestContextKey).(*common.NormalizedRequest) + if req != nil { + method, _ := req.Method() + if method != "" && common.IsEvmWriteMethod(method) { + logger.Debug().Msgf("ignoring hedge for write request: %s", method) + return false + } + } + + // Continue with the next hedge + return true + }) + return builder.Build(), nil } @@ -251,7 +266,7 @@ func createRetryPolicy(scope Scope, component string, cfg *common.RetryPolicyCon builder.HandleIf(func(result *common.NormalizedResponse, err error) bool { // 400 / 404 / 405 / 413 -> No Retry // RPC-RPC client-side error (invalid params) -> No Retry - if common.HasErrorCode(err, common.ErrCodeEndpointClientSideException) { + if common.IsClientError(err) { return false } @@ -336,6 +351,14 @@ func createRetryPolicy(scope Scope, component string, cfg *common.RetryPolicyCon } } + // Must not retry any 'write' methods + if result != nil && result.Request() != nil { + method, _ := result.Request().Method() + if method != "" && common.IsEvmWriteMethod(method) { + return false + } + } + // 5xx -> Retry return err != nil }) @@ -387,19 +410,21 @@ func TranslateFailsafeError(upstreamId, method string, execErr error) error { } if err != nil { - if method != "" { - if ser, ok := execErr.(common.StandardError); ok { - be := ser.Base() - if be != nil { - if upstreamId != "" { - be.Details = map[string]interface{}{ - "upstreamId": upstreamId, - "method": method, - } - } else { - be.Details = map[string]interface{}{ - "method": method, - } + if ser, ok := execErr.(common.StandardError); ok { + be := ser.Base() + if be != nil { + if upstreamId != "" && method != "" { + be.Details = map[string]interface{}{ + "upstreamId": upstreamId, + "method": method, + } + } else if method != "" { + be.Details = map[string]interface{}{ + "method": method, + } + } else if upstreamId != "" { + be.Details = map[string]interface{}{ + "upstreamId": upstreamId, } } } diff --git a/upstream/http_json_json_client_test.go b/upstream/http_json_json_client_test.go index 41fb6b4bd..c78eedeb7 100644 --- a/upstream/http_json_json_client_test.go +++ b/upstream/http_json_json_client_test.go @@ -1,6 +1,7 @@ package upstream import ( + "bytes" "context" "fmt" "net/url" @@ -172,17 +173,17 @@ func TestHttpJsonRpcClient_BatchRequests(t *testing.T) { client, err := NewGenericHttpJsonRpcClient(&logger, &Upstream{ config: &common.UpstreamConfig{ - Endpoint: "http://rpc1.localhost:8545", + Endpoint: "http://rpc1.localhost", JsonRpc: &common.JsonRpcUpstreamConfig{ SupportsBatch: &common.TRUE, BatchMaxSize: 5, BatchMaxWait: "500ms", }, }, - }, &url.URL{Scheme: "http", Host: "rpc1.localhost:8545"}) + }, &url.URL{Scheme: "http", Host: "rpc1.localhost"}) assert.NoError(t, err) - gock.New("http://rpc1.localhost:8545"). + gock.New("http://rpc1.localhost"). Post("/"). Times(5). Reply(200). @@ -196,7 +197,15 @@ func TestHttpJsonRpcClient_BatchRequests(t *testing.T) { req1 := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"eth_blockNumber","params":[]}`)) resp1, err1 := client.SendRequest(context.Background(), req1) assert.NoError(t, err1) - assert.Equal(t, `{"jsonrpc":"2.0","id":1,"result":"0x1"}`, string(resp1.Body())) + if resp1 == nil { + panic(fmt.Sprintf("SeparateBatchRequestsWithSameIDs: resp1 is nil err1: %v", err1)) + } + wr := bytes.NewBuffer([]byte{}) + rdr, werr := resp1.GetReader() + assert.NoError(t, werr) + wr.ReadFrom(rdr) + txt := wr.String() + assert.Equal(t, `{"jsonrpc":"2.0","id":1,"result":"0x1"}`, txt) }() wg.Add(1) go func() { @@ -204,7 +213,12 @@ func TestHttpJsonRpcClient_BatchRequests(t *testing.T) { req6 := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":6,"method":"eth_blockNumber","params":[]}`)) resp6, err6 := client.SendRequest(context.Background(), req6) assert.NoError(t, err6) - assert.Equal(t, `{"jsonrpc":"2.0","id":6,"result":"0x6"}`, string(resp6.Body())) + wr := bytes.NewBuffer([]byte{}) + rdr, werr := resp6.GetReader() + wr.ReadFrom(rdr) + assert.NoError(t, werr) + txt := wr.String() + assert.Equal(t, `{"jsonrpc":"2.0","id":6,"result":"0x6"}`, txt) }() time.Sleep(10 * time.Millisecond) } @@ -346,7 +360,7 @@ func TestHttpJsonRpcClient_BatchRequests(t *testing.T) { req := common.NewNormalizedRequest([]byte(fmt.Sprintf(`{"jsonrpc":"2.0","id":%d,"method":"eth_blockNumber","params":[]}`, id))) _, err := client.SendRequest(context.Background(), req) assert.Error(t, err) - assert.Contains(t, err.Error(), "upstream returned non-JSON response body") + assert.Contains(t, err.Error(), "503 Service Unavailable") }(i + 1) } wg.Wait() @@ -592,8 +606,8 @@ func TestHttpJsonRpcClient_BatchRequestErrors(t *testing.T) { start := time.Now() _, err := client.SendRequest(ctx, req) dur := time.Since(start) - assert.Greater(t, dur, 745*time.Millisecond) - assert.Less(t, dur, 755*time.Millisecond) + assert.Greater(t, dur, 740*time.Millisecond) + assert.Less(t, dur, 780*time.Millisecond) assert.Error(t, err) assert.Contains(t, err.Error(), "remote endpoint request timeout") }(i + 1) diff --git a/upstream/http_json_rpc_client.go b/upstream/http_json_rpc_client.go index 469d62b55..adb6d6f3c 100644 --- a/upstream/http_json_rpc_client.go +++ b/upstream/http_json_rpc_client.go @@ -3,17 +3,15 @@ package upstream import ( "bytes" "context" - "encoding/json" "errors" "fmt" - "io" "net/http" "net/url" "strings" "sync" "time" - "github.com/bytedance/sonic" + "github.com/bytedance/sonic/ast" "github.com/erpc/erpc/common" "github.com/erpc/erpc/util" "github.com/rs/zerolog" @@ -171,16 +169,29 @@ func (c *GenericHttpJsonRpcClient) queueRequest(id interface{}, req *batchReques c.batchDeadline = &ctxd } } - c.logger.Debug().Msgf("queuing request %s for batch (current batch: %d)", id, len(c.batchRequests)) + c.logger.Debug().Msgf("queuing request %+v for batch (current batch size: %d)", id, len(c.batchRequests)) + + if c.logger.GetLevel() == zerolog.TraceLevel { + for _, req := range c.batchRequests { + jrr, _ := req.request.JsonRpcRequest() + jrr.Lock() + rqs, _ := common.SonicCfg.Marshal(jrr) + jrr.Unlock() + c.logger.Trace().Interface("id", req.request.Id()).Str("method", jrr.Method).Msgf("pending batch request: %s", string(rqs)) + } + } if len(c.batchRequests) == 1 { + c.logger.Trace().Interface("id", id).Msgf("starting batch timer") c.batchTimer = time.AfterFunc(c.batchMaxWait, c.processBatch) c.batchMu.Unlock() } else if len(c.batchRequests) >= c.batchMaxSize { + c.logger.Trace().Interface("id", id).Msgf("stopping batch timer to process total of %d requests", len(c.batchRequests)) c.batchTimer.Stop() c.batchMu.Unlock() c.processBatch() } else { + c.logger.Trace().Interface("id", id).Msgf("continue waiting for batch") c.batchMu.Unlock() } } @@ -190,6 +201,7 @@ func (c *GenericHttpJsonRpcClient) processBatch() { var cancelCtx context.CancelFunc c.batchMu.Lock() + c.logger.Debug().Msgf("processing batch with %d requests", len(c.batchRequests)) requests := c.batchRequests if c.batchDeadline != nil { batchCtx, cancelCtx = context.WithDeadline(context.Background(), *c.batchDeadline) @@ -205,11 +217,11 @@ func (c *GenericHttpJsonRpcClient) processBatch() { if ln == 0 { return } - c.logger.Debug().Msgf("processing batch with %d requests", ln) batchReq := make([]common.JsonRpcRequest, 0, ln) for _, req := range requests { jrReq, err := req.request.JsonRpcRequest() + c.logger.Trace().Interface("id", req.request.Id()).Str("method", jrReq.Method).Msgf("preparing batch request") if err != nil { req.err <- common.NewErrUpstreamRequest( err, @@ -221,6 +233,7 @@ func (c *GenericHttpJsonRpcClient) processBatch() { continue } req.request.RLock() + jrReq.RLock() batchReq = append(batchReq, common.JsonRpcRequest{ JSONRPC: jrReq.JSONRPC, Method: jrReq.Method, @@ -229,9 +242,13 @@ func (c *GenericHttpJsonRpcClient) processBatch() { }) } - requestBody, err := sonic.Marshal(batchReq) + requestBody, err := common.SonicCfg.Marshal(batchReq) for _, req := range requests { req.request.RUnlock() + jrReq, _ := req.request.JsonRpcRequest() + if jrReq != nil { + jrReq.RUnlock() + } } if err != nil { for _, req := range requests { @@ -242,62 +259,47 @@ func (c *GenericHttpJsonRpcClient) processBatch() { c.logger.Debug().Msgf("sending batch json rpc POST request to %s: %s", c.Url.Host, requestBody) - httpReq, errReq := http.NewRequestWithContext(batchCtx, "POST", c.Url.String(), bytes.NewBuffer(requestBody)) - httpReq.Header.Set("Content-Type", "application/json") - httpReq.Header.Set("User-Agent", fmt.Sprintf("erpc (Project/%s; Budget/%s)", c.upstream.ProjectId, c.upstream.config.RateLimitBudget)) - if errReq != nil { + reqStartTime := time.Now() + httpReq, err := http.NewRequestWithContext(batchCtx, "POST", c.Url.String(), bytes.NewReader(requestBody)) + if err != nil { for _, req := range requests { - req.err <- errReq + req.err <- &common.BaseError{ + Code: "ErrHttp", + Message: fmt.Sprintf("%v", err), + Details: map[string]interface{}{ + "url": c.Url.String(), + "upstreamId": c.upstream.Config().Id, + "request": requestBody, + }, + } } return } + httpReq.Header.Set("Content-Type", "application/json") + httpReq.Header.Set("User-Agent", fmt.Sprintf("erpc (%s/%s; Project/%s; Budget/%s)", common.ErpcVersion, common.ErpcCommitSha, c.upstream.ProjectId, c.upstream.config.RateLimitBudget)) - batchRespChan := make(chan *http.Response, 1) - batchErrChan := make(chan error, 1) - - startedAt := time.Now() - go func() { - resp, err := c.httpClient.Do(httpReq) - if err != nil { - if resp != nil { - er := resp.Body.Close() - if er != nil { - c.logger.Error().Err(er).Msgf("failed to close response body") - } + // Make the HTTP request + resp, err := c.httpClient.Do(httpReq) + if err != nil { + if errors.Is(err, context.DeadlineExceeded) { + for _, req := range requests { + req.err <- common.NewErrEndpointRequestTimeout(time.Since(reqStartTime)) } - batchErrChan <- err } else { - batchRespChan <- resp - } - }() - - select { - case <-batchCtx.Done(): - for _, req := range requests { - err := batchCtx.Err() - if errors.Is(err, context.DeadlineExceeded) { - err = common.NewErrEndpointRequestTimeout(time.Since(startedAt)) + for _, req := range requests { + req.err <- common.NewErrEndpointTransportFailure(err) } - req.err <- err - } - return - case err := <-batchErrChan: - for _, req := range requests { - req.err <- common.NewErrEndpointServerSideException( - fmt.Errorf(strings.ReplaceAll(err.Error(), c.Url.String(), "")), - nil, - ) } return - case resp := <-batchRespChan: - c.processBatchResponse(requests, resp) - return } + + c.processBatchResponse(requests, resp) } func (c *GenericHttpJsonRpcClient) processBatchResponse(requests map[interface{}]*batchRequest, resp *http.Response) { defer resp.Body.Close() - respBody, err := io.ReadAll(resp.Body) + + bodyBytes, err := util.ReadAll(resp.Body, 128*1024, int(resp.ContentLength)) // 128KB if err != nil { for _, req := range requests { req.err <- err @@ -305,77 +307,140 @@ func (c *GenericHttpJsonRpcClient) processBatchResponse(requests map[interface{} return } - if c.logger.GetLevel() == zerolog.DebugLevel { - c.logger.Debug().Str("body", string(respBody)).Msgf("received batch response") - } + searcher := ast.NewSearcher(util.Mem2Str(bodyBytes)) + searcher.CopyReturn = false + searcher.ConcurrentRead = false + searcher.ValidateJSON = false - // Usually when upstream is dead and returns a non-JSON response body - if respBody[0] == '<' { - for _, req := range requests { - req.err <- common.NewErrEndpointServerSideException( - fmt.Errorf("upstream returned non-JSON response body"), - map[string]interface{}{ - "statusCode": resp.StatusCode, - "headers": resp.Header, - "body": string(respBody), - }, - ) - } - return - } - - var batchResp []json.RawMessage - err = sonic.Unmarshal(respBody, &batchResp) + rootNode, err := searcher.GetByPath() if err != nil { - // Try parsing as single json-rpc object, - // some providers return a single object on some errors even when request is batch. - // this is a workaround to handle those cases. - nr := common.NewNormalizedResponse().WithBody(respBody) - for _, br := range requests { - inr, err := common.CopyResponseForRequest(nr, br.request) - if err != nil { - br.err <- err - continue + jrResp := &common.JsonRpcResponse{} + err = jrResp.ParseError(util.Mem2Str(bodyBytes)) + if err != nil { + for _, req := range requests { + req.err <- err } - err = c.normalizeJsonRpcError(resp, inr) + return + } + for _, req := range requests { + jrr, err := jrResp.Clone() if err != nil { - br.err <- err + req.err <- err } else { - br.response <- nr + nr := common.NewNormalizedResponse(). + WithRequest(req.request). + WithJsonRpcResponse(jrr) + err = c.normalizeJsonRpcError(resp, nr) + req.err <- err } } return } - for _, rawResp := range batchResp { - var jrResp common.JsonRpcResponse - err := sonic.Unmarshal(rawResp, &jrResp) + if rootNode.TypeSafe() == ast.V_ARRAY { + arrNodes, err := rootNode.ArrayUseNode() if err != nil { - continue + for _, req := range requests { + req.err <- err + } + return } - - if req, ok := requests[jrResp.ID]; ok { - nr := common.NewNormalizedResponse().WithRequest(req.request).WithBody(rawResp) + for _, elemNode := range arrNodes { + var id interface{} + jrResp, err := getJsonRpcResponseFromNode(elemNode) + if jrResp != nil { + id = jrResp.ID() + } + if id == nil { + c.logger.Warn().Msgf("unexpected response received without ID: %s", util.Mem2Str(bodyBytes)) + } else if req, ok := requests[id]; ok { + nr := common.NewNormalizedResponse().WithRequest(req.request).WithJsonRpcResponse(jrResp) + if err != nil { + req.err <- err + } else { + err := c.normalizeJsonRpcError(resp, nr) + if err != nil { + req.err <- err + } else { + req.response <- nr + } + } + delete(requests, id) + } else { + c.logger.Warn().Msgf("unexpected response received with ID: %s", id) + } + } + // Handle any remaining requests that didn't receive a response + anyMissingId := false + for _, req := range requests { + req.err <- fmt.Errorf("no response received for request ID: %d", req.request.Id()) + anyMissingId = true + } + if anyMissingId { + c.logger.Error().Str("response", util.Mem2Str(bodyBytes)).Msgf("some requests did not receive a response (matching ID)") + } + } else if rootNode.TypeSafe() == ast.V_OBJECT { + // Single object response + jrResp, err := getJsonRpcResponseFromNode(rootNode) + if err != nil { + for _, req := range requests { + req.err <- err + } + return + } + for _, req := range requests { + nr := common.NewNormalizedResponse().WithRequest(req.request).WithJsonRpcResponse(jrResp) err := c.normalizeJsonRpcError(resp, nr) if err != nil { req.err <- err } else { req.response <- nr } - delete(requests, jrResp.ID) + } + } else { + // Unexpected response type + for _, req := range requests { + req.err <- common.NewErrUpstreamMalformedResponse(fmt.Errorf("unexpected response type (not array nor object): %s", util.Mem2Str(bodyBytes)), c.upstream.Config().Id) } } +} - // Handle any remaining requests that didn't receive a response which is very unexpected - // it means the upstream response did not include any item with request.ID for one or more the requests - for _, req := range requests { - jrReq, err := req.request.JsonRpcRequest() - if err != nil { - req.err <- fmt.Errorf("unexpected no response received for request: %w", err) - } else { - req.err <- fmt.Errorf("unexpected no response received for request %s", jrReq.ID) +func getJsonRpcResponseFromNode(rootNode ast.Node) (*common.JsonRpcResponse, error) { + idNode := rootNode.GetByPath("id") + rawID, _ := idNode.Raw() + resultNode := rootNode.GetByPath("result") + rawResult, rawResultErr := resultNode.Raw() + errorNode := rootNode.GetByPath("error") + rawError, rawErrorErr := errorNode.Raw() + + if rawResultErr != nil && rawErrorErr != nil { + var jrResp *common.JsonRpcResponse + + if rawID != "" { + err := jrResp.SetIDBytes(util.Str2Mem(rawID)) + if err != nil { + return nil, err + } } + + cause := fmt.Sprintf("cannot parse json rpc response from upstream, for result: %s, for error: %s", rawResult, rawError) + jrResp = &common.JsonRpcResponse{ + Result: util.Str2Mem(rawResult), + Error: common.NewErrJsonRpcExceptionExternal( + int(common.JsonRpcErrorParseException), + cause, + "", + ), + } + + return jrResp, nil } + + return common.NewJsonRpcResponseFromBytes( + util.Str2Mem(rawID), + util.Str2Mem(rawResult), + util.Str2Mem(rawError), + ) } func (c *GenericHttpJsonRpcClient) sendSingleRequest(ctx context.Context, req *common.NormalizedRequest) (*common.NormalizedResponse, error) { @@ -386,18 +451,21 @@ func (c *GenericHttpJsonRpcClient) sendSingleRequest(ctx context.Context, req *c c.upstream.Config().Id, req.NetworkId(), jrReq.Method, - 0, 0, 0, 0, + 0, + 0, + 0, + 0, ) } - req.RLock() - requestBody, err := sonic.Marshal(common.JsonRpcRequest{ + jrReq.RLock() + requestBody, err := common.SonicCfg.Marshal(common.JsonRpcRequest{ JSONRPC: jrReq.JSONRPC, Method: jrReq.Method, Params: jrReq.Params, ID: jrReq.ID, }) - req.RUnlock() + jrReq.RUnlock() if err != nil { return nil, err @@ -425,15 +493,13 @@ func (c *GenericHttpJsonRpcClient) sendSingleRequest(ctx context.Context, req *c if errors.Is(err, context.DeadlineExceeded) { return nil, common.NewErrEndpointRequestTimeout(time.Since(reqStartTime)) } - return nil, err - } - defer resp.Body.Close() - respBody, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err + return nil, common.NewErrEndpointTransportFailure(err) } - nr := common.NewNormalizedResponse().WithRequest(req).WithBody(respBody) + nr := common.NewNormalizedResponse(). + WithRequest(req). + WithBody(resp.Body). + WithExpectedSize(int(resp.ContentLength)) return nr, c.normalizeJsonRpcError(resp, nr) } @@ -451,7 +517,6 @@ func (c *GenericHttpJsonRpcClient) normalizeJsonRpcError(r *http.Response, nr *c "upstreamId": c.upstream.Config().Id, "statusCode": r.StatusCode, "headers": r.Header, - "body": string(nr.Body()), }, ) return e @@ -474,7 +539,6 @@ func (c *GenericHttpJsonRpcClient) normalizeJsonRpcError(r *http.Response, nr *c "upstreamId": c.upstream.Config().Id, "statusCode": r.StatusCode, "headers": r.Header, - "body": string(nr.Body()), }, ) @@ -487,7 +551,7 @@ func extractJsonRpcError(r *http.Response, nr *common.NormalizedResponse, jr *co var details map[string]interface{} = make(map[string]interface{}) details["statusCode"] = r.StatusCode - details["headers"] = util.ExtractUsefulHeaders(r.Header) + details["headers"] = util.ExtractUsefulHeaders(r) if ver := getVendorSpecificErrorIfAny(r, nr, jr, details); ver != nil { return ver @@ -495,32 +559,45 @@ func extractJsonRpcError(r *http.Response, nr *common.NormalizedResponse, jr *co code := common.JsonRpcErrorNumber(err.Code) - if err.Data != "" { - // Some providers such as Alchemy prefix the data with this string - // we omit this prefix for standardization. - if strings.HasPrefix(err.Data, "Reverted ") { - details["data"] = err.Data[9:] - } else { - details["data"] = err.Data + switch err.Data.(type) { + case string: + s := err.Data.(string) + if s != "" { + // Some providers such as Alchemy prefix the data with this string + // we omit this prefix for standardization. + if strings.HasPrefix(s, "Reverted ") { + details["data"] = s[9:] + } else { + details["data"] = s + } } + default: + // passthrough error data as is + details["data"] = err.Data } // Infer from known status codes - if r.StatusCode == 401 || r.StatusCode == 403 { - return common.NewErrEndpointUnauthorized( + if r.StatusCode == 415 || code == common.JsonRpcErrorUnsupportedException { + return common.NewErrEndpointUnsupported( common.NewErrJsonRpcExceptionInternal( int(code), - common.JsonRpcErrorUnauthorized, + common.JsonRpcErrorUnsupportedException, err.Message, nil, details, ), ) - } else if r.StatusCode == 415 || code == common.JsonRpcErrorUnsupportedException { - return common.NewErrEndpointUnsupported( + } else if r.StatusCode == 429 || + strings.Contains(err.Message, "requests limited to") || + strings.Contains(err.Message, "has exceeded") || + strings.Contains(err.Message, "Exceeded the quota") || + strings.Contains(err.Message, "Too many requests") || + strings.Contains(err.Message, "Too Many Requests") || + strings.Contains(err.Message, "under too much load") { + return common.NewErrEndpointCapacityExceeded( common.NewErrJsonRpcExceptionInternal( int(code), - common.JsonRpcErrorUnsupportedException, + common.JsonRpcErrorCapacityExceeded, err.Message, nil, details, @@ -537,7 +614,7 @@ func extractJsonRpcError(r *http.Response, nr *common.NormalizedResponse, jr *co strings.Contains(err.Message, "query exceeds limit") || strings.Contains(err.Message, "exceeds the range") || strings.Contains(err.Message, "range limit exceeded") { - return common.NewErrEndpointEvmLargeRange( + return common.NewErrEndpointRequestTooLarge( common.NewErrJsonRpcExceptionInternal( int(code), common.JsonRpcErrorCapacityExceeded, @@ -545,13 +622,10 @@ func extractJsonRpcError(r *http.Response, nr *common.NormalizedResponse, jr *co nil, details, ), + common.EvmBlockRangeTooLarge, ) - } else if r.StatusCode == 429 || - strings.Contains(err.Message, "has exceeded") || - strings.Contains(err.Message, "Exceeded the quota") || - strings.Contains(err.Message, "under too much load") { - - return common.NewErrEndpointCapacityExceeded( + } else if strings.Contains(err.Message, "specify less number of address") { + return common.NewErrEndpointRequestTooLarge( common.NewErrJsonRpcExceptionInternal( int(code), common.JsonRpcErrorCapacityExceeded, @@ -559,6 +633,7 @@ func extractJsonRpcError(r *http.Response, nr *common.NormalizedResponse, jr *co nil, details, ), + common.EvmAddressesTooLarge, ) } else if strings.Contains(err.Message, "reached the free tier") || strings.Contains(err.Message, "Monthly capacity limit") { @@ -576,7 +651,10 @@ func extractJsonRpcError(r *http.Response, nr *common.NormalizedResponse, jr *co strings.Contains(err.Message, "unknown block") || strings.Contains(err.Message, "Unknown block") || strings.Contains(err.Message, "height must be less than or equal") || - strings.Contains(err.Message, "finalized block not found") || + strings.Contains(err.Message, "invalid blockhash finalized") || + strings.Contains(err.Message, "Expect block number from id") || + strings.Contains(err.Message, "block not found") || + strings.Contains(err.Message, "block height passed is invalid") || // Usually happens on Avalanche when querying a pretty recent block: strings.Contains(err.Message, "cannot query unfinalized") || strings.Contains(err.Message, "height is not available") || @@ -587,7 +665,9 @@ func extractJsonRpcError(r *http.Response, nr *common.NormalizedResponse, jr *co (strings.Contains(err.Message, "blocks specified") && strings.Contains(err.Message, "cannot be found")) || strings.Contains(err.Message, "transaction not found") || strings.Contains(err.Message, "cannot find transaction") || - strings.Contains(err.Message, "after last accepted block") { + strings.Contains(err.Message, "after last accepted block") || + strings.Contains(err.Message, "is greater than latest") || + strings.Contains(err.Message, "No state available") { return common.NewErrEndpointMissingData( common.NewErrJsonRpcExceptionInternal( int(code), @@ -607,7 +687,30 @@ func extractJsonRpcError(r *http.Response, nr *common.NormalizedResponse, jr *co details, ), ) - } else if code == -32602 { + } else if code == -32602 || + strings.Contains(err.Message, "param is required") || + strings.Contains(err.Message, "Invalid Request") || + strings.Contains(err.Message, "validation errors") || + strings.Contains(err.Message, "invalid argument") { + if dt, ok := err.Data.(map[string]interface{}); ok { + if msg, ok := dt["message"]; ok { + if strings.Contains(msg.(string), "validation errors in batch") { + // Intentionally return a server-side error for failed requests in a batch + // so they are retried in a different batch. + // TODO Should we split a batch instead on json-rpc client level? + return common.NewErrEndpointServerSideException( + common.NewErrJsonRpcExceptionInternal( + int(code), + common.JsonRpcErrorServerSideException, + err.Message, + nil, + details, + ), + nil, + ) + } + } + } return common.NewErrEndpointClientSideException( common.NewErrJsonRpcExceptionInternal( int(code), @@ -617,7 +720,10 @@ func extractJsonRpcError(r *http.Response, nr *common.NormalizedResponse, jr *co details, ), ) - } else if strings.Contains(err.Message, "reverted") || strings.Contains(err.Message, "VM execution error") { + } else if strings.Contains(err.Message, "reverted") || + strings.Contains(err.Message, "VM execution error") || + strings.Contains(err.Message, "transaction: revert") || + strings.Contains(err.Message, "VM Exception") { return common.NewErrEndpointClientSideException( common.NewErrJsonRpcExceptionInternal( int(code), @@ -627,7 +733,10 @@ func extractJsonRpcError(r *http.Response, nr *common.NormalizedResponse, jr *co details, ), ) - } else if strings.Contains(err.Message, "insufficient funds") || strings.Contains(err.Message, "out of gas") { + } else if strings.Contains(err.Message, "insufficient funds") || + strings.Contains(err.Message, "insufficient balance") || + strings.Contains(err.Message, "out of gas") || + strings.Contains(err.Message, "gas too low") { return common.NewErrEndpointClientSideException( common.NewErrJsonRpcExceptionInternal( int(code), @@ -637,8 +746,14 @@ func extractJsonRpcError(r *http.Response, nr *common.NormalizedResponse, jr *co details, ), ) - } else if strings.Contains(err.Message, "not found") || strings.Contains(err.Message, "does not exist/is not available") { - if strings.Contains(err.Message, "Method") || strings.Contains(err.Message, "method") { + } else if strings.Contains(err.Message, "not found") || + strings.Contains(err.Message, "does not exist") || + strings.Contains(err.Message, "is not available") || + strings.Contains(err.Message, "is disabled") { + if strings.Contains(err.Message, "Method") || + strings.Contains(err.Message, "method") || + strings.Contains(err.Message, "Module") || + strings.Contains(err.Message, "module") { return common.NewErrEndpointUnsupported( common.NewErrJsonRpcExceptionInternal( int(code), @@ -648,7 +763,10 @@ func extractJsonRpcError(r *http.Response, nr *common.NormalizedResponse, jr *co details, ), ) - } else if strings.Contains(err.Message, "header") { + } else if strings.Contains(err.Message, "header") || + strings.Contains(err.Message, "block") || + strings.Contains(err.Message, "Header") || + strings.Contains(err.Message, "Block") { return common.NewErrEndpointMissingData( common.NewErrJsonRpcExceptionInternal( int(code), @@ -672,7 +790,7 @@ func extractJsonRpcError(r *http.Response, nr *common.NormalizedResponse, jr *co } else if strings.Contains(err.Message, "Unsupported method") || strings.Contains(err.Message, "not supported") || strings.Contains(err.Message, "method is not whitelisted") || - strings.Contains(err.Message, "module is disabled") { + strings.Contains(err.Message, "is not included in your current plan") { return common.NewErrEndpointUnsupported( common.NewErrJsonRpcExceptionInternal( int(code), @@ -682,9 +800,7 @@ func extractJsonRpcError(r *http.Response, nr *common.NormalizedResponse, jr *co details, ), ) - } else if strings.Contains(err.Message, "Invalid Request") || - strings.Contains(err.Message, "validation errors") || - strings.Contains(err.Message, "invalid argument") { + } else if code == -32600 { return common.NewErrEndpointClientSideException( common.NewErrJsonRpcExceptionInternal( int(code), @@ -694,6 +810,16 @@ func extractJsonRpcError(r *http.Response, nr *common.NormalizedResponse, jr *co details, ), ) + } else if r.StatusCode == 401 || r.StatusCode == 403 || strings.Contains(err.Message, "not allowed to access") { + return common.NewErrEndpointUnauthorized( + common.NewErrJsonRpcExceptionInternal( + int(code), + common.JsonRpcErrorUnauthorized, + err.Message, + nil, + details, + ), + ) } // By default we consider a problem on the server so that retry/failover mechanisms try other upstreams @@ -710,26 +836,21 @@ func extractJsonRpcError(r *http.Response, nr *common.NormalizedResponse, jr *co } // There's a special case for certain clients that return a normal response for reverts: - if jr != nil && jr.Result != nil { - res, err := jr.ParsedResult() - if err != nil { - return err - } - if dt, ok := res.(string); ok { - // keccak256("Error(string)") - if strings.HasPrefix(dt, "0x08c379a0") { - return common.NewErrEndpointClientSideException( - common.NewErrJsonRpcExceptionInternal( - 0, - common.JsonRpcErrorEvmReverted, - "transaction reverted", - nil, - map[string]interface{}{ - "data": dt, - }, - ), - ) - } + if jr != nil { + dt := util.Mem2Str(jr.Result) + // keccak256("Error(string)") + if strings.HasPrefix(dt, "0x08c379a0") { + return common.NewErrEndpointClientSideException( + common.NewErrJsonRpcExceptionInternal( + 0, + common.JsonRpcErrorEvmReverted, + "transaction reverted", + nil, + map[string]interface{}{ + "data": dt, + }, + ), + ) } } diff --git a/upstream/init_test.go b/upstream/init_test.go new file mode 100644 index 000000000..300aa0f4b --- /dev/null +++ b/upstream/init_test.go @@ -0,0 +1,9 @@ +package upstream + +import ( + "github.com/rs/zerolog" +) + +func init() { + zerolog.SetGlobalLevel(zerolog.Disabled) +} diff --git a/upstream/pimlico_http_json_rpc_client.go b/upstream/pimlico_http_json_rpc_client.go index a4e73d3d9..88c509f2e 100644 --- a/upstream/pimlico_http_json_rpc_client.go +++ b/upstream/pimlico_http_json_rpc_client.go @@ -145,7 +145,12 @@ func (c *PimlicoHttpJsonRpcClient) SupportsNetwork(networkId string) (bool, erro return false, err } - cidh, err := common.NormalizeHex(jrr.Result) + cids, err := jrr.PeekStringByPath() + if err != nil { + return false, err + } + + cidh, err := common.NormalizeHex(cids) if err != nil { return false, err } diff --git a/upstream/registry.go b/upstream/registry.go index ee7abe8a9..a3d723785 100644 --- a/upstream/registry.go +++ b/upstream/registry.go @@ -358,12 +358,13 @@ func (u *UpstreamsRegistry) updateScoresAndSort(networkId, method string, upsLis u.sortUpstreams(networkId, method, upsList) u.sortedUpstreams[networkId][method] = upsList - newSortStr := "" - for _, ups := range upsList { - newSortStr += fmt.Sprintf("%s ", ups.Config().Id) + if u.logger.GetLevel() >= zerolog.TraceLevel { + newSortStr := "" + for _, ups := range upsList { + newSortStr += fmt.Sprintf("%s ", ups.Config().Id) + } + u.logger.Trace().Str("projectId", u.prjId).Str("networkId", networkId).Str("method", method).Str("newSort", newSortStr).Msgf("sorted upstreams") } - - u.logger.Trace().Str("projectId", u.prjId).Str("networkId", networkId).Str("method", method).Str("newSort", newSortStr).Msgf("sorted upstreams") } func (u *UpstreamsRegistry) calculateScore( @@ -459,3 +460,7 @@ func (u *UpstreamsRegistry) GetUpstreamsHealth() (*UpstreamsHealth, error) { UpstreamScores: upstreamScores, }, nil } + +func (u *UpstreamsRegistry) GetMetricsTracker() *health.Tracker { + return u.metricsTracker +} diff --git a/upstream/registry_test.go b/upstream/registry_test.go index e3af29a0e..4956e7943 100644 --- a/upstream/registry_test.go +++ b/upstream/registry_test.go @@ -2,7 +2,6 @@ package upstream import ( "context" - "fmt" "sync" "testing" "time" @@ -522,7 +521,6 @@ func simulateFailedRequests(tracker *health.Tracker, network, upstream, method s func checkUpstreamScoreOrder(t *testing.T, registry *UpstreamsRegistry, networkID, method string, expectedOrder []string) { registry.RefreshUpstreamNetworkMethodScores() scores := registry.upstreamScores - fmt.Printf("Checking recorded scores: %v for order: %s\n", scores, expectedOrder) for i, ups := range expectedOrder { if i+1 < len(expectedOrder) { @@ -538,7 +536,6 @@ func checkUpstreamScoreOrder(t *testing.T, registry *UpstreamsRegistry, networkI } sortedUpstreams, err := registry.GetSortedUpstreams(networkID, method) - fmt.Printf("Checking upstream order: %v\n", sortedUpstreams) assert.NoError(t, err) registry.RLockUpstreams() diff --git a/upstream/request_test.go b/upstream/request_test.go index 76eb3a8bd..25a49ba1f 100644 --- a/upstream/request_test.go +++ b/upstream/request_test.go @@ -7,7 +7,7 @@ import ( ) func TestNormalizedRequest_BodyCannotBeDecoded(t *testing.T) { - normReq := common.NewNormalizedRequest([]byte(`{"method": "test", "params": "invalid"}`)) + normReq := common.NewNormalizedRequest([]byte(`{"method":"test","params":"invalid"}`)) // Call the JsonRpcRequest method to parse and normalize the request jsonRpcReq, err := normReq.JsonRpcRequest() diff --git a/upstream/thirdweb_http_json_rpc_client.go b/upstream/thirdweb_http_json_rpc_client.go index 8cad37409..b8aa8581d 100644 --- a/upstream/thirdweb_http_json_rpc_client.go +++ b/upstream/thirdweb_http_json_rpc_client.go @@ -71,7 +71,12 @@ func (c *ThirdwebHttpJsonRpcClient) SupportsNetwork(networkId string) (bool, err return false, err } - cidh, err := common.NormalizeHex(jrr.Result) + cids, err := jrr.PeekStringByPath() + if err != nil { + return false, err + } + + cidh, err := common.NormalizeHex(cids) if err != nil { return false, err } diff --git a/upstream/upstream.go b/upstream/upstream.go index 25cc3430d..0bb270a62 100644 --- a/upstream/upstream.go +++ b/upstream/upstream.go @@ -143,21 +143,12 @@ func (u *Upstream) prepareRequest(nr *common.NormalizedRequest) error { return common.NewErrJsonRpcExceptionInternal( 0, common.JsonRpcErrorParseException, - "failed to unmarshal jsonrpc request", - err, - nil, - ) - } - err = common.NormalizeEvmHttpJsonRpc(nr, jsonRpcReq) - if err != nil { - return common.NewErrJsonRpcExceptionInternal( - 0, - common.JsonRpcErrorServerSideException, - "failed to normalize jsonrpc request", + "failed to unmarshal json-rpc request", err, nil, ) } + common.NormalizeEvmHttpJsonRpc(nr, jsonRpcReq) } else { return common.NewErrJsonRpcExceptionInternal( 0, @@ -296,7 +287,11 @@ func (u *Upstream) Forward(ctx context.Context, req *common.NormalizedRequest) ( if jrr != nil && jrr.Error == nil { req.SetLastValidResponse(resp) } - lg.Debug().Err(errCall).Str("response", resp.String()).Msgf("upstream call result received") + if lg.GetLevel() == zerolog.TraceLevel { + lg.Debug().Err(errCall).Interface("response", resp).Msgf("upstream call result received") + } else { + lg.Debug().Err(errCall).Msgf("upstream call result received") + } } else { lg.Debug().Err(errCall).Msgf("upstream call result received") } @@ -377,15 +372,19 @@ func (u *Upstream) Forward(ctx context.Context, req *common.NormalizedRequest) ( } func (u *Upstream) Executor() failsafe.Executor[*common.NormalizedResponse] { + // TODO extend this to per-network and/or per-method because of either upstream performance diff + // or if user wants diff policies (retry/cb/integrity) per network/method. return u.failsafeExecutor } func (u *Upstream) EvmGetChainId(ctx context.Context) (string, error) { pr := common.NewNormalizedRequest([]byte(`{"jsonrpc":"2.0","id":75412,"method":"eth_chainId","params":[]}`)) + resp, err := u.Forward(ctx, pr) if err != nil { return "", err } + jrr, err := resp.JsonRpcResponse() if err != nil { return "", err @@ -393,16 +392,15 @@ func (u *Upstream) EvmGetChainId(ctx context.Context) (string, error) { if jrr.Error != nil { return "", jrr.Error } - - res, err := jrr.ParsedResult() + var chainId string + err = common.SonicCfg.Unmarshal(jrr.Result, &chainId) if err != nil { return "", err } - hex, err := common.NormalizeHex(res) + hex, err := common.NormalizeHex(chainId) if err != nil { return "", err } - dec, err := common.HexToUint64(hex) if err != nil { return "", err @@ -438,6 +436,7 @@ func (u *Upstream) IgnoreMethod(method string) { } u.methodCheckResultsMu.Lock() + // TODO make this per-network vs global because some upstreams support multiple networks (e.g. alchemy://) u.config.IgnoreMethods = append(u.config.IgnoreMethods, method) if u.methodCheckResults == nil { u.methodCheckResults = map[string]bool{} diff --git a/util/fastmem.go b/util/fastmem.go new file mode 100644 index 000000000..bd6be8293 --- /dev/null +++ b/util/fastmem.go @@ -0,0 +1,37 @@ +package util + +import ( + "bytes" + "io" + "unsafe" +) + +type GoSlice struct { + Ptr unsafe.Pointer + Len int + Cap int +} + +type GoString struct { + Ptr unsafe.Pointer + Len int +} + +//go:nosplit +func Mem2Str(v []byte) (s string) { + (*GoString)(unsafe.Pointer(&s)).Len = (*GoSlice)(unsafe.Pointer(&v)).Len // #nosec G103 + (*GoString)(unsafe.Pointer(&s)).Ptr = (*GoSlice)(unsafe.Pointer(&v)).Ptr // #nosec G103 + return +} + +//go:nosplit +func Str2Mem(s string) (v []byte) { + (*GoSlice)(unsafe.Pointer(&v)).Cap = (*GoString)(unsafe.Pointer(&s)).Len // #nosec G103 + (*GoSlice)(unsafe.Pointer(&v)).Len = (*GoString)(unsafe.Pointer(&s)).Len // #nosec G103 + (*GoSlice)(unsafe.Pointer(&v)).Ptr = (*GoString)(unsafe.Pointer(&s)).Ptr // #nosec G103 + return +} + +func StringToReaderCloser(b string) io.ReadCloser { + return io.NopCloser(bytes.NewBuffer(Str2Mem(b))) +} diff --git a/util/headers.go b/util/headers.go index 083da2739..c6314d79d 100644 --- a/util/headers.go +++ b/util/headers.go @@ -1,13 +1,14 @@ package util import ( - "net/http" "strings" + + "net/http" ) -func ExtractUsefulHeaders(headers http.Header) map[string]interface{} { +func ExtractUsefulHeaders(r *http.Response) map[string]interface{} { var result = make(map[string]interface{}) - for k, v := range headers { + for k := range r.Header { kl := strings.ToLower(k) if strings.HasPrefix(kl, "x-") || strings.Contains(kl, "trace") || @@ -16,7 +17,7 @@ func ExtractUsefulHeaders(headers http.Header) map[string]interface{} { strings.Contains(kl, "-id") || kl == "content-type" || kl == "content-length" { - result[kl] = v + result[kl] = r.Header.Get(k) } } diff --git a/util/json_rpc.go b/util/json_rpc.go new file mode 100644 index 000000000..0a28a608e --- /dev/null +++ b/util/json_rpc.go @@ -0,0 +1,11 @@ +package util + +import ( + "math" + "math/rand" +) + +// RandomID returns a value appropriate for a JSON-RPC ID field (i.e int64 type but with 32 bit range) to avoid overflow during conversions and reading/sending to upstreams +func RandomID() int64 { + return int64(rand.Intn(math.MaxInt32)) // #nosec G404 +} diff --git a/util/reader.go b/util/reader.go new file mode 100644 index 000000000..1612bf7fa --- /dev/null +++ b/util/reader.go @@ -0,0 +1,32 @@ +package util + +import ( + "bytes" + "io" +) + +func ReadAll(reader io.Reader, chunkSize int64, expectedSize int) ([]byte, error) { + buf := bytes.NewBuffer(make([]byte, 0, 16*1024)) // 16KB default + + if expectedSize > 0 && expectedSize < 50*1024*1024 { // 50MB cap to avoid DDoS by a corrupt/malicious upstream + n := expectedSize - buf.Cap() + if n > 0 { + buf.Grow(n) + } + } + + for { + n, err := io.CopyN(buf, reader, chunkSize) + if err != nil { + if err == io.EOF { + break + } + return nil, err + } + if n == 0 { + break + } + } + + return buf.Bytes(), nil +} diff --git a/util/redact.go b/util/redact.go index 440fb9248..53123cf6b 100644 --- a/util/redact.go +++ b/util/redact.go @@ -10,7 +10,7 @@ import ( func RedactEndpoint(endpoint string) string { // Calculate hash of the entire original endpoint hasher := sha256.New() - hasher.Write([]byte(endpoint)) + hasher.Write(Str2Mem(endpoint)) hash := hex.EncodeToString(hasher.Sum(nil)) // Parse the endpoint URL diff --git a/vendors/drpc.go b/vendors/drpc.go index 7ac0d268e..adc755aff 100644 --- a/vendors/drpc.go +++ b/vendors/drpc.go @@ -54,7 +54,7 @@ func (v *DrpcVendor) GetVendorSpecificErrorIfAny(resp *http.Response, jrr interf details["data"] = err.Data } - if code == 4 && strings.Contains(msg, "token is invalid") { + if strings.Contains(msg, "token is invalid") { return common.NewErrEndpointUnauthorized( common.NewErrJsonRpcExceptionInternal( code, @@ -64,19 +64,20 @@ func (v *DrpcVendor) GetVendorSpecificErrorIfAny(resp *http.Response, jrr interf details, ), ) - } else if code == 32601 && strings.Contains(msg, "does not exist/is not available") { - // Intentionally consider missing methods as server-side exceptions + } else if strings.Contains(msg, "does not exist/is not available") { + // Intentionally consider missing methods as client-side exceptions // because dRPC might give a false error when their underlying nodes - // have issues e.g. you might falsly get "eth_blockNumber not supported" errors. - return common.NewErrEndpointServerSideException( + // have issues e.g. you might falsely get "eth_blockNumber not supported" errors. + // The reason we don't consider this as server-side exceptions is because + // we don't want to trigger the circuit breaker in such cases. + return common.NewErrEndpointClientSideException( common.NewErrJsonRpcExceptionInternal( code, - common.JsonRpcErrorServerSideException, + common.JsonRpcErrorUnsupportedException, msg, nil, details, ), - details, ) } } diff --git a/vendors/quicknode.go b/vendors/quicknode.go index ba845a09e..8292408e3 100644 --- a/vendors/quicknode.go +++ b/vendors/quicknode.go @@ -51,8 +51,9 @@ func (v *QuicknodeVendor) GetVendorSpecificErrorIfAny(resp *http.Response, jrr i } if code == -32602 && strings.Contains(msg, "eth_getLogs") && strings.Contains(msg, "limited") { - return common.NewErrEndpointEvmLargeRange( + return common.NewErrEndpointRequestTooLarge( common.NewErrJsonRpcExceptionInternal(code, common.JsonRpcErrorEvmLogsLargeRange, msg, nil, details), + common.EvmBlockRangeTooLarge, ) } else if code == -32000 { if strings.Contains(msg, "header not found") || strings.Contains(msg, "could not find block") {