feat: update transaction lifecycle tracing

cmd/opera/launcher/launcher.go

@@ -54,6 +54,7 @@ var (
 	legacyRpcFlags   []cli.Flag
 	rpcFlags         []cli.Flag
 	metricsFlags     []cli.Flag
+	tracingFlags     []cli.Flag
 )
 
 func initFlags() {
@@ -157,7 +158,12 @@ func initFlags() {
 		utils.MetricsInfluxDBUsernameFlag,
 		utils.MetricsInfluxDBPasswordFlag,
 		utils.MetricsInfluxDBTagsFlag,
+	}
+
+	tracingFlags = []cli.Flag{
 		tracing.EnableFlag,
+		tracing.AgentEndpointFlag,
+		tracing.EnableDevFlag,
 	}
 
 	nodeFlags = []cli.Flag{}
@@ -213,6 +219,7 @@ func init() {
 	app.Flags = append(app.Flags, consoleFlags...)
 	app.Flags = append(app.Flags, debug.Flags...)
 	app.Flags = append(app.Flags, metricsFlags...)
+	app.Flags = append(app.Flags, tracingFlags...)
 
 	app.Before = func(ctx *cli.Context) error {
 		if err := debug.Setup(ctx); err != nil {
@@ -246,12 +253,11 @@ func lachesisMain(ctx *cli.Context) error {
 		return fmt.Errorf("invalid command: %q", args[0])
 	}
 
-	// TODO: tracing flags
-	//tracingStop, err := tracing.Start(ctx)
-	//if err != nil {
-	//	return err
-	//}
-	//defer tracingStop()
+	tracingStop, err := tracing.Start(ctx)
+	if err != nil {
+		return err
+	}
+	defer tracingStop()
 
 	cfg := makeAllConfigs(ctx)
 	genesisPath := getOperaGenesis(ctx)

cmd/opera/launcher/tracing/tracing.go

@@ -1,7 +1,10 @@
 package tracing
 
 import (
+	"errors"
+	"fmt"
 	opentracing "github.com/opentracing/opentracing-go"
+	"github.com/uber/jaeger-client-go"
 	jaegercfg "github.com/uber/jaeger-client-go/config"
 	jaegerlog "github.com/uber/jaeger-client-go/log"
 	"github.com/uber/jaeger-lib/metrics"
@@ -10,38 +13,65 @@ import (
 	"github.com/Fantom-foundation/go-opera/tracing"
 )
 
-var EnableFlag = cli.BoolFlag{
-	Name:  "tracing",
-	Usage: "Enable traces collection and reporting",
-}
+var (
+	EnableFlag = cli.BoolFlag{
+		Name:  "tracing",
+		Usage: "Enable traces collection and reporting",
+	}
 
-func Start(ctx *cli.Context) (stop func(), err error) {
-	stop = func() {}
+	EnableDevFlag = cli.BoolFlag{
+		Name:  "tracing.dev",
+		Usage: "Enable traces collection and reporting in development mode",
+	}
 
-	if !ctx.Bool(EnableFlag.Name) {
-		return
+	AgentEndpointFlag = cli.StringFlag{
+		Name:  "tracing.agent",
+		Usage: "Jaeger agent endpoint. Default is localhost:6831",
+		Value: "localhost:6831",
 	}
 
-	var cfg *jaegercfg.Configuration
-	cfg, err = jaegercfg.FromEnv()
-	if err != nil {
-		return
+	ErrInvalidEndpoint = errors.New("invalid agent endpoint")
+)
+
+func Start(ctx *cli.Context) (func(), error) {
+	if !ctx.Bool(EnableFlag.Name) && !ctx.Bool(EnableDevFlag.Name) {
+		return func() {}, nil
 	}
 
-	cfg.ServiceName = "opera"
+	agentEndpoint := ctx.String(AgentEndpointFlag.Name)
+	if agentEndpoint == "" {
+		return nil, ErrInvalidEndpoint
+	}
+
+	// Default config recommended for production
+	cfg := jaegercfg.Configuration{
+		ServiceName: "opera",
+		Reporter: &jaegercfg.ReporterConfig{
+			LocalAgentHostPort: agentEndpoint,
+		},
+	}
+
+	if ctx.Bool(EnableDevFlag.Name) {
+		// Makes sampler collect and report all traces
+		cfg.Sampler = &jaegercfg.SamplerConfig{
+			Type:  jaeger.SamplerTypeConst,
+			Param: 1,
+		}
+	}
 
 	tracer, closer, err := cfg.NewTracer(
 		jaegercfg.Logger(jaegerlog.StdLogger),
 		jaegercfg.Metrics(metrics.NullFactory),
 	)
 	if err != nil {
-		return
+		return nil, fmt.Errorf("new tracer: %w", err)
 	}
-	stop = func() {
+	stop := func() {
 		closer.Close()
 	}
 
 	opentracing.SetGlobalTracer(tracer)
 	tracing.SetEnabled(true)
-	return
+
+	return stop, nil
 }

demo/start.sh

@@ -24,6 +24,7 @@ do
 	--fakenet=${ACC}/$N \
 	--port=${PORT} \
 	--nat extip:127.0.0.1 \
+  --tracing.dev \
 	--http --http.addr="127.0.0.1" --http.port=${RPCP} --http.corsdomain="*" --http.api="eth,debug,net,admin,web3,personal,txpool,ftm,dag" \
 	--ws --ws.addr="127.0.0.1" --ws.port=${WSP} --ws.origins="*" --ws.api="eth,debug,net,admin,web3,personal,txpool,ftm,dag" \
 	--nousb --verbosity=3 --tracing &>> opera$i.log)&

evmcore/tx_pool.go

@@ -18,6 +18,7 @@ package evmcore
 
 import (
 	"errors"
+	"github.com/Fantom-foundation/go-opera/tracing"
 	"math"
 	"math/big"
 	"math/rand"
@@ -1828,6 +1829,8 @@ func (t *txLookup) Remove(hash common.Hash) {
 	t.lock.Lock()
 	defer t.lock.Unlock()
 
+	defer tracing.FinishTx(hash, "txLookup.Remove()")
+
 	tx, ok := t.locals[hash]
 	if !ok {
 		tx, ok = t.remotes[hash]

tracing/tx-tracing.go

@@ -2,14 +2,21 @@ package tracing
 
 import (
 	"sync"
+	"time"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/opentracing/opentracing-go"
 )
 
+// txSpan wraps opentracing span and stores additional payload
+type txSpan struct {
+	opentracing.Span
+	begin time.Time
+}
+
 var (
 	enabled   bool
-	txSpans   = make(map[common.Hash]opentracing.Span)
+	txSpans   = make(map[common.Hash]txSpan)
 	txSpansMu sync.RWMutex
 
 	noopSpan = opentracing.NoopTracer{}.StartSpan("")
@@ -35,9 +42,13 @@ func StartTx(tx common.Hash, operation string) {
 		return
 	}
 
-	span := opentracing.StartSpan("lifecycle")
+	span := txSpan{
+		Span:  opentracing.StartSpan("lifecycle"),
+		begin: time.Now(),
+	}
 	span.SetTag("txhash", tx.String())
 	span.SetTag("enter", operation)
+
 	txSpans[tx] = span
 }
 
@@ -55,7 +66,9 @@ func FinishTx(tx common.Hash, operation string) {
 	}
 
 	span.SetTag("exit", operation)
+	span.SetTag("time.millis", time.Since(span.begin).Milliseconds())
 	span.Finish()
+
 	delete(txSpans, tx)
 }