Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions contrib/opentelemetry/tracing_interceptor.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (
"go.temporal.io/sdk/interceptor"
"go.temporal.io/sdk/log"
"go.temporal.io/sdk/temporal"
"go.temporal.io/sdk/workflow"
)

// DefaultTextMapPropagator is the default OpenTelemetry TextMapPropagator used
Expand Down Expand Up @@ -174,6 +175,21 @@ func (t *tracer) ContextWithSpan(ctx context.Context, span interceptor.TracerSpa
return trace.ContextWithSpan(ctx, span.(*tracerSpan).Span)
}

// SpanFromWorkflowContext extracts an OpenTelemetry span from the given
// workflow context. If no span is found, a no-op span is returned.
func SpanFromWorkflowContext(ctx workflow.Context) (trace.Span, bool) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please add a godoc to this function explaining the functionality

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure, godoc added

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why return a bool that is always true?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah this look like a bug

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

May need to be clarified in the godoc that this is unsafe/non-deterministic because it can technically return different values when replaying vs not

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So you'd like a comment warning people they should only use the Span data for observability needs? That's generally the assumption with observability tools, but I can make it explicit.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes. That assumption is reasonable in code where non-deterministic elements are fine if they chose to use span data for other things, but for Temporal, it's usually better to clearly mark any non-deterministic workflow functions we offer as unsafe if they may be.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wonder if SpanContextFromWorkflowContext is needed/better

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We haven't had any need to use SpanContext, but it is another option. The Datadog interceptor only provides Span as well so I followed it's example since it's already in the codebase.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, this is probably good, as I can't think of a situation where you could get a span context but not a span. I will say that query and update handlers will get a new span on non-replay, or the "run workflow" span on replay since they don't create new spans on replay but outer "run workflow" does IIUC.

val := ctx.Value(spanContextKey{})

if val != nil {
if span, ok := val.(*tracerSpan); ok {
return span.Span, true
}
}

// Fallback to OpenTelemetry span extraction behavior
return trace.SpanFromContext(nil), false
}

func (t *tracer) StartSpan(opts *interceptor.TracerStartSpanOptions) (interceptor.TracerSpan, error) {
// Create context with parent
var parent trace.SpanContext
Expand Down
83 changes: 83 additions & 0 deletions contrib/opentelemetry/tracing_interceptor_test.go
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
package opentelemetry_test

import (
"context"
"errors"
"testing"
"time"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
"go.opentelemetry.io/otel/sdk/trace/tracetest"
Expand All @@ -15,6 +18,9 @@ import (
"go.temporal.io/sdk/interceptor"
"go.temporal.io/sdk/internal/interceptortest"
"go.temporal.io/sdk/temporal"
"go.temporal.io/sdk/testsuite"
"go.temporal.io/sdk/worker"
"go.temporal.io/sdk/workflow"
)

func TestSpanPropagation(t *testing.T) {
Expand Down Expand Up @@ -152,3 +158,80 @@ func TestBenignErrorSpanStatus(t *testing.T) {
})
}
}

func setCustomSpanAttrWorkflow(ctx workflow.Context) error {
span, ok := opentelemetry.SpanFromWorkflowContext(ctx)
if !ok {
return errors.New("Did not find span in workflow context")
}

span.SetAttributes(attribute.String("testTag", "testValue"))
return nil
}

func TestSpanFromWorkflowContext(t *testing.T) {
rec := tracetest.NewSpanRecorder()
tracer, err := opentelemetry.NewTracer(opentelemetry.TracerOptions{
Tracer: sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(rec)).Tracer(""),
})
require.NoError(t, err)

var suite testsuite.WorkflowTestSuite
env := suite.NewTestWorkflowEnvironment()
env.RegisterWorkflow(setCustomSpanAttrWorkflow)

// Set tracer interceptor
env.SetWorkerOptions(worker.Options{
Interceptors: []interceptor.WorkerInterceptor{interceptor.NewTracingInterceptor(tracer)},
})

env.ExecuteWorkflow(setCustomSpanAttrWorkflow)

require.True(t, env.IsWorkflowCompleted())

// Verify span was recorded with added attribute
spans := rec.Ended()
require.GreaterOrEqual(t, len(spans), 1)

found := false
for _, s := range spans {
for _, kv := range s.Attributes() {
if string(kv.Key) == "testTag" && kv.Value.AsString() == "testValue" {
found = true
break
}
}
if found {
break
}
}

require.True(t, found, "expected to find attribute 'testTag=testValue' on recorded spans")
}

func TestSpanFromWorkflowContextNoOpSpan(t *testing.T) {
var suite testsuite.WorkflowTestSuite
env := suite.NewTestWorkflowEnvironment()

nilValueWorkflow := func(ctx workflow.Context) error {
span, ok := opentelemetry.SpanFromWorkflowContext(ctx)

if ok {
return errors.New("Expected ok to be false")
}

// Make sure we retain behavior of returning no-op span when no span is present in context
noopSpan := trace.SpanFromContext(context.TODO())
if span != noopSpan {
return errors.New("Expected span to be no-op span")
}

return nil
}

env.RegisterWorkflow(nilValueWorkflow)
env.ExecuteWorkflow(nilValueWorkflow)

require.True(t, env.IsWorkflowCompleted())
require.NoError(t, env.GetWorkflowError())
}
Loading