feat(telemetry): add credential.fetch tracing across all providers

go.mod

@@ -1,8 +1,6 @@
 module go.riptides.io/tokenex
 
-go 1.24.4
-
-toolchain go1.24.6
+go 1.25.0
 
 require (
 	cloud.google.com/go/iam v1.5.2
@@ -28,7 +26,10 @@ require (
 	sigs.k8s.io/controller-runtime v0.21.0
 )
 
-require github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+require (
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+)
 
 require (
 	cloud.google.com/go/auth v0.16.2 // indirect
@@ -44,7 +45,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sso v1.25.5 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3 // indirect
-	github.com/aws/smithy-go v1.22.4 // indirect
+	github.com/aws/smithy-go v1.27.3 // indirect
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/cenkalti/backoff/v5 v5.0.3
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
@@ -63,7 +64,7 @@ require (
 	github.com/google/go-cmp v0.7.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/s2a-go v0.1.9 // indirect
-	github.com/google/uuid v1.6.0 // indirect
+	github.com/google/uuid v1.6.0
 	github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
 	github.com/googleapis/gax-go/v2 v2.14.2 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
@@ -86,12 +87,12 @@ require (
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/ryanuber/go-glob v1.0.0 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
-	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
+	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect
-	go.opentelemetry.io/otel v1.36.0 // indirect
-	go.opentelemetry.io/otel/metric v1.36.0 // indirect
-	go.opentelemetry.io/otel/trace v1.36.0 // indirect
+	go.opentelemetry.io/otel v1.44.0
+	go.opentelemetry.io/otel/metric v1.44.0 // indirect
+	go.opentelemetry.io/otel/trace v1.44.0
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/crypto v0.45.0 // indirect
 	golang.org/x/net v0.47.0 // indirect

go.sum

@@ -68,8 +68,8 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3 h1:BpOxT3yhLwSJ77qIY3DoHAQj
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3/go.mod h1:vq/GQR1gOFLquZMSrxUK/cpvKCNVYibNyJ1m7JrU88E=
 github.com/aws/aws-sdk-go-v2/service/sts v1.34.0 h1:NFOJ/NXEGV4Rq//71Hs1jC/NvPs1ezajK+yQmkwnPV0=
 github.com/aws/aws-sdk-go-v2/service/sts v1.34.0/go.mod h1:7ph2tGpfQvwzgistp2+zga9f+bCjlQJPkPUmMgDSD7w=
-github.com/aws/smithy-go v1.22.4 h1:uqXzVZNuNexwc/xrh6Tb56u89WDlJY6HS+KC0S4QSjw=
-github.com/aws/smithy-go v1.22.4/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI=
+github.com/aws/smithy-go v1.27.3 h1:F3Zb497UhhskkfpJmfkXswyo+t0sh9OTBnIHjogWbVY=
+github.com/aws/smithy-go v1.27.3/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY=
@@ -289,8 +289,8 @@ github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0leargg
 github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
 github.com/redis/go-redis/v9 v9.8.0 h1:q3nRvjrlge/6UD7eTu/DSg2uYiU2mCL0G/uzBWqhicI=
 github.com/redis/go-redis/v9 v9.8.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw=
-github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
-github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
+github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
+github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
 github.com/ryanuber/columnize v2.1.2+incompatible h1:C89EOx/XBWwIXl8wm8OPJBd7kPF25UfsK2X7Ph/zCAk=
 github.com/ryanuber/columnize v2.1.2+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk=
@@ -329,28 +329,28 @@ github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM=
 github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4=
 go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
-go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
-go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
+go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
+go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
 go.opentelemetry.io/contrib/detectors/gcp v1.35.0 h1:bGvFt68+KTiAKFlacHW6AhA56GF2rS0bdD3aJYEnmzA=
 go.opentelemetry.io/contrib/detectors/gcp v1.35.0/go.mod h1:qGWP8/+ILwMRIUf9uIVLloR1uo5ZYAslM4O6OqUi1DA=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 h1:q4XOmH/0opmeuJtPsbFNivyl7bCt7yRBbeEm2sC/XtQ=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0/go.mod h1:snMWehoOh2wsEwnvvwtDyFCxVeDAODenXHtn5vzrKjo=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q=
-go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg=
-go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E=
+go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU=
+go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 h1:Vh5HayB/0HHfOQA7Ctx69E/Y/DcQSMPpKANYVMQ7fBA=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0/go.mod h1:cpgtDBaqD/6ok/UG0jT15/uKjAY8mRA53diogHBg3UI=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 h1:5pojmb1U1AogINhN3SurB+zm/nIcusopeBNp42f45QM=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0/go.mod h1:57gTHJSE5S1tqg+EKsLPlTWhpHMsWlVmer+LA926XiA=
-go.opentelemetry.io/otel/metric v1.36.0 h1:MoWPKVhQvJ+eeXWHFBOPoBOi20jh6Iq2CcCREuTYufE=
-go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs=
+go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc=
+go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo=
 go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs=
 go.opentelemetry.io/otel/sdk v1.36.0/go.mod h1:+lC+mTgD+MUWfjJubi2vvXWcVxyr9rmlshZni72pXeY=
 go.opentelemetry.io/otel/sdk/metric v1.36.0 h1:r0ntwwGosWGaa0CrSt8cuNuTcccMXERFwHX4dThiPis=
 go.opentelemetry.io/otel/sdk/metric v1.36.0/go.mod h1:qTNOhFDfKRwX0yXOqJYegL5WRaW376QbB7P4Pb0qva4=
-go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w=
-go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA=
+go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk=
+go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE=
 go.opentelemetry.io/proto/otlp v1.4.0 h1:TA9WRvW6zMwP+Ssb6fLoUIuirti1gGbP28GcKG1jgeg=
 go.opentelemetry.io/proto/otlp v1.4.0/go.mod h1:PPBWZIP98o2ElSqI35IHfu7hIhSwvc5N38Jw8pXuGFY=
 go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=

pkg/aws/creds.go

@@ -13,9 +13,13 @@ import (
 	"github.com/aws/aws-sdk-go-v2/credentials/stscreds"
 	"github.com/aws/aws-sdk-go-v2/service/sts"
 	"github.com/go-logr/logr"
+	"github.com/google/uuid"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
 
 	"go.riptides.io/tokenex/pkg/credential"
 	"go.riptides.io/tokenex/pkg/option"
+	tokenextelemetry "go.riptides.io/tokenex/pkg/telemetry"
 	"go.riptides.io/tokenex/pkg/token"
 	"go.riptides.io/tokenex/pkg/util"
 )
@@ -26,6 +30,8 @@ type credentialsConfig struct {
 	roleSessionName       string
 	durationSeconds       *int32
 	identityTokenProvider token.IdentityTokenProvider
+
+	tracerProvider trace.TracerProvider
 }
 
 // CredentialsProvider defines the interface for obtaining AWS credentials.
@@ -72,13 +78,61 @@ type Provider interface {
 
 func (cp *credentialsProvider) isAWS() {}
 
+func fetchCredentials(ctx context.Context, tracer trace.Tracer, configAttrs []attribute.KeyValue, provider *stscreds.WebIdentityRoleProvider, retriever *tokenRetriever) (aws.Credentials, error) {
+	fetchCtx, span := tracer.Start(ctx, fetchSpanName, trace.WithAttributes(configAttrs...))
+	defer span.End()
+
+	// we need to store current ctx as provider.Retrieve(fetchCtx) invoked bellow
+	// doesn't pass the fetch to TokenRetriever.GetIdentityToken()
+	retriever.ctx.Store(fetchCtx)
+
+	awsCreds, err := provider.Retrieve(fetchCtx)
+	if err != nil {
+		err = errors.WrapIf(err, "failed to retrieve credentials")
+		tokenextelemetry.RecordResult(span, err)
+
+		return aws.Credentials{}, err
+	}
+
+	span.SetAttributes(fetchSpanResultAttrs(awsCreds)...)
+	tokenextelemetry.RecordResult(span, nil)
+
+	return awsCreds, nil
+}
+
 // refreshCredentialsLoop handles the credential retrieval and refresh loop.
-func (cp *credentialsProvider) refreshCredentialsLoop(ctx context.Context, provider *stscreds.WebIdentityRoleProvider, credsChan chan credential.Result) {
+func (cp *credentialsProvider) refreshCredentialsLoop(ctx context.Context, cfg *credentialsConfig, credsChan chan credential.Result) {
+	tracer := tokenextelemetry.Tracer(ctx, cfg.tracerProvider, instrumentationScopeName)
+
+	// Create WebIdentityRoleProvider options
+	providerOpts := []func(*stscreds.WebIdentityRoleOptions){
+		func(o *stscreds.WebIdentityRoleOptions) {
+			o.RoleSessionName = cfg.roleSessionName
+			if cfg.durationSeconds != nil {
+				o.Duration = time.Duration(*cfg.durationSeconds) * time.Second
+			}
+		},
+	}
+
+	// Create the WebIdentityRoleProvider
+	retriever := &tokenRetriever{
+		provider: cfg.identityTokenProvider,
+	}
+
+	provider := stscreds.NewWebIdentityRoleProvider(
+		cp.stsClient,
+		cfg.roleArn,
+		retriever,
+		providerOpts...,
+	)
+
+	correlationID := uuid.NewString()
+	configAttrs := fetchSpanConfigAttrs(cfg, cp.stsClient.Options().Region, correlationID)
+
 	for {
-		// Get credentials
-		awsCreds, err := provider.Retrieve(ctx)
+		awsCreds, err := fetchCredentials(ctx, tracer, configAttrs, provider, retriever)
 		if err != nil {
-			util.SendErrorToChannel(credsChan, errors.WrapIf(err, "failed to retrieve credentials"))
+			util.SendErrorToChannel(credsChan, err)
 
 			return
 		}
@@ -210,29 +264,11 @@ func (cp *credentialsProvider) GetCredentials(ctx context.Context, tokenProvider
 		return nil, errors.WrapIf(err, "failed to get initial ID token")
 	}
 
-	// Create WebIdentityRoleProvider options
-	providerOpts := []func(*stscreds.WebIdentityRoleOptions){
-		func(o *stscreds.WebIdentityRoleOptions) {
-			o.RoleSessionName = cfg.roleSessionName
-			if cfg.durationSeconds != nil {
-				o.Duration = time.Duration(*cfg.durationSeconds) * time.Second
-			}
-		},
-	}
-
-	// Create the WebIdentityRoleProvider
-	provider := stscreds.NewWebIdentityRoleProvider(
-		cp.stsClient,
-		cfg.roleArn,
-		&tokenRetriever{provider: tokenProvider, ctx: ctx},
-		providerOpts...,
-	)
-
 	credsChan := make(chan credential.Result, 1)
 
 	go func() {
 		defer close(credsChan)
-		cp.refreshCredentialsLoop(ctx, provider, credsChan)
+		cp.refreshCredentialsLoop(ctx, cfg, credsChan)
 	}()
 
 	return credsChan, nil

pkg/aws/option.go

@@ -4,6 +4,8 @@
 package aws
 
 import (
+	"go.opentelemetry.io/otel/trace"
+
 	"go.riptides.io/tokenex/pkg/option"
 	"go.riptides.io/tokenex/pkg/token"
 )
@@ -71,3 +73,11 @@ func WithIdentityTokenProvider(idtp token.IdentityTokenProvider) option.Option {
 		c.identityTokenProvider = idtp
 	})
 }
+
+// WithTracerProvider sets the OTel TracerProvider used to emit credential.fetch spans.
+// If not set, the tracer falls back to the current span's TracerProvider (if any), then the OTel global TracerProvider.
+func WithTracerProvider(tracerProvider trace.TracerProvider) option.Option {
+	return withCredentialsOption(func(c *credentialsConfig) {
+		c.tracerProvider = tracerProvider
+	})
+}

pkg/aws/telemetry.go

@@ -0,0 +1,46 @@
+// Copyright (c) 2026 Riptides Labs, Inc.
+// SPDX-License-Identifier: MIT
+
+package aws
+
+import (
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"go.opentelemetry.io/otel/attribute"
+)
+
+const fetchSpanName = "credential.fetch"
+
+const instrumentationScopeName = "go.riptides.io/tokenex/pkg/aws"
+
+func fetchSpanConfigAttrs(cfg *credentialsConfig, region string, correlationID string) []attribute.KeyValue {
+	attrs := []attribute.KeyValue{
+		attribute.String("cfg.role_arn", cfg.roleArn),
+		attribute.String("cfg.role_session_name", cfg.roleSessionName),
+		attribute.String("correlation_id", correlationID),
+	}
+	if cfg.durationSeconds != nil {
+		attrs = append(attrs, attribute.Int64("cfg.duration_seconds", int64(*cfg.durationSeconds)))
+	}
+	if region != "" {
+		attrs = append(attrs, attribute.String("cfg.region", region))
+	}
+
+	return attrs
+}
+
+func fetchSpanResultAttrs(awsCreds aws.Credentials) []attribute.KeyValue {
+	attrs := []attribute.KeyValue{
+		attribute.Bool("credential.expires", awsCreds.CanExpire),
+		attribute.String("credential.aws.source", awsCreds.Source),
+	}
+	if awsCreds.AccountID != "" {
+		attrs = append(attrs, attribute.String("credential.aws.account_id", awsCreds.AccountID))
+	}
+	if awsCreds.CanExpire {
+		attrs = append(attrs, attribute.String("credential.expires_at", awsCreds.Expires.UTC().Format(time.RFC3339)))
+	}
+
+	return attrs
+}

pkg/aws/token_retriever.go

@@ -4,28 +4,33 @@
 package aws
 
 import (
-	"context"
-
 	"github.com/aws/aws-sdk-go-v2/credentials/stscreds"
+	"go.opentelemetry.io/otel/trace"
 
+	tokenextelemetry "go.riptides.io/tokenex/pkg/telemetry"
 	"go.riptides.io/tokenex/pkg/token"
+	"go.riptides.io/tokenex/pkg/util"
 )
 
 // tokenRetriever implements stscreds.IdentityTokenRetriever.
 type tokenRetriever struct {
 	provider token.IdentityTokenProvider
-	//nolint: containedctx
-	ctx context.Context
+
+	ctx util.ContextHolder
 }
 
 // GetIdentityToken returns the token from the provider after validating it.
 func (t *tokenRetriever) GetIdentityToken() ([]byte, error) {
-	token, err := t.provider.GetToken(t.ctx)
+	ctx := t.ctx.Context()
+
+	idToken, err := t.provider.GetToken(ctx)
 	if err != nil {
 		return nil, err
 	}
 
-	return []byte(token.Token), nil
+	trace.SpanFromContext(ctx).SetAttributes(tokenextelemetry.IdentityTokenAttrs("id_token", idToken.Token, idToken.ExpiresAt)...)
+
+	return []byte(idToken.Token), nil
 }
 
 // Ensure tokenRetriever implements the interface.