From 2b4f0434cbd1bac8be81bcf542d4822b3702235d Mon Sep 17 00:00:00 2001 From: Dragoslav Pavkovic Date: Thu, 29 Oct 2026 13:31:47 +0100 Subject: [PATCH 1/5] Handling opentelemetry tracing --- .../ProcessInstanceLogger.scala | 15 -- core/baker-interface-in-memory/README.md | 47 ++++ core/baker-interface-in-memory/pom.xml | 17 ++ .../baker/runtime/inmemory/InMemoryBaker.kt | 201 +++++++++++++++++- .../inmemory/InMemoryBakerTracingTest.kt | 193 +++++++++++++++++ .../src/test/resources/logback-test.xml | 2 +- .../runtime/model/TracingMdcRegistry.java | 52 +++++ .../baker/runtime/model/BakerLogging.scala | 37 +++- .../recipeinstance/TransitionExecution.scala | 12 +- pom.xml | 3 +- 10 files changed, 550 insertions(+), 29 deletions(-) create mode 100644 core/baker-interface-in-memory/src/test/kotlin/com/ing/baker/runtime/inmemory/InMemoryBakerTracingTest.kt create mode 100644 core/baker-interface/src/main/java/com/ing/baker/runtime/model/TracingMdcRegistry.java diff --git a/core/baker-akka-actors/src/main/scala/com/ing/baker/runtime/akka/actor/process_instance/ProcessInstanceLogger.scala b/core/baker-akka-actors/src/main/scala/com/ing/baker/runtime/akka/actor/process_instance/ProcessInstanceLogger.scala index b47643716..7f39f07a3 100644 --- a/core/baker-akka-actors/src/main/scala/com/ing/baker/runtime/akka/actor/process_instance/ProcessInstanceLogger.scala +++ b/core/baker-akka-actors/src/main/scala/com/ing/baker/runtime/akka/actor/process_instance/ProcessInstanceLogger.scala @@ -9,21 +9,6 @@ import scala.concurrent.duration._ object ProcessInstanceLogger { - import org.joda.time.format.PeriodFormatterBuilder - - val durationFormatter = new PeriodFormatterBuilder() - .appendDays.appendSuffix("d") - .appendSeparator(" ") - .appendHours.appendSuffix("h") - .appendSeparator(" ") - .appendMinutes.appendSuffix("m") - .appendSeparator(" ") - .appendSeconds.appendSuffix("s") - .appendSeparator(" ") - .appendMillis.appendSuffix("ms") - .appendSeparator(" ") - .toFormatter - implicit class LoggingAdapterFns(log: DiagnosticLoggingAdapter) { val bakerLogging: BakerLogging = BakerLogging.default diff --git a/core/baker-interface-in-memory/README.md b/core/baker-interface-in-memory/README.md index 00b600fb3..419ab30e9 100644 --- a/core/baker-interface-in-memory/README.md +++ b/core/baker-interface-in-memory/README.md @@ -37,3 +37,50 @@ fun main() = runBlocking { - `KotlinBaker` mirrors the Kotlin adapter shape from `baker-interface-kotlin`. - It supports `suspend` APIs and selected `CompletableFuture` wrappers such as `javaAsync`, `bakeAsync`, and `awaitCompletedAsync`. +## OpenTelemetry tracing + +The in-memory runtime emits spans using OpenTelemetry when built through `InMemoryBaker`. + +### What is traced + +- Build lifecycle: `baker.build` +- Recipe lifecycle: `baker.recipe-instance.created` +- Event flow: `baker.event.received`, `baker.event.fired` +- Interaction execution: `baker.interaction.` + +Interaction spans are created as children of the latest event span for the same recipe instance. + +### Common attributes + +- `baker.recipe.id` +- `baker.recipe.name` +- `baker.recipe.instance.id` + +Depending on the span type, additional attributes are set: + +- `baker.event.name` +- `baker.interaction.name` +- `baker.correlation.id` +- `baker.failure.count` +- `error.type` (for failed interactions) + +### Injecting a custom tracer + +By default, Baker uses `GlobalOpenTelemetry.getTracer(...)`. You can provide your own tracer: + +```kotlin +import com.ing.baker.runtime.inmemory.InMemoryBaker +import com.ing.baker.runtime.model.BakerConfig +import io.opentelemetry.api.GlobalOpenTelemetry + +val tracer = GlobalOpenTelemetry.getTracer("my-service.baker") +val baker = InMemoryBaker.java(BakerConfig.default(), emptyList(), tracer) +``` + +For async Java factory usage: + +```kotlin +val tracer = GlobalOpenTelemetry.getTracer("my-service.baker") +val bakerFuture = InMemoryBaker.javaAsync(BakerConfig.default(), emptyList(), tracer) +``` + diff --git a/core/baker-interface-in-memory/pom.xml b/core/baker-interface-in-memory/pom.xml index 3a0970d6b..37c1c0bd5 100644 --- a/core/baker-interface-in-memory/pom.xml +++ b/core/baker-interface-in-memory/pom.xml @@ -46,6 +46,11 @@ org.slf4j slf4j-api + + io.opentelemetry + opentelemetry-api + ${opentelemetry.version} + @@ -120,6 +125,18 @@ ${mockito.core.version} test + + io.opentelemetry + opentelemetry-sdk + ${opentelemetry.version} + test + + + io.opentelemetry + opentelemetry-sdk-testing + ${opentelemetry.version} + test + ch.qos.logback logback-classic diff --git a/core/baker-interface-in-memory/src/main/kotlin/com/ing/baker/runtime/inmemory/InMemoryBaker.kt b/core/baker-interface-in-memory/src/main/kotlin/com/ing/baker/runtime/inmemory/InMemoryBaker.kt index b743d69c9..80c583403 100644 --- a/core/baker-interface-in-memory/src/main/kotlin/com/ing/baker/runtime/inmemory/InMemoryBaker.kt +++ b/core/baker-interface-in-memory/src/main/kotlin/com/ing/baker/runtime/inmemory/InMemoryBaker.kt @@ -5,7 +5,22 @@ import com.ing.baker.runtime.model.BakerConfig import com.ing.baker.runtime.model.BakerF import com.ing.baker.runtime.model.BakerLogging import com.ing.baker.runtime.model.InteractionInstance +import com.ing.baker.runtime.model.TracingMdcRegistry +import com.ing.baker.runtime.scaladsl.EventFired +import com.ing.baker.runtime.scaladsl.EventReceived +import com.ing.baker.runtime.scaladsl.InteractionCompleted +import com.ing.baker.runtime.scaladsl.InteractionFailed +import com.ing.baker.runtime.scaladsl.InteractionStarted +import com.ing.baker.runtime.scaladsl.RecipeInstanceCreated +import io.opentelemetry.api.GlobalOpenTelemetry +import io.opentelemetry.api.trace.Span +import io.opentelemetry.api.trace.StatusCode +import io.opentelemetry.api.trace.Tracer +import io.opentelemetry.context.Context +import scala.runtime.AbstractFunction1 +import scala.runtime.BoxedUnit import java.util.concurrent.CompletableFuture +import java.util.concurrent.ConcurrentHashMap import com.ing.baker.runtime.defaultinteractions.`package$`.`MODULE$` as defaultinteractions import com.ing.baker.runtime.javadsl.Baker as JavaBaker import scala.collection.immutable.List as ScalaList @@ -20,25 +35,59 @@ import scala.collection.immutable.List as ScalaList @Suppress("ABSTRACT_MEMBER_NOT_IMPLEMENTED", "UNCHECKED_CAST") class InMemoryBaker( private val bakerConfig: BakerConfig, - components: BakerComponents> + components: BakerComponents>, + private val tracer: Tracer = defaultTracer() ) : BakerF>( components, InMemoryEffects.asyncSupportAny(), InMemoryEffects.asyncSupportAny() ) { + private val interactionSpans: ConcurrentHashMap = ConcurrentHashMap() + private val latestRecipeInstanceContexts: ConcurrentHashMap = ConcurrentHashMap() + + private object TraceAttributes { + const val RECIPE_ID = "baker.recipe.id" + const val RECIPE_NAME = "baker.recipe.name" + const val RECIPE_INSTANCE_ID = "baker.recipe.instance.id" + const val EVENT_NAME = "baker.event.name" + const val INTERACTION_NAME = "baker.interaction.name" + const val CORRELATION_ID = "baker.correlation.id" + const val FAILURE_COUNT = "baker.failure.count" + const val ERROR_TYPE = "error.type" + } + + init { + registerTracingListener() + } + companion object { + private const val INSTRUMENTATION_NAME = "com.ing.baker.runtime.inmemory.InMemoryBaker" + + private fun defaultTracer(): Tracer = + GlobalOpenTelemetry.getTracer(INSTRUMENTATION_NAME) + @JvmStatic fun build(implementations: ScalaList<*>): CompletableFuture>> = - build(BakerConfig.default(), implementations) + build(BakerConfig.default(), implementations, defaultTracer()) @JvmStatic @Suppress("UNCHECKED_CAST") fun build( config: BakerConfig, implementations: ScalaList<*> + ): CompletableFuture>> = + build(config, implementations, defaultTracer()) + + @JvmStatic + @Suppress("UNCHECKED_CAST") + fun build( + config: BakerConfig, + implementations: ScalaList<*>, + tracer: Tracer ): CompletableFuture>> { + val buildSpan = tracer.spanBuilder("baker.build").startSpan() val ioAsyncSupport = InMemoryEffects.asyncSupportAny() val ioClassTag = InMemoryEffects.classTagAny() val builtinInteractions = @@ -61,18 +110,39 @@ class InMemoryBaker( eventStream, BakerLogging.default() ) - return InMemoryEffects.pure( - InMemoryBaker(config, components) - ) + return try { + val bakerFuture: CompletableFuture>> = + InMemoryEffects.pure(InMemoryBaker(config, components, tracer) as BakerF>) + + bakerFuture.whenComplete { _, throwable -> + if (throwable != null) { + buildSpan.recordException(throwable) + buildSpan.setStatus(StatusCode.ERROR) + } else { + buildSpan.setStatus(StatusCode.OK) + } + buildSpan.end() + } + } catch (throwable: Throwable) { + buildSpan.recordException(throwable) + buildSpan.setStatus(StatusCode.ERROR) + buildSpan.end() + throw throwable + } } @JvmStatic fun java(config: BakerConfig, implementations: List): JavaBaker { + return java(config, implementations, defaultTracer()) + } + + @JvmStatic + fun java(config: BakerConfig, implementations: List, tracer: Tracer): JavaBaker { val futureToCompletableFuture = InMemoryEffects.futureToCompletableFuture() val completableFutureToFuture = InMemoryEffects.completableFutureToFuture() val scalaInteractions = InMemoryEffects.toScalaCompletableFutureInteractions(implementations) - return build(config, scalaInteractions) + return build(config, scalaInteractions, tracer) .let { InMemoryEffects.runSync(it) } .asDeprecatedFutureImplementation(completableFutureToFuture, futureToCompletableFuture) .let { JavaBaker(it) } @@ -83,11 +153,16 @@ class InMemoryBaker( */ @JvmStatic fun javaAsync(config: BakerConfig, implementations: List): CompletableFuture { + return javaAsync(config, implementations, defaultTracer()) + } + + @JvmStatic + fun javaAsync(config: BakerConfig, implementations: List, tracer: Tracer): CompletableFuture { val futureToCompletableFuture = InMemoryEffects.futureToCompletableFuture() val completableFutureToFuture = InMemoryEffects.completableFutureToFuture() val scalaInteractions = InMemoryEffects.toScalaCompletableFutureInteractions(implementations) - return build(config, scalaInteractions) + return build(config, scalaInteractions, tracer) .thenApply { bakerF -> JavaBaker(bakerF.asDeprecatedFutureImplementation(completableFutureToFuture, futureToCompletableFuture)) } @@ -115,5 +190,115 @@ class InMemoryBaker( /** * Attempts to gracefully shutdown the baker system. */ - override fun gracefulShutdown(): CompletableFuture = InMemoryEffects.unit() + override fun gracefulShutdown(): CompletableFuture { + interactionSpans.values.forEach { span -> + span.setStatus(StatusCode.ERROR) + span.end() + } + interactionSpans.clear() + latestRecipeInstanceContexts.clear() + return InMemoryEffects.unit() + } + + private fun registerTracingListener() { + try { + val listener = object : AbstractFunction1() { + override fun apply(event: com.ing.baker.runtime.scaladsl.BakerEvent): BoxedUnit { + when (event) { + is RecipeInstanceCreated -> + tracer.spanBuilder("baker.recipe-instance.created") + .startSpan() + .apply { + applyRecipeAttributes(event.recipeId(), event.recipeName(), event.recipeInstanceId()) + setStatus(StatusCode.OK) + end() + } + + is EventReceived -> { + val span = tracer.spanBuilder("baker.event.received") + .startSpan() + .apply { + applyRecipeAttributes(event.recipeId(), event.recipeName(), event.recipeInstanceId()) + setAttribute(TraceAttributes.EVENT_NAME, event.eventName()) + event.correlationId().toNullable()?.let { setAttribute(TraceAttributes.CORRELATION_ID, it) } + setStatus(StatusCode.OK) + } + TracingMdcRegistry.put(event.recipeInstanceId(), span.spanContext.traceId, span.spanContext.spanId) + latestRecipeInstanceContexts[event.recipeInstanceId()] = Context.root().with(span) + span.end() + } + + is EventFired -> { + val span = tracer.spanBuilder("baker.event.fired") + .startSpan() + .apply { + applyRecipeAttributes(event.recipeId(), event.recipeName(), event.recipeInstanceId()) + setAttribute(TraceAttributes.EVENT_NAME, event.eventName()) + setStatus(StatusCode.OK) + } + TracingMdcRegistry.put(event.recipeInstanceId(), span.spanContext.traceId, span.spanContext.spanId) + latestRecipeInstanceContexts[event.recipeInstanceId()] = Context.root().with(span) + span.end() + + if (event.eventName() == "RecipeSuccessful") { + latestRecipeInstanceContexts.remove(event.recipeInstanceId()) + TracingMdcRegistry.remove(event.recipeInstanceId()) + } + } + + is InteractionStarted -> { + val parentContext = latestRecipeInstanceContexts[event.recipeInstanceId()] + val builder = tracer.spanBuilder("baker.interaction.${event.interactionName()}") + if (parentContext != null) { + builder.setParent(parentContext) + } + val span = builder + .startSpan() + span.applyRecipeAttributes(event.recipeId(), event.recipeName(), event.recipeInstanceId()) + span.setAttribute(TraceAttributes.INTERACTION_NAME, event.interactionName()) + TracingMdcRegistry.put(event.recipeInstanceId(), span.spanContext.traceId, span.spanContext.spanId) + interactionSpans[interactionKey(event.recipeInstanceId(), event.interactionName())] = span + } + + is InteractionCompleted -> { + interactionSpans.remove(interactionKey(event.recipeInstanceId(), event.interactionName()))?.let { span -> + event.eventName().toNullable()?.let { span.setAttribute(TraceAttributes.EVENT_NAME, it) } + span.setStatus(StatusCode.OK) + span.end() + } + } + + is InteractionFailed -> { + interactionSpans.remove(interactionKey(event.recipeInstanceId(), event.interactionName()))?.let { span -> + span.setAttribute(TraceAttributes.FAILURE_COUNT, event.failureCount().toLong()) + span.setAttribute(TraceAttributes.ERROR_TYPE, "interaction.failure") + span.recordException(RuntimeException(event.errorMessage())) + span.setStatus(StatusCode.ERROR, event.errorMessage()) + span.end() + } + } + + else -> Unit + } + + return BoxedUnit.UNIT + } + } + + InMemoryEffects.runSync(registerBakerEventListener(listener)) + } catch (_: Throwable) { + // Tracing must never prevent Baker startup. + } + } + + private fun interactionKey(recipeInstanceId: String, interactionName: String): String = + "$recipeInstanceId::$interactionName" + + private fun Span.applyRecipeAttributes(recipeId: String, recipeName: String, recipeInstanceId: String) { + setAttribute(TraceAttributes.RECIPE_ID, recipeId) + setAttribute(TraceAttributes.RECIPE_NAME, recipeName) + setAttribute(TraceAttributes.RECIPE_INSTANCE_ID, recipeInstanceId) + } + + private fun scala.Option.toNullable(): T? = if (isDefined) get() else null } diff --git a/core/baker-interface-in-memory/src/test/kotlin/com/ing/baker/runtime/inmemory/InMemoryBakerTracingTest.kt b/core/baker-interface-in-memory/src/test/kotlin/com/ing/baker/runtime/inmemory/InMemoryBakerTracingTest.kt new file mode 100644 index 000000000..08fcdb907 --- /dev/null +++ b/core/baker-interface-in-memory/src/test/kotlin/com/ing/baker/runtime/inmemory/InMemoryBakerTracingTest.kt @@ -0,0 +1,193 @@ +package com.ing.baker.runtime.inmemory + +import com.google.common.collect.ImmutableList +import com.ing.baker.compiler.RecipeCompiler +import com.ing.baker.il.CompiledRecipe +import com.ing.baker.recipe.kotlindsl.ExperimentalDsl +import com.ing.baker.runtime.common.RecipeRecord +import com.ing.baker.runtime.inmemory.recipe.CustomerInfo +import com.ing.baker.runtime.inmemory.recipe.LogRecipeFailedImpl +import com.ing.baker.runtime.inmemory.recipe.ManufactureGoods +import com.ing.baker.runtime.inmemory.recipe.SendInvoice +import com.ing.baker.runtime.inmemory.recipe.SensoryEvents.CustomerInfoReceived +import com.ing.baker.runtime.inmemory.recipe.SensoryEvents.OrderPlaced +import com.ing.baker.runtime.inmemory.recipe.SensoryEvents.PaymentMade +import com.ing.baker.runtime.inmemory.recipe.ShipGoods +import com.ing.baker.runtime.inmemory.recipe.ValidateOrder +import com.ing.baker.runtime.inmemory.recipe.WebShopRecipe +import com.ing.baker.runtime.javadsl.EventInstance +import com.ing.baker.runtime.model.BakerConfig +import io.opentelemetry.api.common.AttributeKey +import io.opentelemetry.api.trace.StatusCode +import io.opentelemetry.api.trace.Tracer +import io.opentelemetry.sdk.OpenTelemetrySdk +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter +import io.opentelemetry.sdk.trace.SdkTracerProvider +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.mockito.ArgumentMatchers.anyString +import org.mockito.Mock +import org.mockito.Mockito +import org.mockito.Mockito.`when` +import org.mockito.MockitoAnnotations +import java.time.Duration +import java.util.UUID + +class InMemoryBakerTracingTest { + + @Mock + lateinit var manufactureGoods: ManufactureGoods + + @Mock + lateinit var sendInvoice: SendInvoice + + @Mock + lateinit var shipGoods: ShipGoods + + @Mock + lateinit var validateOrder: ValidateOrder + + @OptIn(ExperimentalDsl::class) + val recipe: CompiledRecipe = RecipeCompiler.compileRecipe(WebShopRecipe.recipe) + + @BeforeEach + fun setUp() { + MockitoAnnotations.openMocks(this) + } + + @Test + fun `creates lifecycle and interaction spans for happy flow`() { + TracerFixture.use { fixture -> + val baker = InMemoryBaker.java(BakerConfig.default(), setupMockImplementations(), fixture.tracer) + baker.addRecipe(RecipeRecord.of(recipe, System.currentTimeMillis(), false, true)).get() + + val goods = "goods" + val order = "order" + val trackingId = "trackingId" + val customerInfo = CustomerInfo("name", "address", "email") + + Mockito.doReturn(ValidateOrder.OrderValid()).`when`(validateOrder).apply(anyString(), anyString()) + Mockito.doReturn(ManufactureGoods.GoodsManufactured(goods)).`when`(manufactureGoods).apply(anyString()) + Mockito.doReturn(SendInvoice.InvoiceSent()).`when`(sendInvoice).apply(customerInfo) + Mockito.doReturn(ShipGoods.GoodsShipped(trackingId)).`when`(shipGoods).apply(goods, customerInfo) + + val recipeInstanceId = UUID.randomUUID().toString() + baker.bake(recipe.recipeId(), recipeInstanceId).get() + baker.fireSensoryEventAndAwaitReceived(recipeInstanceId, EventInstance.from(OrderPlaced(order))).get() + baker.fireSensoryEventAndAwaitReceived(recipeInstanceId, EventInstance.from(CustomerInfoReceived(customerInfo))).get() + baker.fireSensoryEventAndAwaitReceived(recipeInstanceId, EventInstance.from(PaymentMade())).get() + baker.awaitCompleted(recipeInstanceId, Duration.ofSeconds(5)).join() + + val spans = fixture.exporter.finishedSpanItems + val names = spans.map { it.name } + + assertTrue(names.contains("baker.build")) + assertTrue(names.contains("baker.recipe-instance.created")) + assertTrue(names.contains("baker.event.received")) + assertTrue(names.contains("baker.event.fired")) + assertTrue(names.contains("baker.interaction.ValidateOrder")) + assertTrue(names.contains("baker.interaction.ShipGoods")) + assertTrue(names.contains("baker.interaction.SendInvoice")) + assertTrue(names.any { it.startsWith("baker.interaction.") }) + + assertTrue( + spans.any { + it.name == "baker.interaction.ValidateOrder" && + it.status.statusCode == StatusCode.OK && + it.attributes.get(AttributeKey.stringKey("baker.recipe.instance.id")) == recipeInstanceId + }, + "Expected traced ValidateOrder span with recipe instance attribute" + ) + + val eventSpanIds = spans + .filter { + (it.name == "baker.event.received" || it.name == "baker.event.fired") && + it.attributes.get(AttributeKey.stringKey("baker.recipe.instance.id")) == recipeInstanceId + } + .map { it.spanContext.spanId } + .toSet() + + val validateOrderSpan = spans.firstOrNull { + it.name == "baker.interaction.ValidateOrder" && + it.attributes.get(AttributeKey.stringKey("baker.recipe.instance.id")) == recipeInstanceId + } + + assertTrue( + validateOrderSpan != null && eventSpanIds.contains(validateOrderSpan.parentSpanContext.spanId), + "Expected interaction span to be linked as child of an event span" + ) + } + } + + @Test + fun `records failed interaction as error span`() { + TracerFixture.use { fixture -> + val baker = InMemoryBaker.java(BakerConfig.default(), setupMockImplementations(), fixture.tracer) + baker.addRecipe(RecipeRecord.of(recipe, System.currentTimeMillis(), false, true)).get() + + val goods = "goods" + val order = "order" + val customerInfo = CustomerInfo("name", "address", "email") + + `when`(validateOrder.apply(anyString(), anyString())).thenReturn(ValidateOrder.OrderValid()) + `when`(manufactureGoods.apply(anyString())).thenReturn(ManufactureGoods.GoodsManufactured(goods)) + `when`(shipGoods.apply(goods, customerInfo)).thenReturn(ShipGoods.GoodsShipped("tracking-id")) + `when`(sendInvoice.apply(customerInfo)).thenThrow(RuntimeException("Expected tracing failure")) + + val recipeInstanceId = UUID.randomUUID().toString() + baker.bake(recipe.recipeId(), recipeInstanceId).get() + baker.fireSensoryEventAndAwaitReceived(recipeInstanceId, EventInstance.from(OrderPlaced(order))).get() + baker.fireSensoryEventAndAwaitReceived(recipeInstanceId, EventInstance.from(CustomerInfoReceived(customerInfo))).get() + baker.fireSensoryEventAndAwaitReceived(recipeInstanceId, EventInstance.from(PaymentMade())).get() + baker.awaitCompleted(recipeInstanceId, Duration.ofSeconds(5)).join() + + val spans = fixture.exporter.finishedSpanItems + + assertTrue( + spans.any { + it.name == "baker.interaction.SendInvoice" && + it.status.statusCode == StatusCode.ERROR && + it.attributes.get(AttributeKey.stringKey("error.type")) == "interaction.failure" + }, + "Expected failed SendInvoice interaction to be traced as an error span" + ) + } + } + + private fun setupMockImplementations(): List { + return ImmutableList.of(manufactureGoods, sendInvoice, shipGoods, validateOrder, LogRecipeFailedImpl()) + } + + private class TracerFixture private constructor( + val exporter: InMemorySpanExporter, + val tracerProvider: SdkTracerProvider, + val tracer: Tracer + ) : AutoCloseable { + + override fun close() { + tracerProvider.close() + exporter.reset() + } + + companion object { + inline fun use(block: (TracerFixture) -> Unit) { + val exporter = InMemorySpanExporter.create() + val tracerProvider = SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(exporter)) + .build() + val openTelemetry = OpenTelemetrySdk.builder() + .setTracerProvider(tracerProvider) + .build() + + TracerFixture( + exporter = exporter, + tracerProvider = tracerProvider, + tracer = openTelemetry.getTracer("test.inmemory.baker") + ).use(block) + } + } + } +} + diff --git a/core/baker-interface-in-memory/src/test/resources/logback-test.xml b/core/baker-interface-in-memory/src/test/resources/logback-test.xml index 5464005f1..82fccfda0 100644 --- a/core/baker-interface-in-memory/src/test/resources/logback-test.xml +++ b/core/baker-interface-in-memory/src/test/resources/logback-test.xml @@ -2,7 +2,7 @@ - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + %d{HH:mm:ss.SSS} %cyan([%-20.20thread]) %highlight(%-5level) %magenta(%-20.20logger{20}) %yellow(recipe=%X{baker.recipe.name:-}) - %msg%nopex%n diff --git a/core/baker-interface/src/main/java/com/ing/baker/runtime/model/TracingMdcRegistry.java b/core/baker-interface/src/main/java/com/ing/baker/runtime/model/TracingMdcRegistry.java new file mode 100644 index 000000000..63c8784a6 --- /dev/null +++ b/core/baker-interface/src/main/java/com/ing/baker/runtime/model/TracingMdcRegistry.java @@ -0,0 +1,52 @@ +package com.ing.baker.runtime.model; + +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Shares best-effort trace/span ids for MDC enrichment across runtime modules. + */ +public final class TracingMdcRegistry { + + private static final ConcurrentHashMap contexts = new ConcurrentHashMap<>(); + + private TracingMdcRegistry() { + } + + public static void put(String recipeInstanceId, String traceId, String spanId) { + if (recipeInstanceId == null || traceId == null || spanId == null) { + return; + } + contexts.put(recipeInstanceId, new TraceContext(traceId, spanId)); + } + + public static Optional get(String recipeInstanceId) { + return Optional.ofNullable(contexts.get(recipeInstanceId)); + } + + public static void remove(String recipeInstanceId) { + if (recipeInstanceId == null) { + return; + } + contexts.remove(recipeInstanceId); + } + + public static final class TraceContext { + private final String traceId; + private final String spanId; + + public TraceContext(String traceId, String spanId) { + this.traceId = traceId; + this.spanId = spanId; + } + + public String getTraceId() { + return traceId; + } + + public String getSpanId() { + return spanId; + } + } +} + diff --git a/core/baker-interface/src/main/scala/com/ing/baker/runtime/model/BakerLogging.scala b/core/baker-interface/src/main/scala/com/ing/baker/runtime/model/BakerLogging.scala index 2b58c42ac..898583e6c 100644 --- a/core/baker-interface/src/main/scala/com/ing/baker/runtime/model/BakerLogging.scala +++ b/core/baker-interface/src/main/scala/com/ing/baker/runtime/model/BakerLogging.scala @@ -8,6 +8,7 @@ import org.joda.time.format.{PeriodFormatter, PeriodFormatterBuilder} import org.slf4j.MDC import scala.concurrent.duration.FiniteDuration +import scala.jdk.OptionConverters._ object BakerLogging { lazy val defaultLogger: Logger = Logger("com.ing.baker") @@ -17,7 +18,7 @@ object BakerLogging { case class BakerLogging(logger: Logger = BakerLogging.defaultLogger) { - val durationFormatter: PeriodFormatter = new PeriodFormatterBuilder() + private val durationFormatter: PeriodFormatter = new PeriodFormatterBuilder() .appendDays.appendSuffix("d") .appendSeparator(" ") .appendHours.appendSuffix("h") @@ -30,10 +31,40 @@ case class BakerLogging(logger: Logger = BakerLogging.defaultLogger) { .appendSeparator(" ") .toFormatter + private val RecipeNameKey = "recipeName" + private val RecipeIdKey = "recipeId" + private val RecipeInstanceIdKey = "recipeInstanceId" + + private val BakerRecipeNameKey = "baker.recipe.name" + private val BakerRecipeIdKey = "baker.recipe.id" + private val BakerRecipeInstanceIdKey = "baker.recipe.instance.id" + private val TraceIdKey = "trace_id" + private val SpanIdKey = "span_id" + + private def enrichMdc(mdc: Map[String, String]): Map[String, String] = { + val withRecipeNameAliases = mdc.get(RecipeNameKey) + .map(v => mdc + (BakerRecipeNameKey -> v)) + .getOrElse(mdc) + + val withRecipeIdAliases = withRecipeNameAliases.get(RecipeIdKey) + .map(v => withRecipeNameAliases + (BakerRecipeIdKey -> v)) + .getOrElse(withRecipeNameAliases) + + val withInstanceAliases = withRecipeIdAliases.get(RecipeInstanceIdKey) + .map(v => withRecipeIdAliases + (BakerRecipeInstanceIdKey -> v)) + .getOrElse(withRecipeIdAliases) + + withInstanceAliases.get(RecipeInstanceIdKey) + .flatMap(id => TracingMdcRegistry.get(id).toScala) + .map(context => withInstanceAliases + (TraceIdKey -> context.getTraceId) + (SpanIdKey -> context.getSpanId)) + .getOrElse(withInstanceAliases) + } + def withMDC(mdc: Map[String, String], log: Logger => Unit): Unit = { - mdc.foreach { case (k, v) => MDC.put(k, v) } + val effectiveMdc = enrichMdc(mdc) + effectiveMdc.foreach { case (k, v) => MDC.put(k, v) } log(logger) - mdc.keys.foreach(MDC.remove) + effectiveMdc.keys.foreach(MDC.remove) } def addedRecipe(recipeAdded: RecipeAdded): Unit = { diff --git a/core/baker-interface/src/main/scala/com/ing/baker/runtime/model/recipeinstance/TransitionExecution.scala b/core/baker-interface/src/main/scala/com/ing/baker/runtime/model/recipeinstance/TransitionExecution.scala index d21ee8134..8f19e83db 100644 --- a/core/baker-interface/src/main/scala/com/ing/baker/runtime/model/recipeinstance/TransitionExecution.scala +++ b/core/baker-interface/src/main/scala/com/ing/baker/runtime/model/recipeinstance/TransitionExecution.scala @@ -7,7 +7,7 @@ import com.ing.baker.il.{CompiledRecipe, IngredientDescriptor} import com.ing.baker.petrinet.api._ import com.ing.baker.runtime.catseffect.AsyncSupport import com.ing.baker.runtime.catseffect.SyncSupport.syntax._ -import com.ing.baker.runtime.model.BakerComponents +import com.ing.baker.runtime.model.{BakerComponents, TracingMdcRegistry} import com.ing.baker.runtime.model.recipeinstance.RecipeInstance.FatalInteractionException import com.ing.baker.runtime.scaladsl._ import com.ing.baker.types.{ListValue, PrimitiveValue, Value} @@ -160,11 +160,21 @@ private[recipeinstance] case class TransitionExecution( def setupMdc: F[Unit] = async.delay { MDC.put("recipeInstanceId", recipeInstanceId) MDC.put("recipeName", recipe.name) + MDC.put("baker.recipe.instance.id", recipeInstanceId) + MDC.put("baker.recipe.name", recipe.name) + TracingMdcRegistry.get(recipeInstanceId).ifPresent(context => { + MDC.put("trace_id", context.getTraceId) + MDC.put("span_id", context.getSpanId) + }) } def cleanMdc: F[Unit] = async.delay { MDC.remove("recipeInstanceId") MDC.remove("recipeName") + MDC.remove("baker.recipe.instance.id") + MDC.remove("baker.recipe.name") + MDC.remove("trace_id") + MDC.remove("span_id") } def execute: F[Option[EventInstance]] = diff --git a/pom.xml b/pom.xml index f7b3307dd..b65cff919 100644 --- a/pom.xml +++ b/pom.xml @@ -46,9 +46,10 @@ 2.3.21 1.11.0 1.10.0 - 1.5.37 + 1.6.5 4.2.39 3.5 + 1.67.0 2.8.3 0.16.0 0.20.0 From 1ff8993e7805c77316b7231cf94c9f71b6d56590 Mon Sep 17 00:00:00 2001 From: Dragoslav Pavkovic Date: Fri, 30 Oct 2026 15:10:17 +0100 Subject: [PATCH 2/5] Fixing OTel propagation --- core/baker-interface-in-memory/README.md | 37 +++ core/baker-interface-in-memory/pom.xml | 5 + .../baker/runtime/inmemory/InMemoryBaker.kt | 83 ++---- .../InMemoryBakerTracingConcurrencyTest.kt | 238 ++++++++++++++++++ .../inmemory/InMemoryBakerTracingTest.kt | 84 ++++++- core/baker-interface-otel/pom.xml | 74 ++++++ .../runtime/otel/BakerTraceAttributes.java | 21 ++ .../otel/OpenTelemetryContextPropagator.java | 34 +++ ...enTelemetryInteractionInstrumentation.java | 119 +++++++++ .../otel/RecipeInstanceTraceContexts.java | 43 ++++ ...del.propagation.ExecutionContextPropagator | 1 + .../otel/BlockingPoolPropagationTest.java | 93 +++++++ .../OpenTelemetryContextPropagatorTest.java | 104 ++++++++ ...lemetryInteractionInstrumentationTest.java | 135 ++++++++++ .../runtime/model/InteractionCallContext.java | 65 +++++ .../model/InteractionInstrumentation.java | 26 ++ .../runtime/model/TracingMdcRegistry.java | 19 +- .../model/propagation/ContextScope.java | 14 ++ .../model/propagation/ContextSnapshot.java | 15 ++ .../ExecutionContextPropagator.java | 20 ++ .../ExecutionContextPropagators.java | 123 +++++++++ .../propagation/MdcContextPropagator.java | 34 +++ .../runtime/catseffect/AsyncSupport.scala | 5 +- .../runtime/catseffect/SyncSupport.scala | 39 ++- .../baker/runtime/model/BakerComponents.scala | 3 +- .../baker/runtime/model/BakerLogging.scala | 24 +- .../runtime/model/InteractionManager.scala | 17 +- .../recipeinstance/TransitionExecution.scala | 49 ++-- pom.xml | 1 + 29 files changed, 1422 insertions(+), 103 deletions(-) create mode 100644 core/baker-interface-in-memory/src/test/kotlin/com/ing/baker/runtime/inmemory/InMemoryBakerTracingConcurrencyTest.kt create mode 100644 core/baker-interface-otel/pom.xml create mode 100644 core/baker-interface-otel/src/main/java/com/ing/baker/runtime/otel/BakerTraceAttributes.java create mode 100644 core/baker-interface-otel/src/main/java/com/ing/baker/runtime/otel/OpenTelemetryContextPropagator.java create mode 100644 core/baker-interface-otel/src/main/java/com/ing/baker/runtime/otel/OpenTelemetryInteractionInstrumentation.java create mode 100644 core/baker-interface-otel/src/main/java/com/ing/baker/runtime/otel/RecipeInstanceTraceContexts.java create mode 100644 core/baker-interface-otel/src/main/resources/META-INF/services/com.ing.baker.runtime.model.propagation.ExecutionContextPropagator create mode 100644 core/baker-interface-otel/src/test/java/com/ing/baker/runtime/otel/BlockingPoolPropagationTest.java create mode 100644 core/baker-interface-otel/src/test/java/com/ing/baker/runtime/otel/OpenTelemetryContextPropagatorTest.java create mode 100644 core/baker-interface-otel/src/test/java/com/ing/baker/runtime/otel/OpenTelemetryInteractionInstrumentationTest.java create mode 100644 core/baker-interface/src/main/java/com/ing/baker/runtime/model/InteractionCallContext.java create mode 100644 core/baker-interface/src/main/java/com/ing/baker/runtime/model/InteractionInstrumentation.java create mode 100644 core/baker-interface/src/main/java/com/ing/baker/runtime/model/propagation/ContextScope.java create mode 100644 core/baker-interface/src/main/java/com/ing/baker/runtime/model/propagation/ContextSnapshot.java create mode 100644 core/baker-interface/src/main/java/com/ing/baker/runtime/model/propagation/ExecutionContextPropagator.java create mode 100644 core/baker-interface/src/main/java/com/ing/baker/runtime/model/propagation/ExecutionContextPropagators.java create mode 100644 core/baker-interface/src/main/java/com/ing/baker/runtime/model/propagation/MdcContextPropagator.java diff --git a/core/baker-interface-in-memory/README.md b/core/baker-interface-in-memory/README.md index 419ab30e9..4a5528630 100644 --- a/core/baker-interface-in-memory/README.md +++ b/core/baker-interface-in-memory/README.md @@ -50,6 +50,29 @@ The in-memory runtime emits spans using OpenTelemetry when built through `InMemo Interaction spans are created as children of the latest event span for the same recipe instance. +### Ambient context inside interactions + +The interaction span is started on the very thread that invokes your interaction implementation, so +`Span.current()` and the SLF4J `MDC` are valid and belong to *that* invocation, also under heavy concurrency: + +```java +public class ValidateOrderImpl implements ValidateOrder { + @Override + public ValidateOrderOutcome apply(String recipeInstanceId, String order) { + Span.current().addEvent("validating"); // the baker.interaction.ValidateOrder span + log.info("validating"); // MDC contains trace_id, span_id and recipeInstanceId + return new OrderValid(); + } +} +``` + +Baker runs interactions on a shared pool. Before and after every invocation the registered context propagators are +cleared, so a pooled thread can never expose the context of a previous execution. If your interaction spawns its own +threads, propagate the context explicitly (for example with `Context.current().wrap(runnable)`). + +For a `CompletableFuture`-returning interaction the span is ended when the future completes; for a blocking +interaction it is ended when the method returns. + ### Common attributes - `baker.recipe.id` @@ -84,3 +107,17 @@ val tracer = GlobalOpenTelemetry.getTracer("my-service.baker") val bakerFuture = InMemoryBaker.javaAsync(BakerConfig.default(), emptyList(), tracer) ``` + +### Where the OpenTelemetry code lives + +The OpenTelemetry-specific code is isolated in the `baker-interface-otel` module: + +- `OpenTelemetryContextPropagator` — propagates the OTel `Context` to Baker's interaction pool. It is discovered + automatically through `META-INF/services`, so simply having the module on the classpath is enough. +- `OpenTelemetryInteractionInstrumentation` — starts, scopes and ends the `baker.interaction.` span. +- `RecipeInstanceTraceContexts` — per-baker mapping of recipe instance to the context the interaction span is parented to. + +`baker-interface` itself stays free of OpenTelemetry. It only exposes the SPIs +`com.ing.baker.runtime.model.propagation.ExecutionContextPropagator` (propagate any thread-bound context) and +`com.ing.baker.runtime.model.InteractionInstrumentation` (wrap every interaction invocation), which you can implement +to plug in a different tracing or diagnostics stack. diff --git a/core/baker-interface-in-memory/pom.xml b/core/baker-interface-in-memory/pom.xml index 37c1c0bd5..484f221b0 100644 --- a/core/baker-interface-in-memory/pom.xml +++ b/core/baker-interface-in-memory/pom.xml @@ -51,6 +51,11 @@ opentelemetry-api ${opentelemetry.version} + + com.ing.baker + baker-interface-otel + ${project.version} + diff --git a/core/baker-interface-in-memory/src/main/kotlin/com/ing/baker/runtime/inmemory/InMemoryBaker.kt b/core/baker-interface-in-memory/src/main/kotlin/com/ing/baker/runtime/inmemory/InMemoryBaker.kt index 80c583403..ad8ec393e 100644 --- a/core/baker-interface-in-memory/src/main/kotlin/com/ing/baker/runtime/inmemory/InMemoryBaker.kt +++ b/core/baker-interface-in-memory/src/main/kotlin/com/ing/baker/runtime/inmemory/InMemoryBaker.kt @@ -5,12 +5,13 @@ import com.ing.baker.runtime.model.BakerConfig import com.ing.baker.runtime.model.BakerF import com.ing.baker.runtime.model.BakerLogging import com.ing.baker.runtime.model.InteractionInstance +import com.ing.baker.runtime.model.InteractionInstrumentation import com.ing.baker.runtime.model.TracingMdcRegistry +import com.ing.baker.runtime.otel.BakerTraceAttributes +import com.ing.baker.runtime.otel.OpenTelemetryInteractionInstrumentation +import com.ing.baker.runtime.otel.RecipeInstanceTraceContexts import com.ing.baker.runtime.scaladsl.EventFired import com.ing.baker.runtime.scaladsl.EventReceived -import com.ing.baker.runtime.scaladsl.InteractionCompleted -import com.ing.baker.runtime.scaladsl.InteractionFailed -import com.ing.baker.runtime.scaladsl.InteractionStarted import com.ing.baker.runtime.scaladsl.RecipeInstanceCreated import io.opentelemetry.api.GlobalOpenTelemetry import io.opentelemetry.api.trace.Span @@ -20,7 +21,6 @@ import io.opentelemetry.context.Context import scala.runtime.AbstractFunction1 import scala.runtime.BoxedUnit import java.util.concurrent.CompletableFuture -import java.util.concurrent.ConcurrentHashMap import com.ing.baker.runtime.defaultinteractions.`package$`.`MODULE$` as defaultinteractions import com.ing.baker.runtime.javadsl.Baker as JavaBaker import scala.collection.immutable.List as ScalaList @@ -36,25 +36,20 @@ import scala.collection.immutable.List as ScalaList class InMemoryBaker( private val bakerConfig: BakerConfig, components: BakerComponents>, - private val tracer: Tracer = defaultTracer() + private val tracer: Tracer = defaultTracer(), + private val traceContexts: RecipeInstanceTraceContexts = RecipeInstanceTraceContexts() ) : BakerF>( components, InMemoryEffects.asyncSupportAny(), InMemoryEffects.asyncSupportAny() ) { - private val interactionSpans: ConcurrentHashMap = ConcurrentHashMap() - private val latestRecipeInstanceContexts: ConcurrentHashMap = ConcurrentHashMap() - private object TraceAttributes { - const val RECIPE_ID = "baker.recipe.id" - const val RECIPE_NAME = "baker.recipe.name" - const val RECIPE_INSTANCE_ID = "baker.recipe.instance.id" - const val EVENT_NAME = "baker.event.name" - const val INTERACTION_NAME = "baker.interaction.name" - const val CORRELATION_ID = "baker.correlation.id" - const val FAILURE_COUNT = "baker.failure.count" - const val ERROR_TYPE = "error.type" + const val RECIPE_ID = BakerTraceAttributes.RECIPE_ID + const val RECIPE_NAME = BakerTraceAttributes.RECIPE_NAME + const val RECIPE_INSTANCE_ID = BakerTraceAttributes.RECIPE_INSTANCE_ID + const val EVENT_NAME = BakerTraceAttributes.EVENT_NAME + const val CORRELATION_ID = BakerTraceAttributes.CORRELATION_ID } init { @@ -103,16 +98,20 @@ class InMemoryBaker( val recipeManager = InMemoryRecipeManager() val eventStream = InMemoryEventStream() val interactions = InMemoryInteractionManager(interactionInstances) + val traceContexts = RecipeInstanceTraceContexts() val components = BakerComponents( interactions, recipeInstanceManager, recipeManager, eventStream, - BakerLogging.default() + BakerLogging.default(), + OpenTelemetryInteractionInstrumentation(tracer, traceContexts) as InteractionInstrumentation ) return try { val bakerFuture: CompletableFuture>> = - InMemoryEffects.pure(InMemoryBaker(config, components, tracer) as BakerF>) + InMemoryEffects.pure( + InMemoryBaker(config, components, tracer, traceContexts) as BakerF> + ) bakerFuture.whenComplete { _, throwable -> if (throwable != null) { @@ -191,12 +190,7 @@ class InMemoryBaker( * Attempts to gracefully shutdown the baker system. */ override fun gracefulShutdown(): CompletableFuture { - interactionSpans.values.forEach { span -> - span.setStatus(StatusCode.ERROR) - span.end() - } - interactionSpans.clear() - latestRecipeInstanceContexts.clear() + traceContexts.clear() return InMemoryEffects.unit() } @@ -224,7 +218,7 @@ class InMemoryBaker( setStatus(StatusCode.OK) } TracingMdcRegistry.put(event.recipeInstanceId(), span.spanContext.traceId, span.spanContext.spanId) - latestRecipeInstanceContexts[event.recipeInstanceId()] = Context.root().with(span) + traceContexts.put(event.recipeInstanceId(), Context.current().with(span)) span.end() } @@ -237,47 +231,15 @@ class InMemoryBaker( setStatus(StatusCode.OK) } TracingMdcRegistry.put(event.recipeInstanceId(), span.spanContext.traceId, span.spanContext.spanId) - latestRecipeInstanceContexts[event.recipeInstanceId()] = Context.root().with(span) + traceContexts.put(event.recipeInstanceId(), Context.current().with(span)) span.end() if (event.eventName() == "RecipeSuccessful") { - latestRecipeInstanceContexts.remove(event.recipeInstanceId()) + traceContexts.remove(event.recipeInstanceId()) TracingMdcRegistry.remove(event.recipeInstanceId()) } } - is InteractionStarted -> { - val parentContext = latestRecipeInstanceContexts[event.recipeInstanceId()] - val builder = tracer.spanBuilder("baker.interaction.${event.interactionName()}") - if (parentContext != null) { - builder.setParent(parentContext) - } - val span = builder - .startSpan() - span.applyRecipeAttributes(event.recipeId(), event.recipeName(), event.recipeInstanceId()) - span.setAttribute(TraceAttributes.INTERACTION_NAME, event.interactionName()) - TracingMdcRegistry.put(event.recipeInstanceId(), span.spanContext.traceId, span.spanContext.spanId) - interactionSpans[interactionKey(event.recipeInstanceId(), event.interactionName())] = span - } - - is InteractionCompleted -> { - interactionSpans.remove(interactionKey(event.recipeInstanceId(), event.interactionName()))?.let { span -> - event.eventName().toNullable()?.let { span.setAttribute(TraceAttributes.EVENT_NAME, it) } - span.setStatus(StatusCode.OK) - span.end() - } - } - - is InteractionFailed -> { - interactionSpans.remove(interactionKey(event.recipeInstanceId(), event.interactionName()))?.let { span -> - span.setAttribute(TraceAttributes.FAILURE_COUNT, event.failureCount().toLong()) - span.setAttribute(TraceAttributes.ERROR_TYPE, "interaction.failure") - span.recordException(RuntimeException(event.errorMessage())) - span.setStatus(StatusCode.ERROR, event.errorMessage()) - span.end() - } - } - else -> Unit } @@ -291,9 +253,6 @@ class InMemoryBaker( } } - private fun interactionKey(recipeInstanceId: String, interactionName: String): String = - "$recipeInstanceId::$interactionName" - private fun Span.applyRecipeAttributes(recipeId: String, recipeName: String, recipeInstanceId: String) { setAttribute(TraceAttributes.RECIPE_ID, recipeId) setAttribute(TraceAttributes.RECIPE_NAME, recipeName) diff --git a/core/baker-interface-in-memory/src/test/kotlin/com/ing/baker/runtime/inmemory/InMemoryBakerTracingConcurrencyTest.kt b/core/baker-interface-in-memory/src/test/kotlin/com/ing/baker/runtime/inmemory/InMemoryBakerTracingConcurrencyTest.kt new file mode 100644 index 000000000..a542eb0f8 --- /dev/null +++ b/core/baker-interface-in-memory/src/test/kotlin/com/ing/baker/runtime/inmemory/InMemoryBakerTracingConcurrencyTest.kt @@ -0,0 +1,238 @@ +package com.ing.baker.runtime.inmemory + +import com.google.common.collect.ImmutableList +import com.ing.baker.compiler.RecipeCompiler +import com.ing.baker.il.CompiledRecipe +import com.ing.baker.recipe.kotlindsl.ExperimentalDsl +import com.ing.baker.runtime.common.RecipeRecord +import com.ing.baker.runtime.inmemory.recipe.CustomerInfo +import com.ing.baker.runtime.inmemory.recipe.LogRecipeFailedImpl +import com.ing.baker.runtime.inmemory.recipe.ManufactureGoodsImpl +import com.ing.baker.runtime.inmemory.recipe.SendInvoiceImpl +import com.ing.baker.runtime.inmemory.recipe.SensoryEvents.CustomerInfoReceived +import com.ing.baker.runtime.inmemory.recipe.SensoryEvents.OrderPlaced +import com.ing.baker.runtime.inmemory.recipe.SensoryEvents.PaymentMade +import com.ing.baker.runtime.inmemory.recipe.ShipGoodsImpl +import com.ing.baker.runtime.inmemory.recipe.ValidateOrder +import com.ing.baker.runtime.inmemory.recipe.WebShopRecipe +import com.ing.baker.runtime.javadsl.Baker +import com.ing.baker.runtime.javadsl.EventInstance +import com.ing.baker.runtime.model.BakerConfig +import io.opentelemetry.api.common.AttributeKey +import io.opentelemetry.api.trace.Span +import io.opentelemetry.api.trace.Tracer +import io.opentelemetry.sdk.OpenTelemetrySdk +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter +import io.opentelemetry.sdk.trace.SdkTracerProvider +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.slf4j.MDC +import java.time.Duration +import java.util.UUID +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit + +/** + * Verifies that the ambient tracing context observed inside an interaction belongs to that very interaction, + * also when many recipe instances execute concurrently on the shared interaction thread pool. + */ +class InMemoryBakerTracingConcurrencyTest { + + private val instanceCount = 32 + private val concurrencyLevel = 8 + private val invalidTraceId = "00000000000000000000000000000000" + + @OptIn(ExperimentalDsl::class) + private val recipe: CompiledRecipe = RecipeCompiler.compileRecipe(WebShopRecipe.recipe) + + class ObservedContext( + val traceId: String, + val spanId: String, + val mdcTraceId: String?, + val mdcSpanId: String?, + val mdcRecipeInstanceId: String? + ) + + /** Records the context that is ambient while the interaction runs. */ + class RecordingValidateOrder( + private val observed: ConcurrentHashMap, + private val started: CountDownLatch + ) : ValidateOrder { + + override fun apply(recipeInstanceId: String, order: String): ValidateOrder.ValidateOrderOutcome { + // Make sure interactions of different recipe instances overlap on the shared pool. + started.countDown() + started.await(10, TimeUnit.SECONDS) + + val spanContext = Span.current().spanContext + observed[recipeInstanceId] = ObservedContext( + traceId = spanContext.traceId, + spanId = spanContext.spanId, + mdcTraceId = MDC.get("trace_id"), + mdcSpanId = MDC.get("span_id"), + mdcRecipeInstanceId = MDC.get("recipeInstanceId") + ) + return ValidateOrder.OrderValid() + } + } + + @Test + fun `concurrent executions keep their own trace inside interactions`() { + val observed = ConcurrentHashMap() + val started = CountDownLatch(concurrencyLevel) + + TracerFixture.use { fixture -> + val baker = InMemoryBaker.java( + BakerConfig.default(), + ImmutableList.of( + ManufactureGoodsImpl(), + SendInvoiceImpl(), + ShipGoodsImpl(), + RecordingValidateOrder(observed, started), + LogRecipeFailedImpl() + ), + fixture.tracer + ) + baker.addRecipe(RecipeRecord.of(recipe, System.currentTimeMillis(), false, true)).get() + + val recipeInstanceIds = (1..instanceCount).map { UUID.randomUUID().toString() } + val executor = Executors.newFixedThreadPool(concurrencyLevel) + try { + recipeInstanceIds.mapIndexed { index, recipeInstanceId -> + executor.submit { runHappyFlow(baker, recipeInstanceId, "order-$index") } + }.forEach { it.get(30, TimeUnit.SECONDS) } + } finally { + executor.shutdownNow() + } + + val spans = fixture.exporter.finishedSpanItems + assertEquals(instanceCount, observed.size, "Expected every execution to record its ambient context") + + val traceIdsPerInstance = recipeInstanceIds.associateWith { recipeInstanceId -> + spans + .filter { it.attributes.get(AttributeKey.stringKey("baker.recipe.instance.id")) == recipeInstanceId } + .map { it.spanContext.traceId } + .toSet() + } + + recipeInstanceIds.forEach { recipeInstanceId -> + val context = observed[recipeInstanceId] + assertNotNull(context, "Expected a recorded context for $recipeInstanceId") + requireNotNull(context) + + assertTrue( + context.traceId != invalidTraceId, + "Expected a valid ambient span inside interaction of $recipeInstanceId" + ) + + val interactionSpan = spans.firstOrNull { + it.name == "baker.interaction.ValidateOrder" && + it.attributes.get(AttributeKey.stringKey("baker.recipe.instance.id")) == recipeInstanceId + } + assertNotNull(interactionSpan, "Expected an interaction span for $recipeInstanceId") + requireNotNull(interactionSpan) + + assertEquals( + interactionSpan.spanContext.spanId, + context.spanId, + "Span.current() inside the interaction of $recipeInstanceId must be its own interaction span" + ) + assertEquals(interactionSpan.spanContext.traceId, context.traceId) + + val otherTraceIds = traceIdsPerInstance + .filterKeys { it != recipeInstanceId } + .values + .flatten() + .toSet() + assertTrue( + !otherTraceIds.contains(context.traceId), + "Execution $recipeInstanceId observed a trace id that belongs to another execution" + ) + } + } + } + + @Test + fun `interaction mdc carries the trace of its own interaction span`() { + val observed = ConcurrentHashMap() + val started = CountDownLatch(concurrencyLevel) + + TracerFixture.use { fixture -> + val baker = InMemoryBaker.java( + BakerConfig.default(), + ImmutableList.of( + ManufactureGoodsImpl(), + SendInvoiceImpl(), + ShipGoodsImpl(), + RecordingValidateOrder(observed, started), + LogRecipeFailedImpl() + ), + fixture.tracer + ) + baker.addRecipe(RecipeRecord.of(recipe, System.currentTimeMillis(), false, true)).get() + + val recipeInstanceIds = (1..instanceCount).map { UUID.randomUUID().toString() } + val executor = Executors.newFixedThreadPool(concurrencyLevel) + try { + recipeInstanceIds.mapIndexed { index, recipeInstanceId -> + executor.submit { runHappyFlow(baker, recipeInstanceId, "order-$index") } + }.forEach { it.get(30, TimeUnit.SECONDS) } + } finally { + executor.shutdownNow() + } + + recipeInstanceIds.forEach { recipeInstanceId -> + val context = requireNotNull(observed[recipeInstanceId]) + assertEquals(context.traceId, context.mdcTraceId, "MDC trace_id must match the ambient span") + assertEquals(context.spanId, context.mdcSpanId, "MDC span_id must match the ambient span") + assertEquals(recipeInstanceId, context.mdcRecipeInstanceId, "MDC must carry the own recipe instance") + } + + assertTrue(MDC.get("trace_id") == null, "Interaction MDC must not leak to the calling thread") + } + } + + private fun runHappyFlow(baker: Baker, recipeInstanceId: String, order: String) { + val customerInfo = CustomerInfo("name", "address", "email") + baker.bake(recipe.recipeId(), recipeInstanceId).get() + baker.fireSensoryEventAndAwaitReceived(recipeInstanceId, EventInstance.from(OrderPlaced(order))).get() + baker.fireSensoryEventAndAwaitReceived(recipeInstanceId, EventInstance.from(CustomerInfoReceived(customerInfo))).get() + baker.fireSensoryEventAndAwaitReceived(recipeInstanceId, EventInstance.from(PaymentMade())).get() + baker.awaitCompleted(recipeInstanceId, Duration.ofSeconds(30)).join() + } + + private class TracerFixture private constructor( + val exporter: InMemorySpanExporter, + val tracerProvider: SdkTracerProvider, + val tracer: Tracer + ) : AutoCloseable { + + override fun close() { + tracerProvider.close() + exporter.reset() + } + + companion object { + inline fun use(block: (TracerFixture) -> Unit) { + val exporter = InMemorySpanExporter.create() + val tracerProvider = SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(exporter)) + .build() + val openTelemetry = OpenTelemetrySdk.builder() + .setTracerProvider(tracerProvider) + .build() + + TracerFixture( + exporter = exporter, + tracerProvider = tracerProvider, + tracer = openTelemetry.getTracer("test.inmemory.baker.concurrency") + ).use(block) + } + } + } +} diff --git a/core/baker-interface-in-memory/src/test/kotlin/com/ing/baker/runtime/inmemory/InMemoryBakerTracingTest.kt b/core/baker-interface-in-memory/src/test/kotlin/com/ing/baker/runtime/inmemory/InMemoryBakerTracingTest.kt index 08fcdb907..85c9d39e5 100644 --- a/core/baker-interface-in-memory/src/test/kotlin/com/ing/baker/runtime/inmemory/InMemoryBakerTracingTest.kt +++ b/core/baker-interface-in-memory/src/test/kotlin/com/ing/baker/runtime/inmemory/InMemoryBakerTracingTest.kt @@ -74,11 +74,7 @@ class InMemoryBakerTracingTest { Mockito.doReturn(ShipGoods.GoodsShipped(trackingId)).`when`(shipGoods).apply(goods, customerInfo) val recipeInstanceId = UUID.randomUUID().toString() - baker.bake(recipe.recipeId(), recipeInstanceId).get() - baker.fireSensoryEventAndAwaitReceived(recipeInstanceId, EventInstance.from(OrderPlaced(order))).get() - baker.fireSensoryEventAndAwaitReceived(recipeInstanceId, EventInstance.from(CustomerInfoReceived(customerInfo))).get() - baker.fireSensoryEventAndAwaitReceived(recipeInstanceId, EventInstance.from(PaymentMade())).get() - baker.awaitCompleted(recipeInstanceId, Duration.ofSeconds(5)).join() + runHappyFlow(baker, recipeInstanceId, order, customerInfo) val spans = fixture.exporter.finishedSpanItems val names = spans.map { it.name } @@ -121,6 +117,71 @@ class InMemoryBakerTracingTest { } } + @Test + fun `assigns a distinct trace to every baker execution`() { + TracerFixture.use { fixture -> + val baker = InMemoryBaker.java(BakerConfig.default(), setupMockImplementations(), fixture.tracer) + baker.addRecipe(RecipeRecord.of(recipe, System.currentTimeMillis(), false, true)).get() + + val goods = "goods" + val customerInfo = CustomerInfo("name", "address", "email") + + Mockito.doReturn(ValidateOrder.OrderValid()).`when`(validateOrder).apply(anyString(), anyString()) + Mockito.doReturn(ManufactureGoods.GoodsManufactured(goods)).`when`(manufactureGoods).apply(anyString()) + Mockito.doReturn(SendInvoice.InvoiceSent()).`when`(sendInvoice).apply(customerInfo) + Mockito.doReturn(ShipGoods.GoodsShipped("trackingId")).`when`(shipGoods).apply(goods, customerInfo) + + val recipeInstanceIds = (1..100).map { UUID.randomUUID().toString() } + recipeInstanceIds.forEachIndexed { index, recipeInstanceId -> + runHappyFlow(baker, recipeInstanceId, "order-$index", customerInfo) + } + + val spans = fixture.exporter.finishedSpanItems + val tracesPerInstance = recipeInstanceIds.associateWith { recipeInstanceId -> + spans + .filter { it.attributes.get(AttributeKey.stringKey("baker.recipe.instance.id")) == recipeInstanceId } + .map { it.spanContext.traceId } + .toSet() + } + + tracesPerInstance.forEach { (recipeInstanceId, traceIds) -> + assertTrue(traceIds.isNotEmpty(), "Expected traced spans for execution $recipeInstanceId") + } + + recipeInstanceIds.forEach { current -> + val otherTraceIds = tracesPerInstance + .filterKeys { it != current } + .values + .flatten() + .toSet() + + assertTrue( + tracesPerInstance.getValue(current).none { otherTraceIds.contains(it) }, + "Expected execution $current to use trace ids that are not shared with other executions" + ) + } + + recipeInstanceIds.forEach { recipeInstanceId -> + val instanceSpans = spans.filter { + it.attributes.get(AttributeKey.stringKey("baker.recipe.instance.id")) == recipeInstanceId + } + val eventTraceIdsBySpanId = instanceSpans + .filter { it.name == "baker.event.received" || it.name == "baker.event.fired" } + .associate { it.spanContext.spanId to it.spanContext.traceId } + + val interactionSpans = instanceSpans.filter { it.name.startsWith("baker.interaction.") } + assertTrue(interactionSpans.isNotEmpty(), "Expected interaction spans for execution $recipeInstanceId") + + assertTrue( + interactionSpans.all { + eventTraceIdsBySpanId[it.parentSpanContext.spanId] == it.spanContext.traceId + }, + "Expected interaction spans of execution $recipeInstanceId to share the trace of their parent event span" + ) + } + } + } + @Test fun `records failed interaction as error span`() { TracerFixture.use { fixture -> @@ -156,6 +217,19 @@ class InMemoryBakerTracingTest { } } + private fun runHappyFlow( + baker: com.ing.baker.runtime.javadsl.Baker, + recipeInstanceId: String, + order: String, + customerInfo: CustomerInfo + ) { + baker.bake(recipe.recipeId(), recipeInstanceId).get() + baker.fireSensoryEventAndAwaitReceived(recipeInstanceId, EventInstance.from(OrderPlaced(order))).get() + baker.fireSensoryEventAndAwaitReceived(recipeInstanceId, EventInstance.from(CustomerInfoReceived(customerInfo))).get() + baker.fireSensoryEventAndAwaitReceived(recipeInstanceId, EventInstance.from(PaymentMade())).get() + baker.awaitCompleted(recipeInstanceId, Duration.ofSeconds(5)).join() + } + private fun setupMockImplementations(): List { return ImmutableList.of(manufactureGoods, sendInvoice, shipGoods, validateOrder, LogRecipeFailedImpl()) } diff --git a/core/baker-interface-otel/pom.xml b/core/baker-interface-otel/pom.xml new file mode 100644 index 000000000..1783e45c4 --- /dev/null +++ b/core/baker-interface-otel/pom.xml @@ -0,0 +1,74 @@ + + + 4.0.0 + + + com.ing.baker + baker + 6.0.0-SNAPSHOT + ../../pom.xml + + + baker-interface-otel + Baker Runtime OpenTelemetry + OpenTelemetry context propagation and interaction instrumentation for Baker runtimes + + + + com.ing.baker + baker-interface + ${project.version} + + + + io.opentelemetry + opentelemetry-api + ${opentelemetry.version} + + + + org.slf4j + slf4j-api + + + + + org.junit.jupiter + junit-jupiter-engine + test + + + io.opentelemetry + opentelemetry-sdk + ${opentelemetry.version} + test + + + io.opentelemetry + opentelemetry-sdk-testing + ${opentelemetry.version} + test + + + ch.qos.logback + logback-classic + test + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + ${maven.surefire.plugin.version} + + false + -XX:+EnableDynamicAgentLoading + + + + + diff --git a/core/baker-interface-otel/src/main/java/com/ing/baker/runtime/otel/BakerTraceAttributes.java b/core/baker-interface-otel/src/main/java/com/ing/baker/runtime/otel/BakerTraceAttributes.java new file mode 100644 index 000000000..aafeb47a4 --- /dev/null +++ b/core/baker-interface-otel/src/main/java/com/ing/baker/runtime/otel/BakerTraceAttributes.java @@ -0,0 +1,21 @@ +package com.ing.baker.runtime.otel; + +/** + * Attribute keys shared by all Baker OpenTelemetry instrumentation. + */ +public final class BakerTraceAttributes { + + public static final String RECIPE_ID = "baker.recipe.id"; + public static final String RECIPE_NAME = "baker.recipe.name"; + public static final String RECIPE_INSTANCE_ID = "baker.recipe.instance.id"; + public static final String EVENT_NAME = "baker.event.name"; + public static final String INTERACTION_NAME = "baker.interaction.name"; + public static final String CORRELATION_ID = "baker.correlation.id"; + public static final String FAILURE_COUNT = "baker.failure.count"; + public static final String ERROR_TYPE = "error.type"; + + public static final String INTERACTION_FAILURE = "interaction.failure"; + + private BakerTraceAttributes() { + } +} diff --git a/core/baker-interface-otel/src/main/java/com/ing/baker/runtime/otel/OpenTelemetryContextPropagator.java b/core/baker-interface-otel/src/main/java/com/ing/baker/runtime/otel/OpenTelemetryContextPropagator.java new file mode 100644 index 000000000..3efc803d0 --- /dev/null +++ b/core/baker-interface-otel/src/main/java/com/ing/baker/runtime/otel/OpenTelemetryContextPropagator.java @@ -0,0 +1,34 @@ +package com.ing.baker.runtime.otel; + +import com.ing.baker.runtime.model.propagation.ContextScope; +import com.ing.baker.runtime.model.propagation.ContextSnapshot; +import com.ing.baker.runtime.model.propagation.ExecutionContextPropagator; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.ContextStorage; +import io.opentelemetry.context.Scope; + +/** + * Moves the OpenTelemetry {@link Context} of the submitting thread to the thread that actually executes the + * work, and removes it again afterwards. + * + *

Registered through {@code META-INF/services}, so simply having this module on the classpath makes + * Baker's internal thread hops context aware. + */ +public final class OpenTelemetryContextPropagator implements ExecutionContextPropagator { + + @Override + public ContextSnapshot capture() { + final Context captured = Context.current(); + return () -> { + final Scope scope = captured.makeCurrent(); + return (ContextScope) scope::close; + }; + } + + @Override + public void clear() { + // Detaches whatever context a previous task left attached to this (pooled) thread. The returned scope + // is intentionally not closed: closing it would restore exactly the leaked context we want gone. + ContextStorage.get().attach(Context.root()); + } +} diff --git a/core/baker-interface-otel/src/main/java/com/ing/baker/runtime/otel/OpenTelemetryInteractionInstrumentation.java b/core/baker-interface-otel/src/main/java/com/ing/baker/runtime/otel/OpenTelemetryInteractionInstrumentation.java new file mode 100644 index 000000000..6b4132e47 --- /dev/null +++ b/core/baker-interface-otel/src/main/java/com/ing/baker/runtime/otel/OpenTelemetryInteractionInstrumentation.java @@ -0,0 +1,119 @@ +package com.ing.baker.runtime.otel; + +import com.ing.baker.runtime.model.InteractionCallContext; +import com.ing.baker.runtime.model.InteractionInstrumentation; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.Scope; +import org.slf4j.MDC; + +import java.util.concurrent.CompletionException; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.ExecutionException; +import java.util.function.Supplier; + +/** + * Starts a span for every interaction invocation and attaches it to the thread that runs the interaction, so + * {@code Span.current()} inside an interaction always belongs to that very invocation, even when + * interactions of different recipe instances share a thread pool. + */ +public final class OpenTelemetryInteractionInstrumentation implements InteractionInstrumentation { + + private static final String TRACE_ID_MDC_KEY = "trace_id"; + private static final String SPAN_ID_MDC_KEY = "span_id"; + + private final Tracer tracer; + private final RecipeInstanceTraceContexts parentContexts; + + public OpenTelemetryInteractionInstrumentation(Tracer tracer, RecipeInstanceTraceContexts parentContexts) { + this.tracer = tracer; + this.parentContexts = parentContexts; + } + + @Override + public A aroundInteraction(InteractionCallContext context, Supplier interaction) { + final Span span; + try { + span = startSpan(context); + } catch (Throwable throwable) { + // Tracing must never break execution. + return interaction.get(); + } + + final String previousTraceId = MDC.get(TRACE_ID_MDC_KEY); + final String previousSpanId = MDC.get(SPAN_ID_MDC_KEY); + boolean endsAsynchronously = false; + + try (Scope ignored = span.makeCurrent()) { + MDC.put(TRACE_ID_MDC_KEY, span.getSpanContext().getTraceId()); + MDC.put(SPAN_ID_MDC_KEY, span.getSpanContext().getSpanId()); + + final A result = interaction.get(); + if (result instanceof CompletionStage) { + endsAsynchronously = true; + ((CompletionStage) result).whenComplete((value, throwable) -> end(span, throwable)); + } else { + span.setStatus(StatusCode.OK); + } + return result; + } catch (Throwable throwable) { + endsAsynchronously = false; + recordFailure(span, throwable); + throw throwable; + } finally { + restore(TRACE_ID_MDC_KEY, previousTraceId); + restore(SPAN_ID_MDC_KEY, previousSpanId); + if (!endsAsynchronously) { + span.end(); + } + } + } + + private Span startSpan(InteractionCallContext context) { + final Context parent = parentContexts.parentOf(context.getRecipeInstanceId()); + final Span span = tracer.spanBuilder("baker.interaction." + context.getInteractionName()) + .setParent(parent) + .startSpan(); + span.setAttribute(BakerTraceAttributes.RECIPE_ID, context.getRecipeId()); + span.setAttribute(BakerTraceAttributes.RECIPE_NAME, context.getRecipeName()); + span.setAttribute(BakerTraceAttributes.RECIPE_INSTANCE_ID, context.getRecipeInstanceId()); + span.setAttribute(BakerTraceAttributes.INTERACTION_NAME, context.getInteractionName()); + context.getCorrelationId() + .ifPresent(correlationId -> span.setAttribute(BakerTraceAttributes.CORRELATION_ID, correlationId)); + return span; + } + + private static void end(Span span, Throwable throwable) { + if (throwable == null) { + span.setStatus(StatusCode.OK); + } else { + recordFailure(span, throwable); + } + span.end(); + } + + private static void recordFailure(Span span, Throwable throwable) { + final Throwable cause = unwrap(throwable); + span.setAttribute(BakerTraceAttributes.ERROR_TYPE, BakerTraceAttributes.INTERACTION_FAILURE); + span.recordException(cause); + span.setStatus(StatusCode.ERROR, cause.getMessage() == null ? cause.toString() : cause.getMessage()); + } + + private static Throwable unwrap(Throwable throwable) { + if ((throwable instanceof CompletionException || throwable instanceof ExecutionException) + && throwable.getCause() != null) { + return throwable.getCause(); + } + return throwable; + } + + private static void restore(String key, String value) { + if (value == null) { + MDC.remove(key); + } else { + MDC.put(key, value); + } + } +} diff --git a/core/baker-interface-otel/src/main/java/com/ing/baker/runtime/otel/RecipeInstanceTraceContexts.java b/core/baker-interface-otel/src/main/java/com/ing/baker/runtime/otel/RecipeInstanceTraceContexts.java new file mode 100644 index 000000000..bb3cf96d9 --- /dev/null +++ b/core/baker-interface-otel/src/main/java/com/ing/baker/runtime/otel/RecipeInstanceTraceContexts.java @@ -0,0 +1,43 @@ +package com.ing.baker.runtime.otel; + +import io.opentelemetry.context.Context; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Keeps the tracing context that interaction spans of a recipe instance should be parented to. + * + *

Each Baker runtime owns its own instance, so contexts never leak between bakers, and entries are + * removed as soon as the recipe instance is done. + */ +public final class RecipeInstanceTraceContexts { + + private final Map contexts = new ConcurrentHashMap<>(); + + public void put(String recipeInstanceId, Context context) { + if (recipeInstanceId == null || context == null) { + return; + } + contexts.put(recipeInstanceId, context); + } + + /** + * Returns the parent context for the given recipe instance, falling back to the context currently + * attached to the calling thread. + */ + public Context parentOf(String recipeInstanceId) { + final Context context = recipeInstanceId == null ? null : contexts.get(recipeInstanceId); + return context != null ? context : Context.current(); + } + + public void remove(String recipeInstanceId) { + if (recipeInstanceId != null) { + contexts.remove(recipeInstanceId); + } + } + + public void clear() { + contexts.clear(); + } +} diff --git a/core/baker-interface-otel/src/main/resources/META-INF/services/com.ing.baker.runtime.model.propagation.ExecutionContextPropagator b/core/baker-interface-otel/src/main/resources/META-INF/services/com.ing.baker.runtime.model.propagation.ExecutionContextPropagator new file mode 100644 index 000000000..d2ceeb0c8 --- /dev/null +++ b/core/baker-interface-otel/src/main/resources/META-INF/services/com.ing.baker.runtime.model.propagation.ExecutionContextPropagator @@ -0,0 +1 @@ +com.ing.baker.runtime.otel.OpenTelemetryContextPropagator diff --git a/core/baker-interface-otel/src/test/java/com/ing/baker/runtime/otel/BlockingPoolPropagationTest.java b/core/baker-interface-otel/src/test/java/com/ing/baker/runtime/otel/BlockingPoolPropagationTest.java new file mode 100644 index 000000000..621fb9248 --- /dev/null +++ b/core/baker-interface-otel/src/test/java/com/ing/baker/runtime/otel/BlockingPoolPropagationTest.java @@ -0,0 +1,93 @@ +package com.ing.baker.runtime.otel; + +import com.ing.baker.runtime.catseffect.SyncSupport; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.ContextStorage; +import io.opentelemetry.context.Scope; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.slf4j.MDC; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Verifies that the propagator registered through {@code META-INF/services} is picked up by Baker's blocking + * pool, which is where interaction implementations run. + */ +class BlockingPoolPropagationTest { + + private static final String INVALID_TRACE_ID = "00000000000000000000000000000000"; + + private InMemorySpanExporter exporter; + private SdkTracerProvider tracerProvider; + private io.opentelemetry.api.trace.Tracer tracer; + + @BeforeEach + void setUp() { + exporter = InMemorySpanExporter.create(); + tracerProvider = SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(exporter)) + .build(); + tracer = OpenTelemetrySdk.builder().setTracerProvider(tracerProvider).build().getTracer("test.otel.blocking"); + } + + @AfterEach + void tearDown() { + tracerProvider.close(); + exporter.reset(); + MDC.clear(); + } + + private static A runBlocking(scala.Function0 thunk) throws Exception { + final CompletableFuture future = + SyncSupport.fromCompletableFuture().blocking(thunk); + return future.get(10, TimeUnit.SECONDS); + } + + @Test + void propagatesSpanAndMdcToTheBlockingPool() throws Exception { + final Span span = tracer.spanBuilder("caller").startSpan(); + MDC.put("recipeInstanceId", "instance-1"); + try (Scope ignored = span.makeCurrent()) { + final String[] observed = runBlocking(() -> new String[]{ + Span.current().getSpanContext().getSpanId(), + MDC.get("recipeInstanceId") + }); + assertEquals(span.getSpanContext().getSpanId(), observed[0]); + assertEquals("instance-1", observed[1]); + } finally { + span.end(); + MDC.clear(); + } + } + + @Test + void doesNotExposeContextOfPreviousWorkOnTheSamePooledThread() throws Exception { + final Span leaked = tracer.spanBuilder("leaked").startSpan(); + + // Simulate badly behaved work that attaches a context on the pooled thread without detaching it. + runBlocking(() -> { + ContextStorage.get().attach(Context.current().with(leaked)); + MDC.put("trace_id", "leaked-trace"); + return null; + }); + + final String[] observed = runBlocking(() -> new String[]{ + Span.current().getSpanContext().getTraceId(), + MDC.get("trace_id") + }); + + assertEquals(INVALID_TRACE_ID, observed[0], "A pooled thread must not expose the context of previous work"); + assertEquals(null, observed[1], "A pooled thread must not expose the MDC of previous work"); + leaked.end(); + } +} diff --git a/core/baker-interface-otel/src/test/java/com/ing/baker/runtime/otel/OpenTelemetryContextPropagatorTest.java b/core/baker-interface-otel/src/test/java/com/ing/baker/runtime/otel/OpenTelemetryContextPropagatorTest.java new file mode 100644 index 000000000..86187257a --- /dev/null +++ b/core/baker-interface-otel/src/test/java/com/ing/baker/runtime/otel/OpenTelemetryContextPropagatorTest.java @@ -0,0 +1,104 @@ +package com.ing.baker.runtime.otel; + +import com.ing.baker.runtime.model.propagation.ContextScope; +import com.ing.baker.runtime.model.propagation.ContextSnapshot; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.ContextStorage; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + +class OpenTelemetryContextPropagatorTest { + + private static final String INVALID_TRACE_ID = "00000000000000000000000000000000"; + + private final OpenTelemetryContextPropagator propagator = new OpenTelemetryContextPropagator(); + + @Test + void movesTheCapturedContextToTheExecutingThread() throws Exception { + try (TracerFixture fixture = TracerFixture.create()) { + final Span span = fixture.tracer().spanBuilder("caller").startSpan(); + final ContextSnapshot snapshot; + try (io.opentelemetry.context.Scope ignored = span.makeCurrent()) { + snapshot = propagator.capture(); + } + + final ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + final Future observed = executor.submit(() -> { + try (ContextScope scope = snapshot.restore()) { + return Span.current().getSpanContext().getSpanId(); + } + }); + assertEquals(span.getSpanContext().getSpanId(), observed.get(5, TimeUnit.SECONDS)); + + final Future afterScope = executor.submit( + () -> Span.current().getSpanContext().getTraceId()); + assertEquals(INVALID_TRACE_ID, afterScope.get(5, TimeUnit.SECONDS), + "The propagated context must not survive the scope it was restored in"); + } finally { + executor.shutdownNow(); + } + span.end(); + } + } + + @Test + void clearsContextLeakedByPreviousWorkOnTheSameThread() throws Exception { + try (TracerFixture fixture = TracerFixture.create()) { + final ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + final Span leaked = fixture.tracer().spanBuilder("leaked").startSpan(); + executor.submit(() -> ContextStorage.get().attach(Context.current().with(leaked))) + .get(5, TimeUnit.SECONDS); + + final Future stillLeaking = executor.submit( + () -> Span.current().getSpanContext().getSpanId()); + assertEquals(leaked.getSpanContext().getSpanId(), stillLeaking.get(5, TimeUnit.SECONDS)); + + final Future afterClear = executor.submit(() -> { + propagator.clear(); + return Span.current().getSpanContext().getTraceId(); + }); + assertEquals(INVALID_TRACE_ID, afterClear.get(5, TimeUnit.SECONDS)); + assertNotEquals(INVALID_TRACE_ID, leaked.getSpanContext().getTraceId()); + assertFalse(Span.current().getSpanContext().isValid(), + "The test thread itself should not have any ambient span"); + leaked.end(); + } finally { + executor.shutdownNow(); + } + } + } + + record TracerFixture(InMemorySpanExporter exporter, SdkTracerProvider tracerProvider, + io.opentelemetry.api.trace.Tracer tracer) implements AutoCloseable { + + static TracerFixture create() { + final InMemorySpanExporter exporter = InMemorySpanExporter.create(); + final SdkTracerProvider tracerProvider = SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(exporter)) + .build(); + final OpenTelemetrySdk sdk = OpenTelemetrySdk.builder().setTracerProvider(tracerProvider).build(); + return new TracerFixture(exporter, tracerProvider, sdk.getTracer("test.otel.propagator")); + } + + @Override + public void close() { + tracerProvider.close(); + exporter.reset(); + } + } +} diff --git a/core/baker-interface-otel/src/test/java/com/ing/baker/runtime/otel/OpenTelemetryInteractionInstrumentationTest.java b/core/baker-interface-otel/src/test/java/com/ing/baker/runtime/otel/OpenTelemetryInteractionInstrumentationTest.java new file mode 100644 index 000000000..7e6b07747 --- /dev/null +++ b/core/baker-interface-otel/src/test/java/com/ing/baker/runtime/otel/OpenTelemetryInteractionInstrumentationTest.java @@ -0,0 +1,135 @@ +package com.ing.baker.runtime.otel; + +import com.ing.baker.runtime.model.InteractionCallContext; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Context; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.data.SpanData; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.slf4j.MDC; + +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class OpenTelemetryInteractionInstrumentationTest { + + private InMemorySpanExporter exporter; + private SdkTracerProvider tracerProvider; + private Tracer tracer; + private RecipeInstanceTraceContexts parentContexts; + private OpenTelemetryInteractionInstrumentation instrumentation; + + @BeforeEach + void setUp() { + exporter = InMemorySpanExporter.create(); + tracerProvider = SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(exporter)) + .build(); + tracer = OpenTelemetrySdk.builder().setTracerProvider(tracerProvider).build().getTracer("test.otel"); + parentContexts = new RecipeInstanceTraceContexts(); + instrumentation = new OpenTelemetryInteractionInstrumentation(tracer, parentContexts); + } + + @AfterEach + void tearDown() { + tracerProvider.close(); + exporter.reset(); + MDC.clear(); + } + + private static InteractionCallContext callContext(String recipeInstanceId) { + return new InteractionCallContext("recipe-id", "recipe", recipeInstanceId, "ValidateOrder", 42L, "correlation-1"); + } + + @Test + void attachesItsOwnSpanToTheExecutingThread() { + final String observedSpanId = instrumentation.aroundInteraction( + callContext("instance-1"), + () -> Span.current().getSpanContext().getSpanId()); + + final List spans = exporter.getFinishedSpanItems(); + assertEquals(1, spans.size()); + final SpanData span = spans.get(0); + assertEquals("baker.interaction.ValidateOrder", span.getName()); + assertEquals(span.getSpanContext().getSpanId(), observedSpanId); + assertEquals(StatusCode.OK, span.getStatus().getStatusCode()); + assertEquals("instance-1", span.getAttributes().get(AttributeKey.stringKey("baker.recipe.instance.id"))); + assertEquals("ValidateOrder", span.getAttributes().get(AttributeKey.stringKey("baker.interaction.name"))); + assertEquals("correlation-1", span.getAttributes().get(AttributeKey.stringKey("baker.correlation.id"))); + assertFalse(Span.current().getSpanContext().isValid(), "The span must be detached after the invocation"); + } + + @Test + void parentsTheSpanToTheRecipeInstanceContext() { + final Span parent = tracer.spanBuilder("baker.event.received").startSpan(); + parentContexts.put("instance-2", Context.root().with(parent)); + parent.end(); + + instrumentation.aroundInteraction(callContext("instance-2"), () -> null); + + final SpanData interactionSpan = exporter.getFinishedSpanItems().stream() + .filter(span -> span.getName().startsWith("baker.interaction.")) + .findFirst() + .orElseThrow(); + assertEquals(parent.getSpanContext().getSpanId(), interactionSpan.getParentSpanId()); + assertEquals(parent.getSpanContext().getTraceId(), interactionSpan.getSpanContext().getTraceId()); + } + + @Test + void setsAndRestoresTraceMdcKeys() { + MDC.put("trace_id", "previous-trace"); + + instrumentation.aroundInteraction(callContext("instance-3"), () -> { + assertEquals(Span.current().getSpanContext().getTraceId(), MDC.get("trace_id")); + assertEquals(Span.current().getSpanContext().getSpanId(), MDC.get("span_id")); + return null; + }); + + assertEquals("previous-trace", MDC.get("trace_id")); + assertNull(MDC.get("span_id")); + } + + @Test + void endsTheSpanWhenAnAsynchronousInteractionFails() { + final CompletableFuture future = new CompletableFuture<>(); + + instrumentation.aroundInteraction(callContext("instance-4"), () -> future); + + assertTrue(exporter.getFinishedSpanItems().isEmpty(), "Span must stay open until the interaction completes"); + + future.completeExceptionally(new RuntimeException("Expected failure")); + + final SpanData span = exporter.getFinishedSpanItems().get(0); + assertEquals(StatusCode.ERROR, span.getStatus().getStatusCode()); + assertEquals("interaction.failure", span.getAttributes().get(AttributeKey.stringKey("error.type"))); + } + + @Test + void recordsSynchronousFailures() { + final RuntimeException failure = new RuntimeException("Expected failure"); + try { + instrumentation.aroundInteraction(callContext("instance-5"), () -> { + throw failure; + }); + } catch (RuntimeException expected) { + assertEquals(failure, expected); + } + + final SpanData span = exporter.getFinishedSpanItems().get(0); + assertEquals(StatusCode.ERROR, span.getStatus().getStatusCode()); + assertEquals("interaction.failure", span.getAttributes().get(AttributeKey.stringKey("error.type"))); + } +} diff --git a/core/baker-interface/src/main/java/com/ing/baker/runtime/model/InteractionCallContext.java b/core/baker-interface/src/main/java/com/ing/baker/runtime/model/InteractionCallContext.java new file mode 100644 index 000000000..b60638ccf --- /dev/null +++ b/core/baker-interface/src/main/java/com/ing/baker/runtime/model/InteractionCallContext.java @@ -0,0 +1,65 @@ +package com.ing.baker.runtime.model; + +import java.util.Optional; + +/** + * Identifies a single interaction invocation, handed to {@link InteractionInstrumentation} on the thread + * that actually invokes the interaction implementation. + */ +public final class InteractionCallContext { + + private final String recipeId; + private final String recipeName; + private final String recipeInstanceId; + private final String interactionName; + private final long executionId; + private final String correlationId; + + public InteractionCallContext(String recipeId, + String recipeName, + String recipeInstanceId, + String interactionName, + long executionId, + String correlationId) { + this.recipeId = recipeId; + this.recipeName = recipeName; + this.recipeInstanceId = recipeInstanceId; + this.interactionName = interactionName; + this.executionId = executionId; + this.correlationId = correlationId; + } + + public String getRecipeId() { + return recipeId; + } + + public String getRecipeName() { + return recipeName; + } + + public String getRecipeInstanceId() { + return recipeInstanceId; + } + + public String getInteractionName() { + return interactionName; + } + + public long getExecutionId() { + return executionId; + } + + public Optional getCorrelationId() { + return Optional.ofNullable(correlationId); + } + + @Override + public String toString() { + return "InteractionCallContext{" + + "recipeName='" + recipeName + '\'' + + ", recipeInstanceId='" + recipeInstanceId + '\'' + + ", interactionName='" + interactionName + '\'' + + ", executionId=" + executionId + + '}'; + } +} diff --git a/core/baker-interface/src/main/java/com/ing/baker/runtime/model/InteractionInstrumentation.java b/core/baker-interface/src/main/java/com/ing/baker/runtime/model/InteractionInstrumentation.java new file mode 100644 index 000000000..b8efd5b3e --- /dev/null +++ b/core/baker-interface/src/main/java/com/ing/baker/runtime/model/InteractionInstrumentation.java @@ -0,0 +1,26 @@ +package com.ing.baker.runtime.model; + +import java.util.function.Supplier; + +/** + * Hook invoked around every interaction invocation, on the thread that runs the interaction. + * + *

This is the extension point used for tracing: an implementation can start a span, attach it to the + * executing thread and detach it again once the invocation returns, guaranteeing that + * {@code Span.current()} inside an interaction belongs to that interaction only. + */ +public interface InteractionInstrumentation { + + InteractionInstrumentation NOOP = new InteractionInstrumentation() { + @Override + public A aroundInteraction(InteractionCallContext context, Supplier interaction) { + return interaction.get(); + } + }; + + static InteractionInstrumentation noop() { + return NOOP; + } + + A aroundInteraction(InteractionCallContext context, Supplier interaction); +} diff --git a/core/baker-interface/src/main/java/com/ing/baker/runtime/model/TracingMdcRegistry.java b/core/baker-interface/src/main/java/com/ing/baker/runtime/model/TracingMdcRegistry.java index 63c8784a6..801e5aca9 100644 --- a/core/baker-interface/src/main/java/com/ing/baker/runtime/model/TracingMdcRegistry.java +++ b/core/baker-interface/src/main/java/com/ing/baker/runtime/model/TracingMdcRegistry.java @@ -1,14 +1,29 @@ package com.ing.baker.runtime.model; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; /** * Shares best-effort trace/span ids for MDC enrichment across runtime modules. + * + *

Only used for log lines emitted by Baker itself, outside of an interaction invocation. Inside an + * interaction the exact trace context is attached to the executing thread by + * {@link InteractionInstrumentation}. Entries are bounded so an abandoned recipe instance can never grow + * this registry indefinitely. */ public final class TracingMdcRegistry { - private static final ConcurrentHashMap contexts = new ConcurrentHashMap<>(); + private static final int MAX_ENTRIES = 10_000; + + private static final Map contexts = Collections.synchronizedMap( + new LinkedHashMap(256, 0.75f, false) { + @Override + protected boolean removeEldestEntry(Map.Entry eldest) { + return size() > MAX_ENTRIES; + } + }); private TracingMdcRegistry() { } diff --git a/core/baker-interface/src/main/java/com/ing/baker/runtime/model/propagation/ContextScope.java b/core/baker-interface/src/main/java/com/ing/baker/runtime/model/propagation/ContextScope.java new file mode 100644 index 000000000..c4d9c4aee --- /dev/null +++ b/core/baker-interface/src/main/java/com/ing/baker/runtime/model/propagation/ContextScope.java @@ -0,0 +1,14 @@ +package com.ing.baker.runtime.model.propagation; + +/** + * Undoes a {@link ContextSnapshot#restore()} on the thread that performed it. + */ +@FunctionalInterface +public interface ContextScope extends AutoCloseable { + + ContextScope NOOP = () -> { + }; + + @Override + void close(); +} diff --git a/core/baker-interface/src/main/java/com/ing/baker/runtime/model/propagation/ContextSnapshot.java b/core/baker-interface/src/main/java/com/ing/baker/runtime/model/propagation/ContextSnapshot.java new file mode 100644 index 000000000..f9314fd36 --- /dev/null +++ b/core/baker-interface/src/main/java/com/ing/baker/runtime/model/propagation/ContextSnapshot.java @@ -0,0 +1,15 @@ +package com.ing.baker.runtime.model.propagation; + +/** + * Immutable capture of ambient context taken on the submitting thread. + */ +@FunctionalInterface +public interface ContextSnapshot { + + ContextSnapshot NOOP = () -> ContextScope.NOOP; + + /** + * Installs the captured context on the calling thread. Must be closed on that same thread. + */ + ContextScope restore(); +} diff --git a/core/baker-interface/src/main/java/com/ing/baker/runtime/model/propagation/ExecutionContextPropagator.java b/core/baker-interface/src/main/java/com/ing/baker/runtime/model/propagation/ExecutionContextPropagator.java new file mode 100644 index 000000000..f28bf99de --- /dev/null +++ b/core/baker-interface/src/main/java/com/ing/baker/runtime/model/propagation/ExecutionContextPropagator.java @@ -0,0 +1,20 @@ +package com.ing.baker.runtime.model.propagation; + +/** + * Propagates ambient thread state (MDC, tracing context, ...) across Baker's internal thread hops. + * + *

Implementations are discovered through {@link java.util.ServiceLoader} and must be thread safe. + * {@link #capture()} is always called on the submitting thread, {@link ContextSnapshot#restore()} and + * {@link ContextScope#close()} on the executing thread. + */ +public interface ExecutionContextPropagator { + + ContextSnapshot capture(); + + /** + * Drops any ambient state left behind on the calling thread, so a pooled thread never exposes + * state of a previously executed task. + */ + default void clear() { + } +} diff --git a/core/baker-interface/src/main/java/com/ing/baker/runtime/model/propagation/ExecutionContextPropagators.java b/core/baker-interface/src/main/java/com/ing/baker/runtime/model/propagation/ExecutionContextPropagators.java new file mode 100644 index 000000000..1a2edf6ee --- /dev/null +++ b/core/baker-interface/src/main/java/com/ing/baker/runtime/model/propagation/ExecutionContextPropagators.java @@ -0,0 +1,123 @@ +package com.ing.baker.runtime.model.propagation; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.ServiceLoader; +import java.util.concurrent.CopyOnWriteArrayList; + +/** + * Registry of the {@link ExecutionContextPropagator}s used whenever Baker hands work to another thread. + * + *

Besides the built-in {@link MdcContextPropagator}, implementations are discovered through + * {@link ServiceLoader}, so runtimes such as {@code baker-interface-otel} can plug in without adding + * dependencies to {@code baker-interface}. + */ +public final class ExecutionContextPropagators { + + private static final Logger logger = LoggerFactory.getLogger(ExecutionContextPropagators.class); + + private static final List propagators = new CopyOnWriteArrayList<>(); + + static { + reload(); + } + + private ExecutionContextPropagators() { + } + + /** + * Captures the ambient context of the calling thread. + */ + public static ContextSnapshot capture() { + if (propagators.isEmpty()) { + return ContextSnapshot.NOOP; + } + final List snapshots = new ArrayList<>(propagators.size()); + for (ExecutionContextPropagator propagator : propagators) { + try { + final ContextSnapshot snapshot = propagator.capture(); + if (snapshot != null) { + snapshots.add(snapshot); + } + } catch (Throwable throwable) { + logger.debug("Context propagator {} failed to capture context", propagator, throwable); + } + } + if (snapshots.isEmpty()) { + return ContextSnapshot.NOOP; + } + return () -> restoreAll(snapshots); + } + + /** + * Drops ambient context left behind on the calling thread by previously executed work. + */ + public static void clear() { + for (ExecutionContextPropagator propagator : propagators) { + try { + propagator.clear(); + } catch (Throwable throwable) { + logger.debug("Context propagator {} failed to clear context", propagator, throwable); + } + } + } + + public static void register(ExecutionContextPropagator propagator) { + if (propagator != null) { + propagators.add(propagator); + } + } + + public static void unregister(ExecutionContextPropagator propagator) { + propagators.remove(propagator); + } + + public static List registered() { + return Collections.unmodifiableList(new ArrayList<>(propagators)); + } + + /** + * Rebuilds the registry from the built-in propagators and the ones found on the classpath. + */ + public static void reload() { + final List discovered = new ArrayList<>(); + discovered.add(new MdcContextPropagator()); + try { + for (ExecutionContextPropagator propagator : ServiceLoader.load( + ExecutionContextPropagator.class, ExecutionContextPropagators.class.getClassLoader())) { + discovered.add(propagator); + } + } catch (Throwable throwable) { + logger.warn("Failed to load context propagators, continuing without them", throwable); + } + propagators.clear(); + propagators.addAll(discovered); + } + + private static ContextScope restoreAll(List snapshots) { + final List scopes = new ArrayList<>(snapshots.size()); + for (ContextSnapshot snapshot : snapshots) { + try { + final ContextScope scope = snapshot.restore(); + if (scope != null) { + scopes.add(scope); + } + } catch (Throwable throwable) { + logger.debug("Failed to restore propagated context", throwable); + } + } + return () -> { + for (int index = scopes.size() - 1; index >= 0; index--) { + try { + scopes.get(index).close(); + } catch (Throwable throwable) { + logger.debug("Failed to close propagated context scope", throwable); + } + } + }; + } +} diff --git a/core/baker-interface/src/main/java/com/ing/baker/runtime/model/propagation/MdcContextPropagator.java b/core/baker-interface/src/main/java/com/ing/baker/runtime/model/propagation/MdcContextPropagator.java new file mode 100644 index 000000000..722989d7b --- /dev/null +++ b/core/baker-interface/src/main/java/com/ing/baker/runtime/model/propagation/MdcContextPropagator.java @@ -0,0 +1,34 @@ +package com.ing.baker.runtime.model.propagation; + +import org.slf4j.MDC; + +import java.util.Map; + +/** + * Built-in propagator moving the SLF4J {@link MDC} across thread hops. + */ +public final class MdcContextPropagator implements ExecutionContextPropagator { + + @Override + public ContextSnapshot capture() { + final Map captured = MDC.getCopyOfContextMap(); + return () -> { + final Map previous = MDC.getCopyOfContextMap(); + apply(captured); + return () -> apply(previous); + }; + } + + @Override + public void clear() { + MDC.clear(); + } + + private static void apply(Map context) { + if (context == null || context.isEmpty()) { + MDC.clear(); + } else { + MDC.setContextMap(context); + } + } +} diff --git a/core/baker-interface/src/main/scala/com/ing/baker/runtime/catseffect/AsyncSupport.scala b/core/baker-interface/src/main/scala/com/ing/baker/runtime/catseffect/AsyncSupport.scala index 6cf5e43c7..51e584188 100644 --- a/core/baker-interface/src/main/scala/com/ing/baker/runtime/catseffect/AsyncSupport.scala +++ b/core/baker-interface/src/main/scala/com/ing/baker/runtime/catseffect/AsyncSupport.scala @@ -76,7 +76,10 @@ object AsyncSupport { override def flatMap[A, B](fa: IO[A])(f: A => IO[B]): IO[B] = fa.flatMap(f) override def raiseError[A](throwable: Throwable): IO[A] = IO.raiseError(throwable) override def delay[A](thunk: => A): IO[A] = IO.delay(thunk) - override def blocking[A](thunk: => A): IO[A] = IO.blocking(thunk) + override def blocking[A](thunk: => A): IO[A] = + IO.delay(SyncSupport.captureContext()).flatMap { snapshot => + IO.blocking(SyncSupport.runWithPropagatedContext(snapshot)(thunk)) + } override def unit: IO[Unit] = IO.unit override def attempt[A](fa: IO[A]): IO[Either[Throwable, A]] = fa.attempt diff --git a/core/baker-interface/src/main/scala/com/ing/baker/runtime/catseffect/SyncSupport.scala b/core/baker-interface/src/main/scala/com/ing/baker/runtime/catseffect/SyncSupport.scala index e6edb24fe..bc33b3920 100644 --- a/core/baker-interface/src/main/scala/com/ing/baker/runtime/catseffect/SyncSupport.scala +++ b/core/baker-interface/src/main/scala/com/ing/baker/runtime/catseffect/SyncSupport.scala @@ -1,6 +1,7 @@ package com.ing.baker.runtime.catseffect import cats.effect.Sync +import com.ing.baker.runtime.model.propagation.{ContextSnapshot, ExecutionContextPropagators} import java.util.concurrent.{CompletableFuture, CompletionStage, Executor, Executors} import java.util.function.{Function, Supplier} @@ -32,6 +33,36 @@ object SyncSupport { thread }) + /** + * Runs the given thunk on the blocking pool while propagating the ambient context (MDC, tracing context) + * of the submitting thread. The context is installed and removed on the executing thread itself, and any + * ambient context is cleared at both task boundaries so a pooled thread never exposes the context of + * another in-flight execution. + */ + private def supplyAsyncWithPropagation[A](thunk: => A): CompletableFuture[A] = { + val snapshot = ExecutionContextPropagators.capture() + CompletableFuture.supplyAsync(new Supplier[A] { + override def get(): A = runWithPropagatedContext(snapshot)(thunk) + }, blockingExecutor) + } + + /** + * Installs a previously captured context on the current thread, runs the thunk and removes it again. + * Ambient context is cleared at both task boundaries, so a pooled thread never exposes the context of + * another in-flight execution. + */ + def runWithPropagatedContext[A](snapshot: ContextSnapshot)(thunk: => A): A = { + ExecutionContextPropagators.clear() + val scope = snapshot.restore() + try thunk + finally { + scope.close() + ExecutionContextPropagators.clear() + } + } + + def captureContext(): ContextSnapshot = ExecutionContextPropagators.capture() + private def failedFuture[A](throwable: Throwable): CompletableFuture[A] = { val future = new CompletableFuture[A]() future.completeExceptionally(throwable) @@ -56,9 +87,7 @@ object SyncSupport { case NonFatal(exception) => raiseError(exception) } override def blocking[A](thunk: => A): CompletionStage[A] = - CompletableFuture.supplyAsync(new Supplier[A] { - override def get(): A = thunk - }, blockingExecutor) + supplyAsyncWithPropagation(thunk) override def unit: CompletionStage[Unit] = pure(()) } @@ -80,9 +109,7 @@ object SyncSupport { case NonFatal(exception) => raiseError(exception) } override def blocking[A](thunk: => A): CompletableFuture[A] = - CompletableFuture.supplyAsync(new Supplier[A] { - override def get(): A = thunk - }, blockingExecutor) + supplyAsyncWithPropagation(thunk) override def unit: CompletableFuture[Unit] = pure(()) } diff --git a/core/baker-interface/src/main/scala/com/ing/baker/runtime/model/BakerComponents.scala b/core/baker-interface/src/main/scala/com/ing/baker/runtime/model/BakerComponents.scala index 664de44d6..8638cc16a 100644 --- a/core/baker-interface/src/main/scala/com/ing/baker/runtime/model/BakerComponents.scala +++ b/core/baker-interface/src/main/scala/com/ing/baker/runtime/model/BakerComponents.scala @@ -4,4 +4,5 @@ case class BakerComponents[F[_]](interactions: InteractionManager[F], recipeInstanceManager: RecipeInstanceManager[F], recipeManager: RecipeManager[F], eventStream: EventStream[F], - logging: BakerLogging = BakerLogging()) + logging: BakerLogging = BakerLogging(), + instrumentation: InteractionInstrumentation = InteractionInstrumentation.noop()) diff --git a/core/baker-interface/src/main/scala/com/ing/baker/runtime/model/BakerLogging.scala b/core/baker-interface/src/main/scala/com/ing/baker/runtime/model/BakerLogging.scala index 898583e6c..d76635b6f 100644 --- a/core/baker-interface/src/main/scala/com/ing/baker/runtime/model/BakerLogging.scala +++ b/core/baker-interface/src/main/scala/com/ing/baker/runtime/model/BakerLogging.scala @@ -54,17 +54,31 @@ case class BakerLogging(logger: Logger = BakerLogging.defaultLogger) { .map(v => withRecipeIdAliases + (BakerRecipeInstanceIdKey -> v)) .getOrElse(withRecipeIdAliases) - withInstanceAliases.get(RecipeInstanceIdKey) - .flatMap(id => TracingMdcRegistry.get(id).toScala) - .map(context => withInstanceAliases + (TraceIdKey -> context.getTraceId) + (SpanIdKey -> context.getSpanId)) + // Trace ids attached to the current thread (set by the interaction instrumentation) are exact, the + // per-recipe-instance registry is only a best effort fallback for log lines emitted by Baker itself. + val currentThreadTrace = for { + traceId <- Option(MDC.get(TraceIdKey)) + spanId <- Option(MDC.get(SpanIdKey)) + } yield (traceId, spanId) + + currentThreadTrace + .orElse( + withInstanceAliases.get(RecipeInstanceIdKey) + .flatMap(id => TracingMdcRegistry.get(id).toScala) + .map(context => (context.getTraceId, context.getSpanId))) + .map { case (traceId, spanId) => withInstanceAliases + (TraceIdKey -> traceId) + (SpanIdKey -> spanId) } .getOrElse(withInstanceAliases) } def withMDC(mdc: Map[String, String], log: Logger => Unit): Unit = { val effectiveMdc = enrichMdc(mdc) + val previous = effectiveMdc.keys.map(key => key -> Option(MDC.get(key))).toList effectiveMdc.foreach { case (k, v) => MDC.put(k, v) } - log(logger) - effectiveMdc.keys.foreach(MDC.remove) + try log(logger) + finally previous.foreach { + case (key, Some(value)) => MDC.put(key, value) + case (key, None) => MDC.remove(key) + } } def addedRecipe(recipeAdded: RecipeAdded): Unit = { diff --git a/core/baker-interface/src/main/scala/com/ing/baker/runtime/model/InteractionManager.scala b/core/baker-interface/src/main/scala/com/ing/baker/runtime/model/InteractionManager.scala index 00d6cc115..d731a66c8 100644 --- a/core/baker-interface/src/main/scala/com/ing/baker/runtime/model/InteractionManager.scala +++ b/core/baker-interface/src/main/scala/com/ing/baker/runtime/model/InteractionManager.scala @@ -79,14 +79,27 @@ trait InteractionManager[F[_]] { def existsFor(interaction: InteractionTransition)(implicit sync: SyncSupport[F]): F[Boolean] = sync.map(findFor(interaction))(_.nonEmpty) - def execute(interaction: InteractionTransition, input: Seq[IngredientInstance], metadata: Option[Map[String, String]])(implicit sync: SyncSupport[F]): F[Option[EventInstance]] = { + def execute(interaction: InteractionTransition, + input: Seq[IngredientInstance], + metadata: Option[Map[String, String]], + callContext: Option[InteractionCallContext] = None, + instrumentation: InteractionInstrumentation = InteractionInstrumentation.noop())(implicit sync: SyncSupport[F]): F[Option[EventInstance]] = { if(interaction.interactionName.startsWith(checkpointEventInteractionPrefix)){ sync.pure(Some(EventInstance(interaction.interactionName.stripPrefix(checkpointEventInteractionPrefix)))) } else{ sync.flatMap(findFor(interaction)) { case Some(implementation) => { // Interaction implementations are often reflection/IO heavy; shift invocation to the blocking pool. - sync.flatMap(sync.blocking(implementation.execute(input, metadata.getOrElse(Map()))))(identity) + // The instrumentation hook runs on that very thread, so tracing context attached by it is strictly + // scoped to this single invocation. + sync.flatMap(sync.blocking { + def invoke: F[Option[EventInstance]] = implementation.execute(input, metadata.getOrElse(Map())) + callContext match { + case Some(context) => + instrumentation.aroundInteraction[F[Option[EventInstance]]](context, () => invoke) + case None => invoke + } + })(identity) } case None => sync.raiseError(new FatalInteractionException(s"No implementation available for interaction ${interaction.interactionName}")) } diff --git a/core/baker-interface/src/main/scala/com/ing/baker/runtime/model/recipeinstance/TransitionExecution.scala b/core/baker-interface/src/main/scala/com/ing/baker/runtime/model/recipeinstance/TransitionExecution.scala index 8f19e83db..b4f6e5195 100644 --- a/core/baker-interface/src/main/scala/com/ing/baker/runtime/model/recipeinstance/TransitionExecution.scala +++ b/core/baker-interface/src/main/scala/com/ing/baker/runtime/model/recipeinstance/TransitionExecution.scala @@ -7,7 +7,7 @@ import com.ing.baker.il.{CompiledRecipe, IngredientDescriptor} import com.ing.baker.petrinet.api._ import com.ing.baker.runtime.catseffect.AsyncSupport import com.ing.baker.runtime.catseffect.SyncSupport.syntax._ -import com.ing.baker.runtime.model.{BakerComponents, TracingMdcRegistry} +import com.ing.baker.runtime.model.{BakerComponents, InteractionCallContext} import com.ing.baker.runtime.model.recipeinstance.RecipeInstance.FatalInteractionException import com.ing.baker.runtime.scaladsl._ import com.ing.baker.types.{ListValue, PrimitiveValue, Value} @@ -157,31 +157,46 @@ private[recipeinstance] case class TransitionExecution( } } - def setupMdc: F[Unit] = async.delay { + def setupMdc(): Unit = { MDC.put("recipeInstanceId", recipeInstanceId) MDC.put("recipeName", recipe.name) MDC.put("baker.recipe.instance.id", recipeInstanceId) MDC.put("baker.recipe.name", recipe.name) - TracingMdcRegistry.get(recipeInstanceId).ifPresent(context => { - MDC.put("trace_id", context.getTraceId) - MDC.put("span_id", context.getSpanId) - }) + MDC.put("baker.recipe.id", recipe.recipeId) } - def cleanMdc: F[Unit] = async.delay { + def cleanMdc(): Unit = { MDC.remove("recipeInstanceId") MDC.remove("recipeName") MDC.remove("baker.recipe.instance.id") MDC.remove("baker.recipe.name") - MDC.remove("trace_id") - MDC.remove("span_id") + MDC.remove("baker.recipe.id") } - def execute: F[Option[EventInstance]] = - components.interactions.execute( - interactionTransition, - buildInteractionInput, - Some(recipeInstanceMetadata)) + val callContext: InteractionCallContext = + new InteractionCallContext( + recipe.recipeId, + recipe.name, + recipeInstanceId, + interactionTransition.interactionName, + id, + correlationId.orNull) + + /** The MDC is set while the interaction effect is built, which is exactly when the invocation is handed to + * the blocking pool. The propagators capture it there and re-install it on the executing thread, after + * which it is removed again on the thread that built the effect. + */ + def execute: F[Option[EventInstance]] = { + setupMdc() + try + components.interactions.execute( + interactionTransition, + buildInteractionInput, + Some(recipeInstanceMetadata), + Some(callContext), + components.instrumentation) + finally cleanMdc() + } for { @@ -191,11 +206,7 @@ private[recipeinstance] case class TransitionExecution( interactionStarted <- async.delay(InteractionStarted(startTime, recipe.name, recipe.recipeId, recipeInstanceId, interactionTransition.interactionName)) _ <- async.delay(components.logging.interactionStarted(interactionStarted)) _ <- components.eventStream.publish(interactionStarted) - interactionOutput <- - setupMdc.map(_ => execute).flatMap(x => { - cleanMdc - x - }) + interactionOutput <- execute _ <- validateInteractionOutput(interactionTransition, interactionOutput) transformedOutput: Option[EventInstance] = interactionOutput.map(_.transformWith(interactionTransition)) diff --git a/pom.xml b/pom.xml index b65cff919..8288dc7d9 100644 --- a/pom.xml +++ b/pom.xml @@ -109,6 +109,7 @@ core/baker-types core/baker-intermediate-language core/baker-interface + core/baker-interface-otel core/baker-interface-kotlin core/baker-interface-in-memory core/baker-annotations From 37eed85b64e9f74328504546d4102d1f09c02e48 Mon Sep 17 00:00:00 2001 From: Dragoslav Pavkovic Date: Sun, 1 Nov 2026 15:34:40 +0100 Subject: [PATCH 3/5] OTel - adding InteractionMdc and BakerEventTracingListener --- .../baker/runtime/inmemory/InMemoryBaker.kt | 78 +------------ .../otel/BakerEventTracingListener.java | 105 ++++++++++++++++++ .../baker/runtime/model/InteractionMdc.java | 61 ++++++++++ .../runtime/model/InteractionManager.scala | 4 +- 4 files changed, 171 insertions(+), 77 deletions(-) create mode 100644 core/baker-interface-otel/src/main/java/com/ing/baker/runtime/otel/BakerEventTracingListener.java create mode 100644 core/baker-interface/src/main/java/com/ing/baker/runtime/model/InteractionMdc.java diff --git a/core/baker-interface-in-memory/src/main/kotlin/com/ing/baker/runtime/inmemory/InMemoryBaker.kt b/core/baker-interface-in-memory/src/main/kotlin/com/ing/baker/runtime/inmemory/InMemoryBaker.kt index ad8ec393e..762972726 100644 --- a/core/baker-interface-in-memory/src/main/kotlin/com/ing/baker/runtime/inmemory/InMemoryBaker.kt +++ b/core/baker-interface-in-memory/src/main/kotlin/com/ing/baker/runtime/inmemory/InMemoryBaker.kt @@ -6,20 +6,12 @@ import com.ing.baker.runtime.model.BakerF import com.ing.baker.runtime.model.BakerLogging import com.ing.baker.runtime.model.InteractionInstance import com.ing.baker.runtime.model.InteractionInstrumentation -import com.ing.baker.runtime.model.TracingMdcRegistry -import com.ing.baker.runtime.otel.BakerTraceAttributes +import com.ing.baker.runtime.otel.BakerEventTracingListener import com.ing.baker.runtime.otel.OpenTelemetryInteractionInstrumentation import com.ing.baker.runtime.otel.RecipeInstanceTraceContexts -import com.ing.baker.runtime.scaladsl.EventFired -import com.ing.baker.runtime.scaladsl.EventReceived -import com.ing.baker.runtime.scaladsl.RecipeInstanceCreated import io.opentelemetry.api.GlobalOpenTelemetry -import io.opentelemetry.api.trace.Span import io.opentelemetry.api.trace.StatusCode import io.opentelemetry.api.trace.Tracer -import io.opentelemetry.context.Context -import scala.runtime.AbstractFunction1 -import scala.runtime.BoxedUnit import java.util.concurrent.CompletableFuture import com.ing.baker.runtime.defaultinteractions.`package$`.`MODULE$` as defaultinteractions import com.ing.baker.runtime.javadsl.Baker as JavaBaker @@ -44,14 +36,6 @@ class InMemoryBaker( InMemoryEffects.asyncSupportAny() ) { - private object TraceAttributes { - const val RECIPE_ID = BakerTraceAttributes.RECIPE_ID - const val RECIPE_NAME = BakerTraceAttributes.RECIPE_NAME - const val RECIPE_INSTANCE_ID = BakerTraceAttributes.RECIPE_INSTANCE_ID - const val EVENT_NAME = BakerTraceAttributes.EVENT_NAME - const val CORRELATION_ID = BakerTraceAttributes.CORRELATION_ID - } - init { registerTracingListener() } @@ -196,68 +180,10 @@ class InMemoryBaker( private fun registerTracingListener() { try { - val listener = object : AbstractFunction1() { - override fun apply(event: com.ing.baker.runtime.scaladsl.BakerEvent): BoxedUnit { - when (event) { - is RecipeInstanceCreated -> - tracer.spanBuilder("baker.recipe-instance.created") - .startSpan() - .apply { - applyRecipeAttributes(event.recipeId(), event.recipeName(), event.recipeInstanceId()) - setStatus(StatusCode.OK) - end() - } - - is EventReceived -> { - val span = tracer.spanBuilder("baker.event.received") - .startSpan() - .apply { - applyRecipeAttributes(event.recipeId(), event.recipeName(), event.recipeInstanceId()) - setAttribute(TraceAttributes.EVENT_NAME, event.eventName()) - event.correlationId().toNullable()?.let { setAttribute(TraceAttributes.CORRELATION_ID, it) } - setStatus(StatusCode.OK) - } - TracingMdcRegistry.put(event.recipeInstanceId(), span.spanContext.traceId, span.spanContext.spanId) - traceContexts.put(event.recipeInstanceId(), Context.current().with(span)) - span.end() - } - - is EventFired -> { - val span = tracer.spanBuilder("baker.event.fired") - .startSpan() - .apply { - applyRecipeAttributes(event.recipeId(), event.recipeName(), event.recipeInstanceId()) - setAttribute(TraceAttributes.EVENT_NAME, event.eventName()) - setStatus(StatusCode.OK) - } - TracingMdcRegistry.put(event.recipeInstanceId(), span.spanContext.traceId, span.spanContext.spanId) - traceContexts.put(event.recipeInstanceId(), Context.current().with(span)) - span.end() - - if (event.eventName() == "RecipeSuccessful") { - traceContexts.remove(event.recipeInstanceId()) - TracingMdcRegistry.remove(event.recipeInstanceId()) - } - } - - else -> Unit - } - - return BoxedUnit.UNIT - } - } - - InMemoryEffects.runSync(registerBakerEventListener(listener)) + InMemoryEffects.runSync(registerBakerEventListener(BakerEventTracingListener(tracer, traceContexts))) } catch (_: Throwable) { // Tracing must never prevent Baker startup. } } - private fun Span.applyRecipeAttributes(recipeId: String, recipeName: String, recipeInstanceId: String) { - setAttribute(TraceAttributes.RECIPE_ID, recipeId) - setAttribute(TraceAttributes.RECIPE_NAME, recipeName) - setAttribute(TraceAttributes.RECIPE_INSTANCE_ID, recipeInstanceId) - } - - private fun scala.Option.toNullable(): T? = if (isDefined) get() else null } diff --git a/core/baker-interface-otel/src/main/java/com/ing/baker/runtime/otel/BakerEventTracingListener.java b/core/baker-interface-otel/src/main/java/com/ing/baker/runtime/otel/BakerEventTracingListener.java new file mode 100644 index 000000000..9c4a54737 --- /dev/null +++ b/core/baker-interface-otel/src/main/java/com/ing/baker/runtime/otel/BakerEventTracingListener.java @@ -0,0 +1,105 @@ +package com.ing.baker.runtime.otel; + +import com.ing.baker.runtime.model.TracingMdcRegistry; +import com.ing.baker.runtime.scaladsl.BakerEvent; +import com.ing.baker.runtime.scaladsl.EventFired; +import com.ing.baker.runtime.scaladsl.EventReceived; +import com.ing.baker.runtime.scaladsl.RecipeInstanceCreated; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanBuilder; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Context; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import scala.Option; +import scala.runtime.AbstractFunction1; +import scala.runtime.BoxedUnit; + +/** + * Emits OpenTelemetry spans for the lifecycle and event {@link BakerEvent}s of a Baker runtime. + * + *

Besides emitting spans this listener maintains the per recipe instance trace context that interaction spans + * are parented to, so it must share its {@link RecipeInstanceTraceContexts} instance with the + * {@link OpenTelemetryInteractionInstrumentation} of the same Baker. + * + *

It is a {@code scala.Function1} so it can be handed directly to + * {@code Baker.registerBakerEventListener}. + */ +public final class BakerEventTracingListener extends AbstractFunction1 { + + private static final Logger LOGGER = LoggerFactory.getLogger(BakerEventTracingListener.class); + + private static final String RECIPE_INSTANCE_CREATED_SPAN = "baker.recipe-instance.created"; + private static final String EVENT_RECEIVED_SPAN = "baker.event.received"; + private static final String EVENT_FIRED_SPAN = "baker.event.fired"; + private static final String RECIPE_SUCCESSFUL_EVENT = "RecipeSuccessful"; + + private final Tracer tracer; + private final RecipeInstanceTraceContexts traceContexts; + + public BakerEventTracingListener(Tracer tracer, RecipeInstanceTraceContexts traceContexts) { + this.tracer = tracer; + this.traceContexts = traceContexts; + } + + @Override + public BoxedUnit apply(BakerEvent event) { + try { + onEvent(event); + } catch (Throwable throwable) { + // Tracing must never break the event stream. + LOGGER.debug("Failed to emit a tracing span for baker event {}", event, throwable); + } + return BoxedUnit.UNIT; + } + + private void onEvent(BakerEvent event) { + if (event instanceof RecipeInstanceCreated) { + final RecipeInstanceCreated created = (RecipeInstanceCreated) event; + final Span span = newSpan(RECIPE_INSTANCE_CREATED_SPAN, + created.recipeId(), created.recipeName(), created.recipeInstanceId()); + span.setStatus(StatusCode.OK); + span.end(); + } else if (event instanceof EventReceived) { + final EventReceived received = (EventReceived) event; + final Span span = newSpan(EVENT_RECEIVED_SPAN, + received.recipeId(), received.recipeName(), received.recipeInstanceId()); + span.setAttribute(BakerTraceAttributes.EVENT_NAME, received.eventName()); + final Option correlationId = received.correlationId(); + if (correlationId.isDefined()) { + span.setAttribute(BakerTraceAttributes.CORRELATION_ID, correlationId.get()); + } + span.setStatus(StatusCode.OK); + record(received.recipeInstanceId(), span); + span.end(); + } else if (event instanceof EventFired) { + final EventFired fired = (EventFired) event; + final Span span = newSpan(EVENT_FIRED_SPAN, + fired.recipeId(), fired.recipeName(), fired.recipeInstanceId()); + span.setAttribute(BakerTraceAttributes.EVENT_NAME, fired.eventName()); + span.setStatus(StatusCode.OK); + record(fired.recipeInstanceId(), span); + span.end(); + + if (RECIPE_SUCCESSFUL_EVENT.equals(fired.eventName())) { + traceContexts.remove(fired.recipeInstanceId()); + TracingMdcRegistry.remove(fired.recipeInstanceId()); + } + } + } + + private Span newSpan(String name, String recipeId, String recipeName, String recipeInstanceId) { + final SpanBuilder builder = tracer.spanBuilder(name); + final Span span = builder.startSpan(); + span.setAttribute(BakerTraceAttributes.RECIPE_ID, recipeId); + span.setAttribute(BakerTraceAttributes.RECIPE_NAME, recipeName); + span.setAttribute(BakerTraceAttributes.RECIPE_INSTANCE_ID, recipeInstanceId); + return span; + } + + private void record(String recipeInstanceId, Span span) { + TracingMdcRegistry.put(recipeInstanceId, span.getSpanContext().getTraceId(), span.getSpanContext().getSpanId()); + traceContexts.put(recipeInstanceId, Context.current().with(span)); + } +} diff --git a/core/baker-interface/src/main/java/com/ing/baker/runtime/model/InteractionMdc.java b/core/baker-interface/src/main/java/com/ing/baker/runtime/model/InteractionMdc.java new file mode 100644 index 000000000..a33c5ef75 --- /dev/null +++ b/core/baker-interface/src/main/java/com/ing/baker/runtime/model/InteractionMdc.java @@ -0,0 +1,61 @@ +package com.ing.baker.runtime.model; + +import com.ing.baker.runtime.model.propagation.ContextScope; +import org.slf4j.MDC; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Installs the diagnostic context of a single interaction invocation on the current thread. + * + *

This is applied on the thread that actually invokes the interaction, which makes it independent of how a + * runtime composes its effects: an effect type that only builds a description on the calling thread (such as + * {@code cats.effect.IO}) would otherwise lose a diagnostic context installed by the caller. + */ +public final class InteractionMdc { + + private static final String[] KEYS = { + "recipeInstanceId", + "recipeName", + "baker.recipe.instance.id", + "baker.recipe.name", + "baker.recipe.id", + "baker.interaction.name" + }; + + private InteractionMdc() { + } + + /** + * Sets the interaction MDC keys and returns a scope restoring the previous values. + */ + public static ContextScope install(InteractionCallContext context) { + if (context == null) { + return ContextScope.NOOP; + } + final Map previous = new LinkedHashMap<>(); + for (String key : KEYS) { + previous.put(key, MDC.get(key)); + } + try { + MDC.put("recipeInstanceId", context.getRecipeInstanceId()); + MDC.put("recipeName", context.getRecipeName()); + MDC.put("baker.recipe.instance.id", context.getRecipeInstanceId()); + MDC.put("baker.recipe.name", context.getRecipeName()); + MDC.put("baker.recipe.id", context.getRecipeId()); + MDC.put("baker.interaction.name", context.getInteractionName()); + } catch (Throwable ignored) { + // Diagnostics must never break an interaction. + } + return () -> { + for (Map.Entry entry : previous.entrySet()) { + if (entry.getValue() == null) { + MDC.remove(entry.getKey()); + } else { + MDC.put(entry.getKey(), entry.getValue()); + } + } + }; + } +} diff --git a/core/baker-interface/src/main/scala/com/ing/baker/runtime/model/InteractionManager.scala b/core/baker-interface/src/main/scala/com/ing/baker/runtime/model/InteractionManager.scala index d731a66c8..c1ec6e696 100644 --- a/core/baker-interface/src/main/scala/com/ing/baker/runtime/model/InteractionManager.scala +++ b/core/baker-interface/src/main/scala/com/ing/baker/runtime/model/InteractionManager.scala @@ -96,7 +96,9 @@ trait InteractionManager[F[_]] { def invoke: F[Option[EventInstance]] = implementation.execute(input, metadata.getOrElse(Map())) callContext match { case Some(context) => - instrumentation.aroundInteraction[F[Option[EventInstance]]](context, () => invoke) + val mdcScope = InteractionMdc.install(context) + try instrumentation.aroundInteraction[F[Option[EventInstance]]](context, () => invoke) + finally mdcScope.close() case None => invoke } })(identity) From a7a110bc71d72c4173a1b1c627566e1d68fd5f59 Mon Sep 17 00:00:00 2001 From: Dragoslav Pavkovic Date: Tue, 3 Nov 2026 16:25:48 +0100 Subject: [PATCH 4/5] Converting baker-interface-otel source to Kotlin --- core/baker-interface-otel/pom.xml | 28 ++++ .../otel/BakerEventTracingListener.java | 105 -------------- .../runtime/otel/BakerTraceAttributes.java | 21 --- .../otel/OpenTelemetryContextPropagator.java | 34 ----- ...enTelemetryInteractionInstrumentation.java | 119 --------------- .../otel/RecipeInstanceTraceContexts.java | 43 ------ .../runtime/otel/BakerEventTracingListener.kt | 116 +++++++++++++++ .../runtime/otel/BakerTraceAttributes.kt | 18 +++ .../otel/OpenTelemetryContextPropagator.kt | 32 +++++ ...OpenTelemetryInteractionInstrumentation.kt | 119 +++++++++++++++ .../otel/RecipeInstanceTraceContexts.kt | 43 ++++++ .../otel/BlockingPoolPropagationTest.java | 93 ------------ .../OpenTelemetryContextPropagatorTest.java | 104 -------------- ...lemetryInteractionInstrumentationTest.java | 135 ------------------ .../otel/BlockingPoolPropagationTest.kt | 104 ++++++++++++++ .../OpenTelemetryContextPropagatorTest.kt | 113 +++++++++++++++ ...TelemetryInteractionInstrumentationTest.kt | 129 +++++++++++++++++ 17 files changed, 702 insertions(+), 654 deletions(-) delete mode 100644 core/baker-interface-otel/src/main/java/com/ing/baker/runtime/otel/BakerEventTracingListener.java delete mode 100644 core/baker-interface-otel/src/main/java/com/ing/baker/runtime/otel/BakerTraceAttributes.java delete mode 100644 core/baker-interface-otel/src/main/java/com/ing/baker/runtime/otel/OpenTelemetryContextPropagator.java delete mode 100644 core/baker-interface-otel/src/main/java/com/ing/baker/runtime/otel/OpenTelemetryInteractionInstrumentation.java delete mode 100644 core/baker-interface-otel/src/main/java/com/ing/baker/runtime/otel/RecipeInstanceTraceContexts.java create mode 100644 core/baker-interface-otel/src/main/kotlin/com/ing/baker/runtime/otel/BakerEventTracingListener.kt create mode 100644 core/baker-interface-otel/src/main/kotlin/com/ing/baker/runtime/otel/BakerTraceAttributes.kt create mode 100644 core/baker-interface-otel/src/main/kotlin/com/ing/baker/runtime/otel/OpenTelemetryContextPropagator.kt create mode 100644 core/baker-interface-otel/src/main/kotlin/com/ing/baker/runtime/otel/OpenTelemetryInteractionInstrumentation.kt create mode 100644 core/baker-interface-otel/src/main/kotlin/com/ing/baker/runtime/otel/RecipeInstanceTraceContexts.kt delete mode 100644 core/baker-interface-otel/src/test/java/com/ing/baker/runtime/otel/BlockingPoolPropagationTest.java delete mode 100644 core/baker-interface-otel/src/test/java/com/ing/baker/runtime/otel/OpenTelemetryContextPropagatorTest.java delete mode 100644 core/baker-interface-otel/src/test/java/com/ing/baker/runtime/otel/OpenTelemetryInteractionInstrumentationTest.java create mode 100644 core/baker-interface-otel/src/test/kotlin/com/ing/baker/runtime/otel/BlockingPoolPropagationTest.kt create mode 100644 core/baker-interface-otel/src/test/kotlin/com/ing/baker/runtime/otel/OpenTelemetryContextPropagatorTest.kt create mode 100644 core/baker-interface-otel/src/test/kotlin/com/ing/baker/runtime/otel/OpenTelemetryInteractionInstrumentationTest.kt diff --git a/core/baker-interface-otel/pom.xml b/core/baker-interface-otel/pom.xml index 1783e45c4..79eafcedb 100644 --- a/core/baker-interface-otel/pom.xml +++ b/core/baker-interface-otel/pom.xml @@ -33,6 +33,11 @@ slf4j-api + + org.jetbrains.kotlin + kotlin-stdlib + + org.junit.jupiter @@ -59,7 +64,30 @@ + src/main/kotlin + src/test/kotlin + + org.jetbrains.kotlin + kotlin-maven-plugin + ${kotlin.version} + + + compile + compile + + compile + + + + test-compile + test-compile + + test-compile + + + + org.apache.maven.plugins maven-surefire-plugin diff --git a/core/baker-interface-otel/src/main/java/com/ing/baker/runtime/otel/BakerEventTracingListener.java b/core/baker-interface-otel/src/main/java/com/ing/baker/runtime/otel/BakerEventTracingListener.java deleted file mode 100644 index 9c4a54737..000000000 --- a/core/baker-interface-otel/src/main/java/com/ing/baker/runtime/otel/BakerEventTracingListener.java +++ /dev/null @@ -1,105 +0,0 @@ -package com.ing.baker.runtime.otel; - -import com.ing.baker.runtime.model.TracingMdcRegistry; -import com.ing.baker.runtime.scaladsl.BakerEvent; -import com.ing.baker.runtime.scaladsl.EventFired; -import com.ing.baker.runtime.scaladsl.EventReceived; -import com.ing.baker.runtime.scaladsl.RecipeInstanceCreated; -import io.opentelemetry.api.trace.Span; -import io.opentelemetry.api.trace.SpanBuilder; -import io.opentelemetry.api.trace.StatusCode; -import io.opentelemetry.api.trace.Tracer; -import io.opentelemetry.context.Context; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import scala.Option; -import scala.runtime.AbstractFunction1; -import scala.runtime.BoxedUnit; - -/** - * Emits OpenTelemetry spans for the lifecycle and event {@link BakerEvent}s of a Baker runtime. - * - *

Besides emitting spans this listener maintains the per recipe instance trace context that interaction spans - * are parented to, so it must share its {@link RecipeInstanceTraceContexts} instance with the - * {@link OpenTelemetryInteractionInstrumentation} of the same Baker. - * - *

It is a {@code scala.Function1} so it can be handed directly to - * {@code Baker.registerBakerEventListener}. - */ -public final class BakerEventTracingListener extends AbstractFunction1 { - - private static final Logger LOGGER = LoggerFactory.getLogger(BakerEventTracingListener.class); - - private static final String RECIPE_INSTANCE_CREATED_SPAN = "baker.recipe-instance.created"; - private static final String EVENT_RECEIVED_SPAN = "baker.event.received"; - private static final String EVENT_FIRED_SPAN = "baker.event.fired"; - private static final String RECIPE_SUCCESSFUL_EVENT = "RecipeSuccessful"; - - private final Tracer tracer; - private final RecipeInstanceTraceContexts traceContexts; - - public BakerEventTracingListener(Tracer tracer, RecipeInstanceTraceContexts traceContexts) { - this.tracer = tracer; - this.traceContexts = traceContexts; - } - - @Override - public BoxedUnit apply(BakerEvent event) { - try { - onEvent(event); - } catch (Throwable throwable) { - // Tracing must never break the event stream. - LOGGER.debug("Failed to emit a tracing span for baker event {}", event, throwable); - } - return BoxedUnit.UNIT; - } - - private void onEvent(BakerEvent event) { - if (event instanceof RecipeInstanceCreated) { - final RecipeInstanceCreated created = (RecipeInstanceCreated) event; - final Span span = newSpan(RECIPE_INSTANCE_CREATED_SPAN, - created.recipeId(), created.recipeName(), created.recipeInstanceId()); - span.setStatus(StatusCode.OK); - span.end(); - } else if (event instanceof EventReceived) { - final EventReceived received = (EventReceived) event; - final Span span = newSpan(EVENT_RECEIVED_SPAN, - received.recipeId(), received.recipeName(), received.recipeInstanceId()); - span.setAttribute(BakerTraceAttributes.EVENT_NAME, received.eventName()); - final Option correlationId = received.correlationId(); - if (correlationId.isDefined()) { - span.setAttribute(BakerTraceAttributes.CORRELATION_ID, correlationId.get()); - } - span.setStatus(StatusCode.OK); - record(received.recipeInstanceId(), span); - span.end(); - } else if (event instanceof EventFired) { - final EventFired fired = (EventFired) event; - final Span span = newSpan(EVENT_FIRED_SPAN, - fired.recipeId(), fired.recipeName(), fired.recipeInstanceId()); - span.setAttribute(BakerTraceAttributes.EVENT_NAME, fired.eventName()); - span.setStatus(StatusCode.OK); - record(fired.recipeInstanceId(), span); - span.end(); - - if (RECIPE_SUCCESSFUL_EVENT.equals(fired.eventName())) { - traceContexts.remove(fired.recipeInstanceId()); - TracingMdcRegistry.remove(fired.recipeInstanceId()); - } - } - } - - private Span newSpan(String name, String recipeId, String recipeName, String recipeInstanceId) { - final SpanBuilder builder = tracer.spanBuilder(name); - final Span span = builder.startSpan(); - span.setAttribute(BakerTraceAttributes.RECIPE_ID, recipeId); - span.setAttribute(BakerTraceAttributes.RECIPE_NAME, recipeName); - span.setAttribute(BakerTraceAttributes.RECIPE_INSTANCE_ID, recipeInstanceId); - return span; - } - - private void record(String recipeInstanceId, Span span) { - TracingMdcRegistry.put(recipeInstanceId, span.getSpanContext().getTraceId(), span.getSpanContext().getSpanId()); - traceContexts.put(recipeInstanceId, Context.current().with(span)); - } -} diff --git a/core/baker-interface-otel/src/main/java/com/ing/baker/runtime/otel/BakerTraceAttributes.java b/core/baker-interface-otel/src/main/java/com/ing/baker/runtime/otel/BakerTraceAttributes.java deleted file mode 100644 index aafeb47a4..000000000 --- a/core/baker-interface-otel/src/main/java/com/ing/baker/runtime/otel/BakerTraceAttributes.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.ing.baker.runtime.otel; - -/** - * Attribute keys shared by all Baker OpenTelemetry instrumentation. - */ -public final class BakerTraceAttributes { - - public static final String RECIPE_ID = "baker.recipe.id"; - public static final String RECIPE_NAME = "baker.recipe.name"; - public static final String RECIPE_INSTANCE_ID = "baker.recipe.instance.id"; - public static final String EVENT_NAME = "baker.event.name"; - public static final String INTERACTION_NAME = "baker.interaction.name"; - public static final String CORRELATION_ID = "baker.correlation.id"; - public static final String FAILURE_COUNT = "baker.failure.count"; - public static final String ERROR_TYPE = "error.type"; - - public static final String INTERACTION_FAILURE = "interaction.failure"; - - private BakerTraceAttributes() { - } -} diff --git a/core/baker-interface-otel/src/main/java/com/ing/baker/runtime/otel/OpenTelemetryContextPropagator.java b/core/baker-interface-otel/src/main/java/com/ing/baker/runtime/otel/OpenTelemetryContextPropagator.java deleted file mode 100644 index 3efc803d0..000000000 --- a/core/baker-interface-otel/src/main/java/com/ing/baker/runtime/otel/OpenTelemetryContextPropagator.java +++ /dev/null @@ -1,34 +0,0 @@ -package com.ing.baker.runtime.otel; - -import com.ing.baker.runtime.model.propagation.ContextScope; -import com.ing.baker.runtime.model.propagation.ContextSnapshot; -import com.ing.baker.runtime.model.propagation.ExecutionContextPropagator; -import io.opentelemetry.context.Context; -import io.opentelemetry.context.ContextStorage; -import io.opentelemetry.context.Scope; - -/** - * Moves the OpenTelemetry {@link Context} of the submitting thread to the thread that actually executes the - * work, and removes it again afterwards. - * - *

Registered through {@code META-INF/services}, so simply having this module on the classpath makes - * Baker's internal thread hops context aware. - */ -public final class OpenTelemetryContextPropagator implements ExecutionContextPropagator { - - @Override - public ContextSnapshot capture() { - final Context captured = Context.current(); - return () -> { - final Scope scope = captured.makeCurrent(); - return (ContextScope) scope::close; - }; - } - - @Override - public void clear() { - // Detaches whatever context a previous task left attached to this (pooled) thread. The returned scope - // is intentionally not closed: closing it would restore exactly the leaked context we want gone. - ContextStorage.get().attach(Context.root()); - } -} diff --git a/core/baker-interface-otel/src/main/java/com/ing/baker/runtime/otel/OpenTelemetryInteractionInstrumentation.java b/core/baker-interface-otel/src/main/java/com/ing/baker/runtime/otel/OpenTelemetryInteractionInstrumentation.java deleted file mode 100644 index 6b4132e47..000000000 --- a/core/baker-interface-otel/src/main/java/com/ing/baker/runtime/otel/OpenTelemetryInteractionInstrumentation.java +++ /dev/null @@ -1,119 +0,0 @@ -package com.ing.baker.runtime.otel; - -import com.ing.baker.runtime.model.InteractionCallContext; -import com.ing.baker.runtime.model.InteractionInstrumentation; -import io.opentelemetry.api.trace.Span; -import io.opentelemetry.api.trace.StatusCode; -import io.opentelemetry.api.trace.Tracer; -import io.opentelemetry.context.Context; -import io.opentelemetry.context.Scope; -import org.slf4j.MDC; - -import java.util.concurrent.CompletionException; -import java.util.concurrent.CompletionStage; -import java.util.concurrent.ExecutionException; -import java.util.function.Supplier; - -/** - * Starts a span for every interaction invocation and attaches it to the thread that runs the interaction, so - * {@code Span.current()} inside an interaction always belongs to that very invocation, even when - * interactions of different recipe instances share a thread pool. - */ -public final class OpenTelemetryInteractionInstrumentation implements InteractionInstrumentation { - - private static final String TRACE_ID_MDC_KEY = "trace_id"; - private static final String SPAN_ID_MDC_KEY = "span_id"; - - private final Tracer tracer; - private final RecipeInstanceTraceContexts parentContexts; - - public OpenTelemetryInteractionInstrumentation(Tracer tracer, RecipeInstanceTraceContexts parentContexts) { - this.tracer = tracer; - this.parentContexts = parentContexts; - } - - @Override - public A aroundInteraction(InteractionCallContext context, Supplier interaction) { - final Span span; - try { - span = startSpan(context); - } catch (Throwable throwable) { - // Tracing must never break execution. - return interaction.get(); - } - - final String previousTraceId = MDC.get(TRACE_ID_MDC_KEY); - final String previousSpanId = MDC.get(SPAN_ID_MDC_KEY); - boolean endsAsynchronously = false; - - try (Scope ignored = span.makeCurrent()) { - MDC.put(TRACE_ID_MDC_KEY, span.getSpanContext().getTraceId()); - MDC.put(SPAN_ID_MDC_KEY, span.getSpanContext().getSpanId()); - - final A result = interaction.get(); - if (result instanceof CompletionStage) { - endsAsynchronously = true; - ((CompletionStage) result).whenComplete((value, throwable) -> end(span, throwable)); - } else { - span.setStatus(StatusCode.OK); - } - return result; - } catch (Throwable throwable) { - endsAsynchronously = false; - recordFailure(span, throwable); - throw throwable; - } finally { - restore(TRACE_ID_MDC_KEY, previousTraceId); - restore(SPAN_ID_MDC_KEY, previousSpanId); - if (!endsAsynchronously) { - span.end(); - } - } - } - - private Span startSpan(InteractionCallContext context) { - final Context parent = parentContexts.parentOf(context.getRecipeInstanceId()); - final Span span = tracer.spanBuilder("baker.interaction." + context.getInteractionName()) - .setParent(parent) - .startSpan(); - span.setAttribute(BakerTraceAttributes.RECIPE_ID, context.getRecipeId()); - span.setAttribute(BakerTraceAttributes.RECIPE_NAME, context.getRecipeName()); - span.setAttribute(BakerTraceAttributes.RECIPE_INSTANCE_ID, context.getRecipeInstanceId()); - span.setAttribute(BakerTraceAttributes.INTERACTION_NAME, context.getInteractionName()); - context.getCorrelationId() - .ifPresent(correlationId -> span.setAttribute(BakerTraceAttributes.CORRELATION_ID, correlationId)); - return span; - } - - private static void end(Span span, Throwable throwable) { - if (throwable == null) { - span.setStatus(StatusCode.OK); - } else { - recordFailure(span, throwable); - } - span.end(); - } - - private static void recordFailure(Span span, Throwable throwable) { - final Throwable cause = unwrap(throwable); - span.setAttribute(BakerTraceAttributes.ERROR_TYPE, BakerTraceAttributes.INTERACTION_FAILURE); - span.recordException(cause); - span.setStatus(StatusCode.ERROR, cause.getMessage() == null ? cause.toString() : cause.getMessage()); - } - - private static Throwable unwrap(Throwable throwable) { - if ((throwable instanceof CompletionException || throwable instanceof ExecutionException) - && throwable.getCause() != null) { - return throwable.getCause(); - } - return throwable; - } - - private static void restore(String key, String value) { - if (value == null) { - MDC.remove(key); - } else { - MDC.put(key, value); - } - } -} diff --git a/core/baker-interface-otel/src/main/java/com/ing/baker/runtime/otel/RecipeInstanceTraceContexts.java b/core/baker-interface-otel/src/main/java/com/ing/baker/runtime/otel/RecipeInstanceTraceContexts.java deleted file mode 100644 index bb3cf96d9..000000000 --- a/core/baker-interface-otel/src/main/java/com/ing/baker/runtime/otel/RecipeInstanceTraceContexts.java +++ /dev/null @@ -1,43 +0,0 @@ -package com.ing.baker.runtime.otel; - -import io.opentelemetry.context.Context; - -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -/** - * Keeps the tracing context that interaction spans of a recipe instance should be parented to. - * - *

Each Baker runtime owns its own instance, so contexts never leak between bakers, and entries are - * removed as soon as the recipe instance is done. - */ -public final class RecipeInstanceTraceContexts { - - private final Map contexts = new ConcurrentHashMap<>(); - - public void put(String recipeInstanceId, Context context) { - if (recipeInstanceId == null || context == null) { - return; - } - contexts.put(recipeInstanceId, context); - } - - /** - * Returns the parent context for the given recipe instance, falling back to the context currently - * attached to the calling thread. - */ - public Context parentOf(String recipeInstanceId) { - final Context context = recipeInstanceId == null ? null : contexts.get(recipeInstanceId); - return context != null ? context : Context.current(); - } - - public void remove(String recipeInstanceId) { - if (recipeInstanceId != null) { - contexts.remove(recipeInstanceId); - } - } - - public void clear() { - contexts.clear(); - } -} diff --git a/core/baker-interface-otel/src/main/kotlin/com/ing/baker/runtime/otel/BakerEventTracingListener.kt b/core/baker-interface-otel/src/main/kotlin/com/ing/baker/runtime/otel/BakerEventTracingListener.kt new file mode 100644 index 000000000..1d867c4cf --- /dev/null +++ b/core/baker-interface-otel/src/main/kotlin/com/ing/baker/runtime/otel/BakerEventTracingListener.kt @@ -0,0 +1,116 @@ +package com.ing.baker.runtime.otel + +import com.ing.baker.runtime.model.TracingMdcRegistry +import com.ing.baker.runtime.scaladsl.BakerEvent +import com.ing.baker.runtime.scaladsl.EventFired +import com.ing.baker.runtime.scaladsl.EventReceived +import com.ing.baker.runtime.scaladsl.RecipeInstanceCreated +import io.opentelemetry.api.trace.Span +import io.opentelemetry.api.trace.SpanBuilder +import io.opentelemetry.api.trace.StatusCode +import io.opentelemetry.api.trace.Tracer +import io.opentelemetry.context.Context +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import scala.runtime.AbstractFunction1 +import scala.runtime.BoxedUnit + +/** + * Emits OpenTelemetry spans for the lifecycle and event [BakerEvent]s of a Baker runtime. + * + * Besides emitting spans this listener maintains the per recipe instance trace context that interaction spans + * are parented to, so it must share its [RecipeInstanceTraceContexts] instance with the + * [OpenTelemetryInteractionInstrumentation] of the same Baker. + * + * It is a `scala.Function1` so it can be handed directly to + * `Baker.registerBakerEventListener`. + */ +class BakerEventTracingListener( + private val tracer: Tracer, + private val traceContexts: RecipeInstanceTraceContexts +) : AbstractFunction1() { + + override fun apply(event: BakerEvent): BoxedUnit { + try { + onEvent(event) + } catch (throwable: Throwable) { + LOGGER.debug("Failed to emit a tracing span for baker event {}", event, throwable) + } + return BoxedUnit.UNIT + } + + private fun onEvent(event: BakerEvent) { + when (event) { + is RecipeInstanceCreated -> { + val span = newSpan( + RECIPE_INSTANCE_CREATED_SPAN, + event.recipeId(), + event.recipeName(), + event.recipeInstanceId() + ) + span.setStatus(StatusCode.OK) + span.end() + } + is EventReceived -> { + val span = newSpan( + EVENT_RECEIVED_SPAN, + event.recipeId(), + event.recipeName(), + event.recipeInstanceId() + ) + span.setAttribute(BakerTraceAttributes.EVENT_NAME, event.eventName()) + val correlationId = event.correlationId() + if (correlationId.isDefined) { + span.setAttribute(BakerTraceAttributes.CORRELATION_ID, correlationId.get()) + } + span.setStatus(StatusCode.OK) + record(event.recipeInstanceId(), span) + span.end() + } + is EventFired -> { + val span = newSpan( + EVENT_FIRED_SPAN, + event.recipeId(), + event.recipeName(), + event.recipeInstanceId() + ) + span.setAttribute(BakerTraceAttributes.EVENT_NAME, event.eventName()) + span.setStatus(StatusCode.OK) + record(event.recipeInstanceId(), span) + span.end() + + if (RECIPE_SUCCESSFUL_EVENT == event.eventName()) { + traceContexts.remove(event.recipeInstanceId()) + TracingMdcRegistry.remove(event.recipeInstanceId()) + } + } + } + } + + private fun newSpan(name: String, recipeId: String, recipeName: String, recipeInstanceId: String): Span { + val builder = tracer.spanBuilder(name) + val span = builder.startSpan() + span.setAttribute(BakerTraceAttributes.RECIPE_ID, recipeId) + span.setAttribute(BakerTraceAttributes.RECIPE_NAME, recipeName) + span.setAttribute(BakerTraceAttributes.RECIPE_INSTANCE_ID, recipeInstanceId) + return span + } + + private fun record(recipeInstanceId: String, span: Span) { + TracingMdcRegistry.put( + recipeInstanceId, + span.spanContext.traceId, + span.spanContext.spanId + ) + traceContexts.put(recipeInstanceId, Context.current().with(span)) + } + + companion object { + private val LOGGER: Logger = LoggerFactory.getLogger(BakerEventTracingListener::class.java) + + private const val RECIPE_INSTANCE_CREATED_SPAN = "baker.recipe-instance.created" + private const val EVENT_RECEIVED_SPAN = "baker.event.received" + private const val EVENT_FIRED_SPAN = "baker.event.fired" + private const val RECIPE_SUCCESSFUL_EVENT = "RecipeSuccessful" + } +} diff --git a/core/baker-interface-otel/src/main/kotlin/com/ing/baker/runtime/otel/BakerTraceAttributes.kt b/core/baker-interface-otel/src/main/kotlin/com/ing/baker/runtime/otel/BakerTraceAttributes.kt new file mode 100644 index 000000000..a7f365638 --- /dev/null +++ b/core/baker-interface-otel/src/main/kotlin/com/ing/baker/runtime/otel/BakerTraceAttributes.kt @@ -0,0 +1,18 @@ +package com.ing.baker.runtime.otel + +/** + * Attribute keys shared by all Baker OpenTelemetry instrumentation. + */ +object BakerTraceAttributes { + + const val RECIPE_ID = "baker.recipe.id" + const val RECIPE_NAME = "baker.recipe.name" + const val RECIPE_INSTANCE_ID = "baker.recipe.instance.id" + const val EVENT_NAME = "baker.event.name" + const val INTERACTION_NAME = "baker.interaction.name" + const val CORRELATION_ID = "baker.correlation.id" + const val FAILURE_COUNT = "baker.failure.count" + const val ERROR_TYPE = "error.type" + + const val INTERACTION_FAILURE = "interaction.failure" +} diff --git a/core/baker-interface-otel/src/main/kotlin/com/ing/baker/runtime/otel/OpenTelemetryContextPropagator.kt b/core/baker-interface-otel/src/main/kotlin/com/ing/baker/runtime/otel/OpenTelemetryContextPropagator.kt new file mode 100644 index 000000000..7f520925f --- /dev/null +++ b/core/baker-interface-otel/src/main/kotlin/com/ing/baker/runtime/otel/OpenTelemetryContextPropagator.kt @@ -0,0 +1,32 @@ +package com.ing.baker.runtime.otel + +import com.ing.baker.runtime.model.propagation.ContextScope +import com.ing.baker.runtime.model.propagation.ContextSnapshot +import com.ing.baker.runtime.model.propagation.ExecutionContextPropagator +import io.opentelemetry.context.Context +import io.opentelemetry.context.ContextStorage +import io.opentelemetry.context.Scope + +/** + * Moves the OpenTelemetry [Context] of the submitting thread to the thread that actually executes the + * work, and removes it again afterwards. + * + * Registered through `META-INF/services`, so simply having this module on the classpath makes + * Baker's internal thread hops context aware. + */ +class OpenTelemetryContextPropagator : ExecutionContextPropagator { + + override fun capture(): ContextSnapshot { + val captured = Context.current() + return object : ContextSnapshot { + override fun restore(): ContextScope { + val scope = captured.makeCurrent() + return ContextScope { scope.close() } + } + } + } + + override fun clear() { + ContextStorage.get().attach(Context.root()) + } +} diff --git a/core/baker-interface-otel/src/main/kotlin/com/ing/baker/runtime/otel/OpenTelemetryInteractionInstrumentation.kt b/core/baker-interface-otel/src/main/kotlin/com/ing/baker/runtime/otel/OpenTelemetryInteractionInstrumentation.kt new file mode 100644 index 000000000..003bd7e9b --- /dev/null +++ b/core/baker-interface-otel/src/main/kotlin/com/ing/baker/runtime/otel/OpenTelemetryInteractionInstrumentation.kt @@ -0,0 +1,119 @@ +package com.ing.baker.runtime.otel + +import com.ing.baker.runtime.model.InteractionCallContext +import com.ing.baker.runtime.model.InteractionInstrumentation +import io.opentelemetry.api.trace.Span +import io.opentelemetry.api.trace.StatusCode +import io.opentelemetry.api.trace.Tracer +import io.opentelemetry.context.Context +import io.opentelemetry.context.Scope +import org.slf4j.MDC +import java.util.concurrent.CompletionException +import java.util.concurrent.CompletionStage +import java.util.concurrent.ExecutionException +import java.util.function.Supplier + +/** + * Starts a span for every interaction invocation and attaches it to the thread that runs the interaction, so + * `Span.current()` inside an interaction always belongs to that very invocation, even when + * interactions of different recipe instances share a thread pool. + */ +class OpenTelemetryInteractionInstrumentation( + private val tracer: Tracer, + private val parentContexts: RecipeInstanceTraceContexts +) : InteractionInstrumentation { + + override fun aroundInteraction(context: InteractionCallContext, interaction: Supplier): A { + val span = try { + startSpan(context) + } catch (throwable: Throwable) { + return interaction.get() + } + + val previousTraceId = MDC.get(TRACE_ID_MDC_KEY) + val previousSpanId = MDC.get(SPAN_ID_MDC_KEY) + var endsAsynchronously = false + + return try { + span.makeCurrent().use { + MDC.put(TRACE_ID_MDC_KEY, span.spanContext.traceId) + MDC.put(SPAN_ID_MDC_KEY, span.spanContext.spanId) + + val result = interaction.get() + if (result is CompletionStage<*>) { + endsAsynchronously = true + result.whenComplete { _, throwable -> end(span, throwable) } + } else { + span.setStatus(StatusCode.OK) + } + result + } + } catch (throwable: Throwable) { + endsAsynchronously = false + recordFailure(span, throwable) + throw throwable + } finally { + restore(TRACE_ID_MDC_KEY, previousTraceId) + restore(SPAN_ID_MDC_KEY, previousSpanId) + if (!endsAsynchronously) { + span.end() + } + } + } + + private fun startSpan(context: InteractionCallContext): Span { + val parent = parentContexts.parentOf(context.recipeInstanceId) + val span = tracer.spanBuilder("baker.interaction." + context.interactionName) + .setParent(parent) + .startSpan() + span.setAttribute(BakerTraceAttributes.RECIPE_ID, context.recipeId) + span.setAttribute(BakerTraceAttributes.RECIPE_NAME, context.recipeName) + span.setAttribute(BakerTraceAttributes.RECIPE_INSTANCE_ID, context.recipeInstanceId) + span.setAttribute(BakerTraceAttributes.INTERACTION_NAME, context.interactionName) + context.correlationId + .ifPresent { correlationId -> span.setAttribute(BakerTraceAttributes.CORRELATION_ID, correlationId) } + return span + } + + companion object { + private const val TRACE_ID_MDC_KEY = "trace_id" + private const val SPAN_ID_MDC_KEY = "span_id" + + private fun end(span: Span, throwable: Throwable?) { + if (throwable == null) { + span.setStatus(StatusCode.OK) + } else { + recordFailure(span, throwable) + } + span.end() + } + + private fun recordFailure(span: Span, throwable: Throwable) { + val cause = unwrap(throwable) + span.setAttribute(BakerTraceAttributes.ERROR_TYPE, BakerTraceAttributes.INTERACTION_FAILURE) + span.recordException(cause) + span.setStatus( + StatusCode.ERROR, + cause.message ?: cause.toString() + ) + } + + private fun unwrap(throwable: Throwable): Throwable { + return if ((throwable is CompletionException || throwable is ExecutionException) + && throwable.cause != null + ) { + throwable.cause!! + } else { + throwable + } + } + + private fun restore(key: String, value: String?) { + if (value == null) { + MDC.remove(key) + } else { + MDC.put(key, value) + } + } + } +} diff --git a/core/baker-interface-otel/src/main/kotlin/com/ing/baker/runtime/otel/RecipeInstanceTraceContexts.kt b/core/baker-interface-otel/src/main/kotlin/com/ing/baker/runtime/otel/RecipeInstanceTraceContexts.kt new file mode 100644 index 000000000..4e84110cb --- /dev/null +++ b/core/baker-interface-otel/src/main/kotlin/com/ing/baker/runtime/otel/RecipeInstanceTraceContexts.kt @@ -0,0 +1,43 @@ +package com.ing.baker.runtime.otel + +import io.opentelemetry.context.Context +import java.util.concurrent.ConcurrentHashMap + +/** + * Keeps the tracing context that interaction spans of a recipe instance should be parented to. + * + * Each Baker runtime owns its own instance, so contexts never leak between bakers, and entries are + * removed as soon as the recipe instance is done. + */ +class RecipeInstanceTraceContexts { + + private val contexts = ConcurrentHashMap() + + fun put(recipeInstanceId: String?, context: Context?) { + if (recipeInstanceId != null && context != null) { + contexts[recipeInstanceId] = context + } + } + + /** + * Returns the parent context for the given recipe instance, falling back to the context currently + * attached to the calling thread. + */ + fun parentOf(recipeInstanceId: String?): Context { + return if (recipeInstanceId != null) { + contexts[recipeInstanceId] ?: Context.current() + } else { + Context.current() + } + } + + fun remove(recipeInstanceId: String?) { + if (recipeInstanceId != null) { + contexts.remove(recipeInstanceId) + } + } + + fun clear() { + contexts.clear() + } +} diff --git a/core/baker-interface-otel/src/test/java/com/ing/baker/runtime/otel/BlockingPoolPropagationTest.java b/core/baker-interface-otel/src/test/java/com/ing/baker/runtime/otel/BlockingPoolPropagationTest.java deleted file mode 100644 index 621fb9248..000000000 --- a/core/baker-interface-otel/src/test/java/com/ing/baker/runtime/otel/BlockingPoolPropagationTest.java +++ /dev/null @@ -1,93 +0,0 @@ -package com.ing.baker.runtime.otel; - -import com.ing.baker.runtime.catseffect.SyncSupport; -import io.opentelemetry.api.trace.Span; -import io.opentelemetry.context.Context; -import io.opentelemetry.context.ContextStorage; -import io.opentelemetry.context.Scope; -import io.opentelemetry.sdk.OpenTelemetrySdk; -import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; -import io.opentelemetry.sdk.trace.SdkTracerProvider; -import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.slf4j.MDC; - -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.TimeUnit; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * Verifies that the propagator registered through {@code META-INF/services} is picked up by Baker's blocking - * pool, which is where interaction implementations run. - */ -class BlockingPoolPropagationTest { - - private static final String INVALID_TRACE_ID = "00000000000000000000000000000000"; - - private InMemorySpanExporter exporter; - private SdkTracerProvider tracerProvider; - private io.opentelemetry.api.trace.Tracer tracer; - - @BeforeEach - void setUp() { - exporter = InMemorySpanExporter.create(); - tracerProvider = SdkTracerProvider.builder() - .addSpanProcessor(SimpleSpanProcessor.create(exporter)) - .build(); - tracer = OpenTelemetrySdk.builder().setTracerProvider(tracerProvider).build().getTracer("test.otel.blocking"); - } - - @AfterEach - void tearDown() { - tracerProvider.close(); - exporter.reset(); - MDC.clear(); - } - - private static A runBlocking(scala.Function0 thunk) throws Exception { - final CompletableFuture future = - SyncSupport.fromCompletableFuture().blocking(thunk); - return future.get(10, TimeUnit.SECONDS); - } - - @Test - void propagatesSpanAndMdcToTheBlockingPool() throws Exception { - final Span span = tracer.spanBuilder("caller").startSpan(); - MDC.put("recipeInstanceId", "instance-1"); - try (Scope ignored = span.makeCurrent()) { - final String[] observed = runBlocking(() -> new String[]{ - Span.current().getSpanContext().getSpanId(), - MDC.get("recipeInstanceId") - }); - assertEquals(span.getSpanContext().getSpanId(), observed[0]); - assertEquals("instance-1", observed[1]); - } finally { - span.end(); - MDC.clear(); - } - } - - @Test - void doesNotExposeContextOfPreviousWorkOnTheSamePooledThread() throws Exception { - final Span leaked = tracer.spanBuilder("leaked").startSpan(); - - // Simulate badly behaved work that attaches a context on the pooled thread without detaching it. - runBlocking(() -> { - ContextStorage.get().attach(Context.current().with(leaked)); - MDC.put("trace_id", "leaked-trace"); - return null; - }); - - final String[] observed = runBlocking(() -> new String[]{ - Span.current().getSpanContext().getTraceId(), - MDC.get("trace_id") - }); - - assertEquals(INVALID_TRACE_ID, observed[0], "A pooled thread must not expose the context of previous work"); - assertEquals(null, observed[1], "A pooled thread must not expose the MDC of previous work"); - leaked.end(); - } -} diff --git a/core/baker-interface-otel/src/test/java/com/ing/baker/runtime/otel/OpenTelemetryContextPropagatorTest.java b/core/baker-interface-otel/src/test/java/com/ing/baker/runtime/otel/OpenTelemetryContextPropagatorTest.java deleted file mode 100644 index 86187257a..000000000 --- a/core/baker-interface-otel/src/test/java/com/ing/baker/runtime/otel/OpenTelemetryContextPropagatorTest.java +++ /dev/null @@ -1,104 +0,0 @@ -package com.ing.baker.runtime.otel; - -import com.ing.baker.runtime.model.propagation.ContextScope; -import com.ing.baker.runtime.model.propagation.ContextSnapshot; -import io.opentelemetry.api.trace.Span; -import io.opentelemetry.context.Context; -import io.opentelemetry.context.ContextStorage; -import io.opentelemetry.sdk.OpenTelemetrySdk; -import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; -import io.opentelemetry.sdk.trace.SdkTracerProvider; -import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; -import org.junit.jupiter.api.Test; - -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; - -class OpenTelemetryContextPropagatorTest { - - private static final String INVALID_TRACE_ID = "00000000000000000000000000000000"; - - private final OpenTelemetryContextPropagator propagator = new OpenTelemetryContextPropagator(); - - @Test - void movesTheCapturedContextToTheExecutingThread() throws Exception { - try (TracerFixture fixture = TracerFixture.create()) { - final Span span = fixture.tracer().spanBuilder("caller").startSpan(); - final ContextSnapshot snapshot; - try (io.opentelemetry.context.Scope ignored = span.makeCurrent()) { - snapshot = propagator.capture(); - } - - final ExecutorService executor = Executors.newSingleThreadExecutor(); - try { - final Future observed = executor.submit(() -> { - try (ContextScope scope = snapshot.restore()) { - return Span.current().getSpanContext().getSpanId(); - } - }); - assertEquals(span.getSpanContext().getSpanId(), observed.get(5, TimeUnit.SECONDS)); - - final Future afterScope = executor.submit( - () -> Span.current().getSpanContext().getTraceId()); - assertEquals(INVALID_TRACE_ID, afterScope.get(5, TimeUnit.SECONDS), - "The propagated context must not survive the scope it was restored in"); - } finally { - executor.shutdownNow(); - } - span.end(); - } - } - - @Test - void clearsContextLeakedByPreviousWorkOnTheSameThread() throws Exception { - try (TracerFixture fixture = TracerFixture.create()) { - final ExecutorService executor = Executors.newSingleThreadExecutor(); - try { - final Span leaked = fixture.tracer().spanBuilder("leaked").startSpan(); - executor.submit(() -> ContextStorage.get().attach(Context.current().with(leaked))) - .get(5, TimeUnit.SECONDS); - - final Future stillLeaking = executor.submit( - () -> Span.current().getSpanContext().getSpanId()); - assertEquals(leaked.getSpanContext().getSpanId(), stillLeaking.get(5, TimeUnit.SECONDS)); - - final Future afterClear = executor.submit(() -> { - propagator.clear(); - return Span.current().getSpanContext().getTraceId(); - }); - assertEquals(INVALID_TRACE_ID, afterClear.get(5, TimeUnit.SECONDS)); - assertNotEquals(INVALID_TRACE_ID, leaked.getSpanContext().getTraceId()); - assertFalse(Span.current().getSpanContext().isValid(), - "The test thread itself should not have any ambient span"); - leaked.end(); - } finally { - executor.shutdownNow(); - } - } - } - - record TracerFixture(InMemorySpanExporter exporter, SdkTracerProvider tracerProvider, - io.opentelemetry.api.trace.Tracer tracer) implements AutoCloseable { - - static TracerFixture create() { - final InMemorySpanExporter exporter = InMemorySpanExporter.create(); - final SdkTracerProvider tracerProvider = SdkTracerProvider.builder() - .addSpanProcessor(SimpleSpanProcessor.create(exporter)) - .build(); - final OpenTelemetrySdk sdk = OpenTelemetrySdk.builder().setTracerProvider(tracerProvider).build(); - return new TracerFixture(exporter, tracerProvider, sdk.getTracer("test.otel.propagator")); - } - - @Override - public void close() { - tracerProvider.close(); - exporter.reset(); - } - } -} diff --git a/core/baker-interface-otel/src/test/java/com/ing/baker/runtime/otel/OpenTelemetryInteractionInstrumentationTest.java b/core/baker-interface-otel/src/test/java/com/ing/baker/runtime/otel/OpenTelemetryInteractionInstrumentationTest.java deleted file mode 100644 index 7e6b07747..000000000 --- a/core/baker-interface-otel/src/test/java/com/ing/baker/runtime/otel/OpenTelemetryInteractionInstrumentationTest.java +++ /dev/null @@ -1,135 +0,0 @@ -package com.ing.baker.runtime.otel; - -import com.ing.baker.runtime.model.InteractionCallContext; -import io.opentelemetry.api.common.AttributeKey; -import io.opentelemetry.api.trace.Span; -import io.opentelemetry.api.trace.StatusCode; -import io.opentelemetry.api.trace.Tracer; -import io.opentelemetry.context.Context; -import io.opentelemetry.sdk.OpenTelemetrySdk; -import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; -import io.opentelemetry.sdk.trace.SdkTracerProvider; -import io.opentelemetry.sdk.trace.data.SpanData; -import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.slf4j.MDC; - -import java.util.List; -import java.util.concurrent.CompletableFuture; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -class OpenTelemetryInteractionInstrumentationTest { - - private InMemorySpanExporter exporter; - private SdkTracerProvider tracerProvider; - private Tracer tracer; - private RecipeInstanceTraceContexts parentContexts; - private OpenTelemetryInteractionInstrumentation instrumentation; - - @BeforeEach - void setUp() { - exporter = InMemorySpanExporter.create(); - tracerProvider = SdkTracerProvider.builder() - .addSpanProcessor(SimpleSpanProcessor.create(exporter)) - .build(); - tracer = OpenTelemetrySdk.builder().setTracerProvider(tracerProvider).build().getTracer("test.otel"); - parentContexts = new RecipeInstanceTraceContexts(); - instrumentation = new OpenTelemetryInteractionInstrumentation(tracer, parentContexts); - } - - @AfterEach - void tearDown() { - tracerProvider.close(); - exporter.reset(); - MDC.clear(); - } - - private static InteractionCallContext callContext(String recipeInstanceId) { - return new InteractionCallContext("recipe-id", "recipe", recipeInstanceId, "ValidateOrder", 42L, "correlation-1"); - } - - @Test - void attachesItsOwnSpanToTheExecutingThread() { - final String observedSpanId = instrumentation.aroundInteraction( - callContext("instance-1"), - () -> Span.current().getSpanContext().getSpanId()); - - final List spans = exporter.getFinishedSpanItems(); - assertEquals(1, spans.size()); - final SpanData span = spans.get(0); - assertEquals("baker.interaction.ValidateOrder", span.getName()); - assertEquals(span.getSpanContext().getSpanId(), observedSpanId); - assertEquals(StatusCode.OK, span.getStatus().getStatusCode()); - assertEquals("instance-1", span.getAttributes().get(AttributeKey.stringKey("baker.recipe.instance.id"))); - assertEquals("ValidateOrder", span.getAttributes().get(AttributeKey.stringKey("baker.interaction.name"))); - assertEquals("correlation-1", span.getAttributes().get(AttributeKey.stringKey("baker.correlation.id"))); - assertFalse(Span.current().getSpanContext().isValid(), "The span must be detached after the invocation"); - } - - @Test - void parentsTheSpanToTheRecipeInstanceContext() { - final Span parent = tracer.spanBuilder("baker.event.received").startSpan(); - parentContexts.put("instance-2", Context.root().with(parent)); - parent.end(); - - instrumentation.aroundInteraction(callContext("instance-2"), () -> null); - - final SpanData interactionSpan = exporter.getFinishedSpanItems().stream() - .filter(span -> span.getName().startsWith("baker.interaction.")) - .findFirst() - .orElseThrow(); - assertEquals(parent.getSpanContext().getSpanId(), interactionSpan.getParentSpanId()); - assertEquals(parent.getSpanContext().getTraceId(), interactionSpan.getSpanContext().getTraceId()); - } - - @Test - void setsAndRestoresTraceMdcKeys() { - MDC.put("trace_id", "previous-trace"); - - instrumentation.aroundInteraction(callContext("instance-3"), () -> { - assertEquals(Span.current().getSpanContext().getTraceId(), MDC.get("trace_id")); - assertEquals(Span.current().getSpanContext().getSpanId(), MDC.get("span_id")); - return null; - }); - - assertEquals("previous-trace", MDC.get("trace_id")); - assertNull(MDC.get("span_id")); - } - - @Test - void endsTheSpanWhenAnAsynchronousInteractionFails() { - final CompletableFuture future = new CompletableFuture<>(); - - instrumentation.aroundInteraction(callContext("instance-4"), () -> future); - - assertTrue(exporter.getFinishedSpanItems().isEmpty(), "Span must stay open until the interaction completes"); - - future.completeExceptionally(new RuntimeException("Expected failure")); - - final SpanData span = exporter.getFinishedSpanItems().get(0); - assertEquals(StatusCode.ERROR, span.getStatus().getStatusCode()); - assertEquals("interaction.failure", span.getAttributes().get(AttributeKey.stringKey("error.type"))); - } - - @Test - void recordsSynchronousFailures() { - final RuntimeException failure = new RuntimeException("Expected failure"); - try { - instrumentation.aroundInteraction(callContext("instance-5"), () -> { - throw failure; - }); - } catch (RuntimeException expected) { - assertEquals(failure, expected); - } - - final SpanData span = exporter.getFinishedSpanItems().get(0); - assertEquals(StatusCode.ERROR, span.getStatus().getStatusCode()); - assertEquals("interaction.failure", span.getAttributes().get(AttributeKey.stringKey("error.type"))); - } -} diff --git a/core/baker-interface-otel/src/test/kotlin/com/ing/baker/runtime/otel/BlockingPoolPropagationTest.kt b/core/baker-interface-otel/src/test/kotlin/com/ing/baker/runtime/otel/BlockingPoolPropagationTest.kt new file mode 100644 index 000000000..232459d93 --- /dev/null +++ b/core/baker-interface-otel/src/test/kotlin/com/ing/baker/runtime/otel/BlockingPoolPropagationTest.kt @@ -0,0 +1,104 @@ +package com.ing.baker.runtime.otel + +import com.ing.baker.runtime.catseffect.SyncSupport +import io.opentelemetry.api.trace.Span +import io.opentelemetry.context.Context +import io.opentelemetry.context.ContextStorage +import io.opentelemetry.context.Scope +import io.opentelemetry.sdk.OpenTelemetrySdk +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter +import io.opentelemetry.sdk.trace.SdkTracerProvider +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.slf4j.MDC +import java.util.concurrent.CompletableFuture +import java.util.concurrent.TimeUnit + +/** + * Verifies that the propagator registered through {@code META-INF/services} is picked up by Baker's blocking + * pool, which is where interaction implementations run. + */ +class BlockingPoolPropagationTest { + + private lateinit var exporter: InMemorySpanExporter + private lateinit var tracerProvider: SdkTracerProvider + private lateinit var tracer: io.opentelemetry.api.trace.Tracer + + @BeforeEach + fun setUp() { + exporter = InMemorySpanExporter.create() + tracerProvider = SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(exporter)) + .build() + tracer = OpenTelemetrySdk.builder().setTracerProvider(tracerProvider).build().getTracer("test.otel.blocking") + } + + @AfterEach + fun tearDown() { + tracerProvider.close() + exporter.reset() + MDC.clear() + } + + @Test + fun propagatesSpanAndMdcToTheBlockingPool() { + val span = tracer.spanBuilder("caller").startSpan() + MDC.put("recipeInstanceId", "instance-1") + try { + span.makeCurrent().use { + val observed = runBlocking { + arrayOf( + Span.current().spanContext.spanId, + MDC.get("recipeInstanceId") + ) + } + assertEquals(span.spanContext.spanId, observed[0]) + assertEquals("instance-1", observed[1]) + } + } finally { + span.end() + MDC.clear() + } + } + + @Test + fun doesNotExposeContextOfPreviousWorkOnTheSamePooledThread() { + val leaked = tracer.spanBuilder("leaked").startSpan() + + // Simulate badly behaved work that attaches a context on the pooled thread without detaching it. + runBlocking { + ContextStorage.get().attach(Context.current().with(leaked)) + MDC.put("trace_id", "leaked-trace") + null + } + + val observed = runBlocking { + arrayOf( + Span.current().spanContext.traceId, + MDC.get("trace_id") + ) + } + + assertEquals(INVALID_TRACE_ID, observed[0], "A pooled thread must not expose the context of previous work") + assertNull(observed[1], "A pooled thread must not expose the MDC of previous work") + leaked.end() + } + + private fun runBlocking(thunk: () -> A): A { + @Suppress("UNCHECKED_CAST") + val future = SyncSupport.fromCompletableFuture().blocking( + object : scala.Function0 { + override fun apply(): A = thunk() + } + ) as CompletableFuture + return future.get(10, TimeUnit.SECONDS) + } + + companion object { + private const val INVALID_TRACE_ID = "00000000000000000000000000000000" + } +} diff --git a/core/baker-interface-otel/src/test/kotlin/com/ing/baker/runtime/otel/OpenTelemetryContextPropagatorTest.kt b/core/baker-interface-otel/src/test/kotlin/com/ing/baker/runtime/otel/OpenTelemetryContextPropagatorTest.kt new file mode 100644 index 000000000..3494fb9a8 --- /dev/null +++ b/core/baker-interface-otel/src/test/kotlin/com/ing/baker/runtime/otel/OpenTelemetryContextPropagatorTest.kt @@ -0,0 +1,113 @@ +package com.ing.baker.runtime.otel + +import com.ing.baker.runtime.model.propagation.ContextScope +import com.ing.baker.runtime.model.propagation.ContextSnapshot +import io.opentelemetry.api.trace.Span +import io.opentelemetry.context.Context +import io.opentelemetry.context.ContextStorage +import io.opentelemetry.sdk.OpenTelemetrySdk +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter +import io.opentelemetry.sdk.trace.SdkTracerProvider +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertNotEquals +import org.junit.jupiter.api.Test +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit + +class OpenTelemetryContextPropagatorTest { + + private val propagator = OpenTelemetryContextPropagator() + + @Test + fun movesTheCapturedContextToTheExecutingThread() { + TracerFixture.create().use { fixture -> + val span = fixture.tracer.spanBuilder("caller").startSpan() + val snapshot: ContextSnapshot = span.makeCurrent().use { + propagator.capture() + } + + val executor = Executors.newSingleThreadExecutor() + try { + val observed = executor.submit { + snapshot.restore().use { _: ContextScope -> + Span.current().spanContext.spanId + } + } + assertEquals(span.spanContext.spanId, observed.get(5, TimeUnit.SECONDS)) + + val afterScope = executor.submit { + Span.current().spanContext.traceId + } + assertEquals( + INVALID_TRACE_ID, + afterScope.get(5, TimeUnit.SECONDS), + "The propagated context must not survive the scope it was restored in" + ) + } finally { + executor.shutdownNow() + } + span.end() + } + } + + @Test + fun clearsContextLeakedByPreviousWorkOnTheSameThread() { + TracerFixture.create().use { fixture -> + val executor = Executors.newSingleThreadExecutor() + try { + val leaked = fixture.tracer.spanBuilder("leaked").startSpan() + executor.submit { + ContextStorage.get().attach(Context.current().with(leaked)) + }.get(5, TimeUnit.SECONDS) + + val stillLeaking = executor.submit { + Span.current().spanContext.spanId + } + assertEquals(leaked.spanContext.spanId, stillLeaking.get(5, TimeUnit.SECONDS)) + + val afterClear = executor.submit { + propagator.clear() + Span.current().spanContext.traceId + } + assertEquals(INVALID_TRACE_ID, afterClear.get(5, TimeUnit.SECONDS)) + assertNotEquals(INVALID_TRACE_ID, leaked.spanContext.traceId) + assertFalse( + Span.current().spanContext.isValid, + "The test thread itself should not have any ambient span" + ) + leaked.end() + } finally { + executor.shutdownNow() + } + } + } + + private data class TracerFixture( + val exporter: InMemorySpanExporter, + val tracerProvider: SdkTracerProvider, + val tracer: io.opentelemetry.api.trace.Tracer + ) : AutoCloseable { + + companion object { + fun create(): TracerFixture { + val exporter = InMemorySpanExporter.create() + val tracerProvider = SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(exporter)) + .build() + val sdk = OpenTelemetrySdk.builder().setTracerProvider(tracerProvider).build() + return TracerFixture(exporter, tracerProvider, sdk.getTracer("test.otel.propagator")) + } + } + + override fun close() { + tracerProvider.close() + exporter.reset() + } + } + + companion object { + private const val INVALID_TRACE_ID = "00000000000000000000000000000000" + } +} diff --git a/core/baker-interface-otel/src/test/kotlin/com/ing/baker/runtime/otel/OpenTelemetryInteractionInstrumentationTest.kt b/core/baker-interface-otel/src/test/kotlin/com/ing/baker/runtime/otel/OpenTelemetryInteractionInstrumentationTest.kt new file mode 100644 index 000000000..a5467c0f1 --- /dev/null +++ b/core/baker-interface-otel/src/test/kotlin/com/ing/baker/runtime/otel/OpenTelemetryInteractionInstrumentationTest.kt @@ -0,0 +1,129 @@ +package com.ing.baker.runtime.otel + +import com.ing.baker.runtime.model.InteractionCallContext +import io.opentelemetry.api.common.AttributeKey +import io.opentelemetry.api.trace.Span +import io.opentelemetry.api.trace.StatusCode +import io.opentelemetry.api.trace.Tracer +import io.opentelemetry.context.Context +import io.opentelemetry.sdk.OpenTelemetrySdk +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter +import io.opentelemetry.sdk.trace.SdkTracerProvider +import io.opentelemetry.sdk.trace.data.SpanData +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.slf4j.MDC +import java.util.concurrent.CompletableFuture + +class OpenTelemetryInteractionInstrumentationTest { + + private lateinit var exporter: InMemorySpanExporter + private lateinit var tracerProvider: SdkTracerProvider + private lateinit var tracer: Tracer + private lateinit var parentContexts: RecipeInstanceTraceContexts + private lateinit var instrumentation: OpenTelemetryInteractionInstrumentation + + @BeforeEach + fun setUp() { + exporter = InMemorySpanExporter.create() + tracerProvider = SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(exporter)) + .build() + tracer = OpenTelemetrySdk.builder().setTracerProvider(tracerProvider).build().getTracer("test.otel") + parentContexts = RecipeInstanceTraceContexts() + instrumentation = OpenTelemetryInteractionInstrumentation(tracer, parentContexts) + } + + @AfterEach + fun tearDown() { + tracerProvider.close() + exporter.reset() + MDC.clear() + } + + @Test + fun attachesItsOwnSpanToTheExecutingThread() { + val observedSpanId = instrumentation.aroundInteraction( + callContext("instance-1") + ) { Span.current().spanContext.spanId } + + val spans: List = exporter.finishedSpanItems + assertEquals(1, spans.size) + val span = spans[0] + assertEquals("baker.interaction.ValidateOrder", span.name) + assertEquals(span.spanContext.spanId, observedSpanId) + assertEquals(StatusCode.OK, span.status.statusCode) + assertEquals("instance-1", span.attributes.get(AttributeKey.stringKey("baker.recipe.instance.id"))) + assertEquals("ValidateOrder", span.attributes.get(AttributeKey.stringKey("baker.interaction.name"))) + assertEquals("correlation-1", span.attributes.get(AttributeKey.stringKey("baker.correlation.id"))) + assertFalse(Span.current().spanContext.isValid, "The span must be detached after the invocation") + } + + @Test + fun parentsTheSpanToTheRecipeInstanceContext() { + val parent = tracer.spanBuilder("baker.event.received").startSpan() + parentContexts.put("instance-2", Context.root().with(parent)) + parent.end() + + instrumentation.aroundInteraction(callContext("instance-2")) { null } + + val interactionSpan = exporter.finishedSpanItems + .first { it.name.startsWith("baker.interaction.") } + assertEquals(parent.spanContext.spanId, interactionSpan.parentSpanId) + assertEquals(parent.spanContext.traceId, interactionSpan.spanContext.traceId) + } + + @Test + fun setsAndRestoresTraceMdcKeys() { + MDC.put("trace_id", "previous-trace") + + instrumentation.aroundInteraction(callContext("instance-3")) { + assertEquals(Span.current().spanContext.traceId, MDC.get("trace_id")) + assertEquals(Span.current().spanContext.spanId, MDC.get("span_id")) + null + } + + assertEquals("previous-trace", MDC.get("trace_id")) + assertNull(MDC.get("span_id")) + } + + @Test + fun endsTheSpanWhenAnAsynchronousInteractionFails() { + val future = CompletableFuture() + + instrumentation.aroundInteraction(callContext("instance-4")) { future } + + assertTrue(exporter.finishedSpanItems.isEmpty(), "Span must stay open until the interaction completes") + + future.completeExceptionally(RuntimeException("Expected failure")) + + val span = exporter.finishedSpanItems[0] + assertEquals(StatusCode.ERROR, span.status.statusCode) + assertEquals("interaction.failure", span.attributes.get(AttributeKey.stringKey("error.type"))) + } + + @Test + fun recordsSynchronousFailures() { + val failure = RuntimeException("Expected failure") + try { + instrumentation.aroundInteraction(callContext("instance-5")) { + throw failure + } + } catch (expected: RuntimeException) { + assertEquals(failure, expected) + } + + val span = exporter.finishedSpanItems[0] + assertEquals(StatusCode.ERROR, span.status.statusCode) + assertEquals("interaction.failure", span.attributes.get(AttributeKey.stringKey("error.type"))) + } + + private fun callContext(recipeInstanceId: String): InteractionCallContext = + InteractionCallContext("recipe-id", "recipe", recipeInstanceId, "ValidateOrder", 42L, "correlation-1") +} From 4add8ef60dc06fb23d9aeffccbcb40a078f5fe76 Mon Sep 17 00:00:00 2001 From: Dragoslav Pavkovic Date: Wed, 4 Nov 2026 15:41:14 +0100 Subject: [PATCH 5/5] Fixing trace id propagation --- .../inmemory/InMemoryBakerTracingTest.kt | 47 ++++++++++++++++++- .../runtime/otel/BakerEventTracingListener.kt | 4 +- .../otel/RecipeInstanceTraceContexts.kt | 6 +++ 3 files changed, 54 insertions(+), 3 deletions(-) diff --git a/core/baker-interface-in-memory/src/test/kotlin/com/ing/baker/runtime/inmemory/InMemoryBakerTracingTest.kt b/core/baker-interface-in-memory/src/test/kotlin/com/ing/baker/runtime/inmemory/InMemoryBakerTracingTest.kt index 85c9d39e5..2c22695c5 100644 --- a/core/baker-interface-in-memory/src/test/kotlin/com/ing/baker/runtime/inmemory/InMemoryBakerTracingTest.kt +++ b/core/baker-interface-in-memory/src/test/kotlin/com/ing/baker/runtime/inmemory/InMemoryBakerTracingTest.kt @@ -24,6 +24,7 @@ import io.opentelemetry.sdk.OpenTelemetrySdk import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter import io.opentelemetry.sdk.trace.SdkTracerProvider import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor +import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test @@ -182,6 +183,51 @@ class InMemoryBakerTracingTest { } } + @Test + fun `keeps existing trace id for complete happy flow`() { + TracerFixture.use { fixture -> + val baker = InMemoryBaker.java(BakerConfig.default(), setupMockImplementations(), fixture.tracer) + baker.addRecipe(RecipeRecord.of(recipe, System.currentTimeMillis(), false, true)).get() + + val goods = "goods" + val order = "order" + val customerInfo = CustomerInfo("name", "address", "email") + + Mockito.doReturn(ValidateOrder.OrderValid()).`when`(validateOrder).apply(anyString(), anyString()) + Mockito.doReturn(ManufactureGoods.GoodsManufactured(goods)).`when`(manufactureGoods).apply(anyString()) + Mockito.doReturn(SendInvoice.InvoiceSent()).`when`(sendInvoice).apply(customerInfo) + Mockito.doReturn(ShipGoods.GoodsShipped("trackingId")).`when`(shipGoods).apply(goods, customerInfo) + + val recipeInstanceId = UUID.randomUUID().toString() + val parentSpan = fixture.tracer.spanBuilder("upstream.request").startSpan() + val expectedTraceId = parentSpan.spanContext.traceId + parentSpan.makeCurrent().use { + runHappyFlow(baker, recipeInstanceId, order, customerInfo) + } + parentSpan.end() + + val instanceSpans = fixture.exporter.finishedSpanItems.filter { + it.attributes.get(AttributeKey.stringKey("baker.recipe.instance.id")) == recipeInstanceId + } + + assertTrue(instanceSpans.isNotEmpty(), "Expected traced spans for execution $recipeInstanceId") + assertTrue( + instanceSpans.all { it.spanContext.traceId == expectedTraceId }, + "Expected all spans of execution $recipeInstanceId to keep the upstream trace id" + ) + + val interactionSpan = instanceSpans.firstOrNull { it.name == "baker.interaction.ValidateOrder" } + assertTrue(interactionSpan != null, "Expected ValidateOrder interaction span") + interactionSpan?.let { + assertEquals( + expectedTraceId, + it.spanContext.traceId, + "Expected interaction span to keep the upstream trace id" + ) + } + } + } + @Test fun `records failed interaction as error span`() { TracerFixture.use { fixture -> @@ -264,4 +310,3 @@ class InMemoryBakerTracingTest { } } } - diff --git a/core/baker-interface-otel/src/main/kotlin/com/ing/baker/runtime/otel/BakerEventTracingListener.kt b/core/baker-interface-otel/src/main/kotlin/com/ing/baker/runtime/otel/BakerEventTracingListener.kt index 1d867c4cf..6e4666fb2 100644 --- a/core/baker-interface-otel/src/main/kotlin/com/ing/baker/runtime/otel/BakerEventTracingListener.kt +++ b/core/baker-interface-otel/src/main/kotlin/com/ing/baker/runtime/otel/BakerEventTracingListener.kt @@ -6,7 +6,6 @@ import com.ing.baker.runtime.scaladsl.EventFired import com.ing.baker.runtime.scaladsl.EventReceived import com.ing.baker.runtime.scaladsl.RecipeInstanceCreated import io.opentelemetry.api.trace.Span -import io.opentelemetry.api.trace.SpanBuilder import io.opentelemetry.api.trace.StatusCode import io.opentelemetry.api.trace.Tracer import io.opentelemetry.context.Context @@ -89,6 +88,7 @@ class BakerEventTracingListener( private fun newSpan(name: String, recipeId: String, recipeName: String, recipeInstanceId: String): Span { val builder = tracer.spanBuilder(name) + .setParent(traceContexts.parentOf(recipeInstanceId)) val span = builder.startSpan() span.setAttribute(BakerTraceAttributes.RECIPE_ID, recipeId) span.setAttribute(BakerTraceAttributes.RECIPE_NAME, recipeName) @@ -102,7 +102,7 @@ class BakerEventTracingListener( span.spanContext.traceId, span.spanContext.spanId ) - traceContexts.put(recipeInstanceId, Context.current().with(span)) + traceContexts.putIfAbsent(recipeInstanceId, Context.current().with(span)) } companion object { diff --git a/core/baker-interface-otel/src/main/kotlin/com/ing/baker/runtime/otel/RecipeInstanceTraceContexts.kt b/core/baker-interface-otel/src/main/kotlin/com/ing/baker/runtime/otel/RecipeInstanceTraceContexts.kt index 4e84110cb..b6967c361 100644 --- a/core/baker-interface-otel/src/main/kotlin/com/ing/baker/runtime/otel/RecipeInstanceTraceContexts.kt +++ b/core/baker-interface-otel/src/main/kotlin/com/ing/baker/runtime/otel/RecipeInstanceTraceContexts.kt @@ -19,6 +19,12 @@ class RecipeInstanceTraceContexts { } } + fun putIfAbsent(recipeInstanceId: String?, context: Context?) { + if (recipeInstanceId != null && context != null) { + contexts.putIfAbsent(recipeInstanceId, context) + } + } + /** * Returns the parent context for the given recipe instance, falling back to the context currently * attached to the calling thread.