diff --git a/core/baker-akka-actors/src/main/scala/com/ing/baker/runtime/akka/actor/process_instance/ProcessInstanceLogger.scala b/core/baker-akka-actors/src/main/scala/com/ing/baker/runtime/akka/actor/process_instance/ProcessInstanceLogger.scala index b47643716..7f39f07a3 100644 --- a/core/baker-akka-actors/src/main/scala/com/ing/baker/runtime/akka/actor/process_instance/ProcessInstanceLogger.scala +++ b/core/baker-akka-actors/src/main/scala/com/ing/baker/runtime/akka/actor/process_instance/ProcessInstanceLogger.scala @@ -9,21 +9,6 @@ import scala.concurrent.duration._ object ProcessInstanceLogger { - import org.joda.time.format.PeriodFormatterBuilder - - val durationFormatter = new PeriodFormatterBuilder() - .appendDays.appendSuffix("d") - .appendSeparator(" ") - .appendHours.appendSuffix("h") - .appendSeparator(" ") - .appendMinutes.appendSuffix("m") - .appendSeparator(" ") - .appendSeconds.appendSuffix("s") - .appendSeparator(" ") - .appendMillis.appendSuffix("ms") - .appendSeparator(" ") - .toFormatter - implicit class LoggingAdapterFns(log: DiagnosticLoggingAdapter) { val bakerLogging: BakerLogging = BakerLogging.default diff --git a/core/baker-interface-in-memory/README.md b/core/baker-interface-in-memory/README.md index 00b600fb3..4a5528630 100644 --- a/core/baker-interface-in-memory/README.md +++ b/core/baker-interface-in-memory/README.md @@ -37,3 +37,87 @@ fun main() = runBlocking { - `KotlinBaker` mirrors the Kotlin adapter shape from `baker-interface-kotlin`. - It supports `suspend` APIs and selected `CompletableFuture` wrappers such as `javaAsync`, `bakeAsync`, and `awaitCompletedAsync`. +## OpenTelemetry tracing + +The in-memory runtime emits spans using OpenTelemetry when built through `InMemoryBaker`. + +### What is traced + +- Build lifecycle: `baker.build` +- Recipe lifecycle: `baker.recipe-instance.created` +- Event flow: `baker.event.received`, `baker.event.fired` +- Interaction execution: `baker.interaction.` + +Interaction spans are created as children of the latest event span for the same recipe instance. + +### Ambient context inside interactions + +The interaction span is started on the very thread that invokes your interaction implementation, so +`Span.current()` and the SLF4J `MDC` are valid and belong to *that* invocation, also under heavy concurrency: + +```java +public class ValidateOrderImpl implements ValidateOrder { + @Override + public ValidateOrderOutcome apply(String recipeInstanceId, String order) { + Span.current().addEvent("validating"); // the baker.interaction.ValidateOrder span + log.info("validating"); // MDC contains trace_id, span_id and recipeInstanceId + return new OrderValid(); + } +} +``` + +Baker runs interactions on a shared pool. Before and after every invocation the registered context propagators are +cleared, so a pooled thread can never expose the context of a previous execution. If your interaction spawns its own +threads, propagate the context explicitly (for example with `Context.current().wrap(runnable)`). + +For a `CompletableFuture`-returning interaction the span is ended when the future completes; for a blocking +interaction it is ended when the method returns. + +### Common attributes + +- `baker.recipe.id` +- `baker.recipe.name` +- `baker.recipe.instance.id` + +Depending on the span type, additional attributes are set: + +- `baker.event.name` +- `baker.interaction.name` +- `baker.correlation.id` +- `baker.failure.count` +- `error.type` (for failed interactions) + +### Injecting a custom tracer + +By default, Baker uses `GlobalOpenTelemetry.getTracer(...)`. You can provide your own tracer: + +```kotlin +import com.ing.baker.runtime.inmemory.InMemoryBaker +import com.ing.baker.runtime.model.BakerConfig +import io.opentelemetry.api.GlobalOpenTelemetry + +val tracer = GlobalOpenTelemetry.getTracer("my-service.baker") +val baker = InMemoryBaker.java(BakerConfig.default(), emptyList(), tracer) +``` + +For async Java factory usage: + +```kotlin +val tracer = GlobalOpenTelemetry.getTracer("my-service.baker") +val bakerFuture = InMemoryBaker.javaAsync(BakerConfig.default(), emptyList(), tracer) +``` + + +### Where the OpenTelemetry code lives + +The OpenTelemetry-specific code is isolated in the `baker-interface-otel` module: + +- `OpenTelemetryContextPropagator` — propagates the OTel `Context` to Baker's interaction pool. It is discovered + automatically through `META-INF/services`, so simply having the module on the classpath is enough. +- `OpenTelemetryInteractionInstrumentation` — starts, scopes and ends the `baker.interaction.` span. +- `RecipeInstanceTraceContexts` — per-baker mapping of recipe instance to the context the interaction span is parented to. + +`baker-interface` itself stays free of OpenTelemetry. It only exposes the SPIs +`com.ing.baker.runtime.model.propagation.ExecutionContextPropagator` (propagate any thread-bound context) and +`com.ing.baker.runtime.model.InteractionInstrumentation` (wrap every interaction invocation), which you can implement +to plug in a different tracing or diagnostics stack. diff --git a/core/baker-interface-in-memory/pom.xml b/core/baker-interface-in-memory/pom.xml index 3a0970d6b..484f221b0 100644 --- a/core/baker-interface-in-memory/pom.xml +++ b/core/baker-interface-in-memory/pom.xml @@ -46,6 +46,16 @@ org.slf4j slf4j-api + + io.opentelemetry + opentelemetry-api + ${opentelemetry.version} + + + com.ing.baker + baker-interface-otel + ${project.version} + @@ -120,6 +130,18 @@ ${mockito.core.version} test + + io.opentelemetry + opentelemetry-sdk + ${opentelemetry.version} + test + + + io.opentelemetry + opentelemetry-sdk-testing + ${opentelemetry.version} + test + ch.qos.logback logback-classic diff --git a/core/baker-interface-in-memory/src/main/kotlin/com/ing/baker/runtime/inmemory/InMemoryBaker.kt b/core/baker-interface-in-memory/src/main/kotlin/com/ing/baker/runtime/inmemory/InMemoryBaker.kt index b743d69c9..762972726 100644 --- a/core/baker-interface-in-memory/src/main/kotlin/com/ing/baker/runtime/inmemory/InMemoryBaker.kt +++ b/core/baker-interface-in-memory/src/main/kotlin/com/ing/baker/runtime/inmemory/InMemoryBaker.kt @@ -5,6 +5,13 @@ import com.ing.baker.runtime.model.BakerConfig import com.ing.baker.runtime.model.BakerF import com.ing.baker.runtime.model.BakerLogging import com.ing.baker.runtime.model.InteractionInstance +import com.ing.baker.runtime.model.InteractionInstrumentation +import com.ing.baker.runtime.otel.BakerEventTracingListener +import com.ing.baker.runtime.otel.OpenTelemetryInteractionInstrumentation +import com.ing.baker.runtime.otel.RecipeInstanceTraceContexts +import io.opentelemetry.api.GlobalOpenTelemetry +import io.opentelemetry.api.trace.StatusCode +import io.opentelemetry.api.trace.Tracer import java.util.concurrent.CompletableFuture import com.ing.baker.runtime.defaultinteractions.`package$`.`MODULE$` as defaultinteractions import com.ing.baker.runtime.javadsl.Baker as JavaBaker @@ -20,25 +27,46 @@ import scala.collection.immutable.List as ScalaList @Suppress("ABSTRACT_MEMBER_NOT_IMPLEMENTED", "UNCHECKED_CAST") class InMemoryBaker( private val bakerConfig: BakerConfig, - components: BakerComponents> + components: BakerComponents>, + private val tracer: Tracer = defaultTracer(), + private val traceContexts: RecipeInstanceTraceContexts = RecipeInstanceTraceContexts() ) : BakerF>( components, InMemoryEffects.asyncSupportAny(), InMemoryEffects.asyncSupportAny() ) { + init { + registerTracingListener() + } + companion object { + private const val INSTRUMENTATION_NAME = "com.ing.baker.runtime.inmemory.InMemoryBaker" + + private fun defaultTracer(): Tracer = + GlobalOpenTelemetry.getTracer(INSTRUMENTATION_NAME) + @JvmStatic fun build(implementations: ScalaList<*>): CompletableFuture>> = - build(BakerConfig.default(), implementations) + build(BakerConfig.default(), implementations, defaultTracer()) @JvmStatic @Suppress("UNCHECKED_CAST") fun build( config: BakerConfig, implementations: ScalaList<*> + ): CompletableFuture>> = + build(config, implementations, defaultTracer()) + + @JvmStatic + @Suppress("UNCHECKED_CAST") + fun build( + config: BakerConfig, + implementations: ScalaList<*>, + tracer: Tracer ): CompletableFuture>> { + val buildSpan = tracer.spanBuilder("baker.build").startSpan() val ioAsyncSupport = InMemoryEffects.asyncSupportAny() val ioClassTag = InMemoryEffects.classTagAny() val builtinInteractions = @@ -54,25 +82,50 @@ class InMemoryBaker( val recipeManager = InMemoryRecipeManager() val eventStream = InMemoryEventStream() val interactions = InMemoryInteractionManager(interactionInstances) + val traceContexts = RecipeInstanceTraceContexts() val components = BakerComponents( interactions, recipeInstanceManager, recipeManager, eventStream, - BakerLogging.default() - ) - return InMemoryEffects.pure( - InMemoryBaker(config, components) + BakerLogging.default(), + OpenTelemetryInteractionInstrumentation(tracer, traceContexts) as InteractionInstrumentation ) + return try { + val bakerFuture: CompletableFuture>> = + InMemoryEffects.pure( + InMemoryBaker(config, components, tracer, traceContexts) as BakerF> + ) + + bakerFuture.whenComplete { _, throwable -> + if (throwable != null) { + buildSpan.recordException(throwable) + buildSpan.setStatus(StatusCode.ERROR) + } else { + buildSpan.setStatus(StatusCode.OK) + } + buildSpan.end() + } + } catch (throwable: Throwable) { + buildSpan.recordException(throwable) + buildSpan.setStatus(StatusCode.ERROR) + buildSpan.end() + throw throwable + } } @JvmStatic fun java(config: BakerConfig, implementations: List): JavaBaker { + return java(config, implementations, defaultTracer()) + } + + @JvmStatic + fun java(config: BakerConfig, implementations: List, tracer: Tracer): JavaBaker { val futureToCompletableFuture = InMemoryEffects.futureToCompletableFuture() val completableFutureToFuture = InMemoryEffects.completableFutureToFuture() val scalaInteractions = InMemoryEffects.toScalaCompletableFutureInteractions(implementations) - return build(config, scalaInteractions) + return build(config, scalaInteractions, tracer) .let { InMemoryEffects.runSync(it) } .asDeprecatedFutureImplementation(completableFutureToFuture, futureToCompletableFuture) .let { JavaBaker(it) } @@ -83,11 +136,16 @@ class InMemoryBaker( */ @JvmStatic fun javaAsync(config: BakerConfig, implementations: List): CompletableFuture { + return javaAsync(config, implementations, defaultTracer()) + } + + @JvmStatic + fun javaAsync(config: BakerConfig, implementations: List, tracer: Tracer): CompletableFuture { val futureToCompletableFuture = InMemoryEffects.futureToCompletableFuture() val completableFutureToFuture = InMemoryEffects.completableFutureToFuture() val scalaInteractions = InMemoryEffects.toScalaCompletableFutureInteractions(implementations) - return build(config, scalaInteractions) + return build(config, scalaInteractions, tracer) .thenApply { bakerF -> JavaBaker(bakerF.asDeprecatedFutureImplementation(completableFutureToFuture, futureToCompletableFuture)) } @@ -115,5 +173,17 @@ class InMemoryBaker( /** * Attempts to gracefully shutdown the baker system. */ - override fun gracefulShutdown(): CompletableFuture = InMemoryEffects.unit() + override fun gracefulShutdown(): CompletableFuture { + traceContexts.clear() + return InMemoryEffects.unit() + } + + private fun registerTracingListener() { + try { + InMemoryEffects.runSync(registerBakerEventListener(BakerEventTracingListener(tracer, traceContexts))) + } catch (_: Throwable) { + // Tracing must never prevent Baker startup. + } + } + } diff --git a/core/baker-interface-in-memory/src/test/kotlin/com/ing/baker/runtime/inmemory/InMemoryBakerTracingConcurrencyTest.kt b/core/baker-interface-in-memory/src/test/kotlin/com/ing/baker/runtime/inmemory/InMemoryBakerTracingConcurrencyTest.kt new file mode 100644 index 000000000..a542eb0f8 --- /dev/null +++ b/core/baker-interface-in-memory/src/test/kotlin/com/ing/baker/runtime/inmemory/InMemoryBakerTracingConcurrencyTest.kt @@ -0,0 +1,238 @@ +package com.ing.baker.runtime.inmemory + +import com.google.common.collect.ImmutableList +import com.ing.baker.compiler.RecipeCompiler +import com.ing.baker.il.CompiledRecipe +import com.ing.baker.recipe.kotlindsl.ExperimentalDsl +import com.ing.baker.runtime.common.RecipeRecord +import com.ing.baker.runtime.inmemory.recipe.CustomerInfo +import com.ing.baker.runtime.inmemory.recipe.LogRecipeFailedImpl +import com.ing.baker.runtime.inmemory.recipe.ManufactureGoodsImpl +import com.ing.baker.runtime.inmemory.recipe.SendInvoiceImpl +import com.ing.baker.runtime.inmemory.recipe.SensoryEvents.CustomerInfoReceived +import com.ing.baker.runtime.inmemory.recipe.SensoryEvents.OrderPlaced +import com.ing.baker.runtime.inmemory.recipe.SensoryEvents.PaymentMade +import com.ing.baker.runtime.inmemory.recipe.ShipGoodsImpl +import com.ing.baker.runtime.inmemory.recipe.ValidateOrder +import com.ing.baker.runtime.inmemory.recipe.WebShopRecipe +import com.ing.baker.runtime.javadsl.Baker +import com.ing.baker.runtime.javadsl.EventInstance +import com.ing.baker.runtime.model.BakerConfig +import io.opentelemetry.api.common.AttributeKey +import io.opentelemetry.api.trace.Span +import io.opentelemetry.api.trace.Tracer +import io.opentelemetry.sdk.OpenTelemetrySdk +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter +import io.opentelemetry.sdk.trace.SdkTracerProvider +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.slf4j.MDC +import java.time.Duration +import java.util.UUID +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit + +/** + * Verifies that the ambient tracing context observed inside an interaction belongs to that very interaction, + * also when many recipe instances execute concurrently on the shared interaction thread pool. + */ +class InMemoryBakerTracingConcurrencyTest { + + private val instanceCount = 32 + private val concurrencyLevel = 8 + private val invalidTraceId = "00000000000000000000000000000000" + + @OptIn(ExperimentalDsl::class) + private val recipe: CompiledRecipe = RecipeCompiler.compileRecipe(WebShopRecipe.recipe) + + class ObservedContext( + val traceId: String, + val spanId: String, + val mdcTraceId: String?, + val mdcSpanId: String?, + val mdcRecipeInstanceId: String? + ) + + /** Records the context that is ambient while the interaction runs. */ + class RecordingValidateOrder( + private val observed: ConcurrentHashMap, + private val started: CountDownLatch + ) : ValidateOrder { + + override fun apply(recipeInstanceId: String, order: String): ValidateOrder.ValidateOrderOutcome { + // Make sure interactions of different recipe instances overlap on the shared pool. + started.countDown() + started.await(10, TimeUnit.SECONDS) + + val spanContext = Span.current().spanContext + observed[recipeInstanceId] = ObservedContext( + traceId = spanContext.traceId, + spanId = spanContext.spanId, + mdcTraceId = MDC.get("trace_id"), + mdcSpanId = MDC.get("span_id"), + mdcRecipeInstanceId = MDC.get("recipeInstanceId") + ) + return ValidateOrder.OrderValid() + } + } + + @Test + fun `concurrent executions keep their own trace inside interactions`() { + val observed = ConcurrentHashMap() + val started = CountDownLatch(concurrencyLevel) + + TracerFixture.use { fixture -> + val baker = InMemoryBaker.java( + BakerConfig.default(), + ImmutableList.of( + ManufactureGoodsImpl(), + SendInvoiceImpl(), + ShipGoodsImpl(), + RecordingValidateOrder(observed, started), + LogRecipeFailedImpl() + ), + fixture.tracer + ) + baker.addRecipe(RecipeRecord.of(recipe, System.currentTimeMillis(), false, true)).get() + + val recipeInstanceIds = (1..instanceCount).map { UUID.randomUUID().toString() } + val executor = Executors.newFixedThreadPool(concurrencyLevel) + try { + recipeInstanceIds.mapIndexed { index, recipeInstanceId -> + executor.submit { runHappyFlow(baker, recipeInstanceId, "order-$index") } + }.forEach { it.get(30, TimeUnit.SECONDS) } + } finally { + executor.shutdownNow() + } + + val spans = fixture.exporter.finishedSpanItems + assertEquals(instanceCount, observed.size, "Expected every execution to record its ambient context") + + val traceIdsPerInstance = recipeInstanceIds.associateWith { recipeInstanceId -> + spans + .filter { it.attributes.get(AttributeKey.stringKey("baker.recipe.instance.id")) == recipeInstanceId } + .map { it.spanContext.traceId } + .toSet() + } + + recipeInstanceIds.forEach { recipeInstanceId -> + val context = observed[recipeInstanceId] + assertNotNull(context, "Expected a recorded context for $recipeInstanceId") + requireNotNull(context) + + assertTrue( + context.traceId != invalidTraceId, + "Expected a valid ambient span inside interaction of $recipeInstanceId" + ) + + val interactionSpan = spans.firstOrNull { + it.name == "baker.interaction.ValidateOrder" && + it.attributes.get(AttributeKey.stringKey("baker.recipe.instance.id")) == recipeInstanceId + } + assertNotNull(interactionSpan, "Expected an interaction span for $recipeInstanceId") + requireNotNull(interactionSpan) + + assertEquals( + interactionSpan.spanContext.spanId, + context.spanId, + "Span.current() inside the interaction of $recipeInstanceId must be its own interaction span" + ) + assertEquals(interactionSpan.spanContext.traceId, context.traceId) + + val otherTraceIds = traceIdsPerInstance + .filterKeys { it != recipeInstanceId } + .values + .flatten() + .toSet() + assertTrue( + !otherTraceIds.contains(context.traceId), + "Execution $recipeInstanceId observed a trace id that belongs to another execution" + ) + } + } + } + + @Test + fun `interaction mdc carries the trace of its own interaction span`() { + val observed = ConcurrentHashMap() + val started = CountDownLatch(concurrencyLevel) + + TracerFixture.use { fixture -> + val baker = InMemoryBaker.java( + BakerConfig.default(), + ImmutableList.of( + ManufactureGoodsImpl(), + SendInvoiceImpl(), + ShipGoodsImpl(), + RecordingValidateOrder(observed, started), + LogRecipeFailedImpl() + ), + fixture.tracer + ) + baker.addRecipe(RecipeRecord.of(recipe, System.currentTimeMillis(), false, true)).get() + + val recipeInstanceIds = (1..instanceCount).map { UUID.randomUUID().toString() } + val executor = Executors.newFixedThreadPool(concurrencyLevel) + try { + recipeInstanceIds.mapIndexed { index, recipeInstanceId -> + executor.submit { runHappyFlow(baker, recipeInstanceId, "order-$index") } + }.forEach { it.get(30, TimeUnit.SECONDS) } + } finally { + executor.shutdownNow() + } + + recipeInstanceIds.forEach { recipeInstanceId -> + val context = requireNotNull(observed[recipeInstanceId]) + assertEquals(context.traceId, context.mdcTraceId, "MDC trace_id must match the ambient span") + assertEquals(context.spanId, context.mdcSpanId, "MDC span_id must match the ambient span") + assertEquals(recipeInstanceId, context.mdcRecipeInstanceId, "MDC must carry the own recipe instance") + } + + assertTrue(MDC.get("trace_id") == null, "Interaction MDC must not leak to the calling thread") + } + } + + private fun runHappyFlow(baker: Baker, recipeInstanceId: String, order: String) { + val customerInfo = CustomerInfo("name", "address", "email") + baker.bake(recipe.recipeId(), recipeInstanceId).get() + baker.fireSensoryEventAndAwaitReceived(recipeInstanceId, EventInstance.from(OrderPlaced(order))).get() + baker.fireSensoryEventAndAwaitReceived(recipeInstanceId, EventInstance.from(CustomerInfoReceived(customerInfo))).get() + baker.fireSensoryEventAndAwaitReceived(recipeInstanceId, EventInstance.from(PaymentMade())).get() + baker.awaitCompleted(recipeInstanceId, Duration.ofSeconds(30)).join() + } + + private class TracerFixture private constructor( + val exporter: InMemorySpanExporter, + val tracerProvider: SdkTracerProvider, + val tracer: Tracer + ) : AutoCloseable { + + override fun close() { + tracerProvider.close() + exporter.reset() + } + + companion object { + inline fun use(block: (TracerFixture) -> Unit) { + val exporter = InMemorySpanExporter.create() + val tracerProvider = SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(exporter)) + .build() + val openTelemetry = OpenTelemetrySdk.builder() + .setTracerProvider(tracerProvider) + .build() + + TracerFixture( + exporter = exporter, + tracerProvider = tracerProvider, + tracer = openTelemetry.getTracer("test.inmemory.baker.concurrency") + ).use(block) + } + } + } +} diff --git a/core/baker-interface-in-memory/src/test/kotlin/com/ing/baker/runtime/inmemory/InMemoryBakerTracingTest.kt b/core/baker-interface-in-memory/src/test/kotlin/com/ing/baker/runtime/inmemory/InMemoryBakerTracingTest.kt new file mode 100644 index 000000000..2c22695c5 --- /dev/null +++ b/core/baker-interface-in-memory/src/test/kotlin/com/ing/baker/runtime/inmemory/InMemoryBakerTracingTest.kt @@ -0,0 +1,312 @@ +package com.ing.baker.runtime.inmemory + +import com.google.common.collect.ImmutableList +import com.ing.baker.compiler.RecipeCompiler +import com.ing.baker.il.CompiledRecipe +import com.ing.baker.recipe.kotlindsl.ExperimentalDsl +import com.ing.baker.runtime.common.RecipeRecord +import com.ing.baker.runtime.inmemory.recipe.CustomerInfo +import com.ing.baker.runtime.inmemory.recipe.LogRecipeFailedImpl +import com.ing.baker.runtime.inmemory.recipe.ManufactureGoods +import com.ing.baker.runtime.inmemory.recipe.SendInvoice +import com.ing.baker.runtime.inmemory.recipe.SensoryEvents.CustomerInfoReceived +import com.ing.baker.runtime.inmemory.recipe.SensoryEvents.OrderPlaced +import com.ing.baker.runtime.inmemory.recipe.SensoryEvents.PaymentMade +import com.ing.baker.runtime.inmemory.recipe.ShipGoods +import com.ing.baker.runtime.inmemory.recipe.ValidateOrder +import com.ing.baker.runtime.inmemory.recipe.WebShopRecipe +import com.ing.baker.runtime.javadsl.EventInstance +import com.ing.baker.runtime.model.BakerConfig +import io.opentelemetry.api.common.AttributeKey +import io.opentelemetry.api.trace.StatusCode +import io.opentelemetry.api.trace.Tracer +import io.opentelemetry.sdk.OpenTelemetrySdk +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter +import io.opentelemetry.sdk.trace.SdkTracerProvider +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.mockito.ArgumentMatchers.anyString +import org.mockito.Mock +import org.mockito.Mockito +import org.mockito.Mockito.`when` +import org.mockito.MockitoAnnotations +import java.time.Duration +import java.util.UUID + +class InMemoryBakerTracingTest { + + @Mock + lateinit var manufactureGoods: ManufactureGoods + + @Mock + lateinit var sendInvoice: SendInvoice + + @Mock + lateinit var shipGoods: ShipGoods + + @Mock + lateinit var validateOrder: ValidateOrder + + @OptIn(ExperimentalDsl::class) + val recipe: CompiledRecipe = RecipeCompiler.compileRecipe(WebShopRecipe.recipe) + + @BeforeEach + fun setUp() { + MockitoAnnotations.openMocks(this) + } + + @Test + fun `creates lifecycle and interaction spans for happy flow`() { + TracerFixture.use { fixture -> + val baker = InMemoryBaker.java(BakerConfig.default(), setupMockImplementations(), fixture.tracer) + baker.addRecipe(RecipeRecord.of(recipe, System.currentTimeMillis(), false, true)).get() + + val goods = "goods" + val order = "order" + val trackingId = "trackingId" + val customerInfo = CustomerInfo("name", "address", "email") + + Mockito.doReturn(ValidateOrder.OrderValid()).`when`(validateOrder).apply(anyString(), anyString()) + Mockito.doReturn(ManufactureGoods.GoodsManufactured(goods)).`when`(manufactureGoods).apply(anyString()) + Mockito.doReturn(SendInvoice.InvoiceSent()).`when`(sendInvoice).apply(customerInfo) + Mockito.doReturn(ShipGoods.GoodsShipped(trackingId)).`when`(shipGoods).apply(goods, customerInfo) + + val recipeInstanceId = UUID.randomUUID().toString() + runHappyFlow(baker, recipeInstanceId, order, customerInfo) + + val spans = fixture.exporter.finishedSpanItems + val names = spans.map { it.name } + + assertTrue(names.contains("baker.build")) + assertTrue(names.contains("baker.recipe-instance.created")) + assertTrue(names.contains("baker.event.received")) + assertTrue(names.contains("baker.event.fired")) + assertTrue(names.contains("baker.interaction.ValidateOrder")) + assertTrue(names.contains("baker.interaction.ShipGoods")) + assertTrue(names.contains("baker.interaction.SendInvoice")) + assertTrue(names.any { it.startsWith("baker.interaction.") }) + + assertTrue( + spans.any { + it.name == "baker.interaction.ValidateOrder" && + it.status.statusCode == StatusCode.OK && + it.attributes.get(AttributeKey.stringKey("baker.recipe.instance.id")) == recipeInstanceId + }, + "Expected traced ValidateOrder span with recipe instance attribute" + ) + + val eventSpanIds = spans + .filter { + (it.name == "baker.event.received" || it.name == "baker.event.fired") && + it.attributes.get(AttributeKey.stringKey("baker.recipe.instance.id")) == recipeInstanceId + } + .map { it.spanContext.spanId } + .toSet() + + val validateOrderSpan = spans.firstOrNull { + it.name == "baker.interaction.ValidateOrder" && + it.attributes.get(AttributeKey.stringKey("baker.recipe.instance.id")) == recipeInstanceId + } + + assertTrue( + validateOrderSpan != null && eventSpanIds.contains(validateOrderSpan.parentSpanContext.spanId), + "Expected interaction span to be linked as child of an event span" + ) + } + } + + @Test + fun `assigns a distinct trace to every baker execution`() { + TracerFixture.use { fixture -> + val baker = InMemoryBaker.java(BakerConfig.default(), setupMockImplementations(), fixture.tracer) + baker.addRecipe(RecipeRecord.of(recipe, System.currentTimeMillis(), false, true)).get() + + val goods = "goods" + val customerInfo = CustomerInfo("name", "address", "email") + + Mockito.doReturn(ValidateOrder.OrderValid()).`when`(validateOrder).apply(anyString(), anyString()) + Mockito.doReturn(ManufactureGoods.GoodsManufactured(goods)).`when`(manufactureGoods).apply(anyString()) + Mockito.doReturn(SendInvoice.InvoiceSent()).`when`(sendInvoice).apply(customerInfo) + Mockito.doReturn(ShipGoods.GoodsShipped("trackingId")).`when`(shipGoods).apply(goods, customerInfo) + + val recipeInstanceIds = (1..100).map { UUID.randomUUID().toString() } + recipeInstanceIds.forEachIndexed { index, recipeInstanceId -> + runHappyFlow(baker, recipeInstanceId, "order-$index", customerInfo) + } + + val spans = fixture.exporter.finishedSpanItems + val tracesPerInstance = recipeInstanceIds.associateWith { recipeInstanceId -> + spans + .filter { it.attributes.get(AttributeKey.stringKey("baker.recipe.instance.id")) == recipeInstanceId } + .map { it.spanContext.traceId } + .toSet() + } + + tracesPerInstance.forEach { (recipeInstanceId, traceIds) -> + assertTrue(traceIds.isNotEmpty(), "Expected traced spans for execution $recipeInstanceId") + } + + recipeInstanceIds.forEach { current -> + val otherTraceIds = tracesPerInstance + .filterKeys { it != current } + .values + .flatten() + .toSet() + + assertTrue( + tracesPerInstance.getValue(current).none { otherTraceIds.contains(it) }, + "Expected execution $current to use trace ids that are not shared with other executions" + ) + } + + recipeInstanceIds.forEach { recipeInstanceId -> + val instanceSpans = spans.filter { + it.attributes.get(AttributeKey.stringKey("baker.recipe.instance.id")) == recipeInstanceId + } + val eventTraceIdsBySpanId = instanceSpans + .filter { it.name == "baker.event.received" || it.name == "baker.event.fired" } + .associate { it.spanContext.spanId to it.spanContext.traceId } + + val interactionSpans = instanceSpans.filter { it.name.startsWith("baker.interaction.") } + assertTrue(interactionSpans.isNotEmpty(), "Expected interaction spans for execution $recipeInstanceId") + + assertTrue( + interactionSpans.all { + eventTraceIdsBySpanId[it.parentSpanContext.spanId] == it.spanContext.traceId + }, + "Expected interaction spans of execution $recipeInstanceId to share the trace of their parent event span" + ) + } + } + } + + @Test + fun `keeps existing trace id for complete happy flow`() { + TracerFixture.use { fixture -> + val baker = InMemoryBaker.java(BakerConfig.default(), setupMockImplementations(), fixture.tracer) + baker.addRecipe(RecipeRecord.of(recipe, System.currentTimeMillis(), false, true)).get() + + val goods = "goods" + val order = "order" + val customerInfo = CustomerInfo("name", "address", "email") + + Mockito.doReturn(ValidateOrder.OrderValid()).`when`(validateOrder).apply(anyString(), anyString()) + Mockito.doReturn(ManufactureGoods.GoodsManufactured(goods)).`when`(manufactureGoods).apply(anyString()) + Mockito.doReturn(SendInvoice.InvoiceSent()).`when`(sendInvoice).apply(customerInfo) + Mockito.doReturn(ShipGoods.GoodsShipped("trackingId")).`when`(shipGoods).apply(goods, customerInfo) + + val recipeInstanceId = UUID.randomUUID().toString() + val parentSpan = fixture.tracer.spanBuilder("upstream.request").startSpan() + val expectedTraceId = parentSpan.spanContext.traceId + parentSpan.makeCurrent().use { + runHappyFlow(baker, recipeInstanceId, order, customerInfo) + } + parentSpan.end() + + val instanceSpans = fixture.exporter.finishedSpanItems.filter { + it.attributes.get(AttributeKey.stringKey("baker.recipe.instance.id")) == recipeInstanceId + } + + assertTrue(instanceSpans.isNotEmpty(), "Expected traced spans for execution $recipeInstanceId") + assertTrue( + instanceSpans.all { it.spanContext.traceId == expectedTraceId }, + "Expected all spans of execution $recipeInstanceId to keep the upstream trace id" + ) + + val interactionSpan = instanceSpans.firstOrNull { it.name == "baker.interaction.ValidateOrder" } + assertTrue(interactionSpan != null, "Expected ValidateOrder interaction span") + interactionSpan?.let { + assertEquals( + expectedTraceId, + it.spanContext.traceId, + "Expected interaction span to keep the upstream trace id" + ) + } + } + } + + @Test + fun `records failed interaction as error span`() { + TracerFixture.use { fixture -> + val baker = InMemoryBaker.java(BakerConfig.default(), setupMockImplementations(), fixture.tracer) + baker.addRecipe(RecipeRecord.of(recipe, System.currentTimeMillis(), false, true)).get() + + val goods = "goods" + val order = "order" + val customerInfo = CustomerInfo("name", "address", "email") + + `when`(validateOrder.apply(anyString(), anyString())).thenReturn(ValidateOrder.OrderValid()) + `when`(manufactureGoods.apply(anyString())).thenReturn(ManufactureGoods.GoodsManufactured(goods)) + `when`(shipGoods.apply(goods, customerInfo)).thenReturn(ShipGoods.GoodsShipped("tracking-id")) + `when`(sendInvoice.apply(customerInfo)).thenThrow(RuntimeException("Expected tracing failure")) + + val recipeInstanceId = UUID.randomUUID().toString() + baker.bake(recipe.recipeId(), recipeInstanceId).get() + baker.fireSensoryEventAndAwaitReceived(recipeInstanceId, EventInstance.from(OrderPlaced(order))).get() + baker.fireSensoryEventAndAwaitReceived(recipeInstanceId, EventInstance.from(CustomerInfoReceived(customerInfo))).get() + baker.fireSensoryEventAndAwaitReceived(recipeInstanceId, EventInstance.from(PaymentMade())).get() + baker.awaitCompleted(recipeInstanceId, Duration.ofSeconds(5)).join() + + val spans = fixture.exporter.finishedSpanItems + + assertTrue( + spans.any { + it.name == "baker.interaction.SendInvoice" && + it.status.statusCode == StatusCode.ERROR && + it.attributes.get(AttributeKey.stringKey("error.type")) == "interaction.failure" + }, + "Expected failed SendInvoice interaction to be traced as an error span" + ) + } + } + + private fun runHappyFlow( + baker: com.ing.baker.runtime.javadsl.Baker, + recipeInstanceId: String, + order: String, + customerInfo: CustomerInfo + ) { + baker.bake(recipe.recipeId(), recipeInstanceId).get() + baker.fireSensoryEventAndAwaitReceived(recipeInstanceId, EventInstance.from(OrderPlaced(order))).get() + baker.fireSensoryEventAndAwaitReceived(recipeInstanceId, EventInstance.from(CustomerInfoReceived(customerInfo))).get() + baker.fireSensoryEventAndAwaitReceived(recipeInstanceId, EventInstance.from(PaymentMade())).get() + baker.awaitCompleted(recipeInstanceId, Duration.ofSeconds(5)).join() + } + + private fun setupMockImplementations(): List { + return ImmutableList.of(manufactureGoods, sendInvoice, shipGoods, validateOrder, LogRecipeFailedImpl()) + } + + private class TracerFixture private constructor( + val exporter: InMemorySpanExporter, + val tracerProvider: SdkTracerProvider, + val tracer: Tracer + ) : AutoCloseable { + + override fun close() { + tracerProvider.close() + exporter.reset() + } + + companion object { + inline fun use(block: (TracerFixture) -> Unit) { + val exporter = InMemorySpanExporter.create() + val tracerProvider = SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(exporter)) + .build() + val openTelemetry = OpenTelemetrySdk.builder() + .setTracerProvider(tracerProvider) + .build() + + TracerFixture( + exporter = exporter, + tracerProvider = tracerProvider, + tracer = openTelemetry.getTracer("test.inmemory.baker") + ).use(block) + } + } + } +} diff --git a/core/baker-interface-in-memory/src/test/resources/logback-test.xml b/core/baker-interface-in-memory/src/test/resources/logback-test.xml index 5464005f1..82fccfda0 100644 --- a/core/baker-interface-in-memory/src/test/resources/logback-test.xml +++ b/core/baker-interface-in-memory/src/test/resources/logback-test.xml @@ -2,7 +2,7 @@ - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + %d{HH:mm:ss.SSS} %cyan([%-20.20thread]) %highlight(%-5level) %magenta(%-20.20logger{20}) %yellow(recipe=%X{baker.recipe.name:-}) - %msg%nopex%n diff --git a/core/baker-interface-otel/pom.xml b/core/baker-interface-otel/pom.xml new file mode 100644 index 000000000..79eafcedb --- /dev/null +++ b/core/baker-interface-otel/pom.xml @@ -0,0 +1,102 @@ + + + 4.0.0 + + + com.ing.baker + baker + 6.0.0-SNAPSHOT + ../../pom.xml + + + baker-interface-otel + Baker Runtime OpenTelemetry + OpenTelemetry context propagation and interaction instrumentation for Baker runtimes + + + + com.ing.baker + baker-interface + ${project.version} + + + + io.opentelemetry + opentelemetry-api + ${opentelemetry.version} + + + + org.slf4j + slf4j-api + + + + org.jetbrains.kotlin + kotlin-stdlib + + + + + org.junit.jupiter + junit-jupiter-engine + test + + + io.opentelemetry + opentelemetry-sdk + ${opentelemetry.version} + test + + + io.opentelemetry + opentelemetry-sdk-testing + ${opentelemetry.version} + test + + + ch.qos.logback + logback-classic + test + + + + + src/main/kotlin + src/test/kotlin + + + org.jetbrains.kotlin + kotlin-maven-plugin + ${kotlin.version} + + + compile + compile + + compile + + + + test-compile + test-compile + + test-compile + + + + + + org.apache.maven.plugins + maven-surefire-plugin + ${maven.surefire.plugin.version} + + false + -XX:+EnableDynamicAgentLoading + + + + + diff --git a/core/baker-interface-otel/src/main/kotlin/com/ing/baker/runtime/otel/BakerEventTracingListener.kt b/core/baker-interface-otel/src/main/kotlin/com/ing/baker/runtime/otel/BakerEventTracingListener.kt new file mode 100644 index 000000000..6e4666fb2 --- /dev/null +++ b/core/baker-interface-otel/src/main/kotlin/com/ing/baker/runtime/otel/BakerEventTracingListener.kt @@ -0,0 +1,116 @@ +package com.ing.baker.runtime.otel + +import com.ing.baker.runtime.model.TracingMdcRegistry +import com.ing.baker.runtime.scaladsl.BakerEvent +import com.ing.baker.runtime.scaladsl.EventFired +import com.ing.baker.runtime.scaladsl.EventReceived +import com.ing.baker.runtime.scaladsl.RecipeInstanceCreated +import io.opentelemetry.api.trace.Span +import io.opentelemetry.api.trace.StatusCode +import io.opentelemetry.api.trace.Tracer +import io.opentelemetry.context.Context +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import scala.runtime.AbstractFunction1 +import scala.runtime.BoxedUnit + +/** + * Emits OpenTelemetry spans for the lifecycle and event [BakerEvent]s of a Baker runtime. + * + * Besides emitting spans this listener maintains the per recipe instance trace context that interaction spans + * are parented to, so it must share its [RecipeInstanceTraceContexts] instance with the + * [OpenTelemetryInteractionInstrumentation] of the same Baker. + * + * It is a `scala.Function1` so it can be handed directly to + * `Baker.registerBakerEventListener`. + */ +class BakerEventTracingListener( + private val tracer: Tracer, + private val traceContexts: RecipeInstanceTraceContexts +) : AbstractFunction1() { + + override fun apply(event: BakerEvent): BoxedUnit { + try { + onEvent(event) + } catch (throwable: Throwable) { + LOGGER.debug("Failed to emit a tracing span for baker event {}", event, throwable) + } + return BoxedUnit.UNIT + } + + private fun onEvent(event: BakerEvent) { + when (event) { + is RecipeInstanceCreated -> { + val span = newSpan( + RECIPE_INSTANCE_CREATED_SPAN, + event.recipeId(), + event.recipeName(), + event.recipeInstanceId() + ) + span.setStatus(StatusCode.OK) + span.end() + } + is EventReceived -> { + val span = newSpan( + EVENT_RECEIVED_SPAN, + event.recipeId(), + event.recipeName(), + event.recipeInstanceId() + ) + span.setAttribute(BakerTraceAttributes.EVENT_NAME, event.eventName()) + val correlationId = event.correlationId() + if (correlationId.isDefined) { + span.setAttribute(BakerTraceAttributes.CORRELATION_ID, correlationId.get()) + } + span.setStatus(StatusCode.OK) + record(event.recipeInstanceId(), span) + span.end() + } + is EventFired -> { + val span = newSpan( + EVENT_FIRED_SPAN, + event.recipeId(), + event.recipeName(), + event.recipeInstanceId() + ) + span.setAttribute(BakerTraceAttributes.EVENT_NAME, event.eventName()) + span.setStatus(StatusCode.OK) + record(event.recipeInstanceId(), span) + span.end() + + if (RECIPE_SUCCESSFUL_EVENT == event.eventName()) { + traceContexts.remove(event.recipeInstanceId()) + TracingMdcRegistry.remove(event.recipeInstanceId()) + } + } + } + } + + private fun newSpan(name: String, recipeId: String, recipeName: String, recipeInstanceId: String): Span { + val builder = tracer.spanBuilder(name) + .setParent(traceContexts.parentOf(recipeInstanceId)) + val span = builder.startSpan() + span.setAttribute(BakerTraceAttributes.RECIPE_ID, recipeId) + span.setAttribute(BakerTraceAttributes.RECIPE_NAME, recipeName) + span.setAttribute(BakerTraceAttributes.RECIPE_INSTANCE_ID, recipeInstanceId) + return span + } + + private fun record(recipeInstanceId: String, span: Span) { + TracingMdcRegistry.put( + recipeInstanceId, + span.spanContext.traceId, + span.spanContext.spanId + ) + traceContexts.putIfAbsent(recipeInstanceId, Context.current().with(span)) + } + + companion object { + private val LOGGER: Logger = LoggerFactory.getLogger(BakerEventTracingListener::class.java) + + private const val RECIPE_INSTANCE_CREATED_SPAN = "baker.recipe-instance.created" + private const val EVENT_RECEIVED_SPAN = "baker.event.received" + private const val EVENT_FIRED_SPAN = "baker.event.fired" + private const val RECIPE_SUCCESSFUL_EVENT = "RecipeSuccessful" + } +} diff --git a/core/baker-interface-otel/src/main/kotlin/com/ing/baker/runtime/otel/BakerTraceAttributes.kt b/core/baker-interface-otel/src/main/kotlin/com/ing/baker/runtime/otel/BakerTraceAttributes.kt new file mode 100644 index 000000000..a7f365638 --- /dev/null +++ b/core/baker-interface-otel/src/main/kotlin/com/ing/baker/runtime/otel/BakerTraceAttributes.kt @@ -0,0 +1,18 @@ +package com.ing.baker.runtime.otel + +/** + * Attribute keys shared by all Baker OpenTelemetry instrumentation. + */ +object BakerTraceAttributes { + + const val RECIPE_ID = "baker.recipe.id" + const val RECIPE_NAME = "baker.recipe.name" + const val RECIPE_INSTANCE_ID = "baker.recipe.instance.id" + const val EVENT_NAME = "baker.event.name" + const val INTERACTION_NAME = "baker.interaction.name" + const val CORRELATION_ID = "baker.correlation.id" + const val FAILURE_COUNT = "baker.failure.count" + const val ERROR_TYPE = "error.type" + + const val INTERACTION_FAILURE = "interaction.failure" +} diff --git a/core/baker-interface-otel/src/main/kotlin/com/ing/baker/runtime/otel/OpenTelemetryContextPropagator.kt b/core/baker-interface-otel/src/main/kotlin/com/ing/baker/runtime/otel/OpenTelemetryContextPropagator.kt new file mode 100644 index 000000000..7f520925f --- /dev/null +++ b/core/baker-interface-otel/src/main/kotlin/com/ing/baker/runtime/otel/OpenTelemetryContextPropagator.kt @@ -0,0 +1,32 @@ +package com.ing.baker.runtime.otel + +import com.ing.baker.runtime.model.propagation.ContextScope +import com.ing.baker.runtime.model.propagation.ContextSnapshot +import com.ing.baker.runtime.model.propagation.ExecutionContextPropagator +import io.opentelemetry.context.Context +import io.opentelemetry.context.ContextStorage +import io.opentelemetry.context.Scope + +/** + * Moves the OpenTelemetry [Context] of the submitting thread to the thread that actually executes the + * work, and removes it again afterwards. + * + * Registered through `META-INF/services`, so simply having this module on the classpath makes + * Baker's internal thread hops context aware. + */ +class OpenTelemetryContextPropagator : ExecutionContextPropagator { + + override fun capture(): ContextSnapshot { + val captured = Context.current() + return object : ContextSnapshot { + override fun restore(): ContextScope { + val scope = captured.makeCurrent() + return ContextScope { scope.close() } + } + } + } + + override fun clear() { + ContextStorage.get().attach(Context.root()) + } +} diff --git a/core/baker-interface-otel/src/main/kotlin/com/ing/baker/runtime/otel/OpenTelemetryInteractionInstrumentation.kt b/core/baker-interface-otel/src/main/kotlin/com/ing/baker/runtime/otel/OpenTelemetryInteractionInstrumentation.kt new file mode 100644 index 000000000..003bd7e9b --- /dev/null +++ b/core/baker-interface-otel/src/main/kotlin/com/ing/baker/runtime/otel/OpenTelemetryInteractionInstrumentation.kt @@ -0,0 +1,119 @@ +package com.ing.baker.runtime.otel + +import com.ing.baker.runtime.model.InteractionCallContext +import com.ing.baker.runtime.model.InteractionInstrumentation +import io.opentelemetry.api.trace.Span +import io.opentelemetry.api.trace.StatusCode +import io.opentelemetry.api.trace.Tracer +import io.opentelemetry.context.Context +import io.opentelemetry.context.Scope +import org.slf4j.MDC +import java.util.concurrent.CompletionException +import java.util.concurrent.CompletionStage +import java.util.concurrent.ExecutionException +import java.util.function.Supplier + +/** + * Starts a span for every interaction invocation and attaches it to the thread that runs the interaction, so + * `Span.current()` inside an interaction always belongs to that very invocation, even when + * interactions of different recipe instances share a thread pool. + */ +class OpenTelemetryInteractionInstrumentation( + private val tracer: Tracer, + private val parentContexts: RecipeInstanceTraceContexts +) : InteractionInstrumentation { + + override fun aroundInteraction(context: InteractionCallContext, interaction: Supplier): A { + val span = try { + startSpan(context) + } catch (throwable: Throwable) { + return interaction.get() + } + + val previousTraceId = MDC.get(TRACE_ID_MDC_KEY) + val previousSpanId = MDC.get(SPAN_ID_MDC_KEY) + var endsAsynchronously = false + + return try { + span.makeCurrent().use { + MDC.put(TRACE_ID_MDC_KEY, span.spanContext.traceId) + MDC.put(SPAN_ID_MDC_KEY, span.spanContext.spanId) + + val result = interaction.get() + if (result is CompletionStage<*>) { + endsAsynchronously = true + result.whenComplete { _, throwable -> end(span, throwable) } + } else { + span.setStatus(StatusCode.OK) + } + result + } + } catch (throwable: Throwable) { + endsAsynchronously = false + recordFailure(span, throwable) + throw throwable + } finally { + restore(TRACE_ID_MDC_KEY, previousTraceId) + restore(SPAN_ID_MDC_KEY, previousSpanId) + if (!endsAsynchronously) { + span.end() + } + } + } + + private fun startSpan(context: InteractionCallContext): Span { + val parent = parentContexts.parentOf(context.recipeInstanceId) + val span = tracer.spanBuilder("baker.interaction." + context.interactionName) + .setParent(parent) + .startSpan() + span.setAttribute(BakerTraceAttributes.RECIPE_ID, context.recipeId) + span.setAttribute(BakerTraceAttributes.RECIPE_NAME, context.recipeName) + span.setAttribute(BakerTraceAttributes.RECIPE_INSTANCE_ID, context.recipeInstanceId) + span.setAttribute(BakerTraceAttributes.INTERACTION_NAME, context.interactionName) + context.correlationId + .ifPresent { correlationId -> span.setAttribute(BakerTraceAttributes.CORRELATION_ID, correlationId) } + return span + } + + companion object { + private const val TRACE_ID_MDC_KEY = "trace_id" + private const val SPAN_ID_MDC_KEY = "span_id" + + private fun end(span: Span, throwable: Throwable?) { + if (throwable == null) { + span.setStatus(StatusCode.OK) + } else { + recordFailure(span, throwable) + } + span.end() + } + + private fun recordFailure(span: Span, throwable: Throwable) { + val cause = unwrap(throwable) + span.setAttribute(BakerTraceAttributes.ERROR_TYPE, BakerTraceAttributes.INTERACTION_FAILURE) + span.recordException(cause) + span.setStatus( + StatusCode.ERROR, + cause.message ?: cause.toString() + ) + } + + private fun unwrap(throwable: Throwable): Throwable { + return if ((throwable is CompletionException || throwable is ExecutionException) + && throwable.cause != null + ) { + throwable.cause!! + } else { + throwable + } + } + + private fun restore(key: String, value: String?) { + if (value == null) { + MDC.remove(key) + } else { + MDC.put(key, value) + } + } + } +} diff --git a/core/baker-interface-otel/src/main/kotlin/com/ing/baker/runtime/otel/RecipeInstanceTraceContexts.kt b/core/baker-interface-otel/src/main/kotlin/com/ing/baker/runtime/otel/RecipeInstanceTraceContexts.kt new file mode 100644 index 000000000..b6967c361 --- /dev/null +++ b/core/baker-interface-otel/src/main/kotlin/com/ing/baker/runtime/otel/RecipeInstanceTraceContexts.kt @@ -0,0 +1,49 @@ +package com.ing.baker.runtime.otel + +import io.opentelemetry.context.Context +import java.util.concurrent.ConcurrentHashMap + +/** + * Keeps the tracing context that interaction spans of a recipe instance should be parented to. + * + * Each Baker runtime owns its own instance, so contexts never leak between bakers, and entries are + * removed as soon as the recipe instance is done. + */ +class RecipeInstanceTraceContexts { + + private val contexts = ConcurrentHashMap() + + fun put(recipeInstanceId: String?, context: Context?) { + if (recipeInstanceId != null && context != null) { + contexts[recipeInstanceId] = context + } + } + + fun putIfAbsent(recipeInstanceId: String?, context: Context?) { + if (recipeInstanceId != null && context != null) { + contexts.putIfAbsent(recipeInstanceId, context) + } + } + + /** + * Returns the parent context for the given recipe instance, falling back to the context currently + * attached to the calling thread. + */ + fun parentOf(recipeInstanceId: String?): Context { + return if (recipeInstanceId != null) { + contexts[recipeInstanceId] ?: Context.current() + } else { + Context.current() + } + } + + fun remove(recipeInstanceId: String?) { + if (recipeInstanceId != null) { + contexts.remove(recipeInstanceId) + } + } + + fun clear() { + contexts.clear() + } +} diff --git a/core/baker-interface-otel/src/main/resources/META-INF/services/com.ing.baker.runtime.model.propagation.ExecutionContextPropagator b/core/baker-interface-otel/src/main/resources/META-INF/services/com.ing.baker.runtime.model.propagation.ExecutionContextPropagator new file mode 100644 index 000000000..d2ceeb0c8 --- /dev/null +++ b/core/baker-interface-otel/src/main/resources/META-INF/services/com.ing.baker.runtime.model.propagation.ExecutionContextPropagator @@ -0,0 +1 @@ +com.ing.baker.runtime.otel.OpenTelemetryContextPropagator diff --git a/core/baker-interface-otel/src/test/kotlin/com/ing/baker/runtime/otel/BlockingPoolPropagationTest.kt b/core/baker-interface-otel/src/test/kotlin/com/ing/baker/runtime/otel/BlockingPoolPropagationTest.kt new file mode 100644 index 000000000..232459d93 --- /dev/null +++ b/core/baker-interface-otel/src/test/kotlin/com/ing/baker/runtime/otel/BlockingPoolPropagationTest.kt @@ -0,0 +1,104 @@ +package com.ing.baker.runtime.otel + +import com.ing.baker.runtime.catseffect.SyncSupport +import io.opentelemetry.api.trace.Span +import io.opentelemetry.context.Context +import io.opentelemetry.context.ContextStorage +import io.opentelemetry.context.Scope +import io.opentelemetry.sdk.OpenTelemetrySdk +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter +import io.opentelemetry.sdk.trace.SdkTracerProvider +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.slf4j.MDC +import java.util.concurrent.CompletableFuture +import java.util.concurrent.TimeUnit + +/** + * Verifies that the propagator registered through {@code META-INF/services} is picked up by Baker's blocking + * pool, which is where interaction implementations run. + */ +class BlockingPoolPropagationTest { + + private lateinit var exporter: InMemorySpanExporter + private lateinit var tracerProvider: SdkTracerProvider + private lateinit var tracer: io.opentelemetry.api.trace.Tracer + + @BeforeEach + fun setUp() { + exporter = InMemorySpanExporter.create() + tracerProvider = SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(exporter)) + .build() + tracer = OpenTelemetrySdk.builder().setTracerProvider(tracerProvider).build().getTracer("test.otel.blocking") + } + + @AfterEach + fun tearDown() { + tracerProvider.close() + exporter.reset() + MDC.clear() + } + + @Test + fun propagatesSpanAndMdcToTheBlockingPool() { + val span = tracer.spanBuilder("caller").startSpan() + MDC.put("recipeInstanceId", "instance-1") + try { + span.makeCurrent().use { + val observed = runBlocking { + arrayOf( + Span.current().spanContext.spanId, + MDC.get("recipeInstanceId") + ) + } + assertEquals(span.spanContext.spanId, observed[0]) + assertEquals("instance-1", observed[1]) + } + } finally { + span.end() + MDC.clear() + } + } + + @Test + fun doesNotExposeContextOfPreviousWorkOnTheSamePooledThread() { + val leaked = tracer.spanBuilder("leaked").startSpan() + + // Simulate badly behaved work that attaches a context on the pooled thread without detaching it. + runBlocking { + ContextStorage.get().attach(Context.current().with(leaked)) + MDC.put("trace_id", "leaked-trace") + null + } + + val observed = runBlocking { + arrayOf( + Span.current().spanContext.traceId, + MDC.get("trace_id") + ) + } + + assertEquals(INVALID_TRACE_ID, observed[0], "A pooled thread must not expose the context of previous work") + assertNull(observed[1], "A pooled thread must not expose the MDC of previous work") + leaked.end() + } + + private fun runBlocking(thunk: () -> A): A { + @Suppress("UNCHECKED_CAST") + val future = SyncSupport.fromCompletableFuture().blocking( + object : scala.Function0 { + override fun apply(): A = thunk() + } + ) as CompletableFuture + return future.get(10, TimeUnit.SECONDS) + } + + companion object { + private const val INVALID_TRACE_ID = "00000000000000000000000000000000" + } +} diff --git a/core/baker-interface-otel/src/test/kotlin/com/ing/baker/runtime/otel/OpenTelemetryContextPropagatorTest.kt b/core/baker-interface-otel/src/test/kotlin/com/ing/baker/runtime/otel/OpenTelemetryContextPropagatorTest.kt new file mode 100644 index 000000000..3494fb9a8 --- /dev/null +++ b/core/baker-interface-otel/src/test/kotlin/com/ing/baker/runtime/otel/OpenTelemetryContextPropagatorTest.kt @@ -0,0 +1,113 @@ +package com.ing.baker.runtime.otel + +import com.ing.baker.runtime.model.propagation.ContextScope +import com.ing.baker.runtime.model.propagation.ContextSnapshot +import io.opentelemetry.api.trace.Span +import io.opentelemetry.context.Context +import io.opentelemetry.context.ContextStorage +import io.opentelemetry.sdk.OpenTelemetrySdk +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter +import io.opentelemetry.sdk.trace.SdkTracerProvider +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertNotEquals +import org.junit.jupiter.api.Test +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit + +class OpenTelemetryContextPropagatorTest { + + private val propagator = OpenTelemetryContextPropagator() + + @Test + fun movesTheCapturedContextToTheExecutingThread() { + TracerFixture.create().use { fixture -> + val span = fixture.tracer.spanBuilder("caller").startSpan() + val snapshot: ContextSnapshot = span.makeCurrent().use { + propagator.capture() + } + + val executor = Executors.newSingleThreadExecutor() + try { + val observed = executor.submit { + snapshot.restore().use { _: ContextScope -> + Span.current().spanContext.spanId + } + } + assertEquals(span.spanContext.spanId, observed.get(5, TimeUnit.SECONDS)) + + val afterScope = executor.submit { + Span.current().spanContext.traceId + } + assertEquals( + INVALID_TRACE_ID, + afterScope.get(5, TimeUnit.SECONDS), + "The propagated context must not survive the scope it was restored in" + ) + } finally { + executor.shutdownNow() + } + span.end() + } + } + + @Test + fun clearsContextLeakedByPreviousWorkOnTheSameThread() { + TracerFixture.create().use { fixture -> + val executor = Executors.newSingleThreadExecutor() + try { + val leaked = fixture.tracer.spanBuilder("leaked").startSpan() + executor.submit { + ContextStorage.get().attach(Context.current().with(leaked)) + }.get(5, TimeUnit.SECONDS) + + val stillLeaking = executor.submit { + Span.current().spanContext.spanId + } + assertEquals(leaked.spanContext.spanId, stillLeaking.get(5, TimeUnit.SECONDS)) + + val afterClear = executor.submit { + propagator.clear() + Span.current().spanContext.traceId + } + assertEquals(INVALID_TRACE_ID, afterClear.get(5, TimeUnit.SECONDS)) + assertNotEquals(INVALID_TRACE_ID, leaked.spanContext.traceId) + assertFalse( + Span.current().spanContext.isValid, + "The test thread itself should not have any ambient span" + ) + leaked.end() + } finally { + executor.shutdownNow() + } + } + } + + private data class TracerFixture( + val exporter: InMemorySpanExporter, + val tracerProvider: SdkTracerProvider, + val tracer: io.opentelemetry.api.trace.Tracer + ) : AutoCloseable { + + companion object { + fun create(): TracerFixture { + val exporter = InMemorySpanExporter.create() + val tracerProvider = SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(exporter)) + .build() + val sdk = OpenTelemetrySdk.builder().setTracerProvider(tracerProvider).build() + return TracerFixture(exporter, tracerProvider, sdk.getTracer("test.otel.propagator")) + } + } + + override fun close() { + tracerProvider.close() + exporter.reset() + } + } + + companion object { + private const val INVALID_TRACE_ID = "00000000000000000000000000000000" + } +} diff --git a/core/baker-interface-otel/src/test/kotlin/com/ing/baker/runtime/otel/OpenTelemetryInteractionInstrumentationTest.kt b/core/baker-interface-otel/src/test/kotlin/com/ing/baker/runtime/otel/OpenTelemetryInteractionInstrumentationTest.kt new file mode 100644 index 000000000..a5467c0f1 --- /dev/null +++ b/core/baker-interface-otel/src/test/kotlin/com/ing/baker/runtime/otel/OpenTelemetryInteractionInstrumentationTest.kt @@ -0,0 +1,129 @@ +package com.ing.baker.runtime.otel + +import com.ing.baker.runtime.model.InteractionCallContext +import io.opentelemetry.api.common.AttributeKey +import io.opentelemetry.api.trace.Span +import io.opentelemetry.api.trace.StatusCode +import io.opentelemetry.api.trace.Tracer +import io.opentelemetry.context.Context +import io.opentelemetry.sdk.OpenTelemetrySdk +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter +import io.opentelemetry.sdk.trace.SdkTracerProvider +import io.opentelemetry.sdk.trace.data.SpanData +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.slf4j.MDC +import java.util.concurrent.CompletableFuture + +class OpenTelemetryInteractionInstrumentationTest { + + private lateinit var exporter: InMemorySpanExporter + private lateinit var tracerProvider: SdkTracerProvider + private lateinit var tracer: Tracer + private lateinit var parentContexts: RecipeInstanceTraceContexts + private lateinit var instrumentation: OpenTelemetryInteractionInstrumentation + + @BeforeEach + fun setUp() { + exporter = InMemorySpanExporter.create() + tracerProvider = SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(exporter)) + .build() + tracer = OpenTelemetrySdk.builder().setTracerProvider(tracerProvider).build().getTracer("test.otel") + parentContexts = RecipeInstanceTraceContexts() + instrumentation = OpenTelemetryInteractionInstrumentation(tracer, parentContexts) + } + + @AfterEach + fun tearDown() { + tracerProvider.close() + exporter.reset() + MDC.clear() + } + + @Test + fun attachesItsOwnSpanToTheExecutingThread() { + val observedSpanId = instrumentation.aroundInteraction( + callContext("instance-1") + ) { Span.current().spanContext.spanId } + + val spans: List = exporter.finishedSpanItems + assertEquals(1, spans.size) + val span = spans[0] + assertEquals("baker.interaction.ValidateOrder", span.name) + assertEquals(span.spanContext.spanId, observedSpanId) + assertEquals(StatusCode.OK, span.status.statusCode) + assertEquals("instance-1", span.attributes.get(AttributeKey.stringKey("baker.recipe.instance.id"))) + assertEquals("ValidateOrder", span.attributes.get(AttributeKey.stringKey("baker.interaction.name"))) + assertEquals("correlation-1", span.attributes.get(AttributeKey.stringKey("baker.correlation.id"))) + assertFalse(Span.current().spanContext.isValid, "The span must be detached after the invocation") + } + + @Test + fun parentsTheSpanToTheRecipeInstanceContext() { + val parent = tracer.spanBuilder("baker.event.received").startSpan() + parentContexts.put("instance-2", Context.root().with(parent)) + parent.end() + + instrumentation.aroundInteraction(callContext("instance-2")) { null } + + val interactionSpan = exporter.finishedSpanItems + .first { it.name.startsWith("baker.interaction.") } + assertEquals(parent.spanContext.spanId, interactionSpan.parentSpanId) + assertEquals(parent.spanContext.traceId, interactionSpan.spanContext.traceId) + } + + @Test + fun setsAndRestoresTraceMdcKeys() { + MDC.put("trace_id", "previous-trace") + + instrumentation.aroundInteraction(callContext("instance-3")) { + assertEquals(Span.current().spanContext.traceId, MDC.get("trace_id")) + assertEquals(Span.current().spanContext.spanId, MDC.get("span_id")) + null + } + + assertEquals("previous-trace", MDC.get("trace_id")) + assertNull(MDC.get("span_id")) + } + + @Test + fun endsTheSpanWhenAnAsynchronousInteractionFails() { + val future = CompletableFuture() + + instrumentation.aroundInteraction(callContext("instance-4")) { future } + + assertTrue(exporter.finishedSpanItems.isEmpty(), "Span must stay open until the interaction completes") + + future.completeExceptionally(RuntimeException("Expected failure")) + + val span = exporter.finishedSpanItems[0] + assertEquals(StatusCode.ERROR, span.status.statusCode) + assertEquals("interaction.failure", span.attributes.get(AttributeKey.stringKey("error.type"))) + } + + @Test + fun recordsSynchronousFailures() { + val failure = RuntimeException("Expected failure") + try { + instrumentation.aroundInteraction(callContext("instance-5")) { + throw failure + } + } catch (expected: RuntimeException) { + assertEquals(failure, expected) + } + + val span = exporter.finishedSpanItems[0] + assertEquals(StatusCode.ERROR, span.status.statusCode) + assertEquals("interaction.failure", span.attributes.get(AttributeKey.stringKey("error.type"))) + } + + private fun callContext(recipeInstanceId: String): InteractionCallContext = + InteractionCallContext("recipe-id", "recipe", recipeInstanceId, "ValidateOrder", 42L, "correlation-1") +} diff --git a/core/baker-interface/src/main/java/com/ing/baker/runtime/model/InteractionCallContext.java b/core/baker-interface/src/main/java/com/ing/baker/runtime/model/InteractionCallContext.java new file mode 100644 index 000000000..b60638ccf --- /dev/null +++ b/core/baker-interface/src/main/java/com/ing/baker/runtime/model/InteractionCallContext.java @@ -0,0 +1,65 @@ +package com.ing.baker.runtime.model; + +import java.util.Optional; + +/** + * Identifies a single interaction invocation, handed to {@link InteractionInstrumentation} on the thread + * that actually invokes the interaction implementation. + */ +public final class InteractionCallContext { + + private final String recipeId; + private final String recipeName; + private final String recipeInstanceId; + private final String interactionName; + private final long executionId; + private final String correlationId; + + public InteractionCallContext(String recipeId, + String recipeName, + String recipeInstanceId, + String interactionName, + long executionId, + String correlationId) { + this.recipeId = recipeId; + this.recipeName = recipeName; + this.recipeInstanceId = recipeInstanceId; + this.interactionName = interactionName; + this.executionId = executionId; + this.correlationId = correlationId; + } + + public String getRecipeId() { + return recipeId; + } + + public String getRecipeName() { + return recipeName; + } + + public String getRecipeInstanceId() { + return recipeInstanceId; + } + + public String getInteractionName() { + return interactionName; + } + + public long getExecutionId() { + return executionId; + } + + public Optional getCorrelationId() { + return Optional.ofNullable(correlationId); + } + + @Override + public String toString() { + return "InteractionCallContext{" + + "recipeName='" + recipeName + '\'' + + ", recipeInstanceId='" + recipeInstanceId + '\'' + + ", interactionName='" + interactionName + '\'' + + ", executionId=" + executionId + + '}'; + } +} diff --git a/core/baker-interface/src/main/java/com/ing/baker/runtime/model/InteractionInstrumentation.java b/core/baker-interface/src/main/java/com/ing/baker/runtime/model/InteractionInstrumentation.java new file mode 100644 index 000000000..b8efd5b3e --- /dev/null +++ b/core/baker-interface/src/main/java/com/ing/baker/runtime/model/InteractionInstrumentation.java @@ -0,0 +1,26 @@ +package com.ing.baker.runtime.model; + +import java.util.function.Supplier; + +/** + * Hook invoked around every interaction invocation, on the thread that runs the interaction. + * + *

This is the extension point used for tracing: an implementation can start a span, attach it to the + * executing thread and detach it again once the invocation returns, guaranteeing that + * {@code Span.current()} inside an interaction belongs to that interaction only. + */ +public interface InteractionInstrumentation { + + InteractionInstrumentation NOOP = new InteractionInstrumentation() { + @Override + public A aroundInteraction(InteractionCallContext context, Supplier interaction) { + return interaction.get(); + } + }; + + static InteractionInstrumentation noop() { + return NOOP; + } + + A aroundInteraction(InteractionCallContext context, Supplier interaction); +} diff --git a/core/baker-interface/src/main/java/com/ing/baker/runtime/model/InteractionMdc.java b/core/baker-interface/src/main/java/com/ing/baker/runtime/model/InteractionMdc.java new file mode 100644 index 000000000..a33c5ef75 --- /dev/null +++ b/core/baker-interface/src/main/java/com/ing/baker/runtime/model/InteractionMdc.java @@ -0,0 +1,61 @@ +package com.ing.baker.runtime.model; + +import com.ing.baker.runtime.model.propagation.ContextScope; +import org.slf4j.MDC; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Installs the diagnostic context of a single interaction invocation on the current thread. + * + *

This is applied on the thread that actually invokes the interaction, which makes it independent of how a + * runtime composes its effects: an effect type that only builds a description on the calling thread (such as + * {@code cats.effect.IO}) would otherwise lose a diagnostic context installed by the caller. + */ +public final class InteractionMdc { + + private static final String[] KEYS = { + "recipeInstanceId", + "recipeName", + "baker.recipe.instance.id", + "baker.recipe.name", + "baker.recipe.id", + "baker.interaction.name" + }; + + private InteractionMdc() { + } + + /** + * Sets the interaction MDC keys and returns a scope restoring the previous values. + */ + public static ContextScope install(InteractionCallContext context) { + if (context == null) { + return ContextScope.NOOP; + } + final Map previous = new LinkedHashMap<>(); + for (String key : KEYS) { + previous.put(key, MDC.get(key)); + } + try { + MDC.put("recipeInstanceId", context.getRecipeInstanceId()); + MDC.put("recipeName", context.getRecipeName()); + MDC.put("baker.recipe.instance.id", context.getRecipeInstanceId()); + MDC.put("baker.recipe.name", context.getRecipeName()); + MDC.put("baker.recipe.id", context.getRecipeId()); + MDC.put("baker.interaction.name", context.getInteractionName()); + } catch (Throwable ignored) { + // Diagnostics must never break an interaction. + } + return () -> { + for (Map.Entry entry : previous.entrySet()) { + if (entry.getValue() == null) { + MDC.remove(entry.getKey()); + } else { + MDC.put(entry.getKey(), entry.getValue()); + } + } + }; + } +} diff --git a/core/baker-interface/src/main/java/com/ing/baker/runtime/model/TracingMdcRegistry.java b/core/baker-interface/src/main/java/com/ing/baker/runtime/model/TracingMdcRegistry.java new file mode 100644 index 000000000..801e5aca9 --- /dev/null +++ b/core/baker-interface/src/main/java/com/ing/baker/runtime/model/TracingMdcRegistry.java @@ -0,0 +1,67 @@ +package com.ing.baker.runtime.model; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; + +/** + * Shares best-effort trace/span ids for MDC enrichment across runtime modules. + * + *

Only used for log lines emitted by Baker itself, outside of an interaction invocation. Inside an + * interaction the exact trace context is attached to the executing thread by + * {@link InteractionInstrumentation}. Entries are bounded so an abandoned recipe instance can never grow + * this registry indefinitely. + */ +public final class TracingMdcRegistry { + + private static final int MAX_ENTRIES = 10_000; + + private static final Map contexts = Collections.synchronizedMap( + new LinkedHashMap(256, 0.75f, false) { + @Override + protected boolean removeEldestEntry(Map.Entry eldest) { + return size() > MAX_ENTRIES; + } + }); + + private TracingMdcRegistry() { + } + + public static void put(String recipeInstanceId, String traceId, String spanId) { + if (recipeInstanceId == null || traceId == null || spanId == null) { + return; + } + contexts.put(recipeInstanceId, new TraceContext(traceId, spanId)); + } + + public static Optional get(String recipeInstanceId) { + return Optional.ofNullable(contexts.get(recipeInstanceId)); + } + + public static void remove(String recipeInstanceId) { + if (recipeInstanceId == null) { + return; + } + contexts.remove(recipeInstanceId); + } + + public static final class TraceContext { + private final String traceId; + private final String spanId; + + public TraceContext(String traceId, String spanId) { + this.traceId = traceId; + this.spanId = spanId; + } + + public String getTraceId() { + return traceId; + } + + public String getSpanId() { + return spanId; + } + } +} + diff --git a/core/baker-interface/src/main/java/com/ing/baker/runtime/model/propagation/ContextScope.java b/core/baker-interface/src/main/java/com/ing/baker/runtime/model/propagation/ContextScope.java new file mode 100644 index 000000000..c4d9c4aee --- /dev/null +++ b/core/baker-interface/src/main/java/com/ing/baker/runtime/model/propagation/ContextScope.java @@ -0,0 +1,14 @@ +package com.ing.baker.runtime.model.propagation; + +/** + * Undoes a {@link ContextSnapshot#restore()} on the thread that performed it. + */ +@FunctionalInterface +public interface ContextScope extends AutoCloseable { + + ContextScope NOOP = () -> { + }; + + @Override + void close(); +} diff --git a/core/baker-interface/src/main/java/com/ing/baker/runtime/model/propagation/ContextSnapshot.java b/core/baker-interface/src/main/java/com/ing/baker/runtime/model/propagation/ContextSnapshot.java new file mode 100644 index 000000000..f9314fd36 --- /dev/null +++ b/core/baker-interface/src/main/java/com/ing/baker/runtime/model/propagation/ContextSnapshot.java @@ -0,0 +1,15 @@ +package com.ing.baker.runtime.model.propagation; + +/** + * Immutable capture of ambient context taken on the submitting thread. + */ +@FunctionalInterface +public interface ContextSnapshot { + + ContextSnapshot NOOP = () -> ContextScope.NOOP; + + /** + * Installs the captured context on the calling thread. Must be closed on that same thread. + */ + ContextScope restore(); +} diff --git a/core/baker-interface/src/main/java/com/ing/baker/runtime/model/propagation/ExecutionContextPropagator.java b/core/baker-interface/src/main/java/com/ing/baker/runtime/model/propagation/ExecutionContextPropagator.java new file mode 100644 index 000000000..f28bf99de --- /dev/null +++ b/core/baker-interface/src/main/java/com/ing/baker/runtime/model/propagation/ExecutionContextPropagator.java @@ -0,0 +1,20 @@ +package com.ing.baker.runtime.model.propagation; + +/** + * Propagates ambient thread state (MDC, tracing context, ...) across Baker's internal thread hops. + * + *

Implementations are discovered through {@link java.util.ServiceLoader} and must be thread safe. + * {@link #capture()} is always called on the submitting thread, {@link ContextSnapshot#restore()} and + * {@link ContextScope#close()} on the executing thread. + */ +public interface ExecutionContextPropagator { + + ContextSnapshot capture(); + + /** + * Drops any ambient state left behind on the calling thread, so a pooled thread never exposes + * state of a previously executed task. + */ + default void clear() { + } +} diff --git a/core/baker-interface/src/main/java/com/ing/baker/runtime/model/propagation/ExecutionContextPropagators.java b/core/baker-interface/src/main/java/com/ing/baker/runtime/model/propagation/ExecutionContextPropagators.java new file mode 100644 index 000000000..1a2edf6ee --- /dev/null +++ b/core/baker-interface/src/main/java/com/ing/baker/runtime/model/propagation/ExecutionContextPropagators.java @@ -0,0 +1,123 @@ +package com.ing.baker.runtime.model.propagation; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.ServiceLoader; +import java.util.concurrent.CopyOnWriteArrayList; + +/** + * Registry of the {@link ExecutionContextPropagator}s used whenever Baker hands work to another thread. + * + *

Besides the built-in {@link MdcContextPropagator}, implementations are discovered through + * {@link ServiceLoader}, so runtimes such as {@code baker-interface-otel} can plug in without adding + * dependencies to {@code baker-interface}. + */ +public final class ExecutionContextPropagators { + + private static final Logger logger = LoggerFactory.getLogger(ExecutionContextPropagators.class); + + private static final List propagators = new CopyOnWriteArrayList<>(); + + static { + reload(); + } + + private ExecutionContextPropagators() { + } + + /** + * Captures the ambient context of the calling thread. + */ + public static ContextSnapshot capture() { + if (propagators.isEmpty()) { + return ContextSnapshot.NOOP; + } + final List snapshots = new ArrayList<>(propagators.size()); + for (ExecutionContextPropagator propagator : propagators) { + try { + final ContextSnapshot snapshot = propagator.capture(); + if (snapshot != null) { + snapshots.add(snapshot); + } + } catch (Throwable throwable) { + logger.debug("Context propagator {} failed to capture context", propagator, throwable); + } + } + if (snapshots.isEmpty()) { + return ContextSnapshot.NOOP; + } + return () -> restoreAll(snapshots); + } + + /** + * Drops ambient context left behind on the calling thread by previously executed work. + */ + public static void clear() { + for (ExecutionContextPropagator propagator : propagators) { + try { + propagator.clear(); + } catch (Throwable throwable) { + logger.debug("Context propagator {} failed to clear context", propagator, throwable); + } + } + } + + public static void register(ExecutionContextPropagator propagator) { + if (propagator != null) { + propagators.add(propagator); + } + } + + public static void unregister(ExecutionContextPropagator propagator) { + propagators.remove(propagator); + } + + public static List registered() { + return Collections.unmodifiableList(new ArrayList<>(propagators)); + } + + /** + * Rebuilds the registry from the built-in propagators and the ones found on the classpath. + */ + public static void reload() { + final List discovered = new ArrayList<>(); + discovered.add(new MdcContextPropagator()); + try { + for (ExecutionContextPropagator propagator : ServiceLoader.load( + ExecutionContextPropagator.class, ExecutionContextPropagators.class.getClassLoader())) { + discovered.add(propagator); + } + } catch (Throwable throwable) { + logger.warn("Failed to load context propagators, continuing without them", throwable); + } + propagators.clear(); + propagators.addAll(discovered); + } + + private static ContextScope restoreAll(List snapshots) { + final List scopes = new ArrayList<>(snapshots.size()); + for (ContextSnapshot snapshot : snapshots) { + try { + final ContextScope scope = snapshot.restore(); + if (scope != null) { + scopes.add(scope); + } + } catch (Throwable throwable) { + logger.debug("Failed to restore propagated context", throwable); + } + } + return () -> { + for (int index = scopes.size() - 1; index >= 0; index--) { + try { + scopes.get(index).close(); + } catch (Throwable throwable) { + logger.debug("Failed to close propagated context scope", throwable); + } + } + }; + } +} diff --git a/core/baker-interface/src/main/java/com/ing/baker/runtime/model/propagation/MdcContextPropagator.java b/core/baker-interface/src/main/java/com/ing/baker/runtime/model/propagation/MdcContextPropagator.java new file mode 100644 index 000000000..722989d7b --- /dev/null +++ b/core/baker-interface/src/main/java/com/ing/baker/runtime/model/propagation/MdcContextPropagator.java @@ -0,0 +1,34 @@ +package com.ing.baker.runtime.model.propagation; + +import org.slf4j.MDC; + +import java.util.Map; + +/** + * Built-in propagator moving the SLF4J {@link MDC} across thread hops. + */ +public final class MdcContextPropagator implements ExecutionContextPropagator { + + @Override + public ContextSnapshot capture() { + final Map captured = MDC.getCopyOfContextMap(); + return () -> { + final Map previous = MDC.getCopyOfContextMap(); + apply(captured); + return () -> apply(previous); + }; + } + + @Override + public void clear() { + MDC.clear(); + } + + private static void apply(Map context) { + if (context == null || context.isEmpty()) { + MDC.clear(); + } else { + MDC.setContextMap(context); + } + } +} diff --git a/core/baker-interface/src/main/scala/com/ing/baker/runtime/catseffect/AsyncSupport.scala b/core/baker-interface/src/main/scala/com/ing/baker/runtime/catseffect/AsyncSupport.scala index 6cf5e43c7..51e584188 100644 --- a/core/baker-interface/src/main/scala/com/ing/baker/runtime/catseffect/AsyncSupport.scala +++ b/core/baker-interface/src/main/scala/com/ing/baker/runtime/catseffect/AsyncSupport.scala @@ -76,7 +76,10 @@ object AsyncSupport { override def flatMap[A, B](fa: IO[A])(f: A => IO[B]): IO[B] = fa.flatMap(f) override def raiseError[A](throwable: Throwable): IO[A] = IO.raiseError(throwable) override def delay[A](thunk: => A): IO[A] = IO.delay(thunk) - override def blocking[A](thunk: => A): IO[A] = IO.blocking(thunk) + override def blocking[A](thunk: => A): IO[A] = + IO.delay(SyncSupport.captureContext()).flatMap { snapshot => + IO.blocking(SyncSupport.runWithPropagatedContext(snapshot)(thunk)) + } override def unit: IO[Unit] = IO.unit override def attempt[A](fa: IO[A]): IO[Either[Throwable, A]] = fa.attempt diff --git a/core/baker-interface/src/main/scala/com/ing/baker/runtime/catseffect/SyncSupport.scala b/core/baker-interface/src/main/scala/com/ing/baker/runtime/catseffect/SyncSupport.scala index e6edb24fe..bc33b3920 100644 --- a/core/baker-interface/src/main/scala/com/ing/baker/runtime/catseffect/SyncSupport.scala +++ b/core/baker-interface/src/main/scala/com/ing/baker/runtime/catseffect/SyncSupport.scala @@ -1,6 +1,7 @@ package com.ing.baker.runtime.catseffect import cats.effect.Sync +import com.ing.baker.runtime.model.propagation.{ContextSnapshot, ExecutionContextPropagators} import java.util.concurrent.{CompletableFuture, CompletionStage, Executor, Executors} import java.util.function.{Function, Supplier} @@ -32,6 +33,36 @@ object SyncSupport { thread }) + /** + * Runs the given thunk on the blocking pool while propagating the ambient context (MDC, tracing context) + * of the submitting thread. The context is installed and removed on the executing thread itself, and any + * ambient context is cleared at both task boundaries so a pooled thread never exposes the context of + * another in-flight execution. + */ + private def supplyAsyncWithPropagation[A](thunk: => A): CompletableFuture[A] = { + val snapshot = ExecutionContextPropagators.capture() + CompletableFuture.supplyAsync(new Supplier[A] { + override def get(): A = runWithPropagatedContext(snapshot)(thunk) + }, blockingExecutor) + } + + /** + * Installs a previously captured context on the current thread, runs the thunk and removes it again. + * Ambient context is cleared at both task boundaries, so a pooled thread never exposes the context of + * another in-flight execution. + */ + def runWithPropagatedContext[A](snapshot: ContextSnapshot)(thunk: => A): A = { + ExecutionContextPropagators.clear() + val scope = snapshot.restore() + try thunk + finally { + scope.close() + ExecutionContextPropagators.clear() + } + } + + def captureContext(): ContextSnapshot = ExecutionContextPropagators.capture() + private def failedFuture[A](throwable: Throwable): CompletableFuture[A] = { val future = new CompletableFuture[A]() future.completeExceptionally(throwable) @@ -56,9 +87,7 @@ object SyncSupport { case NonFatal(exception) => raiseError(exception) } override def blocking[A](thunk: => A): CompletionStage[A] = - CompletableFuture.supplyAsync(new Supplier[A] { - override def get(): A = thunk - }, blockingExecutor) + supplyAsyncWithPropagation(thunk) override def unit: CompletionStage[Unit] = pure(()) } @@ -80,9 +109,7 @@ object SyncSupport { case NonFatal(exception) => raiseError(exception) } override def blocking[A](thunk: => A): CompletableFuture[A] = - CompletableFuture.supplyAsync(new Supplier[A] { - override def get(): A = thunk - }, blockingExecutor) + supplyAsyncWithPropagation(thunk) override def unit: CompletableFuture[Unit] = pure(()) } diff --git a/core/baker-interface/src/main/scala/com/ing/baker/runtime/model/BakerComponents.scala b/core/baker-interface/src/main/scala/com/ing/baker/runtime/model/BakerComponents.scala index 664de44d6..8638cc16a 100644 --- a/core/baker-interface/src/main/scala/com/ing/baker/runtime/model/BakerComponents.scala +++ b/core/baker-interface/src/main/scala/com/ing/baker/runtime/model/BakerComponents.scala @@ -4,4 +4,5 @@ case class BakerComponents[F[_]](interactions: InteractionManager[F], recipeInstanceManager: RecipeInstanceManager[F], recipeManager: RecipeManager[F], eventStream: EventStream[F], - logging: BakerLogging = BakerLogging()) + logging: BakerLogging = BakerLogging(), + instrumentation: InteractionInstrumentation = InteractionInstrumentation.noop()) diff --git a/core/baker-interface/src/main/scala/com/ing/baker/runtime/model/BakerLogging.scala b/core/baker-interface/src/main/scala/com/ing/baker/runtime/model/BakerLogging.scala index 2b58c42ac..d76635b6f 100644 --- a/core/baker-interface/src/main/scala/com/ing/baker/runtime/model/BakerLogging.scala +++ b/core/baker-interface/src/main/scala/com/ing/baker/runtime/model/BakerLogging.scala @@ -8,6 +8,7 @@ import org.joda.time.format.{PeriodFormatter, PeriodFormatterBuilder} import org.slf4j.MDC import scala.concurrent.duration.FiniteDuration +import scala.jdk.OptionConverters._ object BakerLogging { lazy val defaultLogger: Logger = Logger("com.ing.baker") @@ -17,7 +18,7 @@ object BakerLogging { case class BakerLogging(logger: Logger = BakerLogging.defaultLogger) { - val durationFormatter: PeriodFormatter = new PeriodFormatterBuilder() + private val durationFormatter: PeriodFormatter = new PeriodFormatterBuilder() .appendDays.appendSuffix("d") .appendSeparator(" ") .appendHours.appendSuffix("h") @@ -30,10 +31,54 @@ case class BakerLogging(logger: Logger = BakerLogging.defaultLogger) { .appendSeparator(" ") .toFormatter + private val RecipeNameKey = "recipeName" + private val RecipeIdKey = "recipeId" + private val RecipeInstanceIdKey = "recipeInstanceId" + + private val BakerRecipeNameKey = "baker.recipe.name" + private val BakerRecipeIdKey = "baker.recipe.id" + private val BakerRecipeInstanceIdKey = "baker.recipe.instance.id" + private val TraceIdKey = "trace_id" + private val SpanIdKey = "span_id" + + private def enrichMdc(mdc: Map[String, String]): Map[String, String] = { + val withRecipeNameAliases = mdc.get(RecipeNameKey) + .map(v => mdc + (BakerRecipeNameKey -> v)) + .getOrElse(mdc) + + val withRecipeIdAliases = withRecipeNameAliases.get(RecipeIdKey) + .map(v => withRecipeNameAliases + (BakerRecipeIdKey -> v)) + .getOrElse(withRecipeNameAliases) + + val withInstanceAliases = withRecipeIdAliases.get(RecipeInstanceIdKey) + .map(v => withRecipeIdAliases + (BakerRecipeInstanceIdKey -> v)) + .getOrElse(withRecipeIdAliases) + + // Trace ids attached to the current thread (set by the interaction instrumentation) are exact, the + // per-recipe-instance registry is only a best effort fallback for log lines emitted by Baker itself. + val currentThreadTrace = for { + traceId <- Option(MDC.get(TraceIdKey)) + spanId <- Option(MDC.get(SpanIdKey)) + } yield (traceId, spanId) + + currentThreadTrace + .orElse( + withInstanceAliases.get(RecipeInstanceIdKey) + .flatMap(id => TracingMdcRegistry.get(id).toScala) + .map(context => (context.getTraceId, context.getSpanId))) + .map { case (traceId, spanId) => withInstanceAliases + (TraceIdKey -> traceId) + (SpanIdKey -> spanId) } + .getOrElse(withInstanceAliases) + } + def withMDC(mdc: Map[String, String], log: Logger => Unit): Unit = { - mdc.foreach { case (k, v) => MDC.put(k, v) } - log(logger) - mdc.keys.foreach(MDC.remove) + val effectiveMdc = enrichMdc(mdc) + val previous = effectiveMdc.keys.map(key => key -> Option(MDC.get(key))).toList + effectiveMdc.foreach { case (k, v) => MDC.put(k, v) } + try log(logger) + finally previous.foreach { + case (key, Some(value)) => MDC.put(key, value) + case (key, None) => MDC.remove(key) + } } def addedRecipe(recipeAdded: RecipeAdded): Unit = { diff --git a/core/baker-interface/src/main/scala/com/ing/baker/runtime/model/InteractionManager.scala b/core/baker-interface/src/main/scala/com/ing/baker/runtime/model/InteractionManager.scala index 00d6cc115..c1ec6e696 100644 --- a/core/baker-interface/src/main/scala/com/ing/baker/runtime/model/InteractionManager.scala +++ b/core/baker-interface/src/main/scala/com/ing/baker/runtime/model/InteractionManager.scala @@ -79,14 +79,29 @@ trait InteractionManager[F[_]] { def existsFor(interaction: InteractionTransition)(implicit sync: SyncSupport[F]): F[Boolean] = sync.map(findFor(interaction))(_.nonEmpty) - def execute(interaction: InteractionTransition, input: Seq[IngredientInstance], metadata: Option[Map[String, String]])(implicit sync: SyncSupport[F]): F[Option[EventInstance]] = { + def execute(interaction: InteractionTransition, + input: Seq[IngredientInstance], + metadata: Option[Map[String, String]], + callContext: Option[InteractionCallContext] = None, + instrumentation: InteractionInstrumentation = InteractionInstrumentation.noop())(implicit sync: SyncSupport[F]): F[Option[EventInstance]] = { if(interaction.interactionName.startsWith(checkpointEventInteractionPrefix)){ sync.pure(Some(EventInstance(interaction.interactionName.stripPrefix(checkpointEventInteractionPrefix)))) } else{ sync.flatMap(findFor(interaction)) { case Some(implementation) => { // Interaction implementations are often reflection/IO heavy; shift invocation to the blocking pool. - sync.flatMap(sync.blocking(implementation.execute(input, metadata.getOrElse(Map()))))(identity) + // The instrumentation hook runs on that very thread, so tracing context attached by it is strictly + // scoped to this single invocation. + sync.flatMap(sync.blocking { + def invoke: F[Option[EventInstance]] = implementation.execute(input, metadata.getOrElse(Map())) + callContext match { + case Some(context) => + val mdcScope = InteractionMdc.install(context) + try instrumentation.aroundInteraction[F[Option[EventInstance]]](context, () => invoke) + finally mdcScope.close() + case None => invoke + } + })(identity) } case None => sync.raiseError(new FatalInteractionException(s"No implementation available for interaction ${interaction.interactionName}")) } diff --git a/core/baker-interface/src/main/scala/com/ing/baker/runtime/model/recipeinstance/TransitionExecution.scala b/core/baker-interface/src/main/scala/com/ing/baker/runtime/model/recipeinstance/TransitionExecution.scala index d21ee8134..b4f6e5195 100644 --- a/core/baker-interface/src/main/scala/com/ing/baker/runtime/model/recipeinstance/TransitionExecution.scala +++ b/core/baker-interface/src/main/scala/com/ing/baker/runtime/model/recipeinstance/TransitionExecution.scala @@ -7,7 +7,7 @@ import com.ing.baker.il.{CompiledRecipe, IngredientDescriptor} import com.ing.baker.petrinet.api._ import com.ing.baker.runtime.catseffect.AsyncSupport import com.ing.baker.runtime.catseffect.SyncSupport.syntax._ -import com.ing.baker.runtime.model.BakerComponents +import com.ing.baker.runtime.model.{BakerComponents, InteractionCallContext} import com.ing.baker.runtime.model.recipeinstance.RecipeInstance.FatalInteractionException import com.ing.baker.runtime.scaladsl._ import com.ing.baker.types.{ListValue, PrimitiveValue, Value} @@ -157,21 +157,46 @@ private[recipeinstance] case class TransitionExecution( } } - def setupMdc: F[Unit] = async.delay { + def setupMdc(): Unit = { MDC.put("recipeInstanceId", recipeInstanceId) MDC.put("recipeName", recipe.name) + MDC.put("baker.recipe.instance.id", recipeInstanceId) + MDC.put("baker.recipe.name", recipe.name) + MDC.put("baker.recipe.id", recipe.recipeId) } - def cleanMdc: F[Unit] = async.delay { + def cleanMdc(): Unit = { MDC.remove("recipeInstanceId") MDC.remove("recipeName") + MDC.remove("baker.recipe.instance.id") + MDC.remove("baker.recipe.name") + MDC.remove("baker.recipe.id") } - def execute: F[Option[EventInstance]] = - components.interactions.execute( - interactionTransition, - buildInteractionInput, - Some(recipeInstanceMetadata)) + val callContext: InteractionCallContext = + new InteractionCallContext( + recipe.recipeId, + recipe.name, + recipeInstanceId, + interactionTransition.interactionName, + id, + correlationId.orNull) + + /** The MDC is set while the interaction effect is built, which is exactly when the invocation is handed to + * the blocking pool. The propagators capture it there and re-install it on the executing thread, after + * which it is removed again on the thread that built the effect. + */ + def execute: F[Option[EventInstance]] = { + setupMdc() + try + components.interactions.execute( + interactionTransition, + buildInteractionInput, + Some(recipeInstanceMetadata), + Some(callContext), + components.instrumentation) + finally cleanMdc() + } for { @@ -181,11 +206,7 @@ private[recipeinstance] case class TransitionExecution( interactionStarted <- async.delay(InteractionStarted(startTime, recipe.name, recipe.recipeId, recipeInstanceId, interactionTransition.interactionName)) _ <- async.delay(components.logging.interactionStarted(interactionStarted)) _ <- components.eventStream.publish(interactionStarted) - interactionOutput <- - setupMdc.map(_ => execute).flatMap(x => { - cleanMdc - x - }) + interactionOutput <- execute _ <- validateInteractionOutput(interactionTransition, interactionOutput) transformedOutput: Option[EventInstance] = interactionOutput.map(_.transformWith(interactionTransition)) diff --git a/pom.xml b/pom.xml index f7b3307dd..8288dc7d9 100644 --- a/pom.xml +++ b/pom.xml @@ -46,9 +46,10 @@ 2.3.21 1.11.0 1.10.0 - 1.5.37 + 1.6.5 4.2.39 3.5 + 1.67.0 2.8.3 0.16.0 0.20.0 @@ -108,6 +109,7 @@ core/baker-types core/baker-intermediate-language core/baker-interface + core/baker-interface-otel core/baker-interface-kotlin core/baker-interface-in-memory core/baker-annotations