diff --git a/dd-sdk-android-core/api/apiSurface b/dd-sdk-android-core/api/apiSurface index 67f4f7815a..8e71ee8113 100644 --- a/dd-sdk-android-core/api/apiSurface +++ b/dd-sdk-android-core/api/apiSurface @@ -381,8 +381,8 @@ open class com.datadog.android.core.sampling.DeterministicSampler : Samp override fun getSampleRate(): Float companion object const val SAMPLE_ALL_RATE: Float - const val SAMPLER_HASHER: ULong - const val MAX_ID: ULong + DEPRECATED const val SAMPLER_HASHER: ULong + DEPRECATED const val MAX_ID: ULong open class com.datadog.android.core.sampling.RateBasedSampler : Sampler constructor(() -> Float) constructor(Float) @@ -448,6 +448,7 @@ object com.datadog.android.log.LogAttributes const val RUM_SESSION_ID: String const val RUM_VIEW_ID: String const val RUM_ACTION_ID: String + const val RUM_SESSION_SAMPLE_RATE: String const val SERVICE_NAME: String const val SOURCE: String const val STATUS: String diff --git a/dd-sdk-android-core/api/dd-sdk-android-core.api b/dd-sdk-android-core/api/dd-sdk-android-core.api index 7b9f86b628..ad1ed6c2a7 100644 --- a/dd-sdk-android-core/api/dd-sdk-android-core.api +++ b/dd-sdk-android-core/api/dd-sdk-android-core.api @@ -1081,6 +1081,7 @@ public final class com/datadog/android/log/LogAttributes { public static final field RUM_ACTION_ID Ljava/lang/String; public static final field RUM_APPLICATION_ID Ljava/lang/String; public static final field RUM_SESSION_ID Ljava/lang/String; + public static final field RUM_SESSION_SAMPLE_RATE Ljava/lang/String; public static final field RUM_VIEW_ID Ljava/lang/String; public static final field SERVICE Ljava/lang/String; public static final field SERVICE_NAME Ljava/lang/String; diff --git a/dd-sdk-android-core/src/main/kotlin/com/datadog/android/core/sampling/DeterministicSampler.kt b/dd-sdk-android-core/src/main/kotlin/com/datadog/android/core/sampling/DeterministicSampler.kt index 5256e6d160..db58c974b1 100644 --- a/dd-sdk-android-core/src/main/kotlin/com/datadog/android/core/sampling/DeterministicSampler.kt +++ b/dd-sdk-android-core/src/main/kotlin/com/datadog/android/core/sampling/DeterministicSampler.kt @@ -8,6 +8,7 @@ package com.datadog.android.core.sampling import androidx.annotation.FloatRange import com.datadog.android.api.InternalLogger +import com.datadog.android.internal.sampling.computeSamplingDecision /** * [Sampler] with the given sample rate using a deterministic algorithm for a stable @@ -46,19 +47,8 @@ open class DeterministicSampler( ) : this(idConverter, sampleRate.toFloat()) /** @inheritDoc */ - override fun sample(item: T): Boolean { - val sampleRate = getSampleRate() - - return when { - sampleRate >= SAMPLE_ALL_RATE -> true - sampleRate <= 0f -> false - else -> { - val hash = idConverter(item) * SAMPLER_HASHER - val threshold = (MAX_ID.toDouble() * sampleRate / SAMPLE_ALL_RATE).toULong() - hash < threshold - } - } - } + override fun sample(item: T): Boolean = + computeSamplingDecision(getSampleRate(), idConverter(item)) /** @inheritDoc */ override fun getSampleRate(): Float { @@ -93,11 +83,13 @@ open class DeterministicSampler( * within the [DeterministicSampler] implementation. This value is a good number for * Knuth hashing (large, prime, fit in 64 bit long). */ + @Deprecated("RUM-13454: Implementation detail, will be removed in v4 (RUM-15590).") const val SAMPLER_HASHER: ULong = 1111111111111111111u /** * The maximum value used as an upper limit for computing hash-based sampling thresholds. */ + @Deprecated("RUM-13454: Implementation detail, will be removed in v4 (RUM-15590).") const val MAX_ID: ULong = 0xFFFFFFFFFFFFFFFFUL } } diff --git a/dd-sdk-android-core/src/main/kotlin/com/datadog/android/log/LogAttributes.kt b/dd-sdk-android-core/src/main/kotlin/com/datadog/android/log/LogAttributes.kt index d8dc134212..6c29de0755 100644 --- a/dd-sdk-android-core/src/main/kotlin/com/datadog/android/log/LogAttributes.kt +++ b/dd-sdk-android-core/src/main/kotlin/com/datadog/android/log/LogAttributes.kt @@ -279,6 +279,13 @@ object LogAttributes { */ const val RUM_ACTION_ID: String = "user_action.id" + /** + * Key for the RUM session sample rate used for cross-product trace sampling rebasing. + * This key is written to spans by [com.datadog.android.trace.internal.RumContextPropagator] + * and read by [com.datadog.android.trace.DeterministicTraceSampler]. + */ + const val RUM_SESSION_SAMPLE_RATE: String = "session_sample_rate" + /** * The name of the application or service generating the log events. (String) * This value is filled automatically by the [Logger]. diff --git a/dd-sdk-android-internal/api/apiSurface b/dd-sdk-android-internal/api/apiSurface index 4b8e2154b8..148c56f265 100644 --- a/dd-sdk-android-internal/api/apiSurface +++ b/dd-sdk-android-internal/api/apiSurface @@ -164,6 +164,7 @@ sealed class com.datadog.android.internal.profiling.ProfilerStopEvent constructor(TTIDRumContext? = null) data class com.datadog.android.internal.profiling.TTIDRumContext constructor(String, String, String, String?, String?, String?) +fun computeSamplingDecision(Float, ULong): Boolean interface com.datadog.android.internal.system.BuildSdkVersionProvider val version: Int val isAtLeastN: Boolean diff --git a/dd-sdk-android-internal/api/dd-sdk-android-internal.api b/dd-sdk-android-internal/api/dd-sdk-android-internal.api index b985828117..e41ade5993 100644 --- a/dd-sdk-android-internal/api/dd-sdk-android-internal.api +++ b/dd-sdk-android-internal/api/dd-sdk-android-internal.api @@ -310,6 +310,10 @@ public final class com/datadog/android/internal/profiling/TTIDRumContext { public fun toString ()Ljava/lang/String; } +public final class com/datadog/android/internal/sampling/DeterministicSamplingKt { + public static final fun computeSamplingDecision-2TYgG_w (FJ)Z +} + public abstract interface class com/datadog/android/internal/system/BuildSdkVersionProvider { public static final field Companion Lcom/datadog/android/internal/system/BuildSdkVersionProvider$Companion; public abstract fun getVersion ()I diff --git a/dd-sdk-android-internal/src/main/java/com/datadog/android/internal/sampling/DeterministicSampling.kt b/dd-sdk-android-internal/src/main/java/com/datadog/android/internal/sampling/DeterministicSampling.kt new file mode 100644 index 0000000000..e3679dcd4d --- /dev/null +++ b/dd-sdk-android-internal/src/main/java/com/datadog/android/internal/sampling/DeterministicSampling.kt @@ -0,0 +1,30 @@ +/* + * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. + * This product includes software developed at Datadog (https://www.datadoghq.com/). + * Copyright 2016-Present Datadog, Inc. + */ + +package com.datadog.android.internal.sampling + +private const val SAMPLE_ALL_RATE: Float = 100f +private const val SAMPLER_HASHER: ULong = 1111111111111111111u +private const val MAX_ID: ULong = 0xFFFFFFFFFFFFFFFFUL + +/** + * Computes a deterministic sampling decision based on the given sample rate and identifier. + * + * @param sampleRate the sample rate in the range [0, 100]. + * @param id a stable numerical identifier derived from the item being sampled. + * @return true if the item should be sampled, false otherwise. + */ +fun computeSamplingDecision(sampleRate: Float, id: ULong): Boolean { + return when { + sampleRate >= SAMPLE_ALL_RATE -> true + sampleRate <= 0f -> false + else -> { + val hash = id * SAMPLER_HASHER + val threshold = (MAX_ID.toDouble() * sampleRate / SAMPLE_ALL_RATE).toULong() + hash < threshold + } + } +} diff --git a/dd-sdk-android-internal/src/test/java/com/datadog/android/internal/sampling/DeterministicSamplingTest.kt b/dd-sdk-android-internal/src/test/java/com/datadog/android/internal/sampling/DeterministicSamplingTest.kt new file mode 100644 index 0000000000..b34221d087 --- /dev/null +++ b/dd-sdk-android-internal/src/test/java/com/datadog/android/internal/sampling/DeterministicSamplingTest.kt @@ -0,0 +1,67 @@ +/* + * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. + * This product includes software developed at Datadog (https://www.datadoghq.com/). + * Copyright 2016-Present Datadog, Inc. + */ + +package com.datadog.android.internal.sampling + +import fr.xgouchet.elmyr.annotation.FloatForgery +import fr.xgouchet.elmyr.annotation.LongForgery +import fr.xgouchet.elmyr.junit5.ForgeExtension +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.RepeatedTest +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.extension.ExtendWith +import org.junit.jupiter.api.extension.Extensions +import org.mockito.junit.jupiter.MockitoExtension +import org.mockito.junit.jupiter.MockitoSettings +import org.mockito.quality.Strictness + +@Extensions( + ExtendWith(MockitoExtension::class), + ExtendWith(ForgeExtension::class) +) +@MockitoSettings(strictness = Strictness.LENIENT) +internal class DeterministicSamplingTest { + + @RepeatedTest(32) + fun `M always return true W computeSamplingDecision() {sampleRate is 100}`( + @LongForgery fakeId: Long + ) { + assertThat(computeSamplingDecision(100f, fakeId.toULong())).isTrue() + } + + @RepeatedTest(32) + fun `M always return true W computeSamplingDecision() {sampleRate above 100}`( + @FloatForgery(min = 100.01f, max = 200f) fakeSampleRate: Float, + @LongForgery fakeId: Long + ) { + assertThat(computeSamplingDecision(fakeSampleRate, fakeId.toULong())).isTrue() + } + + @RepeatedTest(32) + fun `M always return false W computeSamplingDecision() {sampleRate is 0}`( + @LongForgery fakeId: Long + ) { + assertThat(computeSamplingDecision(0f, fakeId.toULong())).isFalse() + } + + @RepeatedTest(32) + fun `M always return false W computeSamplingDecision() {sampleRate below 0}`( + @FloatForgery(min = -100f, max = -0.01f) fakeSampleRate: Float, + @LongForgery fakeId: Long + ) { + assertThat(computeSamplingDecision(fakeSampleRate, fakeId.toULong())).isFalse() + } + + @Test + fun `M return deterministic result W computeSamplingDecision() {same id same rate}`( + @FloatForgery(min = 1f, max = 99f) fakeSampleRate: Float, + @LongForgery fakeId: Long + ) { + val first = computeSamplingDecision(fakeSampleRate, fakeId.toULong()) + val second = computeSamplingDecision(fakeSampleRate, fakeId.toULong()) + assertThat(first).isEqualTo(second) + } +} diff --git a/detekt_custom_safe_calls.yml b/detekt_custom_safe_calls.yml index 6a10420215..923a1ff35b 100644 --- a/detekt_custom_safe_calls.yml +++ b/detekt_custom_safe_calls.yml @@ -1132,6 +1132,7 @@ datadog: - "kotlin.Double.toInt()" - "kotlin.Double.toLong()" - "kotlin.Double.toULong()" + - "kotlin.Float.coerceAtMost(kotlin.Float)" - "kotlin.Float.fromBits(kotlin.Int)" - "kotlin.Float.percent()" - "kotlin.Float.roundToInt()" diff --git a/features/dd-sdk-android-rum/src/main/kotlin/com/datadog/android/rum/internal/domain/RumContext.kt b/features/dd-sdk-android-rum/src/main/kotlin/com/datadog/android/rum/internal/domain/RumContext.kt index 44e59599ea..d311ec6163 100644 --- a/features/dd-sdk-android-rum/src/main/kotlin/com/datadog/android/rum/internal/domain/RumContext.kt +++ b/features/dd-sdk-android-rum/src/main/kotlin/com/datadog/android/rum/internal/domain/RumContext.kt @@ -25,7 +25,8 @@ internal data class RumContext( val syntheticsResultId: String? = null, val viewTimestamp: Long = 0L, val viewTimestampOffset: Long = 0L, - val hasReplay: Boolean = false + val hasReplay: Boolean = false, + val sessionSampleRate: Float = FULL_SESSION_SAMPLE_RATE ) { fun toMap(): Map { @@ -44,7 +45,8 @@ internal data class RumContext( SYNTHETICS_RESULT_ID to syntheticsResultId, VIEW_TIMESTAMP to viewTimestamp, HAS_REPLAY to hasReplay, - VIEW_TIMESTAMP_OFFSET to viewTimestampOffset + VIEW_TIMESTAMP_OFFSET to viewTimestampOffset, + SESSION_SAMPLE_RATE to sessionSampleRate ) } @@ -68,6 +70,8 @@ internal data class RumContext( const val HAS_REPLAY = "view_has_replay" const val VIEW_TIMESTAMP = "view_timestamp" const val VIEW_TIMESTAMP_OFFSET = "view_timestamp_offset" + const val SESSION_SAMPLE_RATE = "session_sample_rate" + const val FULL_SESSION_SAMPLE_RATE: Float = 100f fun fromFeatureContext(featureContext: Map): RumContext { val applicationId = featureContext[APPLICATION_ID] as? String @@ -89,6 +93,8 @@ internal data class RumContext( val hasReplay = featureContext[HAS_REPLAY] as? Boolean ?: false val viewTimestamp = featureContext[VIEW_TIMESTAMP] as? Long ?: 0L val viewTimestampOffset = featureContext[VIEW_TIMESTAMP_OFFSET] as? Long ?: 0L + val sessionSampleRate = (featureContext[SESSION_SAMPLE_RATE] as? Number) + ?.toFloat() ?: FULL_SESSION_SAMPLE_RATE return RumContext( applicationId = applicationId ?: NULL_UUID, @@ -105,7 +111,8 @@ internal data class RumContext( syntheticsResultId = syntheticsResultId, viewTimestamp = viewTimestamp, viewTimestampOffset = viewTimestampOffset, - hasReplay = hasReplay + hasReplay = hasReplay, + sessionSampleRate = sessionSampleRate ) } } diff --git a/features/dd-sdk-android-rum/src/main/kotlin/com/datadog/android/rum/internal/domain/scope/RumSessionScope.kt b/features/dd-sdk-android-rum/src/main/kotlin/com/datadog/android/rum/internal/domain/scope/RumSessionScope.kt index e8004abe75..f6fe7d4c1f 100644 --- a/features/dd-sdk-android-rum/src/main/kotlin/com/datadog/android/rum/internal/domain/scope/RumSessionScope.kt +++ b/features/dd-sdk-android-rum/src/main/kotlin/com/datadog/android/rum/internal/domain/scope/RumSessionScope.kt @@ -65,6 +65,7 @@ internal class RumSessionScope( internal var sessionId = RumContext.NULL_UUID internal var sessionState: State = State.NOT_TRACKED + internal var sessionSampleRate: Float = sessionSampler.getSampleRate() ?: RumContext.FULL_SESSION_SAMPLE_RATE private var startReason: StartReason = StartReason.USER_APP_LAUNCH internal var isActive: Boolean = true private val sessionStartNs = AtomicLong(sdkCore.timeProvider.getDeviceElapsedTimeNanos()) @@ -211,7 +212,8 @@ internal class RumSessionScope( sessionId = sessionId, sessionState = sessionState, sessionStartReason = startReason, - isSessionActive = isActive + isSessionActive = isActive, + sessionSampleRate = sessionSampleRate ) } diff --git a/features/dd-sdk-android-rum/src/test/kotlin/com/datadog/android/rum/internal/domain/RumContextTest.kt b/features/dd-sdk-android-rum/src/test/kotlin/com/datadog/android/rum/internal/domain/RumContextTest.kt index 74829c4568..6ffdfde5ff 100644 --- a/features/dd-sdk-android-rum/src/test/kotlin/com/datadog/android/rum/internal/domain/RumContextTest.kt +++ b/features/dd-sdk-android-rum/src/test/kotlin/com/datadog/android/rum/internal/domain/RumContextTest.kt @@ -7,11 +7,13 @@ package com.datadog.android.rum.internal.domain import com.datadog.android.rum.utils.forge.Configurator +import fr.xgouchet.elmyr.Forge import fr.xgouchet.elmyr.annotation.Forgery import fr.xgouchet.elmyr.junit5.ForgeConfiguration import fr.xgouchet.elmyr.junit5.ForgeExtension import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.RepeatedTest +import org.junit.jupiter.api.Test import org.junit.jupiter.api.extension.ExtendWith @ExtendWith(ForgeExtension::class) @@ -28,4 +30,19 @@ internal class RumContextTest { // Then assertThat(anotherRumContext).isEqualTo(fakeRumContext) } + + @Test + fun `M parse session sample rate W fromFeatureContext() {value is Double}`(forge: Forge) { + // Given: use an explicit Double to verify that as? Number handles Double (as? Float would return null) + val fakeDouble: Double = forge.aDouble(min = 0.0, max = 100.0) + val featureContext = mapOf( + RumContext.SESSION_SAMPLE_RATE to fakeDouble + ) + + // When + val rumContext = RumContext.fromFeatureContext(featureContext) + + // Then + assertThat(rumContext.sessionSampleRate).isEqualTo(fakeDouble.toFloat()) + } } diff --git a/features/dd-sdk-android-rum/src/test/kotlin/com/datadog/android/rum/internal/domain/scope/RumSessionScopeTest.kt b/features/dd-sdk-android-rum/src/test/kotlin/com/datadog/android/rum/internal/domain/scope/RumSessionScopeTest.kt index 4b372e91c4..3b09645afe 100644 --- a/features/dd-sdk-android-rum/src/test/kotlin/com/datadog/android/rum/internal/domain/scope/RumSessionScopeTest.kt +++ b/features/dd-sdk-android-rum/src/test/kotlin/com/datadog/android/rum/internal/domain/scope/RumSessionScopeTest.kt @@ -266,6 +266,16 @@ internal class RumSessionScopeTest { assertThat(childScope?.sampleRate).isCloseTo(fakeSampleRate, offset(0.001f)) } + @Test + fun `M use full session sample rate W init() { getSampleRate returns null }`() { + // Given + whenever(mockSessionSampler.getSampleRate()).thenReturn(null) + initializeTestedScope(withMockChildScope = false) + + // Then + assertThat(testedScope.sessionSampleRate).isEqualTo(RumContext.FULL_SESSION_SAMPLE_RATE) + } + @Test fun `M delegate events to child scope W handleViewEvent() {TRACKED}`( forge: Forge @@ -478,6 +488,23 @@ internal class RumSessionScopeTest { assertThat(context.viewId).isEqualTo(fakeParentContext.viewId) } + @Test + fun `M use full session sample rate W getRumContext() {getSampleRate returns null}`( + forge: Forge + ) { + // Given + whenever(mockSessionSampler.getSampleRate()).thenReturn(null) + whenever(mockSessionSampler.sample(any())).thenReturn(true) + initializeTestedScope() + + // When + testedScope.handleEvent(forge.startViewEvent(), fakeDatadogContext, mockEventWriteScope, mockWriter) + val context = testedScope.getRumContext() + + // Then + assertThat(context.sessionSampleRate).isEqualTo(RumContext.FULL_SESSION_SAMPLE_RATE) + } + @Test fun `M set TRACKED W renewSession() {sampler returns true}`(forge: Forge) { // Given diff --git a/features/dd-sdk-android-trace/api/apiSurface b/features/dd-sdk-android-trace/api/apiSurface index 6d1366fa7d..c2e8905b40 100644 --- a/features/dd-sdk-android-trace/api/apiSurface +++ b/features/dd-sdk-android-trace/api/apiSurface @@ -22,6 +22,8 @@ open class com.datadog.android.trace.DeterministicTraceSampler : com.datadog.and constructor(() -> Float) constructor(Float) constructor(Double) + override fun sample(com.datadog.android.trace.api.span.DatadogSpan): Boolean + DEPRECATED fun getSampleRate(com.datadog.android.trace.api.span.DatadogSpan): Float annotation com.datadog.android.trace.ExperimentalTraceApi object com.datadog.android.trace.GlobalDatadogTracer fun registerIfAbsent(com.datadog.android.trace.api.tracer.DatadogTracer): Boolean diff --git a/features/dd-sdk-android-trace/api/dd-sdk-android-trace.api b/features/dd-sdk-android-trace/api/dd-sdk-android-trace.api index 8f08cafab3..2dedb32abe 100644 --- a/features/dd-sdk-android-trace/api/dd-sdk-android-trace.api +++ b/features/dd-sdk-android-trace/api/dd-sdk-android-trace.api @@ -34,6 +34,9 @@ public class com/datadog/android/trace/DeterministicTraceSampler : com/datadog/a public fun (D)V public fun (F)V public fun (Lkotlin/jvm/functions/Function0;)V + public final fun getSampleRate (Lcom/datadog/android/trace/api/span/DatadogSpan;)F + public fun sample (Lcom/datadog/android/trace/api/span/DatadogSpan;)Z + public synthetic fun sample (Ljava/lang/Object;)Z } public abstract interface annotation class com/datadog/android/trace/ExperimentalTraceApi : java/lang/annotation/Annotation { diff --git a/features/dd-sdk-android-trace/src/main/kotlin/com/datadog/android/trace/ApmNetworkInstrumentationConfiguration.kt b/features/dd-sdk-android-trace/src/main/kotlin/com/datadog/android/trace/ApmNetworkInstrumentationConfiguration.kt index 56e630cdf9..4364c27e08 100644 --- a/features/dd-sdk-android-trace/src/main/kotlin/com/datadog/android/trace/ApmNetworkInstrumentationConfiguration.kt +++ b/features/dd-sdk-android-trace/src/main/kotlin/com/datadog/android/trace/ApmNetworkInstrumentationConfiguration.kt @@ -234,7 +234,8 @@ class ApmNetworkInstrumentationConfiguration internal constructor( internal companion object { internal const val ALL_IN_SAMPLE_RATE: Double = 100.0 - internal const val DEFAULT_TRACE_SAMPLE_RATE: Float = 100f + private const val DEFAULT_TRACE_SAMPLE_RATE: Float = 100f + internal const val NETWORK_REQUESTS_TRACKING_FEATURE_NAME = "Network Requests" internal fun ApmNetworkInstrumentationConfiguration.createInstrumentation( diff --git a/features/dd-sdk-android-trace/src/main/kotlin/com/datadog/android/trace/DeterministicTraceSampler.kt b/features/dd-sdk-android-trace/src/main/kotlin/com/datadog/android/trace/DeterministicTraceSampler.kt index 3481222e33..68b99b45e6 100644 --- a/features/dd-sdk-android-trace/src/main/kotlin/com/datadog/android/trace/DeterministicTraceSampler.kt +++ b/features/dd-sdk-android-trace/src/main/kotlin/com/datadog/android/trace/DeterministicTraceSampler.kt @@ -8,15 +8,30 @@ package com.datadog.android.trace import androidx.annotation.FloatRange import com.datadog.android.core.sampling.DeterministicSampler +import com.datadog.android.internal.sampling.computeSamplingDecision +import com.datadog.android.log.LogAttributes import com.datadog.android.trace.api.span.DatadogSpan import com.datadog.android.trace.internal.net.SpanSamplingIdProvider /** - * A [com.datadog.android.core.sampling.DeterministicSampler] using the TraceID of a Span to compute the sampling decision. + * A [com.datadog.android.core.sampling.DeterministicSampler] using the TraceID of a Span to compute the sampling + * decision. * - * @param sampleRateProvider Provider for the sample rate value which will be called each time - * the sampling decision needs to be made. All the values should be in the range [0;100]. + * When a span carries a RUM session context (written by + * [com.datadog.android.trace.internal.RumContextPropagator]), the sampling threshold is + * automatically rebased against the RUM session sample rate so that the combined effective + * sampling probability reflects: + * + * ``` + * effectiveRate = traceSampleRate * sessionSampleRate / 100 + * ``` + * + * Spans without a `session_sample_rate` tag are sampled at the raw trace sample rate. + * + * @param sampleRateProvider Provider for the trace sample rate. Called each time the sampling + * decision needs to be made. Values must be in the range [0;100]. */ +// TODO RUM-13454 -> Make this class internal in V4 (RUM-15590) open class DeterministicTraceSampler( sampleRateProvider: () -> Float ) : DeterministicSampler( @@ -25,7 +40,7 @@ open class DeterministicTraceSampler( ) { /** - * Creates a new instance of [DeterministicSampler] with the given sample rate. + * Creates a new instance of [DeterministicTraceSampler] with the given sample rate. * * @param sampleRate Sample rate to use. */ @@ -34,11 +49,53 @@ open class DeterministicTraceSampler( ) : this({ sampleRate }) /** - * Creates a new instance of [DeterministicSampler] with the given sample rate. + * Creates a new instance of [DeterministicTraceSampler] with the given sample rate. * * @param sampleRate Sample rate to use. */ constructor( @FloatRange(from = 0.0, to = 100.0) sampleRate: Double ) : this(sampleRate.toFloat()) + + /** @inheritDoc */ + override fun sample(item: DatadogSpan): Boolean = + computeSamplingDecision(resolveEffectiveSampleRate(item), SpanSamplingIdProvider.provideId(item)) + + /** + * Returns the effective sample rate for [item], applying cross-product rebasing when the + * span carries a `session_sample_rate` tag (written by + * [com.datadog.android.trace.internal.RumContextPropagator]). + * + * When no tag is present the raw trace sample rate is returned unchanged. + */ + // TODO RUM-13454 -> Remove this in V4 when deprecating legacy paths (RUM-15590) + @Deprecated( + "Will be removed in v4.", + level = DeprecationLevel.WARNING + ) + fun getSampleRate(item: DatadogSpan): Float { + return resolveEffectiveSampleRate(item) + } + + // region private + + private fun resolveEffectiveSampleRate(item: DatadogSpan): Float { + val traceSampleRate = super.getSampleRate() + val sessionSampleRate = resolveSessionSampleRate(item) + return if (sessionSampleRate != null && + sessionSampleRate >= 0f && + sessionSampleRate < DeterministicSampler.SAMPLE_ALL_RATE + ) { + (traceSampleRate * sessionSampleRate / DeterministicSampler.SAMPLE_ALL_RATE) + .coerceAtMost(DeterministicSampler.SAMPLE_ALL_RATE) + } else { + traceSampleRate + } + } + + private fun resolveSessionSampleRate(item: DatadogSpan): Float? { + return (item.context().tags[LogAttributes.RUM_SESSION_SAMPLE_RATE] as? Number)?.toFloat() + } + + // endregion } diff --git a/features/dd-sdk-android-trace/src/main/kotlin/com/datadog/android/trace/internal/ApmNetworkInstrumentation.kt b/features/dd-sdk-android-trace/src/main/kotlin/com/datadog/android/trace/internal/ApmNetworkInstrumentation.kt index f721cfb51d..16a4bfa3c1 100644 --- a/features/dd-sdk-android-trace/src/main/kotlin/com/datadog/android/trace/internal/ApmNetworkInstrumentation.kt +++ b/features/dd-sdk-android-trace/src/main/kotlin/com/datadog/android/trace/internal/ApmNetworkInstrumentation.kt @@ -32,6 +32,7 @@ import com.datadog.android.trace.internal.net.RequestTracingState import com.datadog.android.trace.internal.net.TracerProvider import com.datadog.android.trace.internal.net.applyPriority import com.datadog.android.trace.internal.net.buildSpan +import com.datadog.android.trace.internal.net.effectiveSampleRate import com.datadog.android.trace.internal.net.finishRumAware import com.datadog.android.trace.internal.net.sample import java.net.HttpURLConnection @@ -73,6 +74,7 @@ class ApmNetworkInstrumentation internal constructor( val networkTracingScope: ApmNetworkTracingScope = ApmNetworkTracingScope.ALL ) { private val rumContextPropagator = RumContextPropagator { internalSdkCore } + private val internalSdkCore: InternalSdkCore? get() = sdkCoreReference.get() as? InternalSdkCore @@ -148,7 +150,7 @@ class ApmNetworkInstrumentation internal constructor( return RequestTracingState( span = span, isSampled = isSampled, - sampleRate = traceSampler.getSampleRate(), + sampleRate = traceSampler.effectiveSampleRate(span), requestInfoBuilder = tracedRequestInfoBuilder ) } diff --git a/features/dd-sdk-android-trace/src/main/kotlin/com/datadog/android/trace/internal/RumContextPropagator.kt b/features/dd-sdk-android-trace/src/main/kotlin/com/datadog/android/trace/internal/RumContextPropagator.kt index 342e3faf1b..f7fb3ba91e 100644 --- a/features/dd-sdk-android-trace/src/main/kotlin/com/datadog/android/trace/internal/RumContextPropagator.kt +++ b/features/dd-sdk-android-trace/src/main/kotlin/com/datadog/android/trace/internal/RumContextPropagator.kt @@ -62,6 +62,10 @@ class RumContextPropagator(private val sdkCoreProvider: () -> FeatureSdkCore?) { instance.setTag(LogAttributes.RUM_SESSION_ID, rumContext["session_id"]) instance.setTag(LogAttributes.RUM_VIEW_ID, rumContext["view_id"]) instance.setTag(LogAttributes.RUM_ACTION_ID, rumContext["action_id"]) + instance.setTag( + LogAttributes.RUM_SESSION_SAMPLE_RATE, + rumContext[LogAttributes.RUM_SESSION_SAMPLE_RATE] + ) instance.setTag(HttpCodec.RUM_KEY_USER_ID, datadogContext.userInfo.id) instance.setTag(HttpCodec.RUM_KEY_ACCOUNT_ID, datadogContext.accountInfo?.id) } diff --git a/features/dd-sdk-android-trace/src/main/kotlin/com/datadog/android/trace/internal/net/ApmNetworkInstrumentationExt.kt b/features/dd-sdk-android-trace/src/main/kotlin/com/datadog/android/trace/internal/net/ApmNetworkInstrumentationExt.kt index b3bb44280b..e36f1793e9 100644 --- a/features/dd-sdk-android-trace/src/main/kotlin/com/datadog/android/trace/internal/net/ApmNetworkInstrumentationExt.kt +++ b/features/dd-sdk-android-trace/src/main/kotlin/com/datadog/android/trace/internal/net/ApmNetworkInstrumentationExt.kt @@ -9,6 +9,7 @@ import com.datadog.android.api.feature.Feature import com.datadog.android.api.feature.FeatureSdkCore import com.datadog.android.api.instrumentation.network.HttpRequestInfo import com.datadog.android.core.sampling.Sampler +import com.datadog.android.trace.DeterministicTraceSampler import com.datadog.android.trace.api.DatadogTracingConstants.PrioritySampling import com.datadog.android.trace.api.DatadogTracingConstants.Tags import com.datadog.android.trace.api.span.DatadogSpan @@ -35,11 +36,20 @@ internal fun DatadogSpan.applyPriority(isSampled: Boolean, traceSampler: Sampler if (spanContext.setSamplingPriority(samplingPriority)) { spanContext.setMetric( AGENT_PSR_ATTRIBUTE, - (traceSampler.getSampleRate() ?: ZERO_SAMPLE_RATE) / ALL_IN_SAMPLE_RATE + (traceSampler.effectiveSampleRate(this) ?: ZERO_SAMPLE_RATE) / ALL_IN_SAMPLE_RATE ) } } +// TODO RUM-13454 -> Remove the cast to DeterministicTraceSampler - V4 (RUM-15590) +@Suppress("DEPRECATION") +internal fun Sampler.effectiveSampleRate(span: DatadogSpan): Float? { + return when (this) { + is DeterministicTraceSampler -> getSampleRate(span) + else -> getSampleRate() + } +} + internal fun DatadogSpan.sample(request: HttpRequestInfo, traceSampler: Sampler): Boolean { val samplingPriority = samplingPriority return if (samplingPriority != null) { diff --git a/features/dd-sdk-android-trace/src/test/kotlin/com/datadog/android/trace/internal/RumContextPropagatorTest.kt b/features/dd-sdk-android-trace/src/test/kotlin/com/datadog/android/trace/internal/RumContextPropagatorTest.kt index de8fe17124..272494cab5 100644 --- a/features/dd-sdk-android-trace/src/test/kotlin/com/datadog/android/trace/internal/RumContextPropagatorTest.kt +++ b/features/dd-sdk-android-trace/src/test/kotlin/com/datadog/android/trace/internal/RumContextPropagatorTest.kt @@ -24,6 +24,7 @@ import com.datadog.android.trace.internal.RumContextPropagator.Companion.injectR import com.datadog.android.trace.utils.RumContextTestsUtils.RUM_CONTEXT_ACTION_ID import com.datadog.android.trace.utils.RumContextTestsUtils.RUM_CONTEXT_APPLICATION_ID import com.datadog.android.trace.utils.RumContextTestsUtils.RUM_CONTEXT_SESSION_ID +import com.datadog.android.trace.utils.RumContextTestsUtils.RUM_CONTEXT_SESSION_SAMPLE_RATE import com.datadog.android.trace.utils.RumContextTestsUtils.RUM_CONTEXT_VIEW_ID import com.datadog.android.trace.utils.RumContextTestsUtils.aDatadogContextWithRumContext import com.datadog.android.trace.utils.RumContextTestsUtils.aRumContext @@ -293,6 +294,7 @@ class RumContextPropagatorTest { verify(span).setTag(LogAttributes.RUM_SESSION_ID, fakeRumContext[RUM_CONTEXT_SESSION_ID]) verify(span).setTag(LogAttributes.RUM_VIEW_ID, fakeRumContext[RUM_CONTEXT_VIEW_ID]) verify(span).setTag(LogAttributes.RUM_ACTION_ID, fakeRumContext[RUM_CONTEXT_ACTION_ID]) + verify(span).setTag(LogAttributes.RUM_SESSION_SAMPLE_RATE, fakeRumContext[RUM_CONTEXT_SESSION_SAMPLE_RATE]) verify(span).setTag(HttpCodec.RUM_KEY_ACCOUNT_ID, fakeDatadogContext.accountInfo?.id as? Any) verify(span).setTag(HttpCodec.RUM_KEY_USER_ID, fakeDatadogContext.userInfo.id as? Any) verify(span).setTag(DATADOG_INITIAL_CONTEXT, null as Any?) @@ -317,6 +319,9 @@ class RumContextPropagatorTest { verify(span).setTag(LogAttributes.RUM_SESSION_ID, fakeRumContext[RUM_CONTEXT_SESSION_ID]) verify(span).setTag(LogAttributes.RUM_VIEW_ID, fakeRumContext[RUM_CONTEXT_VIEW_ID]) verify(span).setTag(LogAttributes.RUM_ACTION_ID, fakeRumContext[RUM_CONTEXT_ACTION_ID]) + verify( + span + ).setTag(LogAttributes.RUM_SESSION_SAMPLE_RATE, fakeRumContext[RUM_CONTEXT_SESSION_SAMPLE_RATE]) verify(span).setTag(HttpCodec.RUM_KEY_ACCOUNT_ID, fakeDatadogContext.accountInfo?.id as? Any) verify(span).setTag(HttpCodec.RUM_KEY_USER_ID, fakeDatadogContext.userInfo.id as? Any) verify(span).setTag(DATADOG_INITIAL_CONTEXT, null as Any?) @@ -339,6 +344,7 @@ class RumContextPropagatorTest { verify(span).setTag(LogAttributes.RUM_SESSION_ID, null as Any?) verify(span).setTag(LogAttributes.RUM_VIEW_ID, null as Any?) verify(span).setTag(LogAttributes.RUM_ACTION_ID, null as Any?) + verify(span).setTag(LogAttributes.RUM_SESSION_SAMPLE_RATE, null as Any?) verify(span).setTag(HttpCodec.RUM_KEY_ACCOUNT_ID, null as Any?) verify(span).setTag(HttpCodec.RUM_KEY_USER_ID, null as Any?) verify(span).setTag(DATADOG_INITIAL_CONTEXT, null as Any?) @@ -362,6 +368,7 @@ class RumContextPropagatorTest { verify(span).setTag(LogAttributes.RUM_SESSION_ID, null as Any?) verify(span).setTag(LogAttributes.RUM_VIEW_ID, null as Any?) verify(span).setTag(LogAttributes.RUM_ACTION_ID, null as Any?) + verify(span).setTag(LogAttributes.RUM_SESSION_SAMPLE_RATE, null as Any?) verify(span).setTag(HttpCodec.RUM_KEY_ACCOUNT_ID, null as Any?) verify(span).setTag(HttpCodec.RUM_KEY_USER_ID, null as Any?) verify(span).setTag(DATADOG_INITIAL_CONTEXT, null as Any?) diff --git a/features/dd-sdk-android-trace/src/test/kotlin/com/datadog/android/trace/internal/data/CoreTraceWriterTest.kt b/features/dd-sdk-android-trace/src/test/kotlin/com/datadog/android/trace/internal/data/CoreTraceWriterTest.kt index 3333e0e6c9..6f0e738001 100644 --- a/features/dd-sdk-android-trace/src/test/kotlin/com/datadog/android/trace/internal/data/CoreTraceWriterTest.kt +++ b/features/dd-sdk-android-trace/src/test/kotlin/com/datadog/android/trace/internal/data/CoreTraceWriterTest.kt @@ -183,13 +183,10 @@ internal class CoreTraceWriterTest { argumentCaptor { verify(mockLegacyMapper, times(ddSpans.size)).map(eq(fakeDatadogContext), capture()) allValues.forEach { span -> - verify(span).setTag( - LogAttributes.RUM_APPLICATION_ID, - fakeRumContext[RUM_CONTEXT_APPLICATION_ID] as? Any? - ) - verify(span).setTag(LogAttributes.RUM_SESSION_ID, fakeRumContext[RUM_CONTEXT_SESSION_ID] as? Any?) - verify(span).setTag(LogAttributes.RUM_VIEW_ID, fakeRumContext[RUM_CONTEXT_VIEW_ID] as? Any?) - verify(span).setTag(LogAttributes.RUM_ACTION_ID, fakeRumContext[RUM_CONTEXT_ACTION_ID] as? Any?) + verify(span).setTag(LogAttributes.RUM_APPLICATION_ID, fakeRumContext[RUM_CONTEXT_APPLICATION_ID]) + verify(span).setTag(LogAttributes.RUM_SESSION_ID, fakeRumContext[RUM_CONTEXT_SESSION_ID]) + verify(span).setTag(LogAttributes.RUM_VIEW_ID, fakeRumContext[RUM_CONTEXT_VIEW_ID]) + verify(span).setTag(LogAttributes.RUM_ACTION_ID, fakeRumContext[RUM_CONTEXT_ACTION_ID]) verify(span).setTag(RumContextPropagator.DATADOG_INITIAL_CONTEXT, null as Any?) } } diff --git a/features/dd-sdk-android-trace/src/test/kotlin/com/datadog/android/trace/internal/net/ApmInstrumentationConfigurationTest.kt b/features/dd-sdk-android-trace/src/test/kotlin/com/datadog/android/trace/internal/net/ApmInstrumentationConfigurationTest.kt index 01a7e49168..fbf74a1cb7 100644 --- a/features/dd-sdk-android-trace/src/test/kotlin/com/datadog/android/trace/internal/net/ApmInstrumentationConfigurationTest.kt +++ b/features/dd-sdk-android-trace/src/test/kotlin/com/datadog/android/trace/internal/net/ApmInstrumentationConfigurationTest.kt @@ -141,6 +141,21 @@ internal class ApmInstrumentationConfigurationTest { assertThat(result.traceSampler).isSameAs(mockTraceSampler) } + @Test + fun `M use last writer W setTraceSampler() then setTraceSampleRate()`( + @FloatForgery(min = 0f, max = 100f) fakeSampleRate: Float + ) { + // When + val result = testedBuilder + .setTraceSampler(mockTraceSampler) + .setTraceSampleRate(fakeSampleRate) + .createInstrumentation(fakeNetworkLibraryName) + + // Then + assertThat(result.traceSampler).isInstanceOf(DeterministicTraceSampler::class.java) + assertThat(result.traceSampler.getSampleRate()).isEqualTo(fakeSampleRate) + } + @Test fun `M set trace context injection W setTraceContextInjection()`(forge: Forge) { // Given diff --git a/features/dd-sdk-android-trace/src/test/kotlin/com/datadog/android/trace/internal/net/ApmNetworkInstrumentationTest.kt b/features/dd-sdk-android-trace/src/test/kotlin/com/datadog/android/trace/internal/net/ApmNetworkInstrumentationTest.kt index 559df1e4d4..a78a480bfc 100644 --- a/features/dd-sdk-android-trace/src/test/kotlin/com/datadog/android/trace/internal/net/ApmNetworkInstrumentationTest.kt +++ b/features/dd-sdk-android-trace/src/test/kotlin/com/datadog/android/trace/internal/net/ApmNetworkInstrumentationTest.kt @@ -18,8 +18,10 @@ import com.datadog.android.api.instrumentation.network.MutableHttpRequestInfo import com.datadog.android.core.InternalSdkCore import com.datadog.android.core.internal.net.DefaultFirstPartyHostHeaderTypeResolver import com.datadog.android.core.sampling.Sampler +import com.datadog.android.log.LogAttributes import com.datadog.android.trace.ApmNetworkInstrumentationConfiguration import com.datadog.android.trace.ApmNetworkTracingScope +import com.datadog.android.trace.DeterministicTraceSampler import com.datadog.android.trace.NetworkTracedRequestListener import com.datadog.android.trace.TraceContextInjection import com.datadog.android.trace.TracingHeaderType @@ -27,6 +29,7 @@ import com.datadog.android.trace.api.DatadogTracingConstants.Tags import com.datadog.android.trace.api.span.DatadogSpan import com.datadog.android.trace.api.span.DatadogSpanBuilder import com.datadog.android.trace.api.span.DatadogSpanContext +import com.datadog.android.trace.api.trace.DatadogTraceId import com.datadog.android.trace.api.tracer.DatadogTracer import com.datadog.android.trace.api.withMockPropagationHelper import com.datadog.android.trace.internal.ApmNetworkInstrumentation @@ -218,6 +221,69 @@ internal class ApmNetworkInstrumentationTest { } } + @Test + fun `M use rebased trace sample rate in tracing state W onRequest() {session sample rate tag present}`( + @FloatForgery(min = 0f, max = 100f) fakeTraceSampleRate: Float, + @FloatForgery(min = 0f, max = 99f) fakeSessionSampleRate: Float + ) { + // Given + val traceId = mock { + on { toLong() } doReturn 1L + } + whenever(mockSpanContext.traceId).thenReturn(traceId) + whenever(mockSpanContext.tags).thenReturn( + mapOf(LogAttributes.RUM_SESSION_SAMPLE_RATE to fakeSessionSampleRate) + ) + whenever(mockSpanContext.setSamplingPriority(any())).thenReturn(true) + + testedInstrumentation = createInstrumentation( + traceSampler = DeterministicTraceSampler(fakeTraceSampleRate) + ) + + _TraceInternalProxy.withMockPropagationHelper(mockPropagationHelper) { + // When + val result = checkNotNull(testedInstrumentation.onRequest(mockRequestInfo)) + + // Then + val expectedRebasedSampleRate = fakeTraceSampleRate * fakeSessionSampleRate / + ApmNetworkInstrumentation.ALL_IN_SAMPLE_RATE.toFloat() + assertThat(result.sampleRate).isEqualTo(expectedRebasedSampleRate) + } + } + + @Test + fun `M set rebased trace sample rate metric W onRequest() {session sample rate tag present}`( + @FloatForgery(min = 0f, max = 100f) fakeTraceSampleRate: Float, + @FloatForgery(min = 0f, max = 99f) fakeSessionSampleRate: Float + ) { + // Given + val traceId = mock { + on { toLong() } doReturn 1L + } + whenever(mockSpanContext.traceId).thenReturn(traceId) + whenever(mockSpanContext.tags).thenReturn( + mapOf(LogAttributes.RUM_SESSION_SAMPLE_RATE to fakeSessionSampleRate) + ) + whenever(mockSpanContext.setSamplingPriority(any())).thenReturn(true) + + testedInstrumentation = createInstrumentation( + traceSampler = DeterministicTraceSampler(fakeTraceSampleRate) + ) + + _TraceInternalProxy.withMockPropagationHelper(mockPropagationHelper) { + // When + testedInstrumentation.onRequest(mockRequestInfo) + + // Then + val expectedRebasedSampleRate = fakeTraceSampleRate * fakeSessionSampleRate / + ApmNetworkInstrumentation.ALL_IN_SAMPLE_RATE.toFloat() + verify(mockSpanContext).setMetric( + eq(ApmNetworkInstrumentation.AGENT_PSR_ATTRIBUTE), + eq(expectedRebasedSampleRate.toDouble() / 100.0) + ) + } + } + @Test fun `M set span resource name W onRequest()`() { // Given @@ -935,14 +1001,15 @@ internal class ApmNetworkInstrumentationTest { private fun createInstrumentation( canSendSpan: Boolean = true, networkTracingScope: ApmNetworkTracingScope = ApmNetworkTracingScope.EXCLUDE_INTERNAL_REDIRECTS, - redacted404ResourceName: Boolean = true + redacted404ResourceName: Boolean = true, + traceSampler: Sampler = mockTraceSampler ) = ApmNetworkInstrumentation( canSendSpan = canSendSpan, sdkInstanceName = null, traceOrigin = null, tracerProvider = mockTracerProvider, redacted404ResourceName = redacted404ResourceName, - traceSampler = mockTraceSampler, + traceSampler = traceSampler, injectionType = TraceContextInjection.ALL, tracedRequestListener = mockNetworkTracedRequestListener, localFirstPartyHostHeaderTypeResolver = mockLocalFirstPartyHostResolver, diff --git a/features/dd-sdk-android-trace/src/test/kotlin/com/datadog/android/trace/internal/net/DeterministicTraceSamplerTest.kt b/features/dd-sdk-android-trace/src/test/kotlin/com/datadog/android/trace/internal/net/DeterministicTraceSamplerTest.kt index 9bf98da18a..d3b4766d31 100644 --- a/features/dd-sdk-android-trace/src/test/kotlin/com/datadog/android/trace/internal/net/DeterministicTraceSamplerTest.kt +++ b/features/dd-sdk-android-trace/src/test/kotlin/com/datadog/android/trace/internal/net/DeterministicTraceSamplerTest.kt @@ -6,6 +6,7 @@ package com.datadog.android.trace.internal.net +import com.datadog.android.log.LogAttributes import com.datadog.android.trace.DeterministicTraceSampler import com.datadog.android.trace.api.span.DatadogSpan import com.datadog.android.trace.api.span.DatadogSpanContext @@ -23,12 +24,12 @@ import org.junit.jupiter.api.RepeatedTest import org.junit.jupiter.api.Test import org.junit.jupiter.api.extension.ExtendWith import org.junit.jupiter.api.extension.Extensions -import org.mockito.Mock import org.mockito.junit.jupiter.MockitoExtension import org.mockito.junit.jupiter.MockitoSettings import org.mockito.kotlin.doReturn import org.mockito.kotlin.mock import org.mockito.quality.Strictness +import java.util.UUID @Extensions( ExtendWith(MockitoExtension::class), @@ -40,9 +41,6 @@ internal class DeterministicTraceSamplerTest { private lateinit var testedSampler: DeterministicTraceSampler - @Mock - lateinit var mockSpanContext: DatadogSpanContext - private lateinit var fakeSpans: List @BeforeEach @@ -113,7 +111,7 @@ internal class DeterministicTraceSamplerTest { @FloatForgery(min = 0f, max = 100f) fakeSampleRate: Float ) { // Given - testedSampler = DeterministicTraceSampler { fakeSampleRate } + testedSampler = DeterministicTraceSampler(sampleRateProvider = { fakeSampleRate }) var sampledIn = 0 // When @@ -166,7 +164,7 @@ internal class DeterministicTraceSamplerTest { } @Test - fun `M return sample rate W getSampleRate()`( + fun `M return raw sample rate W getSampleRate()`( @FloatForgery(min = 0f, max = 100f) fakeSampleRate: Float ) { // Given @@ -202,4 +200,229 @@ internal class DeterministicTraceSamplerTest { // Then assertThat(results.distinct()).hasSize(1) } + + // region Session rate rebasing + + @Test + fun `M drop all spans W sample() {session sample rate is 50 and rebased rate is 0}`(forge: Forge) { + // Given: traceSampleRate=0, sessionSampleRate=50 -> rebased = 0*50/100 = 0 + val fakeTraceSampleRate = 0f + val fakeSessionSampleRate = 50f + testedSampler = DeterministicTraceSampler(fakeTraceSampleRate) + val spansWithSession = createSpansWithSessionContext(forge, fakeSessionSampleRate) + + var sampledIn = 0 + + // When + spansWithSession.forEach { if (testedSampler.sample(it)) sampledIn++ } + + // Then — none must be sampled (rebased rate = 0) + assertThat(sampledIn).isEqualTo(0) + } + + @Test + fun `M apply rebased rate W sample() {session sample rate is 50 - deterministic}`() { + // Given: traceSampleRate=50, sessionSampleRate=50 -> rebasedRate = 50*50/100 = 25% + // + // Knuth hash: hash = lastHexSegment * SAMPLER_HASHER + // n=1 -> hash = 1_111_111_111_111_111_111 + // n=5 -> hash = 5_555_555_555_555_555_555 + // + // threshold(25%) = MAX_ULONG * 25/100 ~ 4_611_686_018_427_387_903 + // threshold(50%) = MAX_ULONG * 50/100 ~ 9_223_372_036_854_775_807 + // + // n=1: hash(1.1T) < threshold(25%)(4.6T) -> SAMPLED at 25% + // n=5: hash(5.5T) > threshold(25%)(4.6T) -> NOT SAMPLED at 25% <- rebasing dropped it + // hash(5.5T) < threshold(50%)(9.2T) -> would be SAMPLED at raw 50% <- proves rebasing had effect + val traceSampleRate = 50f + val sessionSampleRate = 50f + testedSampler = DeterministicTraceSampler(traceSampleRate) + + val spanSampled = createSpanWithFixedSessionId( + sessionId = "aaaaaaaa-bbbb-cccc-dddd-000000000001", + sessionSampleRate = sessionSampleRate + ) + val spanDropped = createSpanWithFixedSessionId( + sessionId = "aaaaaaaa-bbbb-cccc-dddd-000000000005", + sessionSampleRate = sessionSampleRate + ) + + // When + Then + assertThat(testedSampler.sample(spanSampled)).isTrue() // hash 1.1T < rebased threshold 4.6T + assertThat(testedSampler.sample(spanDropped)).isFalse() // hash 5.5T > rebased threshold 4.6T + } + + @Test + fun `M use raw trace rate W sample() {session sample rate tag is 100}`() { + // Given: sessionSampleRate=100 -> rebased = traceSampleRate*100/100 = traceSampleRate (50%) + // threshold(50%) = MAX_ULONG * 50/100 ~ 9_223_372_036_854_775_807 + // + // n=5: hash(5.5T) < threshold(50%)(9.2T) -> SAMPLED (same as raw 50%) + // n=9: hash(9.9T) > threshold(50%)(9.2T) -> NOT SAMPLED (same as raw 50%) + val traceSampleRate = 50f + val sessionSampleRate = 100f + testedSampler = DeterministicTraceSampler(traceSampleRate) + + val spanSampled = createSpanWithFixedSessionId( + sessionId = "aaaaaaaa-bbbb-cccc-dddd-000000000005", + traceIdValue = 5L, + sessionSampleRate = sessionSampleRate + ) + val spanDropped = createSpanWithFixedSessionId( + sessionId = "aaaaaaaa-bbbb-cccc-dddd-000000000009", + traceIdValue = 9L, + sessionSampleRate = sessionSampleRate + ) + + // When + Then — rebased rate == raw trace rate, identical sampling decisions + assertThat(testedSampler.sample(spanSampled)).isTrue() + assertThat(testedSampler.sample(spanDropped)).isFalse() + } + + @Test + fun `M use raw trace rate W sample() {no session sample rate tag}`() { + // Given: no session_sample_rate tag -> no rebasing -> raw trace rate used + // threshold(50%) = MAX_ULONG * 50/100 ~ 9_223_372_036_854_775_807 + // + // n=5: hash(5.5T) < threshold(50%)(9.2T) -> SAMPLED + // n=9: hash(9.9T) > threshold(50%)(9.2T) -> NOT SAMPLED + val traceSampleRate = 50f + testedSampler = DeterministicTraceSampler(traceSampleRate) + + val spanSampled = createSpanWithFixedTraceId(5L) + val spanDropped = createSpanWithFixedTraceId(9L) + + // When + Then — no tag -> no rebasing -> raw trace rate used + assertThat(testedSampler.sample(spanSampled)).isTrue() + assertThat(testedSampler.sample(spanDropped)).isFalse() + } + + @Test + fun `M drop all spans W sample() {session sample rate is 0}`(forge: Forge) { + // Given: sessionSampleRate=0 -> rebased = traceSampleRate*0/100 = 0 + val fakeTraceSampleRate = 50f + val fakeSessionSampleRate = 0f + testedSampler = DeterministicTraceSampler(fakeTraceSampleRate) + val spansWithSession = createSpansWithSessionContext(forge, fakeSessionSampleRate) + + var sampledIn = 0 + + // When + spansWithSession.forEach { if (testedSampler.sample(it)) sampledIn++ } + + // Then — all spans dropped (effective rate = 0) + assertThat(sampledIn).isEqualTo(0) + } + + // endregion + + // region getSampleRate(item) — deprecated, kept for V3 compatibility + + @Test + // TODO RUM-13454 -> Remove this test when getSampleRate(DatadogSpan) is removed in V4 (RUM-15590) + @Suppress("DEPRECATION") + fun `M return rebased rate W getSampleRate(item) {session sample rate tag present}`( + @FloatForgery(min = 0f, max = 100f) traceSampleRate: Float, + @FloatForgery(min = 0f, max = 99f) sessionSampleRate: Float, + @LongForgery fakeTraceIdLong: Long + ) { + // Given + testedSampler = DeterministicTraceSampler(traceSampleRate) + val expected = traceSampleRate * sessionSampleRate / 100f + + val traceId = mock { + on { toLong() } doReturn fakeTraceIdLong + } + val context = mock { + on { this.traceId } doReturn traceId + on { tags } doReturn mapOf( + LogAttributes.RUM_SESSION_SAMPLE_RATE to sessionSampleRate + ) + } + val span = mock { + on { context() } doReturn context + } + + // When + val result = testedSampler.getSampleRate(span) + + // Then + assertThat(result).isEqualTo(expected) + } + + @Test + // TODO RUM-13454 -> Remove this test when getSampleRate(DatadogSpan) is removed in V4 (RUM-15590) + @Suppress("DEPRECATION") + fun `M return raw trace rate W getSampleRate(item) {session sample rate tag unavailable}`( + @FloatForgery(min = 0f, max = 100f) traceSampleRate: Float, + @LongForgery fakeTraceIdLong: Long + ) { + // Given: no session_sample_rate tag available + testedSampler = DeterministicTraceSampler(traceSampleRate) + + val traceId = mock { + on { toLong() } doReturn fakeTraceIdLong + } + val context = mock { + on { this.traceId } doReturn traceId + on { tags } doReturn emptyMap() + } + val span = mock { + on { context() } doReturn context + } + + // When + val result = testedSampler.getSampleRate(span) + + // Then — no session rate available -> raw trace rate + assertThat(result).isEqualTo(traceSampleRate) + } + + // endregion + + // region Helpers + + private fun createSpanWithFixedSessionId( + sessionId: String? = null, + traceIdValue: Long = 0L, + sessionSampleRate: Float? = null + ): DatadogSpan { + val traceId = mock { on { toLong() } doReturn traceIdValue } + val tagsMap = buildMap { + if (sessionId != null) put(LogAttributes.RUM_SESSION_ID, sessionId) + if (sessionSampleRate != null) put(LogAttributes.RUM_SESSION_SAMPLE_RATE, sessionSampleRate) + } + val context = mock { + on { this.traceId } doReturn traceId + on { tags } doReturn tagsMap + } + return mock { on { context() } doReturn context } + } + + private fun createSpanWithFixedTraceId(traceIdLong: Long): DatadogSpan { + val traceId = mock { on { toLong() } doReturn traceIdLong } + val context = mock { + on { this.traceId } doReturn traceId + on { tags } doReturn emptyMap() + } + return mock { on { context() } doReturn context } + } + + private fun createSpansWithSessionContext(forge: Forge, sessionSampleRate: Float): List { + val listSize = forge.anInt(256, 1024) + return (0 until listSize).map { + val fakeSessionId = UUID.randomUUID().toString() + val traceId = mock { on { toLong() } doReturn forge.aLong() } + val context = mock { + on { this.traceId } doReturn traceId + on { tags } doReturn mapOf( + LogAttributes.RUM_SESSION_ID to fakeSessionId, + LogAttributes.RUM_SESSION_SAMPLE_RATE to sessionSampleRate + ) + } + mock { on { context() } doReturn context } + } + } + + // endregion } diff --git a/features/dd-sdk-android-trace/src/testFixtures/kotlin/com/datadog/android/trace/utils/RumContextTestsUtils.kt b/features/dd-sdk-android-trace/src/testFixtures/kotlin/com/datadog/android/trace/utils/RumContextTestsUtils.kt index 4a547f9af8..984c3c88bd 100644 --- a/features/dd-sdk-android-trace/src/testFixtures/kotlin/com/datadog/android/trace/utils/RumContextTestsUtils.kt +++ b/features/dd-sdk-android-trace/src/testFixtures/kotlin/com/datadog/android/trace/utils/RumContextTestsUtils.kt @@ -19,12 +19,18 @@ object RumContextTestsUtils { const val RUM_CONTEXT_SESSION_ID = "session_id" const val RUM_CONTEXT_APPLICATION_ID = "application_id" + const val RUM_CONTEXT_SESSION_SAMPLE_RATE = "session_sample_rate" + private const val HEX = 16 - fun Forge.aRumContext(sessionId: Long? = null) = buildMap { + fun Forge.aRumContext( + sessionId: Long? = null, + sessionSampleRate: Float? = null + ) = buildMap { put(RUM_CONTEXT_VIEW_ID, getForgery().toString()) put(RUM_CONTEXT_ACTION_ID, getForgery().toString()) put(RUM_CONTEXT_APPLICATION_ID, getForgery().toString()) + put(RUM_CONTEXT_SESSION_SAMPLE_RATE, sessionSampleRate ?: aFloat(min = 0f, max = 100f)) sessionId?.let { put( RUM_CONTEXT_SESSION_ID, diff --git a/integrations/dd-sdk-android-okhttp/src/main/kotlin/com/datadog/android/okhttp/DatadogInterceptor.kt b/integrations/dd-sdk-android-okhttp/src/main/kotlin/com/datadog/android/okhttp/DatadogInterceptor.kt index 3ea8ce7307..a3ecdb9b90 100644 --- a/integrations/dd-sdk-android-okhttp/src/main/kotlin/com/datadog/android/okhttp/DatadogInterceptor.kt +++ b/integrations/dd-sdk-android-okhttp/src/main/kotlin/com/datadog/android/okhttp/DatadogInterceptor.kt @@ -31,6 +31,7 @@ import com.datadog.android.rum._RumInternalProxy import com.datadog.android.rum.internal.monitor.AdvancedNetworkRumMonitor import com.datadog.android.rum.resource.ResourceHeadersExtractor import com.datadog.android.rum.tracking.ViewTrackingStrategy +import com.datadog.android.trace.DeterministicTraceSampler import com.datadog.android.trace.TraceContextInjection import com.datadog.android.trace.TracingHeaderType import com.datadog.android.trace.api.span.DatadogSpan @@ -242,7 +243,16 @@ open class DatadogInterceptor internal constructor( buildMap { put(RumAttributes.TRACE_ID, span.context().traceId.toHexString()) put(RumAttributes.SPAN_ID, span.context().spanId.toString()) - put(RumAttributes.RULE_PSR, (traceSampler.getSampleRate() ?: ZERO_SAMPLE_RATE) / ALL_IN_SAMPLE_RATE) + // TODO RUM-13454 -> Remove the cast to DeterministicTraceSampler - V4 (RUM-15590) + @Suppress("DEPRECATION") + put( + RumAttributes.RULE_PSR, + ( + (traceSampler as? DeterministicTraceSampler)?.getSampleRate(span) + ?: traceSampler.getSampleRate() + ?: ZERO_SAMPLE_RATE + ) / ALL_IN_SAMPLE_RATE + ) putAll(graphqlAttributes) putAll(graphqlErrorAttributes) } diff --git a/integrations/dd-sdk-android-okhttp/src/main/kotlin/com/datadog/android/okhttp/trace/TracingInterceptor.kt b/integrations/dd-sdk-android-okhttp/src/main/kotlin/com/datadog/android/okhttp/trace/TracingInterceptor.kt index fb32829bcb..8246602ca7 100644 --- a/integrations/dd-sdk-android-okhttp/src/main/kotlin/com/datadog/android/okhttp/trace/TracingInterceptor.kt +++ b/integrations/dd-sdk-android-okhttp/src/main/kotlin/com/datadog/android/okhttp/trace/TracingInterceptor.kt @@ -258,9 +258,15 @@ internal constructor( val spanContext = span.context() if (spanContext.setSamplingPriority(samplingPriority)) { + // TODO RUM-13454 -> Remove the cast to DeterministicTraceSampler - V4 (RUM-15590) + @Suppress("DEPRECATION") spanContext.setMetric( AGENT_PSR_ATTRIBUTE, - (traceSampler.getSampleRate() ?: ZERO_SAMPLE_RATE) / ALL_IN_SAMPLE_RATE + ( + (traceSampler as? DeterministicTraceSampler)?.getSampleRate(span) + ?: traceSampler.getSampleRate() + ?: ZERO_SAMPLE_RATE + ) / ALL_IN_SAMPLE_RATE ) } } diff --git a/integrations/dd-sdk-android-okhttp/src/test/kotlin/com/datadog/android/okhttp/DatadogInterceptorTest.kt b/integrations/dd-sdk-android-okhttp/src/test/kotlin/com/datadog/android/okhttp/DatadogInterceptorTest.kt index 846fa8e29c..8518aa0db2 100644 --- a/integrations/dd-sdk-android-okhttp/src/test/kotlin/com/datadog/android/okhttp/DatadogInterceptorTest.kt +++ b/integrations/dd-sdk-android-okhttp/src/test/kotlin/com/datadog/android/okhttp/DatadogInterceptorTest.kt @@ -1381,5 +1381,55 @@ internal class DatadogInterceptorTest : TracingInterceptorNotSendingSpanTest() { assertThat(resHeaders).isEqualTo(fakeActualResponseHeaders) } + // region rule_psr rebasing + + @Test + fun `M set rebased rule_psr W intercept() {DeterministicTraceSampler, span has session_sample_rate tag}`( + @FloatForgery(min = 1f, max = 100f) fakeTraceSampleRate: Float, + @FloatForgery(min = 1f, max = 99f) fakeSessionSampleRate: Float, + @IntForgery(min = 200, max = 300) statusCode: Int + ) { + // Given — force upstream sampling-priority KEEP so isSampled=true regardless of rate + fakeRequest = forgeRequest { + it.addHeader( + TracingInterceptor.DATADOG_SAMPLING_PRIORITY_HEADER, + com.datadog.android.trace.api.DatadogTracingConstants.PrioritySampling.SAMPLER_KEEP.toString() + ) + } + val deterministicSampler = DeterministicTraceSampler(fakeTraceSampleRate) + whenever(mockSpanContext.tags) doReturn mapOf("session_sample_rate" to fakeSessionSampleRate) + stubChain(mockChain, statusCode) + + whenever(rumMonitor.mockSdkCore.getFeature(Feature.RUM_FEATURE_NAME)) doReturn mock() + whenever(rumMonitor.mockSdkCore.firstPartyHostResolver) doReturn mockResolver + val datadogInterceptor = DatadogInterceptor( + sdkInstanceName = null, + tracedHosts = fakeLocalHosts, + tracedRequestListener = mockRequestListener, + rumResourceAttributesProvider = mockRumAttributesProvider, + traceSampler = deterministicSampler, + traceContextInjection = TraceContextInjection.ALL, + redacted404ResourceName = fakeRedacted404Resources, + localTracerFactory = { _, _ -> mockLocalTracer }, + globalTracerProvider = { mockTracer } + ) + + // When + datadogInterceptor.intercept(mockChain) + + // Then + val expectedRulePsr = fakeTraceSampleRate * fakeSessionSampleRate / 100f / 100f + val attrsCaptor = argumentCaptor>() + verify(rumMonitor.mockInstance).stopResource( + any(), + any(), + anyOrNull(), + any(), + attrsCaptor.capture() + ) + assertThat(attrsCaptor.firstValue[RumAttributes.RULE_PSR]) + .isEqualTo(expectedRulePsr) + } + // endregion } diff --git a/integrations/dd-sdk-android-okhttp/src/test/kotlin/com/datadog/android/okhttp/trace/TracingInterceptorTest.kt b/integrations/dd-sdk-android-okhttp/src/test/kotlin/com/datadog/android/okhttp/trace/TracingInterceptorTest.kt index 9e0b7b63a8..1f861ba150 100644 --- a/integrations/dd-sdk-android-okhttp/src/test/kotlin/com/datadog/android/okhttp/trace/TracingInterceptorTest.kt +++ b/integrations/dd-sdk-android-okhttp/src/test/kotlin/com/datadog/android/okhttp/trace/TracingInterceptorTest.kt @@ -1653,6 +1653,120 @@ internal open class TracingInterceptorTest { ) } + // region agent_psr rebasing + + @Test + fun `M set rebased agent_psr W intercept() {DeterministicTraceSampler, span has session_sample_rate tag}`( + @FloatForgery(min = 1f, max = 100f) fakeTraceSampleRate: Float, + @FloatForgery(min = 1f, max = 99f) fakeSessionSampleRate: Float + ) { + // Given + val deterministicSampler = DeterministicTraceSampler(fakeTraceSampleRate) + whenever(mockSpanContext.tags) doReturn mapOf("session_sample_rate" to fakeSessionSampleRate) + whenever(mockSpan.isRootSpan) doReturn true + whenever(mockSpanContext.setSamplingPriority(any())) doReturn true + whenever(mockResolver.isFirstPartyUrl(fakeUrl.toHttpUrl())).thenReturn(true) + stubChain(mockChain) + + testedInterceptor = TracingInterceptor( + sdkInstanceName = null, + tracedHosts = fakeLocalHosts, + tracedRequestListener = mockRequestListener, + traceOrigin = fakeOrigin, + traceSampler = deterministicSampler, + localTracerFactory = { _, _ -> mockLocalTracer }, + redacted404ResourceName = fakeRedacted404Resources, + traceContextInjection = TraceContextInjection.ALL, + globalTracerProvider = { mockTracer } + ) + + // When + testedInterceptor.intercept(mockChain) + + // Then + val expectedRebasedRate = fakeTraceSampleRate * fakeSessionSampleRate / 100f + verify(mockSpanContext).setMetric( + eq("_dd.agent_psr"), + eq(expectedRebasedRate.toDouble() / 100.0) + ) + } + + @Test + fun `M set raw agent_psr W intercept() {DeterministicTraceSampler, span has no session_sample_rate tag}`( + @FloatForgery(min = 1f, max = 100f) fakeTraceSampleRate: Float + ) { + // Given + val deterministicSampler = DeterministicTraceSampler(fakeTraceSampleRate) + whenever(mockSpanContext.tags) doReturn emptyMap() + whenever(mockSpan.isRootSpan) doReturn true + whenever(mockSpanContext.setSamplingPriority(any())) doReturn true + whenever(mockResolver.isFirstPartyUrl(fakeUrl.toHttpUrl())).thenReturn(true) + stubChain(mockChain) + + testedInterceptor = TracingInterceptor( + sdkInstanceName = null, + tracedHosts = fakeLocalHosts, + tracedRequestListener = mockRequestListener, + traceOrigin = fakeOrigin, + traceSampler = deterministicSampler, + localTracerFactory = { _, _ -> mockLocalTracer }, + redacted404ResourceName = fakeRedacted404Resources, + traceContextInjection = TraceContextInjection.ALL, + globalTracerProvider = { mockTracer } + ) + + // When + testedInterceptor.intercept(mockChain) + + // Then + verify(mockSpanContext).setMetric( + eq("_dd.agent_psr"), + eq(fakeTraceSampleRate.toDouble() / 100.0) + ) + } + + @Test + fun `M set raw agent_psr W intercept() {non-DeterministicTraceSampler}`( + @FloatForgery(min = 1f, max = 100f) fakeSampleRate: Float + ) { + // Given + whenever(mockTraceSampler.getSampleRate()) doReturn fakeSampleRate + whenever(mockSpan.isRootSpan) doReturn true + whenever(mockSpanContext.setSamplingPriority(any())) doReturn true + whenever(mockResolver.isFirstPartyUrl(fakeUrl.toHttpUrl())).thenReturn(true) + stubChain(mockChain) + + // When + testedInterceptor.intercept(mockChain) + + // Then + verify(mockSpanContext).setMetric( + eq("_dd.agent_psr"), + eq(fakeSampleRate.toDouble() / 100.0) + ) + } + + @Test + fun `M set zero agent_psr W intercept() {non-DeterministicTraceSampler, null sample rate}`() { + // Given + whenever(mockTraceSampler.getSampleRate()) doReturn null + whenever(mockSpan.isRootSpan) doReturn true + whenever(mockSpanContext.setSamplingPriority(any())) doReturn true + whenever(mockResolver.isFirstPartyUrl(fakeUrl.toHttpUrl())).thenReturn(true) + stubChain(mockChain) + + // When + testedInterceptor.intercept(mockChain) + + // Then + verify(mockSpanContext).setMetric( + eq("_dd.agent_psr"), + eq(0.0) + ) + } + + // endregion + // region Internal internal fun stubChain(chain: Interceptor.Chain, statusCode: Int = forge.anInt(min = 200, max = 600)) { diff --git a/reliability/single-fit/okhttp/src/test/kotlin/com/datadog/android/okhttp/RumContextPropagationTest.kt b/reliability/single-fit/okhttp/src/test/kotlin/com/datadog/android/okhttp/RumContextPropagationTest.kt index 778b63b85d..5aa73489a9 100644 --- a/reliability/single-fit/okhttp/src/test/kotlin/com/datadog/android/okhttp/RumContextPropagationTest.kt +++ b/reliability/single-fit/okhttp/src/test/kotlin/com/datadog/android/okhttp/RumContextPropagationTest.kt @@ -13,8 +13,6 @@ import com.datadog.android.api.context.DatadogContext import com.datadog.android.api.context.UserInfo import com.datadog.android.api.feature.Feature import com.datadog.android.api.feature.SdkFeatureMock -import com.datadog.android.core.sampling.DeterministicSampler.Companion.MAX_ID -import com.datadog.android.core.sampling.DeterministicSampler.Companion.SAMPLER_HASHER import com.datadog.android.core.stub.StubSDKCore import com.datadog.android.okhttp.RumContextPropagationTest.Companion.SAMPLING_THRESHOLD import com.datadog.android.okhttp.tests.elmyr.OkHttpConfigurator @@ -82,7 +80,10 @@ class RumContextPropagationTest { @Test fun `M send rum sessionId in baggage header W call is made`(forge: Forge) { // Given - val rumContext = forge.aRumContext(SAMPLED_IDS.random()) + val rumContext = forge.aRumContext( + sessionId = SAMPLED_IDS.random(), + sessionSampleRate = 100f + ) val accountInfo = forge.getForgery() val userInfo = forge.getForgery() val datadogContext = forge.aDatadogContextWithRumContext(rumContext, accountInfo, userInfo) @@ -96,11 +97,13 @@ class RumContextPropagationTest { // Then assertSentRequest { - assertThat(getHeader(HEADER_BAGGAGE)).isEqualTo( - "account.id=${accountInfo.id}," + - userInfo.id?.let { "user.id=$it," }.orEmpty() + - "session.id=${rumContext[RUM_CONTEXT_SESSION_ID]}" - ) + val baggageEntries = getHeader(HEADER_BAGGAGE)?.split(",").orEmpty() + val expectedEntries = buildList { + add("account.id=${accountInfo.id}") + userInfo.id?.let { add("user.id=$it") } + add("session.id=${rumContext[RUM_CONTEXT_SESSION_ID]}") + } + assertThat(baggageEntries).containsExactlyInAnyOrder(*expectedEntries.toTypedArray()) } } @@ -111,7 +114,10 @@ class RumContextPropagationTest { forge: Forge ) { // Given - val rumContext = forge.aRumContext(sessionId = SAMPLED_IDS.random()) + val rumContext = forge.aRumContext( + sessionId = SAMPLED_IDS.random(), + sessionSampleRate = 100f + ) val datadogContext = forge.aDatadogContextWithRumContext(rumContext) stubSdkCore = forge.prepareStubSdkCore(datadogContext) Trace.enable(TraceConfiguration.Builder().build(), stubSdkCore) @@ -229,6 +235,8 @@ class RumContextPropagationTest { * * isSampled = SAMPLING_THRESHOLD < MAX_ID / (2 * SAMPLE_HASHER). */ + private const val SAMPLER_HASHER: ULong = 1111111111111111111u + private const val MAX_ID: ULong = 0xFFFFFFFFFFFFFFFFUL private val SAMPLING_THRESHOLD: Long = (MAX_ID.toDouble() / (2.0 * SAMPLER_HASHER.toDouble())).toLong() private const val SAMPLE_RATE = 50f private val SAMPLED_IDS = listOf(SAMPLING_THRESHOLD - 1, SAMPLING_THRESHOLD - 2)