From 9546f8f4ce06a95dcdb62cabd5e501e091de6272 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20D=C3=BCmont?= Date: Fri, 29 Aug 2025 10:45:22 +0200 Subject: [PATCH 01/10] Initial --- .../OrchestrationModuleConfig.java | 33 ++++++++- .../filteringLooseRequestStream.json | 73 +++++++++++++++++++ 2 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 orchestration/src/test/resources/filteringLooseRequestStream.json diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationModuleConfig.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationModuleConfig.java index a9038d68f..2b43662d7 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationModuleConfig.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationModuleConfig.java @@ -2,6 +2,7 @@ import com.google.common.annotations.Beta; import com.sap.ai.sdk.orchestration.model.FilteringModuleConfig; +import com.sap.ai.sdk.orchestration.model.FilteringStreamOptions; import com.sap.ai.sdk.orchestration.model.GroundingModuleConfig; import com.sap.ai.sdk.orchestration.model.InputFilteringConfig; import com.sap.ai.sdk.orchestration.model.LLMModelDetails; @@ -100,6 +101,11 @@ public class OrchestrationModuleConfig { @Nullable SAPDocumentTranslation outputTranslationConfig; + /** Configuration of optional streaming options for output filtering. */ + @With(AccessLevel.NONE) + @Nullable + FilteringStreamOptions outputFilteringStreamOptions; + /** * Creates a new configuration with the given LLM configuration. * @@ -203,7 +209,10 @@ public OrchestrationModuleConfig withOutputFiltering( .map(ContentFilter::createOutputFilterConfig) .toList(); - final var outputFilter = OutputFilteringConfig.create().filters(filterConfigs); + final var outputFilter = + OutputFilteringConfig.create() + .filters(filterConfigs) + .streamOptions(outputFilteringStreamOptions); final var newFilteringConfig = FilteringModuleConfig.create() @@ -213,6 +222,28 @@ public OrchestrationModuleConfig withOutputFiltering( return this.withFilteringConfig(newFilteringConfig); } + /** + * Creates a new configuration with the given output filtering stream options. + * + * @param outputFilteringStreamOptions The output filtering stream options to use. + * @return A new configuration with the given output filtering stream options. + */ + public OrchestrationModuleConfig withOutputFilteringStreamOptions( + @Nullable final FilteringStreamOptions outputFilteringStreamOptions) { + if (filteringConfig != null && filteringConfig.getOutput() != null) { + filteringConfig.getOutput().setStreamOptions(outputFilteringStreamOptions); + } + return new OrchestrationModuleConfig( + this.llmConfig, + this.templateConfig, + this.maskingConfig, + this.filteringConfig, + this.groundingConfig, + this.inputTranslationConfig, + this.outputTranslationConfig, + outputFilteringStreamOptions); + } + /** * Creates a new configuration with the given grounding configuration. * diff --git a/orchestration/src/test/resources/filteringLooseRequestStream.json b/orchestration/src/test/resources/filteringLooseRequestStream.json new file mode 100644 index 000000000..1309daf06 --- /dev/null +++ b/orchestration/src/test/resources/filteringLooseRequestStream.json @@ -0,0 +1,73 @@ +{ + "config": { + "modules": { + "prompt_templating": { + "model": { + "name": "gpt-4o", + "params": { + "temperature": 0.1, + "max_tokens": 50, + "frequency_penalty": 0, + "presence_penalty": 0, + "top_p": 1, + "n": 1 + }, + "version": "latest" + }, + "prompt": { + "template": [ + { + "role": "user", + "content": "Hello World! Why is this phrase so famous?" + } + ], + "defaults": {}, + "tools": [] + } + }, + "filtering": { + "input": { + "filters": [ + { + "type": "azure_content_safety", + "config": { + "hate": 4, + "self_harm": 4, + "sexual": 4, + "violence": 4 + } + }, + { + "type": "llama_guard_3_8b", + "config": { + "self_harm": true + } + } + ] + }, + "output": { + "filters": [ + { + "type": "azure_content_safety", + "config": { + "hate": 4, + "self_harm": 4, + "sexual": 4, + "violence": 4 + } + } + ], + "stream_options" : { + "overlap" : 1000 + } + } + } + }, + "stream" : { + "enabled" : true, + "chunk_size" : 100 + } + }, + "placeholder_values": {}, + "messages_history": [] +} \ No newline at end of file From 220af51a69b76462c029756062efce0887c4593b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20D=C3=BCmont?= Date: Fri, 29 Aug 2025 10:45:59 +0200 Subject: [PATCH 02/10] Add test --- .../orchestration/OrchestrationUnitTest.java | 123 ++++++++++-------- 1 file changed, 66 insertions(+), 57 deletions(-) diff --git a/orchestration/src/test/java/com/sap/ai/sdk/orchestration/OrchestrationUnitTest.java b/orchestration/src/test/java/com/sap/ai/sdk/orchestration/OrchestrationUnitTest.java index df9be4417..fbfc52804 100644 --- a/orchestration/src/test/java/com/sap/ai/sdk/orchestration/OrchestrationUnitTest.java +++ b/orchestration/src/test/java/com/sap/ai/sdk/orchestration/OrchestrationUnitTest.java @@ -59,6 +59,7 @@ import com.sap.ai.sdk.orchestration.model.EmbeddingsPostResponse; import com.sap.ai.sdk.orchestration.model.EmbeddingsResponse; import com.sap.ai.sdk.orchestration.model.ErrorResponse; +import com.sap.ai.sdk.orchestration.model.FilteringStreamOptions; import com.sap.ai.sdk.orchestration.model.GenericModuleResult; import com.sap.ai.sdk.orchestration.model.GroundingFilterSearchConfiguration; import com.sap.ai.sdk.orchestration.model.GroundingModuleConfig; @@ -77,6 +78,7 @@ import com.sap.cloud.sdk.cloudplatform.connectivity.ApacheHttpClient5Accessor; import com.sap.cloud.sdk.cloudplatform.connectivity.ApacheHttpClient5Cache; import com.sap.cloud.sdk.cloudplatform.connectivity.DefaultHttpDestination; +import io.vavr.control.Try; import java.io.IOException; import java.io.InputStream; import java.math.BigDecimal; @@ -118,6 +120,8 @@ class OrchestrationUnitTest { private final Function fileLoader = filename -> Objects.requireNonNull(getClass().getClassLoader().getResourceAsStream(filename)); + private final Function fileLoaderStr = + filename -> new String(Try.of(() -> fileLoader.apply(filename).readAllBytes()).get()); private static OrchestrationClient client; private static OrchestrationModuleConfig config; @@ -247,11 +251,9 @@ void testGrounding() throws IOException { "masked_grounding_input", // maskGroundingInput: true will make this field present "[\"What does Joule do?\"]")); - try (var requestInputStream = fileLoader.apply("groundingRequest.json")) { - final String request = new String(requestInputStream.readAllBytes()); - verify( - postRequestedFor(urlPathEqualTo("/v2/completion")).withRequestBody(equalToJson(request))); - } + final String request = fileLoaderStr.apply("groundingRequest.json"); + verify( + postRequestedFor(urlPathEqualTo("/v2/completion")).withRequestBody(equalToJson(request))); } @Test @@ -281,12 +283,10 @@ void testGroundingWithHelpSapCom() throws IOException { "A fuzzy search is a search technique that is designed to be fast and tolerant of errors"); assertThat(response.getContent()).startsWith("A fuzzy search is a search technique"); - try (var requestInputStream = fileLoader.apply("groundingHelpSapComRequest.json")) { - final String request = new String(requestInputStream.readAllBytes()); - verify( - postRequestedFor(urlPathEqualTo("/v2/completion")) - .withRequestBody(equalToJson(request, true, true))); - } + final String request = fileLoaderStr.apply("groundingHelpSapComRequest.json"); + verify( + postRequestedFor(urlPathEqualTo("/v2/completion")) + .withRequestBody(equalToJson(request, true, true))); } @Test @@ -352,10 +352,8 @@ void testTemplating() throws IOException { assertThat(usage.getTotalTokens()).isEqualTo(26); // verify that null fields are absent from the sent request - try (var requestInputStream = fileLoader.apply("templatingRequest.json")) { - final String request = new String(requestInputStream.readAllBytes()); - verify(postRequestedFor(anyUrl()).withRequestBody(equalToJson(request))); - } + final String request = fileLoaderStr.apply("templatingRequest.json"); + verify(postRequestedFor(anyUrl()).withRequestBody(equalToJson(request))); } @Test @@ -420,10 +418,39 @@ void filteringLoose() throws IOException { // the result is asserted in the verify step below // verify that null fields are absent from the sent request - try (var requestInputStream = fileLoader.apply("filteringLooseRequest.json")) { - final String request = new String(requestInputStream.readAllBytes()); - verify(postRequestedFor(anyUrl()).withRequestBody(equalToJson(request, true, true))); - } + final String request = fileLoaderStr.apply("filteringLooseRequest.json"); + verify(postRequestedFor(anyUrl()).withRequestBody(equalToJson(request, true, true))); + } + + @Test + void filteringLooseStream() throws IOException { + final var res = new String(fileLoader.apply("streamChatCompletion.txt").readAllBytes()); + stubFor( + post(anyUrl()) + .willReturn(aResponse().withBody(res).withHeader("Content-Type", "application/json"))); + + final var azureFilter = + new AzureContentFilter() + .hate(ALLOW_SAFE_LOW_MEDIUM) + .selfHarm(ALLOW_SAFE_LOW_MEDIUM) + .sexual(ALLOW_SAFE_LOW_MEDIUM) + .violence(ALLOW_SAFE_LOW_MEDIUM); + + final var llamaFilter = new LlamaGuardFilter().config(LlamaGuard38b.create().selfHarm(true)); + + OrchestrationModuleConfig myConfig = + config + .withInputFiltering(azureFilter, llamaFilter) + .withOutputFiltering(azureFilter) + .withOutputFilteringStreamOptions(FilteringStreamOptions.create().overlap(1_000)); + + Stream result = client.streamChatCompletion(prompt, myConfig); + assertThat(result).containsExactly("", "Sure", "!"); + // the result is asserted in the verify step below + + // verify that null fields are absent from the sent request + final String request = fileLoaderStr.apply("filteringLooseRequestStream.json"); + verify(postRequestedFor(anyUrl()).withRequestBody(equalToJson(request, true, false))); } @Test @@ -560,10 +587,8 @@ void messagesHistory() throws IOException { .isEqualTo("26ea36b5-c196-4806-a9a6-a686f0c6ad91"); // verify that the history is sent correctly - try (var requestInputStream = fileLoader.apply("messagesHistoryRequest.json")) { - final String requestBody = new String(requestInputStream.readAllBytes()); - verify(postRequestedFor(anyUrl()).withRequestBody(equalToJson(requestBody))); - } + final String requestBody = fileLoaderStr.apply("messagesHistoryRequest.json"); + verify(postRequestedFor(anyUrl()).withRequestBody(equalToJson(requestBody))); } @Test @@ -588,10 +613,8 @@ void maskingPseudonymization() throws IOException { assertThat(result.getContent()).contains("Hi Mallory"); // verify that the request is sent correctly - try (var requestInputStream = fileLoader.apply("maskingRequest.json")) { - final String request = new String(requestInputStream.readAllBytes()); - verify(postRequestedFor(anyUrl()).withRequestBody(equalToJson(request, true, true))); - } + final String request = fileLoaderStr.apply("maskingRequest.json"); + verify(postRequestedFor(anyUrl()).withRequestBody(equalToJson(request, true, true))); } private static Runnable[] errorHandlingCalls() { @@ -991,12 +1014,10 @@ void testMultiMessage() throws IOException { assertThat(orchestrationResult.getChoices().get(0).getFinishReason()).isEqualTo("stop"); assertThat(orchestrationResult.getChoices().get(0).getMessage().getRole()).isEqualTo(ASSISTANT); - try (var requestInputStream = fileLoader.apply("multiMessageRequest.json")) { - final String requestBody = new String(requestInputStream.readAllBytes()); - verify( - postRequestedFor(urlPathEqualTo("/v2/completion")) - .withRequestBody(equalToJson(requestBody))); - } + final String requestBody = fileLoaderStr.apply("multiMessageRequest.json"); + verify( + postRequestedFor(urlPathEqualTo("/v2/completion")) + .withRequestBody(equalToJson(requestBody))); } // Example class @@ -1054,10 +1075,8 @@ class TranslationNotStaticNoConstructor { assertThat(translation.language).isEqualTo("German"); assertThat(translation.translation).isEqualTo("Apfel"); - try (var requestInputStream = fileLoader.apply("jsonSchemaRequest.json")) { - final String request = new String(requestInputStream.readAllBytes()); - verify(postRequestedFor(anyUrl()).withRequestBody(equalToJson(request))); - } + final String request = fileLoaderStr.apply("jsonSchemaRequest.json"); + verify(postRequestedFor(anyUrl()).withRequestBody(equalToJson(request))); } @Test @@ -1119,10 +1138,8 @@ void testResponseFormatJsonObject() throws IOException { final var message = client.chatCompletion(prompt, configWithJsonResponse).getContent(); assertThat(message).isEqualTo("{\"language\": \"German\", \"translation\": \"Apfel\"}"); - try (var requestInputStream = fileLoader.apply("jsonObjectRequest.json")) { - final String request = new String(requestInputStream.readAllBytes()); - verify(postRequestedFor(anyUrl()).withRequestBody(equalToJson(request))); - } + final String request = fileLoaderStr.apply("jsonObjectRequest.json"); + verify(postRequestedFor(anyUrl()).withRequestBody(equalToJson(request))); } @Test @@ -1152,10 +1169,8 @@ void testResponseFormatText() throws IOException { .isEqualTo( "```json\n{\n \"word\": \"apple\",\n \"translation\": \"Apfel\",\n \"language\": \"German\"\n}\n```"); - try (var requestInputStream = fileLoader.apply("responseFormatTextRequest.json")) { - final String request = new String(requestInputStream.readAllBytes()); - verify(postRequestedFor(anyUrl()).withRequestBody(equalToJson(request))); - } + final String request = fileLoaderStr.apply("responseFormatTextRequest.json"); + verify(postRequestedFor(anyUrl()).withRequestBody(equalToJson(request))); } @Test @@ -1179,10 +1194,8 @@ void testTemplateFromPromptRegistryById() throws IOException { assertThat(response.getOriginalResponse().getIntermediateResults().getTemplating()) .hasSize(2); - try (var requestInputStream = fileLoader.apply("templateReferenceByIdRequest.json")) { - final String request = new String(requestInputStream.readAllBytes()); - verify(postRequestedFor(anyUrl()).withRequestBody(equalToJson(request))); - } + final String request = fileLoaderStr.apply("templateReferenceByIdRequest.json"); + verify(postRequestedFor(anyUrl()).withRequestBody(equalToJson(request))); } } @@ -1205,10 +1218,8 @@ void testTemplateFromPromptRegistryByScenario() throws IOException { assertThat(response.getContent()).startsWith("I sistemi ERP (Enterprise Resource Planning)"); assertThat(response.getOriginalResponse().getIntermediateResults().getTemplating()).hasSize(2); - try (var requestInputStream = fileLoader.apply("templateReferenceByScenarioRequest.json")) { - final String request = new String(requestInputStream.readAllBytes()); - verify(postRequestedFor(anyUrl()).withRequestBody(equalToJson(request))); - } + final String request = fileLoaderStr.apply("templateReferenceByScenarioRequest.json"); + verify(postRequestedFor(anyUrl()).withRequestBody(equalToJson(request))); } @Test @@ -1231,10 +1242,8 @@ void testTemplateFromInput() throws IOException { final var response = client.chatCompletion(prompt, configWithTemplate); - try (var requestInputStream = fileLoader.apply("localTemplateRequest.json")) { - final String request = new String(requestInputStream.readAllBytes()); - verify(postRequestedFor(anyUrl()).withRequestBody(equalToJson(request))); - } + final String request = fileLoaderStr.apply("localTemplateRequest.json"); + verify(postRequestedFor(anyUrl()).withRequestBody(equalToJson(request))); } @Test From c7d29dca1daba52c81bacb4f3516cb6c0048e884 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20D=C3=BCmont?= Date: Fri, 29 Aug 2025 10:47:14 +0200 Subject: [PATCH 03/10] Add release note --- docs/release_notes.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/release_notes.md b/docs/release_notes.md index 6d043220b..cc9577087 100644 --- a/docs/release_notes.md +++ b/docs/release_notes.md @@ -23,6 +23,7 @@ `OrchestrationAiModel.GEMINI_1_5_FLASH` - Replacement are `GEMINI_2_5_PRO` and `GEMINI_2_5_FLASH`. - [Orchestration] Deprecated `OrchestrationAiModel.IBM_GRANITE_13B_CHAT` with no replacement. +- [Orchestration] OutputFilter configuration for streaming can be conveniently set via `OrchestrationModuleConfig#withOutputFilteringStreamOptions`. - [OpenAI] [Introduced SpringAI integration with our OpenAI client.](https://sap.github.io/ai-sdk/docs/java/spring-ai/openai) - Added `OpenAiChatModel` - [Prompt Registry] [Using Prompt Registry Templates in SpringAI.](https://sap.github.io/ai-sdk/docs/java/ai-core/prompt-registry#using-templates-in-springai) From ea27ee0a5705fcd96934127631803c86593b1f7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20D=C3=BCmont?= Date: Fri, 29 Aug 2025 10:53:24 +0200 Subject: [PATCH 04/10] Add external link to javadoc --- .../sap/ai/sdk/orchestration/OrchestrationModuleConfig.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationModuleConfig.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationModuleConfig.java index 2b43662d7..3a41fe0bc 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationModuleConfig.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationModuleConfig.java @@ -225,6 +225,9 @@ public OrchestrationModuleConfig withOutputFiltering( /** * Creates a new configuration with the given output filtering stream options. * + * @see Orchestration + * documentation on streaming. * @param outputFilteringStreamOptions The output filtering stream options to use. * @return A new configuration with the given output filtering stream options. */ From cc5932a3aab5d544dc96a23b41dc8f41698dadd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20D=C3=BCmont?= Date: Fri, 29 Aug 2025 10:54:10 +0200 Subject: [PATCH 05/10] Fix PMD --- .../com/sap/ai/sdk/orchestration/OrchestrationModuleConfig.java | 1 + 1 file changed, 1 insertion(+) diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationModuleConfig.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationModuleConfig.java index 3a41fe0bc..25f571ce3 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationModuleConfig.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationModuleConfig.java @@ -231,6 +231,7 @@ public OrchestrationModuleConfig withOutputFiltering( * @param outputFilteringStreamOptions The output filtering stream options to use. * @return A new configuration with the given output filtering stream options. */ + @Nonnull public OrchestrationModuleConfig withOutputFilteringStreamOptions( @Nullable final FilteringStreamOptions outputFilteringStreamOptions) { if (filteringConfig != null && filteringConfig.getOutput() != null) { From f1b2d5e5f7d7d2160a6e3b5c37a73c18ad0c7c59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20D=C3=BCmont?= Date: Mon, 22 Sep 2025 17:53:44 +0200 Subject: [PATCH 06/10] Initial convenience class --- .../ConfigToRequestTransformer.java | 5 +- .../OrchestrationModuleConfig.java | 29 +++++++++-- .../OrchestrationStreamConfig.java | 50 +++++++++++++++++++ 3 files changed, 80 insertions(+), 4 deletions(-) create mode 100644 orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationStreamConfig.java diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/ConfigToRequestTransformer.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/ConfigToRequestTransformer.java index ff77c3e5d..d7d64fb8d 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/ConfigToRequestTransformer.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/ConfigToRequestTransformer.java @@ -38,8 +38,11 @@ static CompletionPostRequest toCompletionPostRequest( val moduleConfigs = toModuleConfigs(configCopy); + val reqConfig = + OrchestrationConfig.create().modules(moduleConfigs).stream(config.getGlobalStreamOptions()); + return CompletionPostRequest.create() - .config(OrchestrationConfig.create().modules(moduleConfigs)) + .config(reqConfig) .placeholderValues(prompt.getTemplateParameters()) .messagesHistory(messageHistory); } diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationModuleConfig.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationModuleConfig.java index 25f571ce3..529b2eba6 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationModuleConfig.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationModuleConfig.java @@ -3,6 +3,7 @@ import com.google.common.annotations.Beta; import com.sap.ai.sdk.orchestration.model.FilteringModuleConfig; import com.sap.ai.sdk.orchestration.model.FilteringStreamOptions; +import com.sap.ai.sdk.orchestration.model.GlobalStreamOptions; import com.sap.ai.sdk.orchestration.model.GroundingModuleConfig; import com.sap.ai.sdk.orchestration.model.InputFilteringConfig; import com.sap.ai.sdk.orchestration.model.LLMModelDetails; @@ -18,6 +19,7 @@ import javax.annotation.Nullable; import lombok.AccessLevel; import lombok.AllArgsConstructor; +import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Value; import lombok.With; @@ -102,10 +104,17 @@ public class OrchestrationModuleConfig { @Nullable SAPDocumentTranslation outputTranslationConfig; /** Configuration of optional streaming options for output filtering. */ - @With(AccessLevel.NONE) + @With(AccessLevel.NONE) // may be exposed to public in the future + @Getter(AccessLevel.PACKAGE) @Nullable FilteringStreamOptions outputFilteringStreamOptions; + /** Configuration of optional streaming options for output filtering. */ + @With(AccessLevel.PRIVATE) // may be exposed to public in the future + @Getter(AccessLevel.PACKAGE) + @Nullable + GlobalStreamOptions globalStreamOptions; + /** * Creates a new configuration with the given LLM configuration. * @@ -121,6 +130,19 @@ public OrchestrationModuleConfig withLlmConfig(@Nonnull final OrchestrationAiMod return withLlmConfig(aiModel.createConfig()); } + /** + * Creates a new configuration with the given stream configuration. + * + * @param config The stream configuration to use. + * @return A new configuration with the given stream configuration. + */ + @Nonnull + public OrchestrationModuleConfig withStreamConfig( + @Nonnull final OrchestrationStreamConfig config) { + return this.withOutputFilteringStreamOptions(config.createFilteringStreamOptions()) + .withGlobalStreamOptions(config.createGlobalStreamOptions()); + } + /** * Creates a new configuration with the given Data Masking configuration. * @@ -232,7 +254,7 @@ public OrchestrationModuleConfig withOutputFiltering( * @return A new configuration with the given output filtering stream options. */ @Nonnull - public OrchestrationModuleConfig withOutputFilteringStreamOptions( + OrchestrationModuleConfig withOutputFilteringStreamOptions( @Nullable final FilteringStreamOptions outputFilteringStreamOptions) { if (filteringConfig != null && filteringConfig.getOutput() != null) { filteringConfig.getOutput().setStreamOptions(outputFilteringStreamOptions); @@ -245,7 +267,8 @@ public OrchestrationModuleConfig withOutputFilteringStreamOptions( this.groundingConfig, this.inputTranslationConfig, this.outputTranslationConfig, - outputFilteringStreamOptions); + outputFilteringStreamOptions, + this.globalStreamOptions); } /** diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationStreamConfig.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationStreamConfig.java new file mode 100644 index 000000000..3ca8bb526 --- /dev/null +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationStreamConfig.java @@ -0,0 +1,50 @@ +package com.sap.ai.sdk.orchestration; + +import com.sap.ai.sdk.orchestration.model.FilteringStreamOptions; +import com.sap.ai.sdk.orchestration.model.GlobalStreamOptions; +import java.util.List; +import java.util.Optional; +import javax.annotation.Nullable; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Value; +import lombok.With; +import lombok.val; + +/** + * Configuration for orchestration streaming options. + * + * @since 1.12.0 + */ +@Value +@With +@AllArgsConstructor(access = AccessLevel.PRIVATE) +public class OrchestrationStreamConfig { + /** + * Number of characters that should be additionally sent to content filtering services from + * previous chunks as additional context. + */ + @Nullable Integer overlap; + + /** Size of the chunks the response will be split into when streaming. */ + @Nullable Integer chunkSize = null; + + /** List of delimiters to use for chunking the response when streaming. */ + @Nullable List delimiters = null; + + @Nullable + FilteringStreamOptions createFilteringStreamOptions() { + return overlap == null ? null : FilteringStreamOptions.create().overlap(overlap); + } + + @Nullable + GlobalStreamOptions createGlobalStreamOptions() { + if (chunkSize == null && delimiters == null) { + return null; + } + val opts = GlobalStreamOptions.create(); + Optional.ofNullable(chunkSize).ifPresent(opts::setChunkSize); + Optional.ofNullable(delimiters).ifPresent(d -> opts.setDelimiters(List.copyOf(d))); + return opts; + } +} From 3f6a7b3a8ec1279855a0c3b92150f5db9420ae66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20D=C3=BCmont?= Date: Mon, 22 Sep 2025 18:22:17 +0200 Subject: [PATCH 07/10] Initial convenience class --- .../orchestration/OrchestrationClient.java | 8 ++++++- .../OrchestrationStreamConfig.java | 17 +++++++++----- .../orchestration/OrchestrationUnitTest.java | 22 +++++++++++++++++++ .../filteringLooseRequestStream.json | 4 +++- 4 files changed, 43 insertions(+), 8 deletions(-) diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java index 483503b85..d8cd5cdaa 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationClient.java @@ -219,7 +219,13 @@ public OrchestrationChatResponse executeRequestFromJsonModuleConfig( @Nonnull public Stream streamChatCompletionDeltas( @Nonnull final CompletionPostRequest request) throws OrchestrationClientException { - request.getConfig().setStream(GlobalStreamOptions.create().enabled(true).delimiters(null)); + val config = request.getConfig(); + val stream = config.getStream(); + if (stream == null) { + config.setStream(GlobalStreamOptions.create().enabled(true).delimiters(null)); + } else { + stream.enabled(true); + } return executor.stream(COMPLETION_ENDPOINT, request, customHeaders); } diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationStreamConfig.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationStreamConfig.java index 3ca8bb526..55c9e61dd 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationStreamConfig.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationStreamConfig.java @@ -24,17 +24,22 @@ public class OrchestrationStreamConfig { * Number of characters that should be additionally sent to content filtering services from * previous chunks as additional context. */ - @Nullable Integer overlap; + @Nullable Integer filterOverlap; /** Size of the chunks the response will be split into when streaming. */ - @Nullable Integer chunkSize = null; + @Nullable Integer chunkSize; /** List of delimiters to use for chunking the response when streaming. */ - @Nullable List delimiters = null; + @Nullable List delimiters; + + /** Default constructor for OrchestrationStreamConfig. */ + public OrchestrationStreamConfig() { + this(null, null, null); + } @Nullable FilteringStreamOptions createFilteringStreamOptions() { - return overlap == null ? null : FilteringStreamOptions.create().overlap(overlap); + return filterOverlap == null ? null : FilteringStreamOptions.create().overlap(filterOverlap); } @Nullable @@ -42,9 +47,9 @@ GlobalStreamOptions createGlobalStreamOptions() { if (chunkSize == null && delimiters == null) { return null; } - val opts = GlobalStreamOptions.create(); + val opts = GlobalStreamOptions.create().enabled(true); Optional.ofNullable(chunkSize).ifPresent(opts::setChunkSize); - Optional.ofNullable(delimiters).ifPresent(d -> opts.setDelimiters(List.copyOf(d))); + opts.setDelimiters(delimiters == null ? null : List.copyOf(delimiters)); return opts; } } diff --git a/orchestration/src/test/java/com/sap/ai/sdk/orchestration/OrchestrationUnitTest.java b/orchestration/src/test/java/com/sap/ai/sdk/orchestration/OrchestrationUnitTest.java index e45572dbc..77e360eb2 100644 --- a/orchestration/src/test/java/com/sap/ai/sdk/orchestration/OrchestrationUnitTest.java +++ b/orchestration/src/test/java/com/sap/ai/sdk/orchestration/OrchestrationUnitTest.java @@ -481,6 +481,28 @@ void filteringLooseStream() throws IOException { verify(postRequestedFor(anyUrl()).withRequestBody(equalToJson(request, true, false))); } + @Test + void convenienceConfig() { + final var azureFilter = new AzureContentFilter().hate(ALLOW_SAFE_LOW_MEDIUM); + + OrchestrationModuleConfig myConfig = + config + .withOutputFiltering(azureFilter) + .withOutputFilteringStreamOptions(FilteringStreamOptions.create().overlap(1_000)); + OrchestrationModuleConfig myConfig2 = + config + .withOutputFiltering(azureFilter) + .withStreamConfig(new OrchestrationStreamConfig().withFilterOverlap(1_000)); + assertThat(myConfig).isEqualTo(myConfig2); + + OrchestrationModuleConfig myConfig3 = + config + .withOutputFiltering(azureFilter) + .withStreamConfig( + new OrchestrationStreamConfig().withFilterOverlap(1_000).withChunkSize(10)); + assertThat(myConfig2).isNotEqualTo(myConfig3); + } + @Test void inputFilteringStrict() { stubFor( diff --git a/orchestration/src/test/resources/filteringLooseRequestStream.json b/orchestration/src/test/resources/filteringLooseRequestStream.json index 1309daf06..749125d03 100644 --- a/orchestration/src/test/resources/filteringLooseRequestStream.json +++ b/orchestration/src/test/resources/filteringLooseRequestStream.json @@ -12,7 +12,9 @@ "top_p": 1, "n": 1 }, - "version": "latest" + "version": "latest", + "timeout" : 600, + "max_retries" : 2 }, "prompt": { "template": [ From f5f376fee4c960c12cafba1c65bc8a041b0ac403 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20D=C3=BCmont?= Date: Mon, 22 Sep 2025 18:25:14 +0200 Subject: [PATCH 08/10] Disable default stream --- .../com/sap/ai/sdk/orchestration/OrchestrationStreamConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationStreamConfig.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationStreamConfig.java index 55c9e61dd..bb7d40ccc 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationStreamConfig.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationStreamConfig.java @@ -47,7 +47,7 @@ GlobalStreamOptions createGlobalStreamOptions() { if (chunkSize == null && delimiters == null) { return null; } - val opts = GlobalStreamOptions.create().enabled(true); + val opts = GlobalStreamOptions.create(); Optional.ofNullable(chunkSize).ifPresent(opts::setChunkSize); opts.setDelimiters(delimiters == null ? null : List.copyOf(delimiters)); return opts; From 2dcbd3b1a34636ae0cb3dcddb25ccaf3aeda6608 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20D=C3=BCmont?= Date: Mon, 22 Sep 2025 18:27:02 +0200 Subject: [PATCH 09/10] Update release note --- docs/release_notes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/release_notes.md b/docs/release_notes.md index 6ddfd170d..04b6cff67 100644 --- a/docs/release_notes.md +++ b/docs/release_notes.md @@ -12,7 +12,7 @@ ### ✨ New Functionality -- [Orchestration] OutputFilter configuration for streaming can be conveniently set via `OrchestrationModuleConfig#withOutputFilteringStreamOptions`. +- [Orchestration] For streaming, add convenience configuration AOU for output-filter-overlap, chunk-size, and delimiters via `OrchestrationModuleConfig#withStreamConfig`. ### 📈 Improvements From 5c46a3eda1f6388acde1e9792ccf19defa124a92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20D=C3=BCmont?= Date: Tue, 7 Oct 2025 10:40:25 +0200 Subject: [PATCH 10/10] JavaDoc fix --- .../com/sap/ai/sdk/orchestration/OrchestrationModuleConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationModuleConfig.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationModuleConfig.java index 622ce7af0..bdd8da4d5 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationModuleConfig.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationModuleConfig.java @@ -110,7 +110,7 @@ public class OrchestrationModuleConfig { @Nullable FilteringStreamOptions outputFilteringStreamOptions; - /** Configuration of optional streaming options for output filtering. */ + /** Configuration of optional global streaming options, e.g. chunk-size. */ @With(AccessLevel.PRIVATE) // may be exposed to public in the future @Getter(AccessLevel.PACKAGE) @Nullable