Skip to content

Commit dde0cbc

Browse files
AlbumenJAlbumenJ
authored
feat(model): add cache_control support for OpenAI and DashScope protocols (#985)
Add prompt caching via `cache_control: {"type": "ephemeral"}` to reduce API latency and cost for both OpenAI and DashScope model integrations. - Add `cacheControl` option in GenerateOptions for model-level config - Add `CACHE_CONTROL` metadata key for per-message manual marking - Add `cache_control` field to OpenAIMessage and DashScopeMessage DTOs - Implement automatic strategy in formatters: apply cache_control to system messages and the last message in conversation - Implement metadata-based manual marking in message converters - Keep OpenAI and DashScope implementations fully independent - Add comprehensive unit tests for both protocols 🤖 Generated with [Qoder][https://qoder.com] ## AgentScope-Java Version [The version of AgentScope-Java you are working on, e.g. 1.0.9, check your pom.xml dependency version or run `mvn dependency:tree | grep agentscope-parent:pom`(only mac/linux)] ## Description [Please describe the background, purpose, changes made, and how to test this PR] ## Checklist Please check the following items before code is ready to be reviewed. - [ ] Code has been formatted with `mvn spotless:apply` - [ ] All tests are passing (`mvn test`) - [ ] Javadoc comments are complete and follow project conventions - [ ] Related documentation has been updated (e.g. links, examples, etc.) - [ ] Code is ready for review
1 parent 8f547f3 commit dde0cbc

15 files changed

Lines changed: 994 additions & 9 deletions

‎agentscope-core/src/main/java/io/agentscope/core/formatter/dashscope/DashScopeChatFormatter.java‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import java.time.Instant;
3030
import java.util.ArrayList;
3131
import java.util.List;
32+
import java.util.Map;
3233
import java.util.stream.Collectors;
3334

3435
/**
@@ -41,6 +42,8 @@
4142
public class DashScopeChatFormatter
4243
extends AbstractBaseFormatter<DashScopeMessage, DashScopeResponse, DashScopeRequest> {
4344

45+
private static final Map<String, String> EPHEMERAL_CACHE_CONTROL = Map.of("type", "ephemeral");
46+
4447
private final DashScopeMessageConverter messageConverter;
4548
private final DashScopeResponseParser responseParser;
4649
private final DashScopeToolsHelper toolsHelper;
@@ -168,4 +171,37 @@ public DashScopeRequest buildRequest(
168171

169172
return request;
170173
}
174+
175+
/**
176+
* Apply cache control to DashScope messages.
177+
*
178+
* <p>Adds <code>cache_control: {"type": "ephemeral"}</code> to all system messages and the last
179+
* message in the list. Messages that already have cache_control set (e.g., via manual metadata
180+
* marking) will not be overwritten.
181+
*
182+
* @param messages the list of formatted DashScope messages
183+
*/
184+
public void applyCacheControl(List<DashScopeMessage> messages) {
185+
if (messages == null || messages.isEmpty()) {
186+
return;
187+
}
188+
for (DashScopeMessage msg : messages) {
189+
if ("system".equals(msg.getRole()) && msg.getCacheControl() == null) {
190+
msg.setCacheControl(EPHEMERAL_CACHE_CONTROL);
191+
}
192+
}
193+
DashScopeMessage lastMsg = messages.get(messages.size() - 1);
194+
if (lastMsg.getCacheControl() == null) {
195+
lastMsg.setCacheControl(EPHEMERAL_CACHE_CONTROL);
196+
}
197+
}
198+
199+
/**
200+
* Get the ephemeral cache control constant.
201+
*
202+
* @return unmodifiable map representing ephemeral cache control
203+
*/
204+
static Map<String, String> getEphemeralCacheControl() {
205+
return EPHEMERAL_CACHE_CONTROL;
206+
}
171207
}

‎agentscope-core/src/main/java/io/agentscope/core/formatter/dashscope/DashScopeMessageConverter.java‎

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import io.agentscope.core.message.AudioBlock;
2121
import io.agentscope.core.message.ContentBlock;
2222
import io.agentscope.core.message.ImageBlock;
23+
import io.agentscope.core.message.MessageMetadataKeys;
2324
import io.agentscope.core.message.Msg;
2425
import io.agentscope.core.message.MsgRole;
2526
import io.agentscope.core.message.TextBlock;
@@ -64,11 +65,17 @@ public DashScopeMessageConverter(Function<List<ContentBlock>, String> toolResult
6465
* @return The converted DashScopeMessage
6566
*/
6667
public DashScopeMessage convertToMessage(Msg msg, boolean useMultimodalFormat) {
68+
DashScopeMessage result;
6769
if (useMultimodalFormat) {
68-
return convertToMultimodalContent(msg);
70+
result = convertToMultimodalContent(msg);
6971
} else {
70-
return convertToSimpleContent(msg);
72+
result = convertToSimpleContent(msg);
7173
}
74+
75+
// Apply cache_control from message metadata if manually marked
76+
applyCacheControlFromMetadata(msg, result);
77+
78+
return result;
7279
}
7380

7481
/**
@@ -237,4 +244,20 @@ private String extractTextContent(Msg msg) {
237244
.map(block -> ((TextBlock) block).getText())
238245
.reduce("", (a, b) -> a.isEmpty() ? b : a + "\n" + b);
239246
}
247+
248+
/**
249+
* Apply cache_control from Msg metadata to the converted DashScopeMessage.
250+
*
251+
* @param msg the source message with metadata
252+
* @param result the converted DashScope message
253+
*/
254+
private void applyCacheControlFromMetadata(Msg msg, DashScopeMessage result) {
255+
if (msg.getMetadata() == null) {
256+
return;
257+
}
258+
Object cacheFlag = msg.getMetadata().get(MessageMetadataKeys.CACHE_CONTROL);
259+
if (Boolean.TRUE.equals(cacheFlag)) {
260+
result.setCacheControl(DashScopeChatFormatter.getEphemeralCacheControl());
261+
}
262+
}
240263
}

‎agentscope-core/src/main/java/io/agentscope/core/formatter/dashscope/DashScopeMultiAgentFormatter.java‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
import java.time.Instant;
3434
import java.util.ArrayList;
3535
import java.util.List;
36+
import java.util.Map;
3637

3738
/**
3839
* DashScope formatter for multi-agent conversations.
@@ -363,4 +364,29 @@ private static class MessageGroup {
363364
this.messages = messages;
364365
}
365366
}
367+
368+
/**
369+
* Apply cache control to DashScope messages.
370+
*
371+
* <p>Adds <code>cache_control: {"type": "ephemeral"}</code> to all system messages and the last
372+
* message in the list. Messages that already have cache_control set (e.g., via manual metadata
373+
* marking) will not be overwritten.
374+
*
375+
* @param messages the list of formatted DashScope messages
376+
*/
377+
public void applyCacheControl(List<DashScopeMessage> messages) {
378+
if (messages == null || messages.isEmpty()) {
379+
return;
380+
}
381+
Map<String, String> ephemeral = DashScopeChatFormatter.getEphemeralCacheControl();
382+
for (DashScopeMessage msg : messages) {
383+
if ("system".equals(msg.getRole()) && msg.getCacheControl() == null) {
384+
msg.setCacheControl(ephemeral);
385+
}
386+
}
387+
DashScopeMessage lastMsg = messages.get(messages.size() - 1);
388+
if (lastMsg.getCacheControl() == null) {
389+
lastMsg.setCacheControl(ephemeral);
390+
}
391+
}
366392
}

‎agentscope-core/src/main/java/io/agentscope/core/formatter/dashscope/dto/DashScopeMessage.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import com.fasterxml.jackson.core.type.TypeReference;
2323
import io.agentscope.core.util.JsonUtils;
2424
import java.util.List;
25+
import java.util.Map;
2526

2627
/**
2728
* DashScope message DTO.
@@ -80,6 +81,10 @@ public class DashScopeMessage {
8081
@JsonProperty("reasoning_content")
8182
private String reasoningContent;
8283

84+
/** Cache control configuration for prompt caching. */
85+
@JsonProperty("cache_control")
86+
private Map<String, String> cacheControl;
87+
8388
public DashScopeMessage() {}
8489

8590
public String getRole() {
@@ -177,6 +182,14 @@ public void setReasoningContent(String reasoningContent) {
177182
this.reasoningContent = reasoningContent;
178183
}
179184

185+
public Map<String, String> getCacheControl() {
186+
return cacheControl;
187+
}
188+
189+
public void setCacheControl(Map<String, String> cacheControl) {
190+
this.cacheControl = cacheControl;
191+
}
192+
180193
public static Builder builder() {
181194
return new Builder();
182195
}
@@ -219,6 +232,11 @@ public Builder reasoningContent(String reasoningContent) {
219232
return this;
220233
}
221234

235+
public Builder cacheControl(Map<String, String> cacheControl) {
236+
message.setCacheControl(cacheControl);
237+
return this;
238+
}
239+
222240
public DashScopeMessage build() {
223241
return message;
224242
}

‎agentscope-core/src/main/java/io/agentscope/core/formatter/openai/OpenAIBaseFormatter.java‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import io.agentscope.core.model.ToolSchema;
2626
import java.time.Instant;
2727
import java.util.List;
28+
import java.util.Map;
2829

2930
/**
3031
* Base formatter for OpenAI Chat Completion HTTP API.
@@ -41,6 +42,8 @@
4142
public abstract class OpenAIBaseFormatter
4243
extends AbstractBaseFormatter<OpenAIMessage, OpenAIResponse, OpenAIRequest> {
4344

45+
private static final Map<String, String> EPHEMERAL_CACHE_CONTROL = Map.of("type", "ephemeral");
46+
4447
protected final OpenAIMessageConverter messageConverter;
4548
protected final OpenAIResponseParser responseParser;
4649

@@ -165,4 +168,37 @@ public OpenAIRequest buildRequest(
165168

166169
return request;
167170
}
171+
172+
/**
173+
* Apply cache control to OpenAI messages.
174+
*
175+
* <p>Adds <code>cache_control: {"type": "ephemeral"}</code> to all system messages and the last
176+
* message in the list. Messages that already have cache_control set (e.g., via manual metadata
177+
* marking) will not be overwritten.
178+
*
179+
* @param messages the list of formatted OpenAI messages
180+
*/
181+
public void applyCacheControl(List<OpenAIMessage> messages) {
182+
if (messages == null || messages.isEmpty()) {
183+
return;
184+
}
185+
for (OpenAIMessage msg : messages) {
186+
if ("system".equals(msg.getRole()) && msg.getCacheControl() == null) {
187+
msg.setCacheControl(EPHEMERAL_CACHE_CONTROL);
188+
}
189+
}
190+
OpenAIMessage lastMsg = messages.get(messages.size() - 1);
191+
if (lastMsg.getCacheControl() == null) {
192+
lastMsg.setCacheControl(EPHEMERAL_CACHE_CONTROL);
193+
}
194+
}
195+
196+
/**
197+
* Get the ephemeral cache control constant.
198+
*
199+
* @return unmodifiable map representing ephemeral cache control
200+
*/
201+
static Map<String, String> getEphemeralCacheControl() {
202+
return EPHEMERAL_CACHE_CONTROL;
203+
}
168204
}

‎agentscope-core/src/main/java/io/agentscope/core/formatter/openai/OpenAIMessageConverter.java‎

Lines changed: 31 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import io.agentscope.core.message.Base64Source;
2525
import io.agentscope.core.message.ContentBlock;
2626
import io.agentscope.core.message.ImageBlock;
27+
import io.agentscope.core.message.MessageMetadataKeys;
2728
import io.agentscope.core.message.Msg;
2829
import io.agentscope.core.message.MsgRole;
2930
import io.agentscope.core.message.Source;
@@ -75,16 +76,23 @@ public OpenAIMessageConverter(
7576
*/
7677
public OpenAIMessage convertToMessage(Msg msg, boolean hasMediaContent) {
7778
// Check if SYSTEM message contains tool result - treat as TOOL role
79+
OpenAIMessage result;
7880
if (msg.getRole() == MsgRole.SYSTEM && msg.hasContentBlocks(ToolResultBlock.class)) {
79-
return convertToolMessage(msg);
81+
result = convertToolMessage(msg);
82+
} else {
83+
result =
84+
switch (msg.getRole()) {
85+
case SYSTEM -> convertSystemMessage(msg);
86+
case USER -> convertUserMessage(msg, hasMediaContent);
87+
case ASSISTANT -> convertAssistantMessage(msg);
88+
case TOOL -> convertToolMessage(msg);
89+
};
8090
}
8191

82-
return switch (msg.getRole()) {
83-
case SYSTEM -> convertSystemMessage(msg);
84-
case USER -> convertUserMessage(msg, hasMediaContent);
85-
case ASSISTANT -> convertAssistantMessage(msg);
86-
case TOOL -> convertToolMessage(msg);
87-
};
92+
// Apply cache_control from message metadata if manually marked
93+
applyCacheControlFromMetadata(msg, result);
94+
95+
return result;
8896
}
8997

9098
/**
@@ -468,4 +476,20 @@ private String convertVideoSourceToUrl(Source source) {
468476
private String detectAudioFormat(String mediaType) {
469477
return OpenAIConverterUtils.detectAudioFormat(mediaType);
470478
}
479+
480+
/**
481+
* Apply cache_control from Msg metadata to the converted OpenAIMessage.
482+
*
483+
* @param msg the source message with metadata
484+
* @param result the converted OpenAI message
485+
*/
486+
private void applyCacheControlFromMetadata(Msg msg, OpenAIMessage result) {
487+
if (msg.getMetadata() == null) {
488+
return;
489+
}
490+
Object cacheFlag = msg.getMetadata().get(MessageMetadataKeys.CACHE_CONTROL);
491+
if (Boolean.TRUE.equals(cacheFlag)) {
492+
result.setCacheControl(OpenAIBaseFormatter.getEphemeralCacheControl());
493+
}
494+
}
471495
}

‎agentscope-core/src/main/java/io/agentscope/core/formatter/openai/dto/OpenAIMessage.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import com.fasterxml.jackson.annotation.JsonInclude;
2121
import com.fasterxml.jackson.annotation.JsonProperty;
2222
import java.util.List;
23+
import java.util.Map;
2324

2425
/**
2526
* OpenAI message DTO.
@@ -93,6 +94,10 @@ public class OpenAIMessage {
9394
@JsonProperty("refusal")
9495
private String refusal;
9596

97+
/** Cache control configuration for prompt caching. */
98+
@JsonProperty("cache_control")
99+
private Map<String, String> cacheControl;
100+
96101
public OpenAIMessage() {}
97102

98103
public String getRole() {
@@ -159,6 +164,14 @@ public void setRefusal(String refusal) {
159164
this.refusal = refusal;
160165
}
161166

167+
public Map<String, String> getCacheControl() {
168+
return cacheControl;
169+
}
170+
171+
public void setCacheControl(Map<String, String> cacheControl) {
172+
this.cacheControl = cacheControl;
173+
}
174+
162175
/**
163176
* Get content as String (for text-only messages).
164177
*
@@ -259,6 +272,11 @@ public Builder refusal(String refusal) {
259272
return this;
260273
}
261274

275+
public Builder cacheControl(Map<String, String> cacheControl) {
276+
message.setCacheControl(cacheControl);
277+
return this;
278+
}
279+
262280
public OpenAIMessage build() {
263281
OpenAIMessage result = message;
264282
message = new OpenAIMessage();

‎agentscope-core/src/main/java/io/agentscope/core/message/MessageMetadataKeys.java‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,4 +105,30 @@ private MessageMetadataKeys() {
105105
* }</pre>
106106
*/
107107
public static final String STRUCTURED_OUTPUT = "_structured_output";
108+
109+
/**
110+
* Metadata key to mark a message for prompt caching.
111+
*
112+
* <p>When set to {@code true}, the formatter will add <code>cache_control:
113+
* {"type": "ephemeral"}</code> to this message during formatting. This allows users to manually
114+
* mark specific
115+
* messages for caching, independent of the automatic cache control strategy configured via
116+
* {@link io.agentscope.core.model.GenerateOptions#getCacheControl()}.
117+
*
118+
* <p>Manually marked messages take priority over the automatic strategy — they will not be
119+
* overwritten.
120+
*
121+
* <p><b>Type:</b> Boolean
122+
* <p><b>Example:</b>
123+
* <pre>{@code
124+
* Map<String, Object> metadata = new HashMap<>();
125+
* metadata.put(MessageMetadataKeys.CACHE_CONTROL, true);
126+
* Msg msg = Msg.builder()
127+
* .role(MsgRole.USER)
128+
* .textContent("Important context to cache...")
129+
* .metadata(metadata)
130+
* .build();
131+
* }</pre>
132+
*/
133+
public static final String CACHE_CONTROL = "_cache_control";
108134
}

‎agentscope-core/src/main/java/io/agentscope/core/model/DashScopeChatModel.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -268,6 +268,15 @@ private Flux<ChatResponse> streamWithHttpClient(
268268
// Apply thinking mode if enabled
269269
applyThinkingMode(request, effectiveOptions);
270270

271+
// Apply cache control if enabled (adds cache_control to system msgs + last msg)
272+
if (Boolean.TRUE.equals(effectiveOptions.getCacheControl())) {
273+
if (formatter instanceof DashScopeChatFormatter chatFmt) {
274+
chatFmt.applyCacheControl(request.getInput().getMessages());
275+
} else if (formatter instanceof DashScopeMultiAgentFormatter multiFmt) {
276+
multiFmt.applyCacheControl(request.getInput().getMessages());
277+
}
278+
}
279+
271280
// Set endpoint type for endpoint selection
272281
request.setEndpointType(endpointType);
273282

0 commit comments

Comments
 (0)