From 6383aa65e8d4a552cdf8bfcf0ec2c39cf3dfb994 Mon Sep 17 00:00:00 2001 From: arielnabavian Date: Wed, 5 Aug 2026 23:40:28 +0000 Subject: [PATCH] fix(bedrock): cache system prompt in auto mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CacheConfig(strategy="auto") previously only wrote a cache breakpoint on the last user message, leaving the (usually largest and most static) system prefix uncached. For workloads that share a system prompt across calls with varying user messages, every call wrote to cache and none read — a net cost regression versus no caching. Append a cachePoint to the system prompt when caching resolves to "anthropic" for the model and the caller hasn't already placed one at the end. Both SDKs mirror the change. TS adds a systemTTL knob to BedrockCacheConfig to match toolsTTL/messagesTTL. Fixes #3144. --- strands-py/src/strands/models/bedrock.py | 51 +++++++++++ .../tests/strands/models/test_bedrock.py | 91 +++++++++++++++++++ .../src/models/__tests__/bedrock.test.ts | 57 +++++++++++- strands-ts/src/models/bedrock.ts | 19 ++++ 4 files changed, 214 insertions(+), 4 deletions(-) diff --git a/strands-py/src/strands/models/bedrock.py b/strands-py/src/strands/models/bedrock.py index d3fa0af495..92d9184c40 100644 --- a/strands-py/src/strands/models/bedrock.py +++ b/strands-py/src/strands/models/bedrock.py @@ -288,6 +288,13 @@ def format_request( ) system_blocks.append({"cachePoint": {"type": cache_prompt}}) + # Auto-inject a cachePoint at the end of the system prompt so repeated calls with + # the same static system prefix hit the cache. Anthropic docs describe the + # tools → system → messages prefix chain; caching only messages leaves the + # (usually largest and most static) system prefix uncached. + if self._should_cache_system(system_blocks): + system_blocks.append(self._build_system_cache_point()) + return { "modelId": self.config["model_id"], "messages": self._format_bedrock_messages(messages), @@ -387,6 +394,50 @@ def _get_additional_request_fields(self, tool_choice: ToolChoice | None) -> dict return {"additionalModelRequestFields": additional_fields} + def _should_cache_system(self, system_blocks: list[SystemContentBlock]) -> bool: + """Whether to auto-inject a cache point at the end of the system prompt. + + True only when auto caching is enabled for this model, ``system_blocks`` has cacheable + content, and no cachePoint has already been placed at the end of the system prefix. + + Args: + system_blocks: The system content blocks that will be sent to Bedrock. + + Returns: + True if a cache point should be appended. + """ + cache_config = self.config.get("cache_config") + if not cache_config: + return False + + resolved: str | None = cache_config.strategy + if resolved == "auto": + resolved = self._cache_strategy + if resolved != "anthropic": + return False + + if not system_blocks: + return False + + if "cachePoint" in system_blocks[-1]: + return False + + return True + + def _build_system_cache_point(self) -> SystemContentBlock: + """Build the cache point block appended to the system prompt. + + Uses ``cache_config.ttl`` when set; falls back to Bedrock's ``default`` (5m). + + Returns: + The cache point block. + """ + cache_point: dict[str, Any] = {"type": "default"} + cache_config = self.config.get("cache_config") + if cache_config and cache_config.ttl: + cache_point["ttl"] = cache_config.ttl + return cast(SystemContentBlock, {"cachePoint": cache_point}) + def _build_tools_cache_point(self) -> list[dict[str, Any]]: """Build the cache point block appended to ``toolConfig.tools`` if ``cache_tools`` is configured. diff --git a/strands-py/tests/strands/models/test_bedrock.py b/strands-py/tests/strands/models/test_bedrock.py index 3d12f84ecb..3dc2d08b04 100644 --- a/strands-py/tests/strands/models/test_bedrock.py +++ b/strands-py/tests/strands/models/test_bedrock.py @@ -3892,3 +3892,94 @@ def test_format_request_cache_tools_string_backward_compat(model, messages, mode exp_cache_point = {"cachePoint": {"type": cache_type}} assert tru_request["toolConfig"]["tools"][-1] == exp_cache_point + + +def test_format_request_auto_appends_system_cache_point(bedrock_client, messages): + """Auto mode appends a cachePoint after the system prompt for a Claude model. + + Regression guard for https://github.com/strands-agents/harness-sdk/issues/3144. + """ + model = BedrockModel( + model_id="us.anthropic.claude-sonnet-4-20250514-v1:0", + cache_config=CacheConfig(strategy="auto"), + ) + + tru_request = model.format_request(messages, system_prompt_content=[{"text": "you are helpful"}]) + + assert tru_request["system"] == [ + {"text": "you are helpful"}, + {"cachePoint": {"type": "default"}}, + ] + + +def test_format_request_auto_system_cache_point_honors_ttl(bedrock_client, messages): + """Auto mode carries cache_config.ttl into the appended system cache point.""" + model = BedrockModel( + model_id="us.anthropic.claude-sonnet-4-20250514-v1:0", + cache_config=CacheConfig(strategy="auto", ttl="1h"), + ) + + tru_request = model.format_request(messages, system_prompt_content=[{"text": "static"}]) + + assert tru_request["system"][-1] == {"cachePoint": {"type": "default", "ttl": "1h"}} + + +def test_format_request_auto_skips_system_cache_point_when_empty(bedrock_client, messages): + """Auto mode does not inject a system cache point when the system prompt is empty.""" + model = BedrockModel( + model_id="us.anthropic.claude-sonnet-4-20250514-v1:0", + cache_config=CacheConfig(strategy="auto"), + ) + + tru_request = model.format_request(messages) + + assert tru_request["system"] == [] + + +def test_format_request_auto_skips_system_cache_point_for_non_claude(bedrock_client, messages): + """Auto mode does not inject a system cache point when the model has no auto strategy.""" + model = BedrockModel( + model_id="amazon.nova-pro-v1:0", + cache_config=CacheConfig(strategy="auto"), + ) + + tru_request = model.format_request(messages, system_prompt_content=[{"text": "static"}]) + + assert tru_request["system"] == [{"text": "static"}] + + +def test_format_request_auto_preserves_caller_placed_system_cache_point(bedrock_client, messages): + """Auto mode does not double-append when the caller already placed a trailing cachePoint.""" + model = BedrockModel( + model_id="us.anthropic.claude-sonnet-4-20250514-v1:0", + cache_config=CacheConfig(strategy="auto"), + ) + + system_blocks = [{"text": "static"}, {"cachePoint": {"type": "default", "ttl": "1h"}}] + tru_request = model.format_request(messages, system_prompt_content=system_blocks) + + assert tru_request["system"] == [ + {"text": "static"}, + {"cachePoint": {"type": "default", "ttl": "1h"}}, + ] + + +def test_format_request_no_cache_config_leaves_system_untouched(bedrock_client, messages): + """With no cache_config, the system prompt is passed through unchanged.""" + model = BedrockModel(model_id="us.anthropic.claude-sonnet-4-20250514-v1:0") + + tru_request = model.format_request(messages, system_prompt_content=[{"text": "static"}]) + + assert tru_request["system"] == [{"text": "static"}] + + +def test_format_request_anthropic_strategy_appends_system_cache_point(bedrock_client, messages): + """Explicit anthropic strategy also appends a system cache point, mirroring auto.""" + model = BedrockModel( + model_id="arn:aws:bedrock:us-east-1:123:application-inference-profile/abc", + cache_config=CacheConfig(strategy="anthropic"), + ) + + tru_request = model.format_request(messages, system_prompt_content=[{"text": "static"}]) + + assert tru_request["system"][-1] == {"cachePoint": {"type": "default"}} diff --git a/strands-ts/src/models/__tests__/bedrock.test.ts b/strands-ts/src/models/__tests__/bedrock.test.ts index f97922f86d..a74d138dbe 100644 --- a/strands-ts/src/models/__tests__/bedrock.test.ts +++ b/strands-ts/src/models/__tests__/bedrock.test.ts @@ -495,7 +495,7 @@ describe('BedrockModel', () => { content: [{ text: 'Hello' }, { cachePoint: { type: 'default' } }], }, ], - system: [{ text: 'You are a helpful assistant' }], + system: [{ text: 'You are a helpful assistant' }, { cachePoint: { type: 'default' } }], toolConfig: { toolChoice: { auto: {} }, tools: [ @@ -1519,7 +1519,8 @@ describe('BedrockModel', () => { vi.clearAllMocks() }) - it('does not add cache points to string system prompt with cacheConfig', async () => { + it('appends a cache point to a string system prompt with cacheConfig', async () => { + // Regression guard for https://github.com/strands-agents/harness-sdk/issues/3144. const provider = new BedrockModel({ cacheConfig: { strategy: 'auto' } }) const messages = [new Message({ role: 'user', content: [new TextBlock('Hello')] })] const options: StreamOptions = { @@ -1536,7 +1537,7 @@ describe('BedrockModel', () => { content: [{ text: 'Hello' }, { cachePoint: { type: 'default' } }], }, ], - system: [{ text: 'You are a helpful assistant' }], + system: [{ text: 'You are a helpful assistant' }, { cachePoint: { type: 'default' } }], }) }) @@ -1682,7 +1683,7 @@ describe('BedrockModel', () => { collectIterator(provider.stream(messages, options)) const call = mockConverseStreamCommand.mock.lastCall?.[0] - expect(call?.system).toStrictEqual([{ text: 'You are a helpful assistant' }]) + expect(call?.system).toStrictEqual([{ text: 'You are a helpful assistant' }, { cachePoint: { type: 'default' } }]) expect(call?.toolConfig?.tools).toStrictEqual([ { toolSpec: { @@ -2135,6 +2136,54 @@ describe('BedrockModel', () => { ], }) }) + + it('carries systemTTL from cacheConfig into the appended system cache point', async () => { + const provider = new BedrockModel({ cacheConfig: { strategy: 'auto', systemTTL: '1h' } }) + const messages = [new Message({ role: 'user', content: [new TextBlock('Hello')] })] + const options: StreamOptions = { systemPrompt: 'static prompt' } + + collectIterator(provider.stream(messages, options)) + + const call = mockConverseStreamCommand.mock.lastCall?.[0] + expect(call?.system).toStrictEqual([{ text: 'static prompt' }, { cachePoint: { type: 'default', ttl: '1h' } }]) + }) + + it('does not duplicate the system cache point when caller placed one at the end', async () => { + const provider = new BedrockModel({ cacheConfig: { strategy: 'auto' } }) + const messages = [new Message({ role: 'user', content: [new TextBlock('Hello')] })] + const options: StreamOptions = { + systemPrompt: [new TextBlock('static prompt'), new CachePointBlock({ cacheType: 'default', ttl: '1h' })], + } + + collectIterator(provider.stream(messages, options)) + + const call = mockConverseStreamCommand.mock.lastCall?.[0] + expect(call?.system).toStrictEqual([{ text: 'static prompt' }, { cachePoint: { type: 'default', ttl: '1h' } }]) + }) + + it('skips the system cache point when the system prompt is absent', async () => { + const provider = new BedrockModel({ cacheConfig: { strategy: 'auto' } }) + const messages = [new Message({ role: 'user', content: [new TextBlock('Hello')] })] + + collectIterator(provider.stream(messages)) + + const call = mockConverseStreamCommand.mock.lastCall?.[0] + expect(call?.system).toBeUndefined() + }) + + it('appends a system cache point under explicit anthropic strategy for ARN inference profiles', async () => { + const provider = new BedrockModel({ + modelId: 'arn:aws:bedrock:us-east-1:123:application-inference-profile/abc', + cacheConfig: { strategy: 'anthropic' }, + }) + const messages = [new Message({ role: 'user', content: [new TextBlock('Hello')] })] + const options: StreamOptions = { systemPrompt: 'static prompt' } + + collectIterator(provider.stream(messages, options)) + + const call = mockConverseStreamCommand.mock.lastCall?.[0] + expect(call?.system).toStrictEqual([{ text: 'static prompt' }, { cachePoint: { type: 'default' } }]) + }) }) describe('guard content in messages', async () => { diff --git a/strands-ts/src/models/bedrock.ts b/strands-ts/src/models/bedrock.ts index 04b608f87e..b1f4e273aa 100644 --- a/strands-ts/src/models/bedrock.ts +++ b/strands-ts/src/models/bedrock.ts @@ -153,6 +153,9 @@ export interface BedrockCacheConfig extends CacheConfig { /** TTL applied to the auto-injected cache point appended to the last user message. */ messagesTTL?: BedrockCacheTTL + + /** TTL applied to the auto-injected cache point appended to the system prompt. */ + systemTTL?: BedrockCacheTTL } /** @@ -682,6 +685,22 @@ export class BedrockModel extends Model { } } + // Auto-inject a cachePoint at the end of the system prompt so repeated calls with the + // same static system prefix hit the cache. Bedrock (Anthropic) documents the prefix + // chain as tools → system → messages; caching only messages leaves the (usually largest + // and most static) system prefix uncached. + if (request.system && request.system.length > 0 && this._shouldEnableCaching()) { + const lastBlock = request.system[request.system.length - 1] + if (!lastBlock || !('cachePoint' in lastBlock)) { + const cachePoint: BedrockCachePointBlock = { type: 'default' } + const ttl = this._config.cacheConfig?.systemTTL + if (ttl !== undefined) { + cachePoint.ttl = ttl as BedrockSdkCacheTTL + } + request.system.push({ cachePoint }) + } + } + // Add tool configuration // Bedrock requires toolConfig when messages contain tool use/result blocks. // When no tools were provided but messages reference past tool usage (e.g. during