Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 51 additions & 0 deletions strands-py/src/strands/models/bedrock.py
Original file line number Diff line number Diff line change
Expand Up @@ -288,6 +288,13 @@ def format_request(
)
system_blocks.append({"cachePoint": {"type": cache_prompt}})

# Auto-inject a cachePoint at the end of the system prompt so repeated calls with
# the same static system prefix hit the cache. Anthropic docs describe the
# tools → system → messages prefix chain; caching only messages leaves the
# (usually largest and most static) system prefix uncached.
if self._should_cache_system(system_blocks):
system_blocks.append(self._build_system_cache_point())

return {
"modelId": self.config["model_id"],
"messages": self._format_bedrock_messages(messages),
Expand Down Expand Up @@ -387,6 +394,50 @@ def _get_additional_request_fields(self, tool_choice: ToolChoice | None) -> dict

return {"additionalModelRequestFields": additional_fields}

def _should_cache_system(self, system_blocks: list[SystemContentBlock]) -> bool:
"""Whether to auto-inject a cache point at the end of the system prompt.

True only when auto caching is enabled for this model, ``system_blocks`` has cacheable
content, and no cachePoint has already been placed at the end of the system prefix.

Args:
system_blocks: The system content blocks that will be sent to Bedrock.

Returns:
True if a cache point should be appended.
"""
cache_config = self.config.get("cache_config")
if not cache_config:
return False

resolved: str | None = cache_config.strategy
if resolved == "auto":
resolved = self._cache_strategy
if resolved != "anthropic":
return False

if not system_blocks:
return False

if "cachePoint" in system_blocks[-1]:
return False

return True

def _build_system_cache_point(self) -> SystemContentBlock:
"""Build the cache point block appended to the system prompt.

Uses ``cache_config.ttl`` when set; falls back to Bedrock's ``default`` (5m).

Returns:
The cache point block.
"""
cache_point: dict[str, Any] = {"type": "default"}
cache_config = self.config.get("cache_config")
if cache_config and cache_config.ttl:
cache_point["ttl"] = cache_config.ttl
return cast(SystemContentBlock, {"cachePoint": cache_point})

def _build_tools_cache_point(self) -> list[dict[str, Any]]:
"""Build the cache point block appended to ``toolConfig.tools`` if ``cache_tools`` is configured.

Expand Down
91 changes: 91 additions & 0 deletions strands-py/tests/strands/models/test_bedrock.py
Original file line number Diff line number Diff line change
Expand Up @@ -3892,3 +3892,94 @@ def test_format_request_cache_tools_string_backward_compat(model, messages, mode

exp_cache_point = {"cachePoint": {"type": cache_type}}
assert tru_request["toolConfig"]["tools"][-1] == exp_cache_point


def test_format_request_auto_appends_system_cache_point(bedrock_client, messages):
"""Auto mode appends a cachePoint after the system prompt for a Claude model.

Regression guard for https://github.com/strands-agents/harness-sdk/issues/3144.
"""
model = BedrockModel(
model_id="us.anthropic.claude-sonnet-4-20250514-v1:0",
cache_config=CacheConfig(strategy="auto"),
)

tru_request = model.format_request(messages, system_prompt_content=[{"text": "you are helpful"}])

assert tru_request["system"] == [
{"text": "you are helpful"},
{"cachePoint": {"type": "default"}},
]


def test_format_request_auto_system_cache_point_honors_ttl(bedrock_client, messages):
"""Auto mode carries cache_config.ttl into the appended system cache point."""
model = BedrockModel(
model_id="us.anthropic.claude-sonnet-4-20250514-v1:0",
cache_config=CacheConfig(strategy="auto", ttl="1h"),
)

tru_request = model.format_request(messages, system_prompt_content=[{"text": "static"}])

assert tru_request["system"][-1] == {"cachePoint": {"type": "default", "ttl": "1h"}}


def test_format_request_auto_skips_system_cache_point_when_empty(bedrock_client, messages):
"""Auto mode does not inject a system cache point when the system prompt is empty."""
model = BedrockModel(
model_id="us.anthropic.claude-sonnet-4-20250514-v1:0",
cache_config=CacheConfig(strategy="auto"),
)

tru_request = model.format_request(messages)

assert tru_request["system"] == []


def test_format_request_auto_skips_system_cache_point_for_non_claude(bedrock_client, messages):
"""Auto mode does not inject a system cache point when the model has no auto strategy."""
model = BedrockModel(
model_id="amazon.nova-pro-v1:0",
cache_config=CacheConfig(strategy="auto"),
)

tru_request = model.format_request(messages, system_prompt_content=[{"text": "static"}])

assert tru_request["system"] == [{"text": "static"}]


def test_format_request_auto_preserves_caller_placed_system_cache_point(bedrock_client, messages):
"""Auto mode does not double-append when the caller already placed a trailing cachePoint."""
model = BedrockModel(
model_id="us.anthropic.claude-sonnet-4-20250514-v1:0",
cache_config=CacheConfig(strategy="auto"),
)

system_blocks = [{"text": "static"}, {"cachePoint": {"type": "default", "ttl": "1h"}}]
tru_request = model.format_request(messages, system_prompt_content=system_blocks)

assert tru_request["system"] == [
{"text": "static"},
{"cachePoint": {"type": "default", "ttl": "1h"}},
]


def test_format_request_no_cache_config_leaves_system_untouched(bedrock_client, messages):
"""With no cache_config, the system prompt is passed through unchanged."""
model = BedrockModel(model_id="us.anthropic.claude-sonnet-4-20250514-v1:0")

tru_request = model.format_request(messages, system_prompt_content=[{"text": "static"}])

assert tru_request["system"] == [{"text": "static"}]


def test_format_request_anthropic_strategy_appends_system_cache_point(bedrock_client, messages):
"""Explicit anthropic strategy also appends a system cache point, mirroring auto."""
model = BedrockModel(
model_id="arn:aws:bedrock:us-east-1:123:application-inference-profile/abc",
cache_config=CacheConfig(strategy="anthropic"),
)

tru_request = model.format_request(messages, system_prompt_content=[{"text": "static"}])

assert tru_request["system"][-1] == {"cachePoint": {"type": "default"}}
57 changes: 53 additions & 4 deletions strands-ts/src/models/__tests__/bedrock.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -495,7 +495,7 @@ describe('BedrockModel', () => {
content: [{ text: 'Hello' }, { cachePoint: { type: 'default' } }],
},
],
system: [{ text: 'You are a helpful assistant' }],
system: [{ text: 'You are a helpful assistant' }, { cachePoint: { type: 'default' } }],
toolConfig: {
toolChoice: { auto: {} },
tools: [
Expand Down Expand Up @@ -1519,7 +1519,8 @@ describe('BedrockModel', () => {
vi.clearAllMocks()
})

it('does not add cache points to string system prompt with cacheConfig', async () => {
it('appends a cache point to a string system prompt with cacheConfig', async () => {
// Regression guard for https://github.com/strands-agents/harness-sdk/issues/3144.
const provider = new BedrockModel({ cacheConfig: { strategy: 'auto' } })
const messages = [new Message({ role: 'user', content: [new TextBlock('Hello')] })]
const options: StreamOptions = {
Expand All @@ -1536,7 +1537,7 @@ describe('BedrockModel', () => {
content: [{ text: 'Hello' }, { cachePoint: { type: 'default' } }],
},
],
system: [{ text: 'You are a helpful assistant' }],
system: [{ text: 'You are a helpful assistant' }, { cachePoint: { type: 'default' } }],
})
})

Expand Down Expand Up @@ -1682,7 +1683,7 @@ describe('BedrockModel', () => {
collectIterator(provider.stream(messages, options))

const call = mockConverseStreamCommand.mock.lastCall?.[0]
expect(call?.system).toStrictEqual([{ text: 'You are a helpful assistant' }])
expect(call?.system).toStrictEqual([{ text: 'You are a helpful assistant' }, { cachePoint: { type: 'default' } }])
expect(call?.toolConfig?.tools).toStrictEqual([
{
toolSpec: {
Expand Down Expand Up @@ -2135,6 +2136,54 @@ describe('BedrockModel', () => {
],
})
})

it('carries systemTTL from cacheConfig into the appended system cache point', async () => {
const provider = new BedrockModel({ cacheConfig: { strategy: 'auto', systemTTL: '1h' } })
const messages = [new Message({ role: 'user', content: [new TextBlock('Hello')] })]
const options: StreamOptions = { systemPrompt: 'static prompt' }

collectIterator(provider.stream(messages, options))

const call = mockConverseStreamCommand.mock.lastCall?.[0]
expect(call?.system).toStrictEqual([{ text: 'static prompt' }, { cachePoint: { type: 'default', ttl: '1h' } }])
})

it('does not duplicate the system cache point when caller placed one at the end', async () => {
const provider = new BedrockModel({ cacheConfig: { strategy: 'auto' } })
const messages = [new Message({ role: 'user', content: [new TextBlock('Hello')] })]
const options: StreamOptions = {
systemPrompt: [new TextBlock('static prompt'), new CachePointBlock({ cacheType: 'default', ttl: '1h' })],
}

collectIterator(provider.stream(messages, options))

const call = mockConverseStreamCommand.mock.lastCall?.[0]
expect(call?.system).toStrictEqual([{ text: 'static prompt' }, { cachePoint: { type: 'default', ttl: '1h' } }])
})

it('skips the system cache point when the system prompt is absent', async () => {
const provider = new BedrockModel({ cacheConfig: { strategy: 'auto' } })
const messages = [new Message({ role: 'user', content: [new TextBlock('Hello')] })]

collectIterator(provider.stream(messages))

const call = mockConverseStreamCommand.mock.lastCall?.[0]
expect(call?.system).toBeUndefined()
})

it('appends a system cache point under explicit anthropic strategy for ARN inference profiles', async () => {
const provider = new BedrockModel({
modelId: 'arn:aws:bedrock:us-east-1:123:application-inference-profile/abc',
cacheConfig: { strategy: 'anthropic' },
})
const messages = [new Message({ role: 'user', content: [new TextBlock('Hello')] })]
const options: StreamOptions = { systemPrompt: 'static prompt' }

collectIterator(provider.stream(messages, options))

const call = mockConverseStreamCommand.mock.lastCall?.[0]
expect(call?.system).toStrictEqual([{ text: 'static prompt' }, { cachePoint: { type: 'default' } }])
})
})

describe('guard content in messages', async () => {
Expand Down
19 changes: 19 additions & 0 deletions strands-ts/src/models/bedrock.ts
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,9 @@ export interface BedrockCacheConfig extends CacheConfig {

/** TTL applied to the auto-injected cache point appended to the last user message. */
messagesTTL?: BedrockCacheTTL

/** TTL applied to the auto-injected cache point appended to the system prompt. */
systemTTL?: BedrockCacheTTL
}

/**
Expand Down Expand Up @@ -682,6 +685,22 @@ export class BedrockModel extends Model<BedrockModelConfig> {
}
}

// Auto-inject a cachePoint at the end of the system prompt so repeated calls with the
// same static system prefix hit the cache. Bedrock (Anthropic) documents the prefix
// chain as tools → system → messages; caching only messages leaves the (usually largest
// and most static) system prefix uncached.
if (request.system && request.system.length > 0 && this._shouldEnableCaching()) {
const lastBlock = request.system[request.system.length - 1]
if (!lastBlock || !('cachePoint' in lastBlock)) {
const cachePoint: BedrockCachePointBlock = { type: 'default' }
const ttl = this._config.cacheConfig?.systemTTL
if (ttl !== undefined) {
cachePoint.ttl = ttl as BedrockSdkCacheTTL
}
request.system.push({ cachePoint })
}
}

// Add tool configuration
// Bedrock requires toolConfig when messages contain tool use/result blocks.
// When no tools were provided but messages reference past tool usage (e.g. during
Expand Down
Loading