Skip to content

Commit 699396d

Browse files
E2E tests (#35)
* first pass at playwright tests * first basic test * fixes, including openai tooling * more small fixes * ci: apply automated fixes --------- Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
1 parent 21d1d9b commit 699396d

29 files changed

Lines changed: 1266 additions & 67 deletions

assets/CleanShot_2025-11-27_at_08.28.46_2x-4a965c47-7c20-4a7c-acd5-317a2c7876cd.png

Loading

knip.json

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,31 @@
11
{
22
"$schema": "https://unpkg.com/knip@5/schema.json",
3-
"ignoreDependencies": ["@faker-js/faker"],
3+
"ignoreDependencies": ["@faker-js/faker", "@playwright/test"],
44
"ignoreWorkspaces": ["examples/**"],
5+
"ignore": [
6+
"packages/typescript/ai-openai/live-tests/**",
7+
"packages/typescript/ai-openai/src/**/*.test.ts",
8+
"packages/typescript/ai-openai/src/audio/audio-provider-options.ts",
9+
"packages/typescript/ai-openai/src/audio/transcribe-provider-options.ts",
10+
"packages/typescript/ai-openai/src/image/image-provider-options.ts",
11+
"packages/typescript/smoke-tests/adapters/src/**",
12+
"packages/typescript/smoke-tests/e2e/playwright.config.ts",
13+
"packages/typescript/smoke-tests/e2e/src/**",
14+
"packages/typescript/smoke-tests/e2e/vite.config.ts"
15+
],
16+
"ignoreExportsUsedInFile": true,
517
"workspaces": {
618
"packages/react-ai": {
719
"ignore": []
20+
},
21+
"packages/typescript/ai-anthropic": {
22+
"ignore": ["src/tools/**"]
23+
},
24+
"packages/typescript/ai-gemini": {
25+
"ignore": ["src/tools/**"]
26+
},
27+
"packages/typescript/ai-openai": {
28+
"ignore": ["src/tools/**"]
829
}
930
}
1031
}

package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
"clean": "find . -name 'dist' -type d -prune -exec rm -rf {} +",
2121
"clean:node_modules": "find . -name 'node_modules' -type d -prune -exec rm -rf {} +",
2222
"clean:all": "pnpm run clean && pnpm run clean:node_modules",
23-
"copy:readme": "cp README.md packages/typescript/ai/README.md && cp README.md packages/typescript/ai-devtools/README.md && cp README.md packages/typescript/ai-client/README.md && cp README.md packages/typescript/ai-gemini/README.md && cp README.md packages/typescript/ai-ollama/README.md && cp README.md packages/typescript/ai-openai/README.md && cp README.md packages/typescript/ai-react/README.md && cp README.md packages/typescript/ai-react-ui/README.md && cp README.md packages/typescript/react-ai-devtools/README.md && cp README.md packages/typescript/solid-ai-devtools/README.md && cp README.md packages/typescript/tests-adapters/README.md",
23+
"copy:readme": "cp README.md packages/typescript/ai/README.md && cp README.md packages/typescript/ai-devtools/README.md && cp README.md packages/typescript/ai-client/README.md && cp README.md packages/typescript/ai-gemini/README.md && cp README.md packages/typescript/ai-ollama/README.md && cp README.md packages/typescript/ai-openai/README.md && cp README.md packages/typescript/ai-react/README.md && cp README.md packages/typescript/ai-react-ui/README.md && cp README.md packages/typescript/react-ai-devtools/README.md && cp README.md packages/typescript/solid-ai-devtools/README.md",
2424
"dev": "pnpm run watch",
2525
"docs:generate": "node scripts/generateDocs.js && pnpm run copy:readme",
2626
"format": "pnpm run prettier:write",
Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
# OpenAI Adapter Live Tests
2+
3+
This directory contains live integration tests for the OpenAI adapter using the Responses API.
4+
5+
## Setup
6+
7+
1. Create a `.env.local` file in this directory with your OpenAI API key:
8+
9+
```
10+
OPENAI_API_KEY=sk-...
11+
```
12+
13+
2. Install dependencies from the workspace root:
14+
15+
```bash
16+
pnpm install
17+
```
18+
19+
## Running Tests
20+
21+
Run individual tests:
22+
23+
```bash
24+
pnpm test # Test with required parameters
25+
pnpm test:optional # Test with optional parameters
26+
```
27+
28+
Run all tests:
29+
30+
```bash
31+
pnpm test:all
32+
```
33+
34+
## Test Scripts
35+
36+
### `tool-test.ts`
37+
38+
Tests tool calling with all required parameters. Verifies that:
39+
40+
- Tool calls are properly detected in the stream
41+
- Function names are correctly captured
42+
- Arguments are passed as JSON strings
43+
- Tools can be executed with the parsed arguments
44+
45+
### `tool-test-optional.ts`
46+
47+
Tests tool calling with optional parameters. Verifies that:
48+
49+
- Tools with optional parameters work correctly
50+
- The strict mode is disabled when not all parameters are required
51+
- Default values can be applied for missing optional parameters
52+
53+
## Key Findings
54+
55+
The OpenAI Responses API has different behavior compared to the Chat Completions API:
56+
57+
1. **Strict Mode**: When `strict: true`, ALL properties must be in the `required` array
58+
2. **Tool Metadata**: Function names come from `response.output_item.added` events, not from `response.function_call_arguments.done`
59+
3. **Finish Reason**: The Responses API doesn't have a `finish_reason` field; it must be inferred from the output content
Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
{
2+
"name": "ai-openai-live-tests",
3+
"version": "0.0.0",
4+
"private": true,
5+
"type": "module",
6+
"scripts": {
7+
"test": "tsx tool-test.ts",
8+
"test:optional": "tsx tool-test-optional.ts",
9+
"test:all": "tsx tool-test.ts && tsx tool-test-optional.ts"
10+
},
11+
"dependencies": {
12+
"@tanstack/ai": "workspace:*",
13+
"@tanstack/ai-openai": "workspace:*"
14+
},
15+
"devDependencies": {
16+
"tsx": "^4.19.2"
17+
}
18+
}
Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,182 @@
1+
import { createOpenAI } from '../src/index'
2+
import { readFileSync } from 'fs'
3+
import { join, dirname } from 'path'
4+
import { fileURLToPath } from 'url'
5+
6+
// Load environment variables from .env.local manually
7+
const __dirname = dirname(fileURLToPath(import.meta.url))
8+
try {
9+
const envContent = readFileSync(join(__dirname, '.env.local'), 'utf-8')
10+
envContent.split('\n').forEach((line) => {
11+
const match = line.match(/^([^=]+)=(.*)$/)
12+
if (match) {
13+
process.env[match[1].trim()] = match[2].trim()
14+
}
15+
})
16+
} catch (e) {
17+
// .env.local not found, will use process.env
18+
}
19+
20+
const apiKey = process.env.OPENAI_API_KEY
21+
22+
if (!apiKey) {
23+
console.error('❌ OPENAI_API_KEY not found in .env.local')
24+
process.exit(1)
25+
}
26+
27+
async function testToolWithOptionalParameters() {
28+
console.log('🚀 Testing OpenAI tool calling with OPTIONAL parameters\n')
29+
30+
const adapter = createOpenAI(apiKey)
31+
32+
// Create a tool with optional parameters (unit is optional)
33+
const getTemperatureTool = {
34+
type: 'function' as const,
35+
function: {
36+
name: 'get_temperature',
37+
description: 'Get the current temperature for a specific location',
38+
parameters: {
39+
type: 'object',
40+
properties: {
41+
location: {
42+
type: 'string',
43+
description: 'The city or location to get the temperature for',
44+
},
45+
unit: {
46+
type: 'string',
47+
enum: ['celsius', 'fahrenheit'],
48+
description:
49+
'The temperature unit (optional, defaults to fahrenheit)',
50+
},
51+
},
52+
required: ['location'], // unit is optional
53+
},
54+
},
55+
execute: async (args: any) => {
56+
console.log(
57+
'✅ Tool executed with arguments:',
58+
JSON.stringify(args, null, 2),
59+
)
60+
61+
if (!args || !args.location) {
62+
console.error('❌ ERROR: Location argument is missing!')
63+
return 'Error: Location is required'
64+
}
65+
66+
const unit = args.unit || 'fahrenheit'
67+
console.log(` - location: "${args.location}"`)
68+
console.log(
69+
` - unit: "${unit}" (${args.unit ? 'provided' : 'defaulted'})`,
70+
)
71+
72+
return `The temperature in ${args.location} is 72°${unit === 'celsius' ? 'C' : 'F'}`
73+
},
74+
}
75+
76+
const messages = [
77+
{
78+
role: 'user' as const,
79+
content:
80+
'What is the temperature in Paris? Use the get_temperature tool.',
81+
},
82+
]
83+
84+
console.log('📤 Sending request with tool:')
85+
console.log(' Tool name:', getTemperatureTool.function.name)
86+
console.log(
87+
' Required params:',
88+
getTemperatureTool.function.parameters.required,
89+
)
90+
console.log(' Optional params:', ['unit'])
91+
console.log(' User message:', messages[0].content)
92+
console.log()
93+
94+
try {
95+
console.log('📥 Streaming response...\n')
96+
97+
let toolCallFound = false
98+
let toolCallArguments: any = null
99+
let toolExecuted = false
100+
let finalResponse = ''
101+
102+
// @ts-ignore - using internal chat method
103+
const stream = adapter.chatStream({
104+
model: 'gpt-4o-mini',
105+
messages,
106+
tools: [getTemperatureTool],
107+
})
108+
109+
for await (const chunk of stream) {
110+
if (chunk.type === 'tool_call') {
111+
toolCallFound = true
112+
toolCallArguments = chunk.toolCall.function.arguments
113+
console.log('🔧 Tool call detected!')
114+
console.log(' Name:', chunk.toolCall.function.name)
115+
console.log(' Arguments (raw):', toolCallArguments)
116+
117+
// Parse if it's a string
118+
if (typeof toolCallArguments === 'string') {
119+
try {
120+
const parsed = JSON.parse(toolCallArguments)
121+
toolCallArguments = parsed
122+
} catch (e) {
123+
console.error(' ❌ Failed to parse arguments:', e)
124+
}
125+
}
126+
127+
// Execute the tool
128+
if (getTemperatureTool.execute) {
129+
console.log('\n🔨 Executing tool...')
130+
try {
131+
const result = await getTemperatureTool.execute(toolCallArguments)
132+
toolExecuted = true
133+
console.log(' Result:', result)
134+
} catch (error) {
135+
console.error(' ❌ Tool execution error:', error)
136+
}
137+
}
138+
}
139+
140+
if (chunk.type === 'content') {
141+
finalResponse += chunk.delta
142+
}
143+
}
144+
145+
console.log('\n' + '='.repeat(60))
146+
console.log('📊 Test Summary:')
147+
console.log(' Tool call found:', toolCallFound ? '✅' : '❌')
148+
console.log(' Arguments received:', toolCallArguments ? '✅' : '❌')
149+
console.log(' Tool executed:', toolExecuted ? '✅' : '❌')
150+
console.log(
151+
' Location provided:',
152+
toolCallArguments?.location ? '✅' : '❌',
153+
)
154+
console.log('='.repeat(60))
155+
156+
if (!toolCallFound) {
157+
console.error('\n❌ FAIL: No tool call was detected')
158+
process.exit(1)
159+
}
160+
161+
if (!toolCallArguments || !toolCallArguments.location) {
162+
console.error('\n❌ FAIL: Tool arguments missing or invalid')
163+
process.exit(1)
164+
}
165+
166+
if (!toolExecuted) {
167+
console.error('\n❌ FAIL: Tool was not executed successfully')
168+
process.exit(1)
169+
}
170+
171+
console.log(
172+
'\n✅ SUCCESS: Tool calling with optional parameters works correctly!',
173+
)
174+
process.exit(0)
175+
} catch (error: any) {
176+
console.error('\n❌ ERROR:', error.message)
177+
console.error('Stack:', error.stack)
178+
process.exit(1)
179+
}
180+
}
181+
182+
testToolWithOptionalParameters()

0 commit comments

Comments
 (0)