diff --git a/.changeset/little-cougars-bow.md b/.changeset/little-cougars-bow.md new file mode 100644 index 00000000..7d1d3514 --- /dev/null +++ b/.changeset/little-cougars-bow.md @@ -0,0 +1,5 @@ +--- +"@instructor-ai/instructor": minor +--- + +adding a new mode to support parsing thinking blocks out of markdown json responses (R1) diff --git a/.github/workflows/test-pr.yml b/.github/workflows/test-pr.yml index 287a0471..234bd3c1 100644 --- a/.github/workflows/test-pr.yml +++ b/.github/workflows/test-pr.yml @@ -23,6 +23,7 @@ jobs: TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0f8ad89b..f978ca27 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,7 +15,7 @@ jobs: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }} GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} - + GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} steps: - uses: actions/checkout@v3 with: diff --git a/bun.lockb b/bun.lockb index aed4d109..78d3d862 100755 Binary files a/bun.lockb and b/bun.lockb differ diff --git a/package.json b/package.json index 824da231..06fd4239 100644 --- a/package.json +++ b/package.json @@ -44,14 +44,17 @@ "outputs", "zod" ], - "author": "Jason Liu", + "contributors": [ + "Dimitri Kennedy", + "Jason Liu" + ], "license": "MIT", "bugs": { "url": "https://github.com/instructor-ai/instructor-js/issues" }, "homepage": "https://github.com/instructor-ai/instructor-js#readme", "dependencies": { - "zod-stream": "2.0.0", + "zod-stream": "3.0.0", "zod-validation-error": "^3.4.0" }, "peerDependencies": { diff --git a/src/constants/providers.ts b/src/constants/providers.ts index 123b0cc2..5696e7d6 100644 --- a/src/constants/providers.ts +++ b/src/constants/providers.ts @@ -1,9 +1,16 @@ import { omit } from "@/lib" import OpenAI from "openai" import { z } from "zod" -import { withResponseModel, MODE as ZMODE, type Mode } from "zod-stream" +import { thinkingJsonParser, withResponseModel, MODE as ZMODE } from "zod-stream" + +import { Mode } from "../types" + +export const MODE: typeof ZMODE = ZMODE + +export const MODE_TO_RESPONSE_PARSER = { + [MODE.THINKING_MD_JSON]: thinkingJsonParser +} -export const MODE = ZMODE export const PROVIDERS = { OAI: "OAI", ANYSCALE: "ANYSCALE", @@ -12,6 +19,7 @@ export const PROVIDERS = { GROQ: "GROQ", OTHER: "OTHER" } as const + export type Provider = keyof typeof PROVIDERS export const PROVIDER_SUPPORTED_MODES: { @@ -98,7 +106,8 @@ export const PROVIDER_SUPPORTED_MODES_BY_MODEL = { [MODE.TOOLS]: ["*"], [MODE.JSON]: ["*"], [MODE.MD_JSON]: ["*"], - [MODE.JSON_SCHEMA]: ["*"] + [MODE.JSON_SCHEMA]: ["*"], + [MODE.THINKING_MD_JSON]: ["*"] }, [PROVIDERS.OAI]: { [MODE.FUNCTIONS]: ["*"], @@ -122,6 +131,7 @@ export const PROVIDER_SUPPORTED_MODES_BY_MODEL = { }, [PROVIDERS.GROQ]: { [MODE.TOOLS]: ["*"], - [MODE.MD_JSON]: ["*"] + [MODE.MD_JSON]: ["*"], + [MODE.THINKING_MD_JSON]: ["deepseek-r1-distill-llama-70b"] } } diff --git a/src/instructor.ts b/src/instructor.ts index f9c749a3..c4aa49c9 100644 --- a/src/instructor.ts +++ b/src/instructor.ts @@ -11,10 +11,11 @@ import { import OpenAI from "openai" import { Stream } from "openai/streaming" import { z, ZodError } from "zod" -import ZodStream, { OAIResponseParser, OAIStream, withResponseModel, type Mode } from "zod-stream" +import ZodStream, { OAIResponseParser, OAIStream, withResponseModel } from "zod-stream" import { fromZodError } from "zod-validation-error" import { + MODE_TO_RESPONSE_PARSER, NON_OAI_PROVIDER_URLS, Provider, PROVIDER_PARAMS_TRANSFORMERS, @@ -22,7 +23,7 @@ import { PROVIDERS } from "./constants/providers" import { iterableTee } from "./lib" -import { ClientTypeChatCompletionParams, CompletionMeta } from "./types" +import { ClientTypeChatCompletionParams, CompletionMeta, Mode } from "./types" const MAX_RETRIES_DEFAULT = 0 @@ -68,6 +69,7 @@ class Instructor { : this.client?.baseURL.includes(NON_OAI_PROVIDER_URLS.TOGETHER) ? PROVIDERS.TOGETHER : this.client?.baseURL.includes(NON_OAI_PROVIDER_URLS.OAI) ? PROVIDERS.OAI : this.client?.baseURL.includes(NON_OAI_PROVIDER_URLS.ANTHROPIC) ? PROVIDERS.ANTHROPIC + : this.client?.baseURL.includes(NON_OAI_PROVIDER_URLS.GROQ) ? PROVIDERS.GROQ : PROVIDERS.OTHER : PROVIDERS.OTHER @@ -187,13 +189,22 @@ class Instructor { throw error } - const parsedCompletion = OAIResponseParser( - completion as OpenAI.Chat.Completions.ChatCompletion - ) + const responseParser = MODE_TO_RESPONSE_PARSER?.[this.mode] ?? OAIResponseParser + const parsedCompletion = responseParser(completion as OpenAI.Chat.Completions.ChatCompletion) try { - const data = JSON.parse(parsedCompletion) as z.infer & { _meta?: CompletionMeta } - return { ...data, _meta: { usage: completion?.usage ?? undefined } } + const responseJson = parsedCompletion.json ?? parsedCompletion + const data = JSON.parse(responseJson) as z.infer & { + _meta?: CompletionMeta + thinking?: string + } + return { + ...data, + _meta: { + usage: completion?.usage ?? undefined, + thinking: parsedCompletion?.thinking ?? undefined + } + } } catch (error) { this.log( "error", diff --git a/src/types/index.ts b/src/types/index.ts index 717fc6bf..ceab5d5a 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -61,9 +61,10 @@ export type LogLevel = "debug" | "info" | "warn" | "error" export type CompletionMeta = Partial & { usage?: OpenAI.CompletionUsage + thinking?: string } -export type Mode = ZMode +export type Mode = ZMode | "THINKING_MD_JSON" export type ResponseModel = ZResponseModel diff --git a/tests/deepseek.test.ts b/tests/deepseek.test.ts new file mode 100644 index 00000000..c8cdc988 --- /dev/null +++ b/tests/deepseek.test.ts @@ -0,0 +1,80 @@ +import Instructor from "@/index" +import { describe, expect, test } from "bun:test" +import OpenAI from "openai" +import { z } from "zod" + +const textBlock = ` +In our recent online meeting, participants from various backgrounds joined to discuss the upcoming tech conference. The names and contact details of the participants were as follows: + +- Name: John Doe, Email: johndoe@email.com, Twitter: @TechGuru44 +- Name: Jane Smith, Email: janesmith@email.com, Twitter: @DigitalDiva88 +- Name: Alex Johnson, Email: alexj@email.com, Twitter: @CodeMaster2023 + +During the meeting, we agreed on several key points. The conference will be held on March 15th, 2024, at the Grand Tech Arena located at 4521 Innovation Drive. Dr. Emily Johnson, a renowned AI researcher, will be our keynote speaker. + +The budget for the event is set at $50,000, covering venue costs, speaker fees, and promotional activities. Each participant is expected to contribute an article to the conference blog by February 20th. + +A follow-up meeting is scheduled for January 25th at 3 PM GMT to finalize the agenda and confirm the list of speakers. +` + +const ExtractionValuesSchema = z.object({ + users: z + .array( + z.object({ + name: z.string(), + email: z.string(), + twitter: z.string() + }) + ) + .min(3), + conference: z.object({ + date: z.string(), + venue: z.string(), + budget: z.number(), + keynoteSpeaker: z.string() + }), + nextMeeting: z.object({ + date: z.string(), + time: z.string(), + timezone: z.string() + }) +}) + +describe("thinking parser - live tests", () => { + test("should parse r1 response with thinking tags", async () => { + const groq = new OpenAI({ + apiKey: process.env["GROQ_API_KEY"] ?? undefined, + baseURL: "https://api.groq.com/openai/v1" + }) + + const client = Instructor({ + client: groq, + mode: "THINKING_MD_JSON", + debug: true + }) + + const result = await client.chat.completions.create({ + messages: [{ role: "user", content: textBlock }], + model: "deepseek-r1-distill-llama-70b", + response_model: { schema: ExtractionValuesSchema, name: "Extract" }, + max_retries: 4 + }) + + console.log("result", result) + + expect(result._meta?.thinking).toBeDefined() + expect(typeof result._meta?.thinking).toBe("string") + + expect(result.users).toHaveLength(3) + expect(result.users[0]).toHaveProperty("name") + expect(result.users[0]).toHaveProperty("email") + expect(result.users[0]).toHaveProperty("twitter") + + expect(result.conference).toBeDefined() + expect(result.conference.budget).toBe(50000) + expect(result.conference.keynoteSpeaker).toBe("Dr. Emily Johnson") + + expect(result.nextMeeting).toBeDefined() + expect(result.nextMeeting.timezone).toBe("GMT") + }) +}) diff --git a/tests/mode.test.ts b/tests/mode.test.ts index 30f94824..ed576bfd 100644 --- a/tests/mode.test.ts +++ b/tests/mode.test.ts @@ -128,23 +128,11 @@ describe("Modes", async () => { const testCases = createTestCases() for await (const { model, mode, provider, defaultMessage } of testCases) { - if (provider !== PROVIDERS.GROQ) { - test(`${provider}: Should return extracted name and age for model ${model} and mode ${mode}`, async () => { - const user = await extractUser(model, mode, provider, defaultMessage) - - expect(user.name).toEqual("Jason Liu") - expect(user.age).toEqual(30) - }) - } else { - test.todo( - `${provider}: Should return extracted name and age for model ${model} and mode ${mode}`, - async () => { - const user = await extractUser(model, mode, provider, defaultMessage) - - expect(user.name).toEqual("Jason Liu") - expect(user.age).toEqual(30) - } - ) - } + test(`${provider}: Should return extracted name and age for model ${model} and mode ${mode}`, async () => { + const user = await extractUser(model, mode, provider, defaultMessage) + + expect(user.name).toEqual("Jason Liu") + expect(user.age).toEqual(30) + }) } })