diff --git a/engine/Cargo.lock b/engine/Cargo.lock index 02a170e0f..c9a9327ec 100644 --- a/engine/Cargo.lock +++ b/engine/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "Inflector" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" + [[package]] name = "addr2line" version = "0.22.0" @@ -26,6 +32,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "aliasable" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" + [[package]] name = "ambassador" version = "0.4.0" @@ -948,6 +960,7 @@ dependencies = [ "internal-baml-codegen", "internal-baml-core", "internal-baml-jinja", + "internal-llm-client", "js-sys", "json_comments", "jsonish", @@ -1011,6 +1024,7 @@ dependencies = [ "indoc", "internal-baml-codegen", "internal-baml-core", + "internal-llm-client", "itertools 0.13.0", "js-sys", "jsonish", @@ -1040,6 +1054,7 @@ dependencies = [ "indexmap 2.2.6", "log", "minijinja", + "once_cell", "serde", "serde_json", "strum", @@ -1106,14 +1121,14 @@ dependencies = [ [[package]] name = "bindgen" -version = "0.69.4" +version = "0.69.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" dependencies = [ "bitflags 2.6.0", "cexpr", "clang-sys", - "itertools 0.12.1", + "itertools 0.10.5", "lazy_static", "lazycell", "proc-macro2", @@ -2588,6 +2603,7 @@ dependencies = [ "internal-baml-parser-database", "internal-baml-prompt-parser", "internal-baml-schema-ast", + "internal-llm-client", "itertools 0.13.0", "log", "minijinja", @@ -2670,8 +2686,10 @@ dependencies = [ "internal-baml-jinja-types", "internal-baml-prompt-parser", "internal-baml-schema-ast", + "internal-llm-client", "itertools 0.13.0", "log", + "ouroboros", "regex", "rustc-hash", "serde", @@ -2707,6 +2725,23 @@ dependencies = [ "test-log", ] +[[package]] +name = "internal-llm-client" +version = "0.68.0" +dependencies = [ + "anyhow", + "baml-types", + "derive_more", + "either", + "enum_dispatch", + "env_logger", + "indexmap 2.2.6", + "log", + "serde", + "serde_json", + "strum", +] + [[package]] name = "io-lifetimes" version = "1.0.11" @@ -2758,15 +2793,6 @@ dependencies = [ "either", ] -[[package]] -name = "itertools" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.13.0" @@ -3362,6 +3388,30 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "ouroboros" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84236d64f1718c387232287cf036eb6632a5ecff226f4ff9dccb8c2b79ba0bde" +dependencies = [ + "aliasable", + "ouroboros_macro", + "stable_deref_trait", +] + +[[package]] +name = "ouroboros_macro" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f463857a6eb96c0136b1d56e56c718350cef30412ec065b48294799a088bca68" +dependencies = [ + "Inflector", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "outref" version = "0.5.1" @@ -3623,6 +3673,30 @@ dependencies = [ "toml_edit", ] +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + [[package]] name = "proc-macro2" version = "1.0.86" @@ -3789,18 +3863,18 @@ dependencies = [ [[package]] name = "rb-sys" -version = "0.9.99" +version = "0.9.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d83151cfea2b67db2444f68c53b119ff77cff235ad711c765072e4daf8f3185b" +checksum = "91dbe37ab6ac2fba187480fb6544b92445e41e5c6f553bf0c33743f3c450a1df" dependencies = [ "rb-sys-build", ] [[package]] name = "rb-sys-build" -version = "0.9.99" +version = "0.9.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32d038214c118ad4a75db555ccb78672e17e1c5c10f344456cd129008dbaa7de" +checksum = "c4d56a49dcb646b70b758789c0d16c055a386a4f2a3346333abb69850fa860ce" dependencies = [ "bindgen", "lazy_static", @@ -4485,6 +4559,12 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "static_assertions" version = "1.1.0" diff --git a/engine/baml-lib/baml-core/Cargo.toml b/engine/baml-lib/baml-core/Cargo.toml index 8c660f510..1190dd03f 100644 --- a/engine/baml-lib/baml-core/Cargo.toml +++ b/engine/baml-lib/baml-core/Cargo.toml @@ -22,6 +22,7 @@ enumflags2 = "0.7" log = "0.4.20" indexmap.workspace = true internal-baml-diagnostics = { path = "../diagnostics" } +internal-llm-client = { path = "../llm-client" } internal-baml-jinja-types = { path = "../jinja" } internal-baml-parser-database = { path = "../parser-database" } internal-baml-prompt-parser = { path = "../prompt-parser" } diff --git a/engine/baml-lib/baml-core/src/ir/mod.rs b/engine/baml-lib/baml-core/src/ir/mod.rs index 2f7eb4dc3..c3bc74ac2 100644 --- a/engine/baml-lib/baml-core/src/ir/mod.rs +++ b/engine/baml-lib/baml-core/src/ir/mod.rs @@ -17,8 +17,6 @@ pub type EnumValue = repr::Node; pub type Class = repr::Node; pub type Field = repr::Node; pub type FieldType = baml_types::FieldType; -pub type Expression = repr::Expression; -pub type Identifier = repr::Identifier; pub type TypeValue = baml_types::TypeValue; pub type FunctionNode = repr::Node; #[allow(dead_code)] diff --git a/engine/baml-lib/baml-core/src/ir/repr.rs b/engine/baml-lib/baml-core/src/ir/repr.rs index fb6d71fa1..74f7822f0 100644 --- a/engine/baml-lib/baml-core/src/ir/repr.rs +++ b/engine/baml-lib/baml-core/src/ir/repr.rs @@ -1,21 +1,22 @@ use std::collections::HashSet; use anyhow::{anyhow, Result}; -use baml_types::{Constraint, ConstraintLevel, FieldType}; +use baml_types::{Constraint, ConstraintLevel, FieldType, StringOr, UnresolvedValue}; use either::Either; use indexmap::{IndexMap, IndexSet}; use internal_baml_parser_database::{ walkers::{ - ClassWalker, ClientSpec as AstClientSpec, ClientWalker, ConfigurationWalker, + ClassWalker, ClientWalker, ConfigurationWalker, EnumValueWalker, EnumWalker, FieldWalker, FunctionWalker, TemplateStringWalker, Walker as AstWalker, }, - Attributes, ParserDatabase, PromptAst, RetryPolicyStrategy, + Attributes, ParserDatabase, PromptAst, RetryPolicyStrategy }; use internal_baml_schema_ast::ast::{SubType, ValExpId}; use baml_types::JinjaExpression; use internal_baml_schema_ast::ast::{self, FieldArity, WithName, WithSpan}; +use internal_llm_client::{ClientProvider, ClientSpec, UnresolvedClientProperty}; use serde::Serialize; use crate::Configuration; @@ -24,7 +25,7 @@ use crate::Configuration; /// It is a representation of the BAML AST that is easier to work with than the /// raw BAML AST, and should include all information necessary to generate /// code in any target language. -#[derive(serde::Serialize, Debug)] +#[derive(Debug)] pub struct IntermediateRepr { enums: Vec>, classes: Vec>, @@ -35,7 +36,6 @@ pub struct IntermediateRepr { retry_policies: Vec>, template_strings: Vec>, - #[serde(skip)] configuration: Configuration, } @@ -223,7 +223,7 @@ impl IntermediateRepr { // [x] rename lockfile/mod.rs to ir/mod.rs // [x] wire Result<> type through, need this to be more sane -#[derive(Debug, serde::Serialize)] +#[derive(Debug)] pub struct NodeAttributes { /// Map of attributes on the corresponding IR node. /// @@ -231,18 +231,16 @@ pub struct NodeAttributes { /// /// - @skip becomes ("skip", bool) /// - @alias(...) becomes ("alias", ...) - #[serde(with = "indexmap::map::serde_seq")] - meta: IndexMap, + meta: IndexMap>, pub constraints: Vec, // Spans - #[serde(skip)] pub span: Option, } impl NodeAttributes { - pub fn get(&self, key: &str) -> Option<&Expression> { + pub fn get(&self, key: &str) -> Option<&UnresolvedValue<()>> { self.meta.get(key) } } @@ -260,7 +258,7 @@ impl Default for NodeAttributes { fn to_ir_attributes( db: &ParserDatabase, maybe_ast_attributes: Option<&Attributes>, -) -> (IndexMap, Vec) { +) -> (IndexMap>, Vec) { let null_result = (IndexMap::new(), Vec::new()); maybe_ast_attributes.map_or(null_result, |attributes| { let Attributes { @@ -270,35 +268,25 @@ fn to_ir_attributes( skip, constraints, } = attributes; - let description = description.as_ref().and_then(|d| { - let name = "description".to_string(); - match d { - ast::Expression::StringValue(s, _) => Some((name, Expression::String(s.clone()))), - ast::Expression::RawStringValue(s) => { - Some((name, Expression::RawString(s.value().to_string()))) - } - ast::Expression::JinjaExpressionValue(j, _) => { - Some((name, Expression::JinjaExpression(j.clone()))) - } - _ => { - eprintln!("Warning, encountered an unexpected description attribute"); - None - } - } - }); + + let description = description + .as_ref() + .map(|d| ("description".to_string(), d.without_meta())); + let alias = alias .as_ref() - .map(|v| ("alias".to_string(), Expression::String(db[*v].to_string()))); + .map(|v| ("alias".to_string(), v.without_meta())); + let dynamic_type = dynamic_type.as_ref().and_then(|v| { if *v { - Some(("dynamic_type".to_string(), Expression::Bool(true))) + Some(("dynamic_type".to_string(), UnresolvedValue::Bool(true, ()))) } else { None } }); let skip = skip.as_ref().and_then(|v| { if *v { - Some(("skip".to_string(), Expression::Bool(true))) + Some(("skip".to_string(), UnresolvedValue::Bool(true, ()))) } else { None } @@ -313,7 +301,7 @@ fn to_ir_attributes( } /// Nodes allow attaching metadata to a given IR entity: attributes, source location, etc -#[derive(serde::Serialize, Debug)] +#[derive(Debug)] pub struct Node { pub attributes: NodeAttributes, pub elem: T, @@ -493,108 +481,33 @@ impl WithRepr for ast::FieldType { } } -#[derive(serde::Serialize, Debug)] -pub enum Identifier { - /// Starts with env.* - ENV(String), - /// The path to a Local Identifer + the local identifer. Separated by '.' - #[allow(dead_code)] - Ref(Vec), - /// A string without spaces or '.' Always starts with a letter. May contain numbers - Local(String), - /// Special types (always lowercase). - Primitive(baml_types::TypeValue), -} - -impl Identifier { - pub fn name(&self) -> String { - match self { - Identifier::ENV(k) => k.clone(), - Identifier::Ref(r) => r.join("."), - Identifier::Local(l) => l.clone(), - Identifier::Primitive(p) => p.to_string(), - } - } -} - -#[derive(serde::Serialize, Debug)] -pub enum Expression { - Identifier(Identifier), - Bool(bool), - Numeric(String), - String(String), - RawString(String), - List(Vec), - Map(Vec<(Expression, Expression)>), - JinjaExpression(JinjaExpression), -} - -impl Expression { - pub fn required_env_vars<'a>(&'a self) -> Vec { - match self { - Expression::Identifier(Identifier::ENV(k)) => vec![k.to_string()], - Expression::List(l) => l.iter().flat_map(Expression::required_env_vars).collect(), - Expression::Map(m) => m - .iter() - .flat_map(|(k, v)| { - let mut keys = k.required_env_vars(); - keys.extend(v.required_env_vars()); - keys - }) - .collect(), - _ => vec![], - } - } -} - -impl WithRepr for ast::Expression { - fn repr(&self, db: &ParserDatabase) -> Result { - Ok(match self { - ast::Expression::BoolValue(val, _) => Expression::Bool(val.clone()), - ast::Expression::NumericValue(val, _) => Expression::Numeric(val.clone()), - ast::Expression::StringValue(val, _) => Expression::String(val.clone()), - ast::Expression::RawStringValue(val) => Expression::RawString(val.value().to_string()), - ast::Expression::JinjaExpressionValue(val, _) => { - Expression::JinjaExpression(val.clone()) - } - ast::Expression::Identifier(idn) => match idn { - ast::Identifier::ENV(k, _) => { - Ok(Expression::Identifier(Identifier::ENV(k.clone()))) - } - ast::Identifier::String(s, _) => Ok(Expression::String(s.clone())), - ast::Identifier::Local(l, _) => { - Ok(Expression::Identifier(Identifier::Local(l.clone()))) - } - ast::Identifier::Ref(r, _) => { - // NOTE(sam): this feels very very wrong, but per vbv, we don't really use refs - // right now, so this should be safe. this is done to ensure that - // "options { model gpt-3.5-turbo }" is represented correctly in the resulting IR, - // specifically that "gpt-3.5-turbo" is actually modelled as Expression::String - // - // this does not impact the handling of "options { api_key env.OPENAI_API_KEY }" - // because that's modelled as Identifier::ENV, not Identifier::Ref - Ok(Expression::String(r.full_name.clone())) - } - - ast::Identifier::Invalid(_, _) => { - Err(anyhow!("Cannot represent an invalid parser-AST identifier")) - } - }?, - ast::Expression::Array(arr, _) => { - Expression::List(arr.iter().map(|e| e.repr(db)).collect::>>()?) - } - ast::Expression::Map(arr, _) => Expression::Map( - arr.iter() - .map(|(k, v)| Ok((k.repr(db)?, v.repr(db)?))) - .collect::>>()?, - ), - }) - } -} +// #[derive(serde::Serialize, Debug)] +// pub enum Identifier { +// /// Starts with env.* +// ENV(String), +// /// The path to a Local Identifer + the local identifer. Separated by '.' +// #[allow(dead_code)] +// Ref(Vec), +// /// A string without spaces or '.' Always starts with a letter. May contain numbers +// Local(String), +// /// Special types (always lowercase). +// Primitive(baml_types::TypeValue), +// } + +// impl Identifier { +// pub fn name(&self) -> String { +// match self { +// Identifier::ENV(k) => k.clone(), +// Identifier::Ref(r) => r.join("."), +// Identifier::Local(l) => l.clone(), +// Identifier::Primitive(p) => p.to_string(), +// } +// } +// } type TemplateStringId = String; -#[derive(serde::Serialize, Debug)] +#[derive(Debug)] pub struct TemplateString { pub name: TemplateStringId, pub params: Vec, @@ -637,7 +550,7 @@ type EnumId = String; #[derive(serde::Serialize, Debug)] pub struct EnumValue(pub String); -#[derive(serde::Serialize, Debug)] +#[derive(Debug)] pub struct Enum { pub name: EnumId, pub values: Vec<(Node, Option)>, @@ -692,7 +605,7 @@ impl WithRepr for EnumWalker<'_> { #[derive(serde::Serialize, Debug)] pub struct Docstring(pub String); -#[derive(serde::Serialize, Debug)] +#[derive(Debug)] pub struct Field { pub name: String, pub r#type: Node, @@ -734,7 +647,7 @@ impl WithRepr for FieldWalker<'_> { type ClassId = String; /// A BAML Class. -#[derive(serde::Serialize, Debug)] +#[derive(Debug)] pub struct Class { /// User defined class name. pub name: ClassId, @@ -794,7 +707,7 @@ impl Class { pub enum OracleType { LLM, } -#[derive(serde::Serialize, Debug)] +#[derive(Debug)] pub struct AliasOverride { pub name: String, // This is used to generate deserializers with aliased keys (see .overload in python deserializer) @@ -802,15 +715,15 @@ pub struct AliasOverride { } // TODO, also add skips -#[derive(serde::Serialize, Debug)] +#[derive(Debug)] pub struct AliasedKey { pub key: String, - pub alias: Expression, + pub alias: UnresolvedValue<()>, } type ImplementationId = String; -#[derive(serde::Serialize, Debug)] +#[derive(Debug)] pub struct Implementation { r#type: OracleType, pub name: ImplementationId, @@ -818,10 +731,8 @@ pub struct Implementation { pub prompt: Prompt, - #[serde(with = "indexmap::map::serde_seq")] pub input_replacers: IndexMap, - #[serde(with = "indexmap::map::serde_seq")] pub output_replacers: IndexMap, pub client: ClientId, @@ -867,7 +778,7 @@ impl Function { } } -#[derive(serde::Serialize, Debug)] +#[derive(Debug)] pub struct Function { pub name: FunctionId, pub inputs: Vec<(String, FieldType)>, @@ -877,66 +788,23 @@ pub struct Function { pub default_config: String, } -#[derive(serde::Serialize, Debug)] +#[derive(Debug)] pub struct FunctionConfig { pub name: String, pub prompt_template: String, - #[serde(skip)] pub prompt_span: ast::Span, pub client: ClientSpec, } -// NB(sam): we used to use this to bridge the wasm layer, but -// I don't think we do anymore. -#[derive(serde::Serialize, Clone, Debug)] -pub enum ClientSpec { - Named(String), - /// Shorthand for "/" - Shorthand(String, String), -} - -impl ClientSpec { - pub fn as_str(&self) -> String { - match self { - ClientSpec::Named(n) => n.clone(), - ClientSpec::Shorthand(provider, model) => format!("{provider}/{model}"), - } - } - - pub fn new_from_id(arg: String) -> Self { - if arg.contains("/") { - let (provider, model) = arg.split_once("/").unwrap(); - ClientSpec::Shorthand(provider.to_string(), model.to_string()) - } else { - ClientSpec::Named(arg) - } - } - - pub fn required_env_vars(&self) -> HashSet { - match self { - ClientSpec::Named(n) => HashSet::new(), - ClientSpec::Shorthand(_, _) => HashSet::new(), - } - } -} - -impl From for ClientSpec { - fn from(spec: AstClientSpec) -> Self { - match spec { - AstClientSpec::Named(n) => ClientSpec::Named(n.to_string()), - AstClientSpec::Shorthand(provider, model) => ClientSpec::Shorthand(provider, model), - } - } -} -impl std::fmt::Display for ClientSpec { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.as_str()) - } -} +// impl std::fmt::Display for ClientSpec { +// fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +// write!(f, "{}", self.as_str()) +// } +// } fn process_field( - overrides: &IndexMap<(String, String), IndexMap>, // Adjust the type according to your actual field type + overrides: &IndexMap<(String, String), IndexMap>>, // Adjust the type according to your actual field type original_name: &str, function_name: &str, impl_name: &str, @@ -944,31 +812,31 @@ fn process_field( // This feeds into deserializer.overload; the registerEnumDeserializer counterpart is in generate_ts_client.rs match overrides.get(&((*function_name).to_string(), (*impl_name).to_string())) { Some(overrides) => { - if let Some(Expression::String(alias)) = overrides.get("alias") { - if let Some(Expression::String(description)) = overrides.get("description") { + if let Some(UnresolvedValue::String(alias, ..)) = overrides.get("alias") { + if let Some(UnresolvedValue::String(description, ..)) = overrides.get("description") { // "alias" and "alias: description" vec![ AliasedKey { key: original_name.to_string(), - alias: Expression::String(alias.clone()), - }, - AliasedKey { - key: original_name.to_string(), - alias: Expression::String(format!("{}: {}", alias, description)), + alias: UnresolvedValue::String(alias.clone(), ()), }, + // AliasedKey { + // key: original_name.to_string(), + // alias: UnresolvedValue::String(format!("{}: {}", alias, description)), + // }, ] } else { // "alias" vec![AliasedKey { key: original_name.to_string(), - alias: Expression::String(alias.clone()), + alias: UnresolvedValue::String(alias.clone(), ()), }] } - } else if let Some(Expression::String(description)) = overrides.get("description") { + } else if let Some(UnresolvedValue::String(description, ..)) = overrides.get("description") { // "description" vec![AliasedKey { key: original_name.to_string(), - alias: Expression::String(description.clone()), + alias: UnresolvedValue::String(description.clone(), ()), }] } else { // no overrides @@ -1028,12 +896,12 @@ impl WithRepr for FunctionWalker<'_> { type ClientId = String; -#[derive(serde::Serialize, Debug)] +#[derive(Debug)] pub struct Client { pub name: ClientId, - pub provider: String, + pub provider: ClientProvider, pub retry_policy_id: Option, - pub options: Vec<(String, Expression)>, + pub options: UnresolvedClientProperty<()>, } impl WithRepr for ClientWalker<'_> { @@ -1052,9 +920,7 @@ impl WithRepr for ClientWalker<'_> { options: self .properties() .options - .iter() - .map(|(k, v)| Ok((k.clone(), v.repr(db)?))) - .collect::>>()?, + .without_meta(), retry_policy_id: self .properties() .retry_policy @@ -1067,14 +933,14 @@ impl WithRepr for ClientWalker<'_> { #[derive(serde::Serialize, Debug)] pub struct RetryPolicyId(pub String); -#[derive(serde::Serialize, Debug)] +#[derive(Debug)] pub struct RetryPolicy { pub name: RetryPolicyId, pub max_retries: u32, pub strategy: RetryPolicyStrategy, // NB: the parser DB has a notion of "empty options" vs "no options"; we collapse // those here into an empty vec - options: Vec<(String, Expression)>, + options: Vec<(String, UnresolvedValue<()>)>, } impl WithRepr for ConfigurationWalker<'_> { @@ -1092,10 +958,7 @@ impl WithRepr for ConfigurationWalker<'_> { max_retries: self.retry_policy().max_retries, strategy: self.retry_policy().strategy, options: match &self.retry_policy().options { - Some(o) => o - .iter() - .map(|((k, _), v)| Ok((k.clone(), v.repr(db)?))) - .collect::>>()?, + Some(o) => o.iter().map(|(k, (_, v))| Ok((k.clone(), v.without_meta()))).collect::>>()?, None => vec![], }, }) @@ -1111,11 +974,11 @@ impl TestCaseFunction { } } -#[derive(serde::Serialize, Debug)] +#[derive(Debug)] pub struct TestCase { pub name: String, pub functions: Vec>, - pub args: IndexMap, + pub args: IndexMap>, pub constraints: Vec, } @@ -1170,7 +1033,7 @@ impl WithRepr for ConfigurationWalker<'_> { .test_case() .args .iter() - .map(|(k, (_, v))| Ok((k.clone(), v.repr(db)?))) + .map(|(k, (_, v))| Ok((k.clone(), v.without_meta()))) .collect::>>()?, functions, constraints: as WithRepr>::attributes( diff --git a/engine/baml-lib/baml-core/src/ir/walker.rs b/engine/baml-lib/baml-core/src/ir/walker.rs index 86e09bc61..7a83a9acb 100644 --- a/engine/baml-lib/baml-core/src/ir/walker.rs +++ b/engine/baml-lib/baml-core/src/ir/walker.rs @@ -1,15 +1,17 @@ use anyhow::Result; -use baml_types::BamlValue; +use baml_types::{BamlValue, EvaluationContext, UnresolvedValue}; use indexmap::IndexMap; +use internal_baml_diagnostics::Span; use internal_baml_parser_database::RetryPolicyStrategy; +use internal_llm_client::ClientSpec; use std::collections::{HashMap, HashSet}; use super::{ repr::{self, FunctionConfig, WithRepr}, - Class, Client, Enum, EnumValue, Expression, Field, FunctionNode, IRHelper, Identifier, Impl, - RetryPolicy, TemplateString, TestCase, Walker, + Class, Client, Enum, EnumValue, Field, FunctionNode, IRHelper, Impl, RetryPolicy, + TemplateString, TestCase, Walker, }; use crate::ir::jinja_helpers::render_expression; @@ -62,13 +64,19 @@ impl<'a> Walker<'a, &'a FunctionNode> { pub fn required_env_vars(&'a self) -> Result> { if let Some(c) = self.elem().configs.first() { match &c.client { - repr::ClientSpec::Named(n) => { + ClientSpec::Named(n) => { let client: super::ClientWalker<'a> = self.db.find_client(n)?; Ok(client.required_env_vars()) } - repr::ClientSpec::Shorthand(provider, _) => { - let env_vars = provider_to_env_vars(provider); - Ok(env_vars.into_iter().map(|(_, v)| v.to_string()).collect()) + ClientSpec::Shorthand(provider, model) => { + let options = IndexMap::from_iter([("model".to_string(), ((), baml_types::UnresolvedValue::String(baml_types::StringOr::Value(model.clone()), ())))]); + let properties = internal_llm_client::PropertyHandler::<()>::new(options, ()); + if let Ok(client) = provider.parse_client_property(properties) { + Ok(client.required_env_vars()) + } else { + // We likely can't make a shorthand client from the given provider + Ok(HashSet::new()) + } } } } else { @@ -122,11 +130,11 @@ impl<'a> Walker<'a, &'a Enum> { &self.elem().name } - pub fn alias(&self, env_values: &HashMap) -> Result> { + pub fn alias(&self, ctx: &EvaluationContext<'_>) -> Result> { self.item .attributes .get("alias") - .map(|v| v.as_string_value(env_values)) + .map(|v| v.resolve_string(ctx)) .transpose() } @@ -159,11 +167,11 @@ impl<'a> Walker<'a, &'a Enum> { } impl<'a> Walker<'a, &'a EnumValue> { - pub fn skip(&self, env_values: &HashMap) -> Result { + pub fn skip(&self, ctx: &EvaluationContext<'_>) -> Result { self.item .attributes .get("skip") - .map(|v| v.as_bool(env_values)) + .map(|v| v.resolve_bool(ctx)) .unwrap_or(Ok(false)) } @@ -171,106 +179,23 @@ impl<'a> Walker<'a, &'a EnumValue> { &self.item.elem.0 } - pub fn alias(&self, env_values: &HashMap) -> Result> { + pub fn alias(&self, ctx: &EvaluationContext<'_>) -> Result> { self.item .attributes .get("alias") - .map(|v| v.as_string_value(env_values)) + .map(|v| v.resolve_string(ctx)) .transpose() } - pub fn description(&self, env_values: &HashMap) -> Result> { + pub fn description(&self, ctx: &EvaluationContext<'_>) -> Result> { self.item .attributes .get("description") - .map(|v| v.as_string_value(env_values)) + .map(|v| v.resolve_string(ctx)) .transpose() } } -impl Expression { - pub fn as_bool(&self, env_values: &HashMap) -> Result { - match self { - Expression::Bool(b) => Ok(*b), - Expression::Identifier(Identifier::ENV(s)) => Ok(env_values.contains_key(s)), - _ => anyhow::bail!("Expected bool value, got {:?}", self), - } - } - - pub fn as_string_value(&self, env_values: &HashMap) -> Result { - match self { - Expression::String(s) => Ok(s.clone()), - Expression::RawString(s) => Ok(s.clone()), - Expression::Identifier(Identifier::ENV(s)) => match env_values.get(s) { - Some(v) => Ok(v.clone()), - None => anyhow::bail!("Environment variable {} not found", s), - }, - Expression::Identifier(idn) => Ok(idn.name().to_string()), - _ => anyhow::bail!("Expected string value, got {:?}", self), - } - } - - /// Normalize an `Expression` into a `BamlValue` in a given context. - /// - /// TODO: Modify the context, rename it to `env` and make it a map - /// from `String` to `BamlValue`. This generalizes the context from - /// known environment variables to variables set by other means. For - /// example, we will eventually want to normalize `JinjaExpressions` found - /// inside an `assert` by augmenting the context with the LLM response. - pub fn normalize(&self, env_values: &HashMap) -> Result { - match self { - Expression::Identifier(idn) => match idn { - repr::Identifier::ENV(s) => match env_values.get(s) { - Some(v) => Ok(BamlValue::String(v.clone())), - None => anyhow::bail!("Environment variable {} not found", s), - }, - repr::Identifier::Ref(r) => Ok(BamlValue::String(r.join(".").to_string())), - repr::Identifier::Local(r) => match r.as_str() { - "true" => Ok(BamlValue::Bool(true)), - "false" => Ok(BamlValue::Bool(false)), - "null" => Ok(BamlValue::Null), - _ => Ok(BamlValue::String(r.to_string())), - }, - repr::Identifier::Primitive(t) => Ok(BamlValue::String(t.to_string())), - }, - Expression::Bool(b) => Ok(BamlValue::Bool(*b)), - Expression::Map(m) => { - let mut map = baml_types::BamlMap::new(); - for (k, v) in m { - map.insert(k.as_string_value(env_values)?, v.normalize(env_values)?); - } - Ok(BamlValue::Map(map)) - } - Expression::List(l) => { - let mut list = Vec::new(); - for v in l { - list.push(v.normalize(env_values)?); - } - Ok(BamlValue::List(list)) - } - Expression::RawString(s) | Expression::String(s) => Ok(BamlValue::String(s.clone())), - repr::Expression::Numeric(n) => { - if let Ok(n) = n.parse::() { - Ok(BamlValue::Int(n)) - } else if let Ok(n) = n.parse::() { - Ok(BamlValue::Float(n)) - } else { - anyhow::bail!("Invalid numeric value: {}", n) - } - } - Expression::JinjaExpression(expr) => { - // TODO: do not coerce all context values to strings. - let jinja_context: HashMap = env_values - .iter() - .map(|(k, v)| (k.clone(), v.clone().into())) - .collect(); - let res_string = render_expression(&expr, &jinja_context)?; - Ok(BamlValue::String(res_string)) - } - } - } -} - impl<'a> Walker<'a, (&'a FunctionNode, &'a Impl)> { #[allow(dead_code)] pub fn function(&'a self) -> Walker<'a, &'a FunctionNode> { @@ -294,7 +219,7 @@ impl<'a> Walker<'a, (&'a FunctionNode, &'a TestCase)> { format!("{}::{}", self.item.0.elem.name(), self.item.1.elem.name) } - pub fn args(&self) -> &IndexMap { + pub fn args(&self) -> &IndexMap> { &self.item.1.elem.args } @@ -308,11 +233,11 @@ impl<'a> Walker<'a, (&'a FunctionNode, &'a TestCase)> { pub fn test_case_params( &self, - env_values: &HashMap, + ctx: &EvaluationContext<'_>, ) -> Result>> { self.args() .iter() - .map(|(k, v)| Ok((k.clone(), v.normalize(env_values)))) + .map(|(k, v)| Ok((k.clone(), v.resolve_serde::(ctx)))) .collect() } @@ -329,11 +254,11 @@ impl<'a> Walker<'a, &'a Class> { &self.elem().name } - pub fn alias(&self, env_values: &HashMap) -> Result> { + pub fn alias(&self, ctx: &EvaluationContext<'_>) -> Result> { self.item .attributes .get("alias") - .map(|v| v.as_string_value(env_values)) + .map(|v| v.resolve_string(ctx)) .transpose() } @@ -386,26 +311,12 @@ impl<'a> Walker<'a, &'a Client> { self.item.attributes.span.as_ref() } - pub fn options(&'a self) -> &'a Vec<(String, Expression)> { + pub fn options(&'a self) -> &'a internal_llm_client::UnresolvedClientProperty<()> { &self.elem().options } pub fn required_env_vars(&'a self) -> HashSet { - let mut env_vars = self - .options() - .iter() - .flat_map(|(_, expr)| expr.required_env_vars()) - .collect::>(); - - let options = self.options(); - for (k, v) in provider_to_env_vars(self.elem().provider.as_str()) { - match k { - Some(k) if !options.iter().any(|(k2, _)| k2 == k) => env_vars.insert(v.to_string()), - None => env_vars.insert(v.to_string()), - _ => false, - }; - } - env_vars + self.options().required_env_vars() } } @@ -466,25 +377,19 @@ impl<'a> Walker<'a, &'a Field> { &self.item.elem } - pub fn alias(&self, env_values: &HashMap) -> Result> { + pub fn alias(&self, ctx: &EvaluationContext<'_>) -> Result> { self.item .attributes .get("alias") - .map(|v| v.as_string_value(env_values)) + .map(|v| v.resolve_string(ctx)) .transpose() } - pub fn description(&self, env_values: &HashMap) -> Result> { + pub fn description(&self, ctx: &EvaluationContext<'_>) -> Result> { self.item .attributes .get("description") - .map(|v| { - let normalized = v.normalize(env_values)?; - let baml_value = normalized - .as_str() - .ok_or(anyhow::anyhow!("Unexpected: Evaluated to non-string value"))?; - Ok(String::from(baml_value)) - }) + .map(|v| v.resolve_string(ctx)) .transpose() } @@ -492,22 +397,3 @@ impl<'a> Walker<'a, &'a Field> { self.item.attributes.span.as_ref() } } - -#[cfg(test)] -mod tests { - use super::*; - use baml_types::JinjaExpression; - - #[test] - fn basic_jinja_normalization() { - let expr = Expression::JinjaExpression(JinjaExpression("this == 'hello'".to_string())); - let env = vec![("this".to_string(), "hello".to_string())] - .into_iter() - .collect(); - let normalized = expr.normalize(&env).unwrap(); - match normalized { - BamlValue::String(s) => assert_eq!(&s, "true"), - _ => panic!("Expected String Expression"), - } - } -} diff --git a/engine/baml-lib/baml-core/src/lib.rs b/engine/baml-lib/baml-core/src/lib.rs index b47803d02..7b473a1e8 100644 --- a/engine/baml-lib/baml-core/src/lib.rs +++ b/engine/baml-lib/baml-core/src/lib.rs @@ -5,6 +5,7 @@ pub use internal_baml_diagnostics; pub use internal_baml_parser_database::{self}; + pub use internal_baml_schema_ast::{self, ast}; use rayon::prelude::*; diff --git a/engine/baml-lib/baml-core/src/validate/generator_loader/v2.rs b/engine/baml-lib/baml-core/src/validate/generator_loader/v2.rs index 061276bd0..feb9d035b 100644 --- a/engine/baml-lib/baml-core/src/validate/generator_loader/v2.rs +++ b/engine/baml-lib/baml-core/src/validate/generator_loader/v2.rs @@ -104,6 +104,7 @@ pub(crate) fn parse_generator( .chain(vec![BOUNDARY_CLOUD_OUTPUT_TYPE].iter()) .map(|s| s.to_string()) .collect(), + false, )); } } diff --git a/engine/baml-lib/baml-core/src/validate/mod.rs b/engine/baml-lib/baml-core/src/validate/mod.rs index 712335faf..a77c317a3 100644 --- a/engine/baml-lib/baml-core/src/validate/mod.rs +++ b/engine/baml-lib/baml-core/src/validate/mod.rs @@ -1,5 +1,4 @@ pub(crate) mod generator_loader; - mod validation_pipeline; pub(crate) use validation_pipeline::validate; diff --git a/engine/baml-lib/baml-core/src/validate/validation_pipeline/validations/clients.rs b/engine/baml-lib/baml-core/src/validate/validation_pipeline/validations/clients.rs index 1092daa04..55c5825a9 100644 --- a/engine/baml-lib/baml-core/src/validate/validation_pipeline/validations/clients.rs +++ b/engine/baml-lib/baml-core/src/validate/validation_pipeline/validations/clients.rs @@ -1,77 +1,59 @@ -use internal_baml_diagnostics::DatamodelError; +use anyhow::Result; +use baml_types::StringOr; +use internal_baml_diagnostics::{DatamodelError, Span}; +use internal_llm_client::{ClientProvider, ClientSpec, PropertyHandler, StrategyClientProperty}; use crate::validate::validation_pipeline::context::Context; -use internal_baml_schema_ast::ast::Expression; -pub(super) fn validate(ctx: &mut Context<'_>) { - // required props are already validated in visit_client. No other validations here. - ctx.db.walk_clients().for_each(|f| { - let (provider, span) = &f.properties().provider; - let allowed_providers = [ - "baml-openai-chat", - "openai", - "openai-generic", - "baml-azure-chat", - "azure-openai", - "baml-anthropic-chat", - "anthropic", - "baml-ollama-chat", - "ollama", - "baml-round-robin", - "round-robin", - "baml-fallback", - "fallback", - "google-ai", - "vertex-ai", - "aws-bedrock", - ]; - - let suggestions: Vec = allowed_providers - .iter() - .filter(|&&p| !p.starts_with("baml-")) - .map(|&p| p.to_string()) - .collect(); +use internal_baml_schema_ast::ast::Expression; - if !allowed_providers.contains(&provider.as_str()) { - ctx.push_error(DatamodelError::not_found_error( - "client provider", - provider, - span.clone(), - suggestions, - )); - } +pub(super) fn validate(ctx: &mut Context<'_>) { + let valid_clients = ctx.db.valid_client_names(); - if provider.as_str() == "fallback" || provider.as_str() == "round-robin" { - let strategy_option = f.properties().options.iter().find(|(k, _)| k == "strategy"); - if let Some((_, strategy_expr)) = strategy_option { - if let Expression::Array(strategy_vec, _span) = strategy_expr { - if strategy_vec.is_empty() { - ctx.push_error(DatamodelError::new_validation_error( - "The strategy array cannot be empty.", - span.clone(), - )); - } - } else { - ctx.push_error(DatamodelError::new_validation_error( - "The strategy must be an array.", - span.clone(), - )); - } - } else { - ctx.push_error(DatamodelError::new_validation_error( - "The strategy key is missing in options.", - span.clone(), - )); - } - } + // required props are already validated in visit_client. No other validations here. + for f in ctx.db.walk_clients() { if let Some((retry_policy, span)) = &f.properties().retry_policy { if ctx.db.find_retry_policy(retry_policy).is_none() { ctx.push_error(DatamodelError::new_type_not_found_error( retry_policy, ctx.db.valid_retry_policy_names(), span.clone(), - )) + )); + } + } + + + // Do any additional validation here for providers that need it. + match &f.properties().options { + internal_llm_client::UnresolvedClientProperty::OpenAI(_) | + internal_llm_client::UnresolvedClientProperty::Anthropic(_) | + internal_llm_client::UnresolvedClientProperty::AWSBedrock(_) | + internal_llm_client::UnresolvedClientProperty::Vertex(_) | + internal_llm_client::UnresolvedClientProperty::GoogleAI(_) => {}, + internal_llm_client::UnresolvedClientProperty::RoundRobin(options) => { + validate_strategy(options, ctx); + }, + internal_llm_client::UnresolvedClientProperty::Fallback(options) => { + validate_strategy(options, ctx); + }, + } + } +} + +fn validate_strategy(options: &impl StrategyClientProperty, ctx: &mut Context<'_>) { + let valid_clients = ctx.db.valid_client_names(); + + for (client, span) in options.strategy() { + if let either::Either::Right(ClientSpec::Named(s)) = client { + if !valid_clients.contains(s) { + ctx.push_error( + DatamodelError::new_client_not_found_error( + s, + span.clone(), + &valid_clients + ) + ); } } - }) + } } diff --git a/engine/baml-lib/baml-core/src/validate/validation_pipeline/validations/functions.rs b/engine/baml-lib/baml-core/src/validate/validation_pipeline/validations/functions.rs index 2296e4191..01f0091d8 100644 --- a/engine/baml-lib/baml-core/src/validate/validation_pipeline/validations/functions.rs +++ b/engine/baml-lib/baml-core/src/validate/validation_pipeline/validations/functions.rs @@ -134,6 +134,7 @@ pub(super) fn validate(ctx: &mut Context<'_>) { &client.0, client.1.clone(), clients.clone(), + false, )) } } diff --git a/engine/baml-lib/baml-types/Cargo.toml b/engine/baml-lib/baml-types/Cargo.toml index 5b50cbb4d..64c564a22 100644 --- a/engine/baml-lib/baml-types/Cargo.toml +++ b/engine/baml-lib/baml-types/Cargo.toml @@ -20,6 +20,7 @@ serde_json.workspace = true strum.workspace = true minijinja.workspace = true log.workspace = true +once_cell = "1" [dependencies.indexmap] workspace = true diff --git a/engine/baml-lib/baml-types/src/lib.rs b/engine/baml-lib/baml-types/src/lib.rs index fb721ff8d..cc29fc8b4 100644 --- a/engine/baml-lib/baml-types/src/lib.rs +++ b/engine/baml-lib/baml-types/src/lib.rs @@ -6,6 +6,7 @@ mod minijinja; mod baml_value; mod field_type; mod generator; +mod value_expr; pub use baml_value::{BamlValue, BamlValueWithMeta}; pub use constraint::*; @@ -14,3 +15,4 @@ pub use generator::{GeneratorDefaultClientMode, GeneratorOutputType}; pub use map::Map as BamlMap; pub use media::{BamlMedia, BamlMediaContent, BamlMediaType, MediaBase64, MediaUrl}; pub use minijinja::JinjaExpression; +pub use value_expr::{ EvaluationContext, ResolvedValue, StringOr, UnresolvedValue, GetEnvVar }; \ No newline at end of file diff --git a/engine/baml-lib/baml-types/src/minijinja.rs b/engine/baml-lib/baml-types/src/minijinja.rs index 36aa7a5a4..c157a5572 100644 --- a/engine/baml-lib/baml-types/src/minijinja.rs +++ b/engine/baml-lib/baml-types/src/minijinja.rs @@ -4,7 +4,7 @@ use crate::{BamlMedia, BamlValue}; /// A wrapper around a jinja expression. The inner `String` should not contain /// the interpolation brackets `{{ }}`; it should be a bare expression like /// `"this|length < something"`. -#[derive(Clone, Debug, PartialEq, serde::Serialize)] +#[derive(Clone, Debug, PartialEq, serde::Serialize, Hash, Eq)] pub struct JinjaExpression(pub String); diff --git a/engine/baml-lib/baml-types/src/value_expr.rs b/engine/baml-lib/baml-types/src/value_expr.rs new file mode 100644 index 000000000..8d34a78b5 --- /dev/null +++ b/engine/baml-lib/baml-types/src/value_expr.rs @@ -0,0 +1,435 @@ +use anyhow::Result; +use std::{collections::{HashMap, HashSet}, str::FromStr}; + +use crate::JinjaExpression; +use indexmap::{IndexMap, IndexSet}; + +#[derive(Debug)] +pub enum Resolvable { + // Enums go into here. + String(Id, Meta), + // Repred as a string, but guaranteed to be a number. + Numeric(String, Meta), + Bool(bool, Meta), + Array(Vec>, Meta), + // This includes key-value pairs for classes + Map(IndexMap)>, Meta), + Null(Meta), +} + +impl Resolvable { + pub fn to_str(self) -> Result<(Id, Meta), Resolvable> { + match self { + Self::String(s, meta) => Ok((s, meta)), + other => Err(other) + } + } + + pub fn to_array(self) -> Result<(Vec>, Meta), Resolvable> { + match self { + Self::Array(a, meta) => Ok((a, meta)), + other => Err(other) + } + } + + pub fn to_map(self) -> Result<(IndexMap)>, Meta), Resolvable> { + match self { + Self::Map(m, meta) => Ok((m, meta)), + other => Err(other) + } + } + + pub fn to_bool(self) -> Result<(bool, Meta), Resolvable> { + match self { + Self::Bool(b, meta) => Ok((b, meta)), + other => Err(other) + } + } + + pub fn to_numeric(self) -> Result<(String, Meta), Resolvable> { + match self { + Self::Numeric(n, meta) => Ok((n, meta)), + other => Err(other) + } + } + + pub fn as_str(&self) -> Option<&Id> { + match self { + Self::String(s, ..) => Some(s), + _ => None + } + } + + pub fn as_null(&self) -> Option<()> { + match self { + Self::Null(..) => Some(()), + _ => None + } + } + + pub fn as_array(&self) -> Option<&Vec>> { + match self { + Self::Array(a, ..) => Some(a), + _ => None + } + } + + pub fn as_map(&self) -> Option<&IndexMap)>> { + match self { + Self::Map(m, ..) => Some(m), + _ => None + } + } + + pub fn as_bool(&self) -> Option { + match self { + Self::Bool(b, ..) => Some(*b), + _ => None + } + } + + pub fn as_numeric(&self) -> Option<&String> { + match self { + Self::Numeric(n, ..) => Some(n), + _ => None + } + } + + pub fn meta(&self) -> &Meta { + match self { + Resolvable::String(_, meta) => meta, + Resolvable::Numeric(_, meta) => meta, + Resolvable::Bool(_, meta) => meta, + Resolvable::Array(_, meta) => meta, + Resolvable::Map(_, meta) => meta, + Resolvable::Null(meta) => meta, + } + } + + pub fn r#type(&self) -> String { + match self { + Resolvable::String(..) => format!("string"), + Resolvable::Numeric(..) => format!("number"), + Resolvable::Bool(..) => format!("bool"), + Resolvable::Array(vec, ..) => { + let parts = vec.iter() + .map(|v| v.r#type()) + .collect::>() + .into_iter() + .collect::>(); + match parts.len() { + 0 => "[]".to_string(), + 1 => format!("{}[]", parts[0]), + _ => format!("({})[]", parts.join(" | ")) + } + }, + Resolvable::Map(index_map, ..) => { + let content = index_map.iter().map(|(k, (_, v))| { + format!("{k}: {}", v.r#type()) + }).collect::>().join(",\n"); + format!("{{\n{content}\n}}") + } + Resolvable::Null(..) => format!("null") + } + } +} + +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +pub enum StringOr { + EnvVar(String), + Value(String), + JinjaExpression(JinjaExpression), +} + +impl StringOr { + pub fn required_env_vars(&self) -> HashSet { + match self { + Self::EnvVar(name) => HashSet::from([name.clone()]), + Self::Value(_) => HashSet::new(), + Self::JinjaExpression(_) => HashSet::new(), + } + } + + pub fn maybe_eq(&self, other: &StringOr) -> bool { + match (self, other) { + (Self::Value(s), Self::Value(o)) => s == o, + (Self::Value(_), _) | (_, Self::Value(_)) => true, + (Self::EnvVar(_), Self::JinjaExpression(_)) | (Self::JinjaExpression(_), Self::EnvVar(_)) => true, + (Self::JinjaExpression(_), Self::JinjaExpression(_)) => true, + (Self::EnvVar(s), Self::EnvVar(o)) => s == o, + } + } +} + +impl std::fmt::Display for StringOr { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Value(s) => write!(f, "{s}"), + Self::EnvVar(s) => write!(f, "${s}"), + Self::JinjaExpression(j) => write!(f, "{{ {} }}", j), + } + } +} + +pub type UnresolvedValue = Resolvable; +pub type ResolvedValue = Resolvable; + +impl UnresolvedValue { + pub fn without_meta(&self) -> UnresolvedValue<()> { + match self { + Self::String(s, ..) => Resolvable::String(s.clone(), ()), + Self::Numeric(n, ..) => Resolvable::Numeric(n.clone(), ()), + Self::Bool(b, ..) => Resolvable::Bool(*b, ()), + Self::Array(a, ..) => Resolvable::Array(a.iter().map(|v| v.without_meta()).collect(), ()), + Self::Map(m, ..) => Resolvable::Map(m.iter().map(|(k, (_, v))| (k.clone(), ((), v.without_meta()))).collect(), ()), + Self::Null(..) => Resolvable::Null(()), + } + } +} + +pub trait GetEnvVar { + fn get_env_var(&self, key: &str) -> Result; + fn set_allow_missing_env_var(&self, allow: bool) -> Self; +} + +pub struct EvaluationContext<'a> { + env_vars: Option<&'a HashMap>, + fill_missing_env_vars: bool, +} + +impl<'a> GetEnvVar for EvaluationContext<'a> { + fn get_env_var(&self, key: &str) -> Result { + match self + .env_vars + .as_ref() + .and_then(|env_vars| env_vars.get(key)) + { + Some(v) => Ok(v.to_string()), + None => { + if self.fill_missing_env_vars { + Ok(format!("${key}")) + } else { + Err(anyhow::anyhow!("Environment variable {key} not set")) + } + } + } + } + + fn set_allow_missing_env_var(&self, allow: bool) -> Self { + Self { + env_vars: self.env_vars, + fill_missing_env_vars: allow, + } + } +} + +impl<'a> EvaluationContext<'a> { + pub fn new(env_vars: &'a HashMap, fill_missing_env_vars: bool) -> Self { + Self { + env_vars: Some(env_vars), + fill_missing_env_vars, + } + } +} + +impl<'db> Default for EvaluationContext<'db> { + fn default() -> Self { + Self { + env_vars: None, + fill_missing_env_vars: true, + } + } +} + +impl<'db> StringOr { + pub fn resolve(&self, ctx: &impl GetEnvVar) -> Result { + match self { + Self::EnvVar(name) => ctx.get_env_var(name), + Self::Value(value) => Ok(value.to_string()), + Self::JinjaExpression(_) => todo!("Jinja expressions cannot yet be resolved"), + } + } +} + +impl UnresolvedValue { + pub fn as_static_str(&self) -> Result<&str> { + match self { + Self::String(StringOr::Value(v), ..) => Ok(v.as_str()), + Self::String(StringOr::EnvVar(..), ..) => anyhow::bail!("Expected a statically defined string, not env variable"), + Self::String(StringOr::JinjaExpression(..), ..) => anyhow::bail!("Expected a statically defined string, not expression"), + Self::Numeric(num, ..) => Ok(num.as_str()), + Self::Array(..) => anyhow::bail!("Expected a string, not an array"), + Self::Bool(..) => anyhow::bail!("Expected a string, not a bool"), + Self::Map(..) => anyhow::bail!("Expected a string, not a map"), + Self::Null(..) => anyhow::bail!("Expected a string, not null"), + } + } + + pub fn resolve_string(&self, ctx: &impl GetEnvVar) -> Result { + match self.resolve(ctx) { + Ok(ResolvedValue::String(s, ..)) => Ok(s), + _ => Err(anyhow::anyhow!("Expected a string")), + } + } + + pub fn resolve_bool(&self, ctx: &impl GetEnvVar) -> Result { + match self.resolve(ctx) { + Ok(ResolvedValue::Bool(b, ..)) => Ok(b), + _ => Err(anyhow::anyhow!("Expected a boolean")), + } + } + + pub fn resolve_array(&self, ctx: &impl GetEnvVar) -> Result> { + match self.resolve(ctx) { + Ok(ResolvedValue::Array(a, ..)) => Ok(a), + _ => Err(anyhow::anyhow!("Expected an array")), + } + } + + pub fn resolve_map( + &self, + ctx: &impl GetEnvVar, + ) -> Result> { + match self.resolve(ctx) { + Ok(ResolvedValue::Map(m, ..)) => Ok(m.into_iter().map(|(k, (_, v))| (k, v)).collect()), + _ => Err(anyhow::anyhow!("Expected a map")), + } + } + + pub fn resolve_numeric(&self, ctx: &impl GetEnvVar) -> Result { + match self.resolve(ctx) { + Ok(ResolvedValue::Numeric(n, ..)) => Ok(n), + _ => Err(anyhow::anyhow!("Expected a numeric value")), + } + } + + pub fn resolve_null(&self, ctx: &impl GetEnvVar) -> Result<()> { + match self.resolve(ctx) { + Ok(ResolvedValue::Null(..)) => Ok(()), + _ => Err(anyhow::anyhow!("Expected a null value")), + } + } + + pub fn resolve_serde( + &self, + ctx: &impl GetEnvVar, + ) -> Result { + let value = self.resolve(ctx)?; + let value: serde_json::Value = value.try_into()?; + match serde_json::from_value(value) { + Ok(v) => Ok(v), + Err(e) => Err(anyhow::anyhow!("Failed to deserialize value: {e}")), + } + } + + /// Resolve the value to a [`ResolvedValue`]. + fn resolve(&self, ctx: &impl GetEnvVar) -> Result { + match self { + Self::String(string_or, ..) => string_or.resolve(ctx).map(|v| ResolvedValue::String(v, ())), + Self::Numeric(numeric, ..) => Ok(ResolvedValue::Numeric(numeric.clone(), ())), + Self::Bool(bool, ..) => Ok(ResolvedValue::Bool(*bool, ())), + Self::Array(array, ..) => { + let values = array + .iter() + .map(|item| item.resolve(ctx)) + .collect::>>()?; + Ok(ResolvedValue::Array(values, ())) + } + Self::Map(map, ..) => { + let values = map + .iter() + .map(|(k, (_, v))| Ok((k.to_string(), ((), v.resolve(ctx)?)))) + .collect::>()?; + Ok(ResolvedValue::Map(values, ())) + } + Self::Null(..) => Ok(ResolvedValue::Null(())), + } + } + + pub fn required_env_vars(&self) -> HashSet { + let mut env_vars = HashSet::new(); + let mut stack = vec![self]; + + while let Some(current) = stack.pop() { + match current { + Self::String(s, ..) => { + env_vars.extend(s.required_env_vars()); + } + Self::Array(array, ..) => { + stack.extend(array); + } + Self::Map(map, ..) => { + stack.extend(map.values().map(|(_, v)| v)); + } + _ => {} + } + } + + env_vars + } +} + +// ResolvedValue -> serde_json::Value +impl TryFrom for serde_json::Value { + type Error = anyhow::Error; + + fn try_from(value: ResolvedValue) -> Result { + Ok(match value { + ResolvedValue::String(s, ..) => serde_json::Value::String(s), + ResolvedValue::Numeric(n, ..) => serde_json::Value::Number(serde_json::Number::from_str(n.as_str())?), + ResolvedValue::Bool(b, ..) => serde_json::Value::Bool(b), + ResolvedValue::Array(a, ..) => serde_json::Value::Array( + a.into_iter().map(|v| serde_json::Value::try_from(v)).collect::>()?, + ), + ResolvedValue::Map(m, ..) => serde_json::Value::Object( + m.into_iter() + .map(|(k, (_, v))| Ok((k.clone(), serde_json::Value::try_from(v)?))) + .collect::>()?, + ), + ResolvedValue::Null(..) => serde_json::Value::Null, + }) + } +} + +impl crate::BamlValue { + pub fn to_resolvable(&self) -> Result> { + Ok(match self { + crate::BamlValue::Enum(_, s) | crate::BamlValue::String(s) => Resolvable::String(StringOr::Value(s.clone()), ()), + crate::BamlValue::Int(i) => Resolvable::Numeric(i.to_string(), ()), + crate::BamlValue::Float(f) => Resolvable::Numeric(f.to_string(), ()), + crate::BamlValue::Bool(b) => Resolvable::Bool(*b, ()), + crate::BamlValue::Class(_, index_map) | crate::BamlValue::Map(index_map) => { + let values = index_map + .iter() + .map(|(k, v)| Ok((k.clone(), ((), v.to_resolvable()?)))) + .collect::>()?; + Resolvable::Map(values, ()) + } + crate::BamlValue::List(vec) => { + let values = vec + .iter() + .map(|v| v.to_resolvable()) + .collect::>()?; + Resolvable::Array(values, ()) + } + crate::BamlValue::Media(m) => m.to_resolvable()?, + crate::BamlValue::Null => Resolvable::Null(()), + }) + } +} + +impl crate::BamlMedia { + pub fn to_resolvable(&self) -> Result> { + let mut index_map = IndexMap::default(); + if let Some(mime_type) = &self.mime_type { + index_map.insert("mime_type".to_string(), ((), Resolvable::String(StringOr::Value(mime_type.clone()), ()))); + } + let (key, value) = match &self.content { + crate::BamlMediaContent::File(f) => ("file", f.path()?.to_string_lossy().to_string()), + crate::BamlMediaContent::Url(u) => ("url", u.url.clone()), + crate::BamlMediaContent::Base64(b) => ("base64", b.base64.clone()), + }; + index_map.insert(key.to_string(), ((), Resolvable::String(StringOr::Value(value), ()))); + Ok(Resolvable::Map(index_map, ())) + } +} diff --git a/engine/baml-lib/baml/tests/validation_files/tests/bad_syntax.baml b/engine/baml-lib/baml/tests/validation_files/tests/bad_syntax.baml index 44a2330bb..d87a6007b 100644 --- a/engine/baml-lib/baml/tests/validation_files/tests/bad_syntax.baml +++ b/engine/baml-lib/baml/tests/validation_files/tests/bad_syntax.baml @@ -16,16 +16,6 @@ test Foo { ] } -// warning: Direct values are not supported. Please pass in parameters by name -// --> tests/bad_syntax.baml:12 -// | -// 11 | functions [Foo] -// 12 | input [ -// 13 | { -// 14 | str_field "hello" -// 15 | },, -// 16 | ] -// | // error: Error validating: Invalid array syntax detected. // --> tests/bad_syntax.baml:15 // | @@ -33,3 +23,15 @@ test Foo { // 15 | },, // 16 | ] // | +// error: Property not known: "input". Did you mean one of these: "args", "functions"? +// --> tests/bad_syntax.baml:12 +// | +// 11 | functions [Foo] +// 12 | input [ +// | +// error: Error validating: Missing `args` property +// --> tests/bad_syntax.baml:10 +// | +// 9 | +// 10 | test Foo { +// | diff --git a/engine/baml-lib/baml/tests/validation_files/tests/values.baml b/engine/baml-lib/baml/tests/validation_files/tests/values.baml index 36a20ae3c..fb003397d 100644 --- a/engine/baml-lib/baml/tests/validation_files/tests/values.baml +++ b/engine/baml-lib/baml/tests/validation_files/tests/values.baml @@ -24,4 +24,17 @@ test Foo { int_array_field [1, 2, 3] bool_array_field [true, false] } -} \ No newline at end of file +} + +// error: Property not known: "input". Did you mean one of these: "args", "functions"? +// --> tests/values.baml:18 +// | +// 17 | functions [Foo] +// 18 | input { +// | +// error: Error validating: Missing `args` property +// --> tests/values.baml:16 +// | +// 15 | +// 16 | test Foo { +// | diff --git a/engine/baml-lib/diagnostics/src/error.rs b/engine/baml-lib/diagnostics/src/error.rs index e79830925..3b37e7c58 100644 --- a/engine/baml-lib/diagnostics/src/error.rs +++ b/engine/baml-lib/diagnostics/src/error.rs @@ -102,6 +102,10 @@ impl DatamodelError { Self::new(msg, span) } + pub fn new_client_error(message: impl Into>, span: Span) -> DatamodelError { + Self::new(message, span) + } + pub fn new_attribute_argument_not_found_error( argument_name: &str, attribute_name: &str, @@ -408,18 +412,21 @@ impl DatamodelError { name: &str, span: Span, mut names: Vec, + include_primitives: bool, ) -> DatamodelError { // Include a list of primitives in the names - let primitives = vec![ - "int".to_string(), - "float".to_string(), - "bool".to_string(), - "string".to_string(), - "image".to_string(), - "audio".to_string(), - "null".to_string(), - ]; - names.extend(primitives); + if include_primitives { + let primitives = vec![ + "int".to_string(), + "float".to_string(), + "bool".to_string(), + "string".to_string(), + "image".to_string(), + "audio".to_string(), + "null".to_string(), + ]; + names.extend(primitives); + } let close_names = sort_by_match(name, &names, Some(3)); let suggestions = if names.is_empty() { @@ -492,6 +499,21 @@ impl DatamodelError { Self::new(format!("{}{}", prefix, suggestions), span) } + pub fn new_client_not_found_error(client_name: &str, span: Span, valid_clients: &[String]) -> DatamodelError { + let names = valid_clients.iter().map(|s| s.to_string()).collect::>(); + let close_names = sort_by_match(client_name, &names, Some(10)); + + let msg = if close_names.is_empty() { + format!("client `{}` does not exist.", client_name) + } else if close_names.len() == 1 { + format!("client `{}` does not exist. Did you mean `{}`?", client_name, close_names[0]) + } else { + format!("client `{}` does not exist. Did you mean one of these: `{}`?", client_name, close_names.join("`, `")) + }; + + Self::new(msg, span) + } + pub fn new_type_not_found_error( type_name: &str, names: Vec, diff --git a/engine/baml-lib/jinja-runtime/src/baml_value_to_jinja_value.rs b/engine/baml-lib/jinja-runtime/src/baml_value_to_jinja_value.rs index 286b3b173..9c0628cfc 100644 --- a/engine/baml-lib/jinja-runtime/src/baml_value_to_jinja_value.rs +++ b/engine/baml-lib/jinja-runtime/src/baml_value_to_jinja_value.rs @@ -1,5 +1,6 @@ use std::collections::HashMap; +use baml_types::EvaluationContext; use indexmap::IndexMap; use internal_baml_core::ir::repr::IntermediateRepr; use internal_baml_core::ir::IRHelper; @@ -10,7 +11,7 @@ pub trait IntoMiniJinjaValue { fn into_minijinja_value( &self, ir: &IntermediateRepr, - env_vars: &HashMap, + eval_ctx: &EvaluationContext<'_>, ) -> minijinja::Value; } @@ -18,7 +19,7 @@ impl IntoMiniJinjaValue for BamlValue { fn into_minijinja_value( &self, ir: &IntermediateRepr, - env_vars: &HashMap, + eval_ctx: &EvaluationContext<'_>, ) -> minijinja::Value { match self { BamlValue::String(s) => minijinja::Value::from(s.clone()), @@ -28,17 +29,17 @@ impl IntoMiniJinjaValue for BamlValue { BamlValue::Map(m) => { let map = m .into_iter() - .map(|(k, v)| (k.as_str(), v.into_minijinja_value(ir, env_vars))); + .map(|(k, v)| (k.as_str(), v.into_minijinja_value(ir, eval_ctx))); minijinja::Value::from_iter(map) } BamlValue::List(l) => { let list: Vec = l .into_iter() - .map(|v| v.into_minijinja_value(ir, env_vars)) + .map(|v| v.into_minijinja_value(ir, eval_ctx)) .collect(); minijinja::Value::from(list) } - BamlValue::Media(i) => i.into_minijinja_value(ir, env_vars), + BamlValue::Media(i) => i.into_minijinja_value(ir, eval_ctx), // For enums and classes we compute the aliases from the IR, and generate custom jinja structs that print out the alias if stringified. BamlValue::Enum(_name, value) => { minijinja::Value::from(value.clone()) @@ -60,14 +61,14 @@ impl IntoMiniJinjaValue for BamlValue { BamlValue::Class(name, m) => { let map = m .into_iter() - .map(|(k, v)| (k.as_str(), v.into_minijinja_value(ir, env_vars))); + .map(|(k, v)| (k.as_str(), v.into_minijinja_value(ir, eval_ctx))); let mut key_to_alias = IndexMap::new(); match ir.find_class(name) { Ok(c) => { for field in c.walk_fields() { let key = field - .alias(&env_vars) + .alias(eval_ctx) .ok() .and_then(|a| a) .unwrap_or_else(|| field.name().to_string()); @@ -101,7 +102,7 @@ impl IntoMiniJinjaValue for BamlMedia { fn into_minijinja_value( &self, _ir: &IntermediateRepr, - _env_vars: &HashMap, + _eval_ctx: &EvaluationContext<'_>, ) -> minijinja::Value { minijinja::Value::from_object(MinijinjaBamlMedia::from(self.clone())) } diff --git a/engine/baml-lib/jinja-runtime/src/lib.rs b/engine/baml-lib/jinja-runtime/src/lib.rs index 3d1b89b19..03ff0652e 100644 --- a/engine/baml-lib/jinja-runtime/src/lib.rs +++ b/engine/baml-lib/jinja-runtime/src/lib.rs @@ -1,4 +1,4 @@ -use baml_types::{BamlMedia, BamlValue}; +use baml_types::{BamlMedia, BamlValue, EvaluationContext}; use colored::*; mod chat_message_part; @@ -408,8 +408,8 @@ pub fn render_prompt( if !matches!(args, BamlValue::Map(_)) { anyhow::bail!("args must be a map"); } - - let minijinja_args: minijinja::Value = args.clone().into_minijinja_value(&ir, env_vars); + let eval_ctx = EvaluationContext::new(env_vars, false); + let minijinja_args: minijinja::Value = args.clone().into_minijinja_value(&ir, &eval_ctx); let default_role = ctx.client.default_role.clone(); let rendered = render_minijinja( template, diff --git a/engine/baml-lib/jsonish/src/tests/mod.rs b/engine/baml-lib/jsonish/src/tests/mod.rs index ce568b4d4..9e2d14ead 100644 --- a/engine/baml-lib/jsonish/src/tests/mod.rs +++ b/engine/baml-lib/jsonish/src/tests/mod.rs @@ -22,7 +22,7 @@ use std::{ path::PathBuf, }; -use baml_types::BamlValue; +use baml_types::{BamlValue, EvaluationContext}; use internal_baml_core::{ internal_baml_diagnostics::SourceFile, ir::{repr::IntermediateRepr, ClassWalker, EnumWalker, FieldType, IRHelper, TypeValue}, @@ -53,7 +53,7 @@ fn load_test_ir(file_content: &str) -> IntermediateRepr { fn render_output_format( ir: &IntermediateRepr, output: &FieldType, - env_values: &HashMap, + env_values: &EvaluationContext<'_>, ) -> Result { let (enums, classes, recursive_classes) = relevant_data_models(ir, output, env_values)?; @@ -68,7 +68,7 @@ fn find_existing_class_field<'a>( class_name: &str, field_name: &str, class_walker: &Result>, - env_values: &HashMap, + env_values: &EvaluationContext<'_>, ) -> Result<(Name, FieldType, Option)> { let Ok(class_walker) = class_walker else { anyhow::bail!("Class {} does not exist", class_name); @@ -88,7 +88,7 @@ fn find_enum_value( enum_name: &str, value_name: &str, enum_walker: &Result>, - env_values: &HashMap, + env_values: &EvaluationContext<'_>, ) -> Result)>> { if enum_walker.is_err() { anyhow::bail!("Enum {} does not exist", enum_name); @@ -124,7 +124,7 @@ fn find_enum_value( fn relevant_data_models<'a>( ir: &'a IntermediateRepr, output: &'a FieldType, - env_values: &HashMap, + env_values: &EvaluationContext<'_>, ) -> Result<(Vec, Vec, IndexSet)> { let mut checked_types: HashSet = HashSet::new(); let mut enums = Vec::new(); diff --git a/engine/baml-lib/llm-client/Cargo.toml b/engine/baml-lib/llm-client/Cargo.toml new file mode 100644 index 000000000..92f7debd7 --- /dev/null +++ b/engine/baml-lib/llm-client/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "internal-llm-client" +edition = "2021" +version.workspace = true +authors.workspace = true +description.workspace = true +license-file.workspace = true + +[lints.rust] +dead_code = "deny" +unused_imports = "deny" +unused_variables = "deny" + +[dependencies] +baml-types = { path = "../baml-types" } +# internal-baml-core = { path = "../baml-core" } +# internal-baml-parser-database = { path = "../parser-database" } +anyhow.workspace = true +indexmap.workspace = true +log.workspace = true +serde.workspace = true +serde_json.workspace = true +enum_dispatch = "0.3.13" +strum.workspace = true +derive_more.workspace = true +either.workspace = true + +[dev-dependencies] +env_logger = "0.11.3" diff --git a/engine/baml-lib/llm-client/src/clients/anthropic.rs b/engine/baml-lib/llm-client/src/clients/anthropic.rs new file mode 100644 index 000000000..dcdaa391a --- /dev/null +++ b/engine/baml-lib/llm-client/src/clients/anthropic.rs @@ -0,0 +1,161 @@ +use std::collections::HashSet; + +use crate::{AllowedRoleMetadata, SupportedRequestModes, UnresolvedAllowedRoleMetadata}; +use anyhow::Result; + +use baml_types::{EvaluationContext, StringOr, UnresolvedValue}; +use indexmap::IndexMap; + +use super::helpers::{Error, PropertyHandler, UnresolvedUrl}; + +#[derive(Debug)] +pub struct UnresolvedAnthropic { + base_url: UnresolvedUrl, + api_key: StringOr, + allowed_roles: Vec, + default_role: Option, + allowed_metadata: UnresolvedAllowedRoleMetadata, + supported_request_modes: SupportedRequestModes, + headers: IndexMap, + properties: IndexMap)>, +} + +impl UnresolvedAnthropic { + pub fn without_meta(&self) -> UnresolvedAnthropic<()> { + UnresolvedAnthropic { + base_url: self.base_url.clone(), + api_key: self.api_key.clone(), + allowed_roles: self.allowed_roles.clone(), + default_role: self.default_role.clone(), + allowed_metadata: self.allowed_metadata.clone(), + supported_request_modes: self.supported_request_modes.clone(), + headers: self.headers.iter().map(|(k, v)| (k.clone(), v.clone())).collect(), + properties: self.properties.iter().map(|(k, (_, v))| (k.clone(), ((), v.without_meta()))).collect(), + } + } +} + +pub struct ResolvedAnthropic { + pub base_url: String, + pub api_key: String, + pub allowed_roles: Vec, + pub default_role: String, + pub allowed_metadata: AllowedRoleMetadata, + pub supported_request_modes: SupportedRequestModes, + pub headers: IndexMap, + pub properties: IndexMap, + pub proxy_url: Option, +} + +impl UnresolvedAnthropic { + pub fn required_env_vars(&self) -> HashSet { + let mut env_vars = HashSet::new(); + env_vars.extend(self.base_url.required_env_vars()); + env_vars.extend(self.api_key.required_env_vars()); + env_vars.extend(self.allowed_roles.iter().map(|r| r.required_env_vars()).flatten()); + self.default_role.as_ref().map(|r| env_vars.extend(r.required_env_vars())); + env_vars.extend(self.allowed_metadata.required_env_vars()); + env_vars.extend(self.supported_request_modes.required_env_vars()); + env_vars.extend(self.headers.values().map(|v| v.required_env_vars()).flatten()); + env_vars.extend(self.properties.values().map(|(_, v)| v.required_env_vars()).flatten()); + + env_vars + } + + pub fn resolve(&self, ctx: &EvaluationContext<'_>) -> Result { + let allowed_roles = self + .allowed_roles + .iter() + .map(|role| role.resolve(ctx)) + .collect::>>()?; + + let Some(default_role) = self.default_role.as_ref() else { + return Err(anyhow::anyhow!("default_role must be provided")); + }; + let default_role = default_role.resolve(ctx)?; + + if !allowed_roles.contains(&default_role) { + return Err(anyhow::anyhow!( + "default_role must be in allowed_roles: {} not in {:?}", + default_role, + allowed_roles + )); + } + + let base_url = self.base_url.resolve(ctx)?; + + let mut headers = self + .headers + .iter() + .map(|(k, v)| Ok((k.clone(), v.resolve(ctx)?))) + .collect::>>()?; + + // Add default Anthropic version header if not present + headers + .entry("anthropic-version".to_string()) + .or_insert_with(|| "2023-06-01".to_string()); + + let properties = { + let mut properties = self + .properties + .iter() + .map(|(k, (_, v))| Ok((k.clone(), v.resolve_serde::(ctx)?))) + .collect::>>()?; + + properties + .entry("max_tokens".to_string()) + .or_insert(serde_json::json!(4096)); + + properties + }; + + Ok(ResolvedAnthropic { + base_url, + api_key: self.api_key.resolve(ctx)?, + allowed_roles, + default_role, + allowed_metadata: self.allowed_metadata.resolve(ctx)?, + supported_request_modes: self.supported_request_modes.clone(), + headers, + properties, + proxy_url: super::helpers::get_proxy_url(ctx), + }) + } + + pub fn create_from( + mut properties: PropertyHandler, + ) -> Result>> { + let base_url = properties.ensure_base_url_with_default(UnresolvedUrl::new_static("https://api.anthropic.com")); + let api_key = properties + .ensure_string("api_key", false) + .map(|(_, v, _)| v.clone()) + .unwrap_or(StringOr::EnvVar("ANTHROPIC_API_KEY".to_string())); + + let allowed_roles = properties.ensure_allowed_roles().unwrap_or(vec![ + StringOr::Value("system".to_string()), + StringOr::Value("user".to_string()), + StringOr::Value("assistant".to_string()), + ]); + + let default_role = properties.ensure_default_role(allowed_roles.as_slice(), 1); + let allowed_metadata = properties.ensure_allowed_metadata(); + let supported_request_modes = properties.ensure_supported_request_modes(); + let headers = properties.ensure_headers().unwrap_or_default(); + + let (properties, errors) = properties.finalize(); + if !errors.is_empty() { + return Err(errors); + } + + Ok(Self { + base_url, + api_key, + allowed_roles, + default_role, + allowed_metadata, + supported_request_modes, + headers, + properties, + }) + } +} diff --git a/engine/baml-lib/llm-client/src/clients/aws_bedrock.rs b/engine/baml-lib/llm-client/src/clients/aws_bedrock.rs new file mode 100644 index 000000000..34deada2a --- /dev/null +++ b/engine/baml-lib/llm-client/src/clients/aws_bedrock.rs @@ -0,0 +1,264 @@ +use std::collections::HashSet; + +use crate::{AllowedRoleMetadata, SupportedRequestModes, UnresolvedAllowedRoleMetadata}; +use anyhow::Result; + +use baml_types::{EvaluationContext, StringOr}; + +use super::helpers::{Error, PropertyHandler}; + +#[derive(Debug, Clone)] +pub struct UnresolvedAwsBedrock { + model: Option, + region: StringOr, + access_key_id: StringOr, + secret_access_key: StringOr, + allowed_roles: Vec, + default_role: Option, + allowed_role_metadata: UnresolvedAllowedRoleMetadata, + supported_request_modes: SupportedRequestModes, + inference_config: Option, +} + +#[derive(Debug, Clone)] +struct UnresolvedInferenceConfiguration { + max_tokens: Option, + temperature: Option, + top_p: Option, + stop_sequences: Option>, +} + +impl UnresolvedInferenceConfiguration { + pub fn resolve(&self, ctx: &EvaluationContext<'_>) -> Result { + Ok(InferenceConfiguration { + max_tokens: self.max_tokens, + temperature: self.temperature, + top_p: self.top_p, + stop_sequences: self + .stop_sequences + .as_ref() + .map(|s| s.iter().map(|s| s.resolve(ctx)).collect::>>()) + .transpose()?, + }) + } + + pub fn required_env_vars(&self) -> HashSet { + self.stop_sequences + .as_ref() + .map(|s| s.iter().map(|s| s.required_env_vars()).flatten().collect()) + .unwrap_or_default() + } +} + +#[derive(Debug)] +pub struct InferenceConfiguration { + pub max_tokens: Option, + pub temperature: Option, + pub top_p: Option, + pub stop_sequences: Option>, +} + +pub struct ResolvedAwsBedrock { + pub model: String, + pub region: Option, + pub access_key_id: Option, + pub secret_access_key: Option, + pub inference_config: Option, + pub allowed_roles: Vec, + pub default_role: String, + pub allowed_role_metadata: AllowedRoleMetadata, + pub supported_request_modes: SupportedRequestModes, +} + +impl UnresolvedAwsBedrock { + pub fn required_env_vars(&self) -> HashSet { + let mut env_vars = HashSet::new(); + self.model + .as_ref() + .map(|m| env_vars.extend(m.required_env_vars())); + env_vars.extend(self.region.required_env_vars()); + env_vars.extend(self.access_key_id.required_env_vars()); + env_vars.extend(self.secret_access_key.required_env_vars()); + env_vars.extend( + self.allowed_roles + .iter() + .map(|r| r.required_env_vars()) + .flatten(), + ); + self.default_role + .as_ref() + .map(|r| env_vars.extend(r.required_env_vars())); + env_vars.extend(self.allowed_role_metadata.required_env_vars()); + env_vars.extend(self.supported_request_modes.required_env_vars()); + self.inference_config + .as_ref() + .map(|c| env_vars.extend(c.required_env_vars())); + env_vars + } + + pub fn resolve(&self, ctx: &EvaluationContext<'_>) -> Result { + let Some(model) = self.model.as_ref() else { + return Err(anyhow::anyhow!("model must be provided")); + }; + + let allowed_roles = self + .allowed_roles + .iter() + .map(|role| role.resolve(ctx)) + .collect::>>()?; + + let Some(default_role) = self.default_role.as_ref() else { + return Err(anyhow::anyhow!("default_role must be provided")); + }; + let default_role = default_role.resolve(ctx)?; + + if !allowed_roles.contains(&default_role) { + return Err(anyhow::anyhow!( + "default_role must be in allowed_roles: {} not in {:?}", + default_role, + allowed_roles + )); + } + + Ok(ResolvedAwsBedrock { + model: model.resolve(ctx)?, + region: self.region.resolve(ctx).ok(), + access_key_id: self.access_key_id.resolve(ctx).ok(), + secret_access_key: self.secret_access_key.resolve(ctx).ok(), + allowed_roles, + default_role, + allowed_role_metadata: self.allowed_role_metadata.resolve(ctx)?, + supported_request_modes: self.supported_request_modes.clone(), + inference_config: self + .inference_config + .as_ref() + .map(|c| c.resolve(ctx)) + .transpose()?, + }) + } + + pub fn create_from( + mut properties: PropertyHandler, + ) -> Result>> { + let model = { + // Add AWS Bedrock-specific validation logic here + let model_id = properties.ensure_string("model_id", false); + let model = properties.ensure_string("model", false); + + match (model_id, model) { + (Some((model_id_key_meta, _, _)), Some((model_key_meta, _, _))) => { + properties.push_error( + "model_id and model cannot both be provided", + model_id_key_meta, + ); + properties + .push_error("model_id and model cannot both be provided", model_key_meta); + None + } + (Some((_, model, _)), None) | (None, Some((_, model, _))) => Some(model), + (None, None) => { + properties.push_option_error("model_id is required"); + None + } + } + }; + + let region = properties + .ensure_string("region", false) + .map(|(_, v, _)| v.clone()) + .unwrap_or_else(|| baml_types::StringOr::EnvVar("AWS_REGION".to_string())); + let access_key_id = properties + .ensure_string("access_key_id", false) + .map(|(_, v, _)| v.clone()) + .unwrap_or_else(|| baml_types::StringOr::EnvVar("AWS_ACCESS_KEY_ID".to_string())); + let secret_access_key = properties + .ensure_string("secret_access_key", false) + .map(|(_, v, _)| v.clone()) + .unwrap_or_else(|| baml_types::StringOr::EnvVar("AWS_SECRET_ACCESS_KEY".to_string())); + + let allowed_roles = properties.ensure_allowed_roles().unwrap_or(vec![ + StringOr::Value("system".to_string()), + StringOr::Value("user".to_string()), + StringOr::Value("assistant".to_string()), + ]); + let default_role = properties.ensure_default_role(allowed_roles.as_slice(), 1); + let allowed_metadata = properties.ensure_allowed_metadata(); + let supported_request_modes = properties.ensure_supported_request_modes(); + + let inference_config = { + let mut inference_config = UnresolvedInferenceConfiguration { + max_tokens: None, + temperature: None, + top_p: None, + stop_sequences: None, + }; + let raw = properties.ensure_map("inference_configuration", false); + if let Some((_, map, _)) = raw { + for (k, (key_span, v)) in map.into_iter() { + match k.as_str() { + "max_tokens" => inference_config.max_tokens = v.as_numeric().and_then(|val| match val.parse() { + Ok(v) => Some(v), + Err(e) => { + properties.push_error(format!("max_tokens must be a number: {}", e), v.meta().clone()); + None + } + }), + "temperature" => inference_config.temperature = v.as_numeric().and_then(|val| match val.parse() { + Ok(v) => Some(v), + Err(e) => { + properties.push_error(format!("temperature must be a number: {}", e), v.meta().clone()); + None + } + }), + "top_p" => inference_config.top_p = v.as_numeric().and_then(|val| match val.parse() { + Ok(v) => Some(v), + Err(e) => { + properties.push_error(format!("top_p must be a number: {}", e), v.meta().clone()); + None + } + }), + "stop_sequences" => inference_config.stop_sequences = match v.to_array() { + Ok((stop_sequences, _)) => Some(stop_sequences.into_iter().filter_map(|s| match s.to_str() { + Ok((s, _)) => Some(s), + Err(e) => { + properties.push_error(format!("stop_sequences values must be a string: got {}", e.r#type()), e.meta().clone()); + None + } + }) + .collect::>()), + Err(e) => { + properties.push_error( + format!("stop_sequences must be an array: {}", e.r#type()), + e.meta().clone(), + ); + None + } + }, + _ => { + properties.push_error(format!("unknown inference_config key: {}", k), key_span.clone()); + }, + } + } + } + Some(inference_config) + }; + + // TODO: Handle inference_configuration + let errors = properties.finalize_empty(); + if !errors.is_empty() { + return Err(errors); + } + + Ok(Self { + model, + region, + access_key_id, + secret_access_key, + allowed_roles, + default_role, + allowed_role_metadata: allowed_metadata, + supported_request_modes, + inference_config, + }) + } +} diff --git a/engine/baml-lib/llm-client/src/clients/fallback.rs b/engine/baml-lib/llm-client/src/clients/fallback.rs new file mode 100644 index 000000000..0f9b15ca9 --- /dev/null +++ b/engine/baml-lib/llm-client/src/clients/fallback.rs @@ -0,0 +1,64 @@ +use std::collections::HashSet; + +use anyhow::Result; +use baml_types::{EvaluationContext, StringOr}; + +use crate::ClientSpec; + +use super::helpers::{Error, PropertyHandler}; + +#[derive(Debug)] +pub struct UnresolvedFallback { + strategy: Vec<(either::Either, Meta)>, +} + +pub struct ResolvedFallback { + pub strategy: Vec, +} + +impl UnresolvedFallback { + pub fn without_meta(&self) -> UnresolvedFallback<()> { + UnresolvedFallback { + strategy: self.strategy.iter().map(|(s, _)| (s.clone(), ())).collect(), + } + } + + pub fn required_env_vars(&self) -> HashSet { + self.strategy.iter().map(|(s, _)| { + match s { + either::Either::Left(s) => s.required_env_vars(), + either::Either::Right(_) => Default::default(), + } + }).flatten().collect() + } + + pub fn resolve(&self, ctx: &EvaluationContext<'_>) -> Result { + let strategy = self.strategy.iter().map(|(s, _)| match s { + either::Either::Left(s) => ClientSpec::new_from_id(s.resolve(ctx)?.as_str()), + either::Either::Right(s) => Ok(s.clone()), + }).collect::>>()?; + Ok(ResolvedFallback { + strategy + }) + } + + pub fn create_from(mut properties: PropertyHandler) -> Result>> { + let strategy = properties.ensure_strategy(); + let errors = properties.finalize_empty(); + + if !errors.is_empty() { + return Err(errors); + } + + let strategy = strategy.expect("strategy is required"); + + Ok(Self { strategy }) + } +} + + +impl super::StrategyClientProperty for UnresolvedFallback { + fn strategy(&self) -> &Vec<(either::Either, Meta)> { + &self.strategy + } +} diff --git a/engine/baml-lib/llm-client/src/clients/google_ai.rs b/engine/baml-lib/llm-client/src/clients/google_ai.rs new file mode 100644 index 000000000..ab8b34d42 --- /dev/null +++ b/engine/baml-lib/llm-client/src/clients/google_ai.rs @@ -0,0 +1,157 @@ +use std::collections::HashSet; + +use anyhow::Result; +use crate::{ + AllowedRoleMetadata, SupportedRequestModes, UnresolvedAllowedRoleMetadata, +}; + +use baml_types::{EvaluationContext, StringOr, UnresolvedValue}; +use indexmap::IndexMap; + +use super::helpers::{Error, PropertyHandler, UnresolvedUrl}; + +#[derive(Debug)] +pub struct UnresolvedGoogleAI { + api_key: StringOr, + base_url: UnresolvedUrl, + headers: IndexMap, + allowed_roles: Vec, + default_role: Option, + model: Option, + allowed_metadata: UnresolvedAllowedRoleMetadata, + supported_request_modes: SupportedRequestModes, + properties: IndexMap)>, +} + + +impl UnresolvedGoogleAI { + pub fn without_meta(&self) -> UnresolvedGoogleAI<()> { + UnresolvedGoogleAI { + allowed_roles: self.allowed_roles.clone(), + default_role: self.default_role.clone(), + api_key: self.api_key.clone(), + model: self.model.clone(), + base_url: self.base_url.clone(), + headers: self.headers.iter().map(|(k, v)| (k.clone(), v.clone())).collect(), + allowed_metadata: self.allowed_metadata.clone(), + supported_request_modes: self.supported_request_modes.clone(), + properties: self.properties.iter().map(|(k, (_, v))| (k.clone(), ((), v.without_meta()))).collect::>(), + } + } +} + +pub struct ResolvedGoogleAI { + pub allowed_roles: Vec, + pub default_role: String, + pub api_key: String, + pub model: String, + pub base_url: String, + pub headers: IndexMap, + pub allowed_metadata: AllowedRoleMetadata, + pub supported_request_modes: SupportedRequestModes, + pub properties: IndexMap, + pub proxy_url: Option, +} + +impl UnresolvedGoogleAI { + pub fn required_env_vars(&self) -> HashSet { + let mut env_vars = HashSet::new(); + env_vars.extend(self.api_key.required_env_vars()); + env_vars.extend(self.base_url.required_env_vars()); + env_vars.extend(self.headers.values().map(|v| v.required_env_vars()).flatten()); + env_vars.extend(self.allowed_roles.iter().map(|r| r.required_env_vars()).flatten()); + self.default_role.as_ref().map(|r| env_vars.extend(r.required_env_vars())); + self.model.as_ref().map(|m| env_vars.extend(m.required_env_vars())); + env_vars.extend(self.allowed_metadata.required_env_vars()); + env_vars.extend(self.supported_request_modes.required_env_vars()); + env_vars.extend(self.properties.values().map(|(_, v)| v.required_env_vars()).flatten()); + env_vars + } + + pub fn resolve(&self, ctx: &EvaluationContext<'_>) -> Result { + let api_key = self.api_key.resolve(ctx)?; + let Some(default_role) = self.default_role.as_ref() else { + return Err(anyhow::anyhow!("default_role must be provided")); + }; + let default_role = default_role.resolve(ctx)?; + + let allowed_roles = self.allowed_roles.iter().map(|r| r.resolve(ctx)).collect::>>()?; + if !allowed_roles.contains(&default_role) { + return Err(anyhow::anyhow!( + "default_role must be in allowed_roles: {} not in {:?}", + default_role, + allowed_roles + )); + } + + + let model = self + .model + .as_ref() + .map(|m| m.resolve(ctx)) + .transpose()? + .unwrap_or_else(|| "gemini-1.5-flash".to_string()); + + let base_url = self.base_url.resolve(ctx)?; + + let headers = self + .headers + .iter() + .map(|(k, v)| Ok((k.clone(), v.resolve(ctx)?))) + .collect::>>()?; + + Ok(ResolvedGoogleAI { + default_role, + api_key, + model, + base_url, + headers, + allowed_roles, + allowed_metadata: self.allowed_metadata.resolve(ctx)?, + supported_request_modes: self.supported_request_modes.clone(), + properties: self + .properties + .iter() + .map(|(k, (_, v))| Ok((k.clone(), v.resolve_serde::(ctx)?))) + .collect::>>()?, + proxy_url: super::helpers::get_proxy_url(ctx), + }) + } + + pub fn create_from(mut properties: PropertyHandler) -> Result>> { + let allowed_roles = properties.ensure_allowed_roles().unwrap_or(vec![ + StringOr::Value("system".to_string()), + StringOr::Value("user".to_string()), + StringOr::Value("assistant".to_string()), + ]); + let default_role = properties.ensure_default_role(allowed_roles.as_slice(), 1); + + let api_key = properties.ensure_api_key().map(|v| v.clone()).unwrap_or(StringOr::EnvVar("GOOGLE_API_KEY".to_string())); + + let model = properties.ensure_string("model", false).map(|(_, v, _)| v.clone()); + + let base_url = properties.ensure_base_url_with_default(UnresolvedUrl::new_static("https://generativelanguage.googleapis.com/v1beta")); + + let allowed_metadata = properties.ensure_allowed_metadata(); + let supported_request_modes = properties.ensure_supported_request_modes(); + let headers = properties.ensure_headers().unwrap_or_default(); + + let (properties, errors) = properties.finalize(); + + if !errors.is_empty() { + return Err(errors); + } + + Ok(Self { + allowed_roles, + default_role, + api_key, + model, + base_url, + headers, + allowed_metadata, + supported_request_modes, + properties, + }) + } +} diff --git a/engine/baml-lib/llm-client/src/clients/helpers.rs b/engine/baml-lib/llm-client/src/clients/helpers.rs new file mode 100644 index 000000000..c45c5ea51 --- /dev/null +++ b/engine/baml-lib/llm-client/src/clients/helpers.rs @@ -0,0 +1,469 @@ +use std::{borrow::Cow, collections::HashSet}; + +use baml_types::{GetEnvVar, StringOr, UnresolvedValue}; +use indexmap::IndexMap; + +use crate::{SupportedRequestModes, UnresolvedAllowedRoleMetadata}; + +#[derive(Debug, Clone)] +pub struct UnresolvedUrl(StringOr); + +impl UnresolvedUrl { + pub fn resolve(&self, ctx: &impl GetEnvVar) -> anyhow::Result { + let mut url = self.0.resolve(ctx)?; + // Strip trailing slash + if url.ends_with('/') { + url.pop(); + } + Ok(url) + } + + pub fn new_static(url: impl Into) -> Self { + Self(StringOr::Value(url.into())) + } + + pub fn required_env_vars(&self) -> HashSet { + self.0.required_env_vars() + } +} + +pub struct Error { + pub message: String, + pub span: Meta, +} + +impl Error { + pub fn new(message: impl Into>, span: Meta) -> Self { + Self { + message: message.into().to_string(), + span, + } + } +} + +pub struct PropertyHandler { + options: IndexMap)>, + span: Meta, + errors: Vec>, +} + +impl PropertyHandler { + pub fn new(options: IndexMap)>, span: Meta) -> Self { + Self { + options, + span, + errors: Vec::new(), + } + } + + pub fn push_option_error(&mut self, message: impl Into>) { + self.errors.push(Error::new(message, self.span.clone())); + } + + pub fn push_error(&mut self, message: impl Into>, span: Meta) { + self.errors.push(Error::new(message, span)); + } + + pub fn ensure_string(&mut self, key: &str, required: bool) -> Option<(Meta, StringOr, Meta)> { + let result = match ensure_string(&mut self.options, key) { + Ok(result) => { + if required && result.is_none() { + self.push_option_error(format!("Missing required property: {}", key)); + } + result + } + Err(e) => { + self.errors.push(e); + return None; + } + }; + + let final_result = + result.map(|(key_span, value, meta)| (key_span.clone(), value, meta.clone())); + + final_result + } + + pub fn ensure_map( + &mut self, + key: &str, + required: bool, + ) -> Option<(Meta, IndexMap)>, Meta)> { + let result = match ensure_map(&mut self.options, key) { + Ok(result) => { + if required && result.is_none() { + self.push_option_error(format!("Missing required property: {}", key)); + } + result + } + Err(e) => { + self.errors.push(e); + return None; + } + }; + + let final_result = + result.map(|(key_span, value, meta)| (key_span.clone(), value, meta.clone())); + + final_result + } + + pub fn ensure_array( + &mut self, + key: &str, + required: bool, + ) -> Option<(Meta, Vec>, Meta)> { + let result = match ensure_array(&mut self.options, key) { + Ok(result) => { + if required && result.is_none() { + self.push_option_error(format!("Missing required property: {}", key)); + } + result + } + Err(e) => { + self.errors.push(e); + return None; + } + }; + + let final_result = + result.map(|(key_span, value, meta)| (key_span.clone(), value, meta.clone())); + + final_result + } + + pub fn ensure_bool(&mut self, key: &str, required: bool) -> Option<(Meta, bool, Meta)> { + let result = match ensure_bool(&mut self.options, key) { + Ok(result) => { + if required && result.is_none() { + self.push_option_error(format!("Missing required property: {}", key)); + } + result + } + Err(e) => { + self.errors.push(e); + return None; + } + }; + + let final_result = + result.map(|(key_span, value, meta)| (key_span.clone(), value, meta.clone())); + + final_result + } + + pub fn ensure_int(&mut self, key: &str, required: bool) -> Option<(Meta, i32, Meta)> { + let result = match ensure_int(&mut self.options, key) { + Ok(result) => { + if required && result.is_none() { + self.push_option_error(format!("Missing required property: {}", key)); + } + result + } + Err(e) => { + self.errors.push(e); + return None; + } + }; + + let final_result = + result.map(|(key_span, value, meta)| (key_span.clone(), value, meta.clone())); + + final_result + } + + pub fn ensure_allowed_roles(&mut self) -> Option> { + self.ensure_array("allowed_roles", false) + .map(|(_, value, _)| { + value + .into_iter() + .filter_map(|v| match v.as_str() { + Some(s) => Some(s.clone()), + None => { + self.push_error( + format!( + "values in allowed_roles must be strings. Got: {}", + v.r#type() + ), + v.meta().clone(), + ); + None + } + }) + .collect() + }) + } + + pub fn ensure_default_role( + &mut self, + allowed_roles: &[StringOr], + default_role_index: usize, + ) -> Option { + self.ensure_string("default_role", false) + .and_then(|(_, value, span)| { + if allowed_roles.iter().any(|v| value.maybe_eq(v)) { + Some(value) + } else { + let allowed_roles_str = allowed_roles + .iter() + .map(|v| format!("{:?}", v)) + .collect::>() + .join(", "); + self.push_error( + format!( + "default_role must be one of {}. Got: {}", + allowed_roles_str, value + ), + span, + ); + None + } + }) + .or_else(|| allowed_roles.get(default_role_index).cloned()) + } + + pub fn ensure_api_key(&mut self) -> Option { + self.ensure_string("api_key", false) + .map(|(_, value, _)| value) + } + + pub fn ensure_base_url_with_default(&mut self, default: UnresolvedUrl) -> UnresolvedUrl { + self.ensure_string("base_url", false) + .map(|(_, value, _)| UnresolvedUrl(value)) + .unwrap_or(default) + } + + pub fn ensure_base_url(&mut self, required: bool) -> Option<(Meta, UnresolvedUrl, Meta)> { + self.ensure_string("base_url", required) + .map(|(key_span, value, meta)| (key_span, UnresolvedUrl(value), meta)) + } + + pub fn ensure_supported_request_modes(&mut self) -> SupportedRequestModes { + let result = self.ensure_bool("supports_streaming", false); + match result { + Some((_, value, _)) => SupportedRequestModes { + stream: Some(value), + }, + None => SupportedRequestModes { stream: None }, + } + } + + pub fn ensure_any(&mut self, key: &str) -> Option<(Meta, UnresolvedValue)> { + self.options.shift_remove(key) + } + + pub fn ensure_allowed_metadata(&mut self) -> UnresolvedAllowedRoleMetadata { + if let Some((_, value)) = self.options.shift_remove("allowed_role_metadata") { + if let Some(allowed_metadata) = value.as_array() { + let allowed_metadata = allowed_metadata + .iter() + .filter_map(|v| match v.as_str() { + Some(s) => Some(s.clone()), + None => { + self.push_error( + "values in allowed_role_metadata must be strings.", + v.meta().clone(), + ); + None + } + }) + .collect(); + return UnresolvedAllowedRoleMetadata::Only(allowed_metadata); + } else if let Some(allowed_metadata) = value.as_str() { + return UnresolvedAllowedRoleMetadata::Value(allowed_metadata.clone()); + } else { + self.push_error( + "allowed_role_metadata must be an array of keys or \"all\" or \"none\". For example: ['key1', 'key2']", + value.meta().clone(), + ); + } + } + UnresolvedAllowedRoleMetadata::None + } + + pub fn ensure_headers(&mut self) -> Option> { + self.ensure_map("headers", false).map(|(_, value, _)| { + value + .into_iter() + .filter_map(|(k, (_, v))| match v.as_str() { + Some(s) => Some((k, s.clone())), + None => { + self.push_error( + format!( + "Header key {} must have a string value. Got: {}", + k, + v.r#type() + ), + v.meta().clone(), + ); + None + } + }) + .collect() + }) + } + + pub fn ensure_strategy( + &mut self, + ) -> Option, Meta)>> { + self.ensure_array("strategy", true) + .map(|(_, value, value_span)| { + if value.is_empty() { + self.push_error("strategy must not be empty", value_span); + } + value + .into_iter() + .filter_map(|v| match v.as_str() { + Some(s) => { + if let StringOr::Value(value) = s { + if let Ok(client_spec) = + crate::ClientSpec::new_from_id(value.as_str()).map_err(|e| { + self.push_error( + format!("Invalid strategy: {}", e), + v.meta().clone(), + ); + }) + { + Some((either::Either::Right(client_spec), v.meta().clone())) + } else { + Some((either::Either::Left(s.clone()), v.meta().clone())) + } + } else { + Some((either::Either::Left(s.clone()), v.meta().clone())) + } + } + None => { + self.push_error( + format!("values in strategy must be strings. Got: {}", v.r#type()), + v.meta().clone(), + ); + None + } + }) + .collect() + }) + } + + pub fn finalize_empty(self) -> Vec> { + let mut errors = self.errors; + for (k, (key_span, _)) in self.options { + errors.push(Error::new(format!("Unsupported property: {}", k), key_span)); + } + errors + } + + pub fn finalize( + self, + ) -> ( + IndexMap)>, + Vec>, + ) { + (self.options, self.errors) + } +} + +fn ensure_string( + options: &mut IndexMap)>, + key: &str, +) -> Result, Error> { + if let Some((key_span, value)) = options.shift_remove(key) { + match value.to_str() { + Ok((s, meta)) => Ok(Some((key_span, s, meta))), + Err(other) => Err(Error { + message: format!("{} must be a string. Got: {}", key, other.r#type()), + span: other.meta().clone(), + }), + } + } else { + Ok(None) + } +} + +fn ensure_array( + options: &mut IndexMap)>, + key: &str, +) -> Result>, Meta)>, Error> { + if let Some((key_span, value)) = options.shift_remove(key) { + match value.to_array() { + Ok((a, meta)) => Ok(Some((key_span, a, meta))), + Err(other) => Err(Error { + message: format!("{} must be an array. Got: {}", key, other.r#type()), + span: other.meta().clone(), + }), + } + } else { + Ok(None) + } +} + +fn ensure_map( + options: &mut IndexMap)>, + key: &str, +) -> Result)>, Meta)>, Error> { + if let Some((key_span, value)) = options.shift_remove(key) { + match value.to_map() { + Ok((m, meta)) => Ok(Some((key_span, m, meta))), + Err(other) => Err(Error { + message: format!("{} must be a map. Got: {}", key, other.r#type()), + span: other.meta().clone(), + }), + } + } else { + Ok(None) + } +} + +fn ensure_bool( + options: &mut IndexMap)>, + key: &str, +) -> Result, Error> { + if let Some((key_span, value)) = options.shift_remove(key) { + match value.to_bool() { + Ok((b, meta)) => Ok(Some((key_span, b, meta))), + Err(other) => Err(Error { + message: format!("{} must be a bool. Got: {}", key, other.r#type()), + span: other.meta().clone(), + }), + } + } else { + Ok(None) + } +} + +fn ensure_int( + options: &mut IndexMap)>, + key: &str, +) -> Result, Error> { + if let Some((key_span, value)) = options.shift_remove(key) { + match value.to_numeric() { + Ok((i, meta)) => { + if let Ok(i) = i.parse::() { + Ok(Some((key_span, i, meta))) + } else { + Err(Error { + message: format!("{} must be an integer. Got: {}", key, i), + span: meta, + }) + } + } + Err(other) => Err(Error { + message: format!("{} must be an integer. Got: {}", key, other.r#type()), + span: other.meta().clone(), + }), + } + } else { + Ok(None) + } +} + +pub(crate) fn get_proxy_url(ctx: &impl GetEnvVar) -> Option { + if cfg!(target_arch = "wasm32") { + // We don't want to accidentally set this unless the user explicitly + // specifies it, so we enforce allow_missing_env_var=false here + StringOr::EnvVar("BOUNDARY_PROXY_URL".to_string()) + .resolve(&ctx.set_allow_missing_env_var(false)) + .ok() + } else { + None + } +} diff --git a/engine/baml-lib/llm-client/src/clients/mod.rs b/engine/baml-lib/llm-client/src/clients/mod.rs new file mode 100644 index 000000000..e44f26867 --- /dev/null +++ b/engine/baml-lib/llm-client/src/clients/mod.rs @@ -0,0 +1,139 @@ +use std::collections::HashSet; + +use baml_types::{EvaluationContext, StringOr}; +pub use helpers::PropertyHandler; + +use crate::ClientSpec; + +mod helpers; + +pub mod openai; +pub mod anthropic; +pub mod aws_bedrock; +pub mod fallback; +pub mod vertex; +pub mod google_ai; +pub mod round_robin; + + +#[derive(Debug)] +pub enum UnresolvedClientProperty { + OpenAI(openai::UnresolvedOpenAI), + Anthropic(anthropic::UnresolvedAnthropic), + AWSBedrock(aws_bedrock::UnresolvedAwsBedrock), + Vertex(vertex::UnresolvedVertex), + GoogleAI(google_ai::UnresolvedGoogleAI), + RoundRobin(round_robin::UnresolvedRoundRobin), + Fallback(fallback::UnresolvedFallback), +} + +pub enum ResolvedClientProperty { + OpenAI(openai::ResolvedOpenAI), + Anthropic(anthropic::ResolvedAnthropic), + AWSBedrock(aws_bedrock::ResolvedAwsBedrock), + Vertex(vertex::ResolvedVertex), + GoogleAI(google_ai::ResolvedGoogleAI), + RoundRobin(round_robin::ResolvedRoundRobin), + Fallback(fallback::ResolvedFallback), +} + +impl ResolvedClientProperty { + pub fn name(&self) -> &str { + match self { + ResolvedClientProperty::RoundRobin(_) => "round-robin", + ResolvedClientProperty::Fallback(_) => "fallback", + ResolvedClientProperty::OpenAI(_) => "openai", + ResolvedClientProperty::Anthropic(_) => "anthropic", + ResolvedClientProperty::AWSBedrock(_) => "aws-bedrock", + ResolvedClientProperty::Vertex(_) => "vertex", + ResolvedClientProperty::GoogleAI(_) => "google-ai", + } + } +} + +impl UnresolvedClientProperty { + pub fn required_env_vars(&self) -> HashSet { + match self { + UnresolvedClientProperty::OpenAI(o) => o.required_env_vars(), + UnresolvedClientProperty::Anthropic(a) => a.required_env_vars(), + UnresolvedClientProperty::AWSBedrock(a) => a.required_env_vars(), + UnresolvedClientProperty::Vertex(v) => v.required_env_vars(), + UnresolvedClientProperty::GoogleAI(g) => g.required_env_vars(), + UnresolvedClientProperty::RoundRobin(r) => r.required_env_vars(), + UnresolvedClientProperty::Fallback(f) => f.required_env_vars(), + } + } + + pub fn resolve(&self, provider: &crate::ClientProvider, ctx: &EvaluationContext<'_>) -> anyhow::Result { + match self { + UnresolvedClientProperty::OpenAI(o) => o.resolve(&provider, ctx).map(ResolvedClientProperty::OpenAI), + UnresolvedClientProperty::Anthropic(a) => a.resolve(ctx).map(ResolvedClientProperty::Anthropic), + UnresolvedClientProperty::AWSBedrock(a) => a.resolve(ctx).map(ResolvedClientProperty::AWSBedrock), + UnresolvedClientProperty::Vertex(v) => v.resolve(ctx).map(ResolvedClientProperty::Vertex), + UnresolvedClientProperty::GoogleAI(g) => g.resolve(ctx).map(ResolvedClientProperty::GoogleAI), + UnresolvedClientProperty::RoundRobin(r) => r.resolve(ctx).map(ResolvedClientProperty::RoundRobin), + UnresolvedClientProperty::Fallback(f) => f.resolve(ctx).map(ResolvedClientProperty::Fallback), + } + } + + pub fn without_meta(&self) -> UnresolvedClientProperty<()> { + match self { + UnresolvedClientProperty::OpenAI(o) => UnresolvedClientProperty::OpenAI(o.without_meta()), + UnresolvedClientProperty::Anthropic(a) => UnresolvedClientProperty::Anthropic(a.without_meta()), + UnresolvedClientProperty::AWSBedrock(a) => UnresolvedClientProperty::AWSBedrock(a.clone()), + UnresolvedClientProperty::Vertex(v) => UnresolvedClientProperty::Vertex(v.without_meta()), + UnresolvedClientProperty::GoogleAI(g) => UnresolvedClientProperty::GoogleAI(g.without_meta()), + UnresolvedClientProperty::RoundRobin(r) => UnresolvedClientProperty::RoundRobin(r.without_meta()), + UnresolvedClientProperty::Fallback(f) => UnresolvedClientProperty::Fallback(f.without_meta()), + } + } +} + +impl crate::ClientProvider { + pub fn parse_client_property(&self, properties: PropertyHandler) -> Result, Vec>> { + Ok(match self { + crate::ClientProvider::OpenAI(open_aiclient_provider_variant) => { + UnresolvedClientProperty::OpenAI(open_aiclient_provider_variant.create_from(properties)?) + }, + crate::ClientProvider::Anthropic => { + UnresolvedClientProperty::Anthropic(anthropic::UnresolvedAnthropic::create_from(properties)?) + }, + crate::ClientProvider::AwsBedrock => { + UnresolvedClientProperty::AWSBedrock(aws_bedrock::UnresolvedAwsBedrock::create_from(properties)?) + }, + crate::ClientProvider::GoogleAi => { + UnresolvedClientProperty::GoogleAI(google_ai::UnresolvedGoogleAI::create_from(properties)?) + }, + crate::ClientProvider::Vertex => { + UnresolvedClientProperty::Vertex(vertex::UnresolvedVertex::create_from(properties)?) + }, + crate::ClientProvider::Strategy(s) => { + s.create_from(properties)? + }, + }) + } +} + +impl crate::OpenAIClientProviderVariant { + fn create_from(&self, properties: PropertyHandler) -> Result, Vec>> { + match self { + crate::OpenAIClientProviderVariant::Base => openai::UnresolvedOpenAI::create_standard(properties), + crate::OpenAIClientProviderVariant::Ollama => openai::UnresolvedOpenAI::create_ollama(properties), + crate::OpenAIClientProviderVariant::Azure => openai::UnresolvedOpenAI::create_azure(properties), + crate::OpenAIClientProviderVariant::Generic => openai::UnresolvedOpenAI::create_generic(properties), + } + } +} + +impl crate::StrategyClientProvider { + fn create_from(&self, properties: PropertyHandler) -> Result, Vec>> { + match self { + crate::StrategyClientProvider::Fallback => Ok(UnresolvedClientProperty::Fallback(fallback::UnresolvedFallback::create_from(properties)?)), + crate::StrategyClientProvider::RoundRobin => Ok(UnresolvedClientProperty::RoundRobin(round_robin::UnresolvedRoundRobin::create_from(properties)?)), + } + } +} + +pub trait StrategyClientProperty { + fn strategy(&self) -> &Vec<(either::Either, Meta)>; +} \ No newline at end of file diff --git a/engine/baml-lib/llm-client/src/clients/openai.rs b/engine/baml-lib/llm-client/src/clients/openai.rs new file mode 100644 index 000000000..e9609e4cd --- /dev/null +++ b/engine/baml-lib/llm-client/src/clients/openai.rs @@ -0,0 +1,321 @@ +use std::collections::HashSet; + +use crate::{AllowedRoleMetadata, SupportedRequestModes, UnresolvedAllowedRoleMetadata}; +use anyhow::Result; + +use baml_types::{GetEnvVar, StringOr, UnresolvedValue}; +use indexmap::IndexMap; + +use super::helpers::{Error, PropertyHandler, UnresolvedUrl}; + +#[derive(Debug)] +pub struct UnresolvedOpenAI { + base_url: Option>, + api_key: Option, + allowed_roles: Vec, + default_role: Option, + allowed_role_metadata: UnresolvedAllowedRoleMetadata, + supported_request_modes: SupportedRequestModes, + headers: IndexMap, + properties: IndexMap)>, + query_params: IndexMap, +} + +impl UnresolvedOpenAI { + pub fn without_meta(&self) -> UnresolvedOpenAI<()> { + UnresolvedOpenAI { + base_url: self.base_url.clone(), + api_key: self.api_key.clone(), + allowed_roles: self.allowed_roles.clone(), + default_role: self.default_role.clone(), + allowed_role_metadata: self.allowed_role_metadata.clone(), + supported_request_modes: self.supported_request_modes.clone(), + headers: self + .headers + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect(), + properties: self + .properties + .iter() + .map(|(k, (_, v))| (k.clone(), ((), v.without_meta()))) + .collect::>(), + query_params: self + .query_params + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect(), + } + } +} + +pub struct ResolvedOpenAI { + pub base_url: String, + pub api_key: Option, + pub allowed_roles: Vec, + pub default_role: String, + pub allowed_metadata: AllowedRoleMetadata, + pub supported_request_modes: SupportedRequestModes, + pub headers: IndexMap, + pub properties: IndexMap, + pub query_params: IndexMap, + pub proxy_url: Option, +} + +impl UnresolvedOpenAI { + pub fn required_env_vars(&self) -> HashSet { + let mut env_vars = HashSet::new(); + + self.base_url.as_ref().map(|url| match url { + either::Either::Left(url) => { + env_vars.extend(url.required_env_vars()); + } + either::Either::Right((resource_name, deployment_id)) => { + env_vars.extend(resource_name.required_env_vars()); + env_vars.extend(deployment_id.required_env_vars()); + } + }); + self.api_key + .as_ref() + .map(|key| env_vars.extend(key.required_env_vars())); + self.allowed_roles + .iter() + .for_each(|role| env_vars.extend(role.required_env_vars())); + self.default_role + .as_ref() + .map(|role| env_vars.extend(role.required_env_vars())); + env_vars.extend(self.allowed_role_metadata.required_env_vars()); + env_vars.extend(self.supported_request_modes.required_env_vars()); + self.headers + .iter() + .for_each(|(_, v)| env_vars.extend(v.required_env_vars())); + self.properties + .iter() + .for_each(|(_, (_, v))| env_vars.extend(v.required_env_vars())); + self.query_params + .iter() + .for_each(|(_, v)| env_vars.extend(v.required_env_vars())); + + env_vars + } + + pub fn resolve(&self, provider: &crate::ClientProvider, ctx: &impl GetEnvVar) -> Result { + let base_url = self + .base_url + .as_ref() + .map(|url| match url { + either::Either::Left(url) => url.resolve(ctx), + either::Either::Right((resource_name, deployment_id)) => { + let resource_name = resource_name.resolve(ctx)?; + let deployment_id = deployment_id.resolve(ctx)?; + Ok(format!( + "https://{}.openai.azure.com/openai/deployments/{}", + resource_name, deployment_id + )) + } + }) + .transpose()?; + + let Some(base_url) = base_url else { + return Err(anyhow::anyhow!("base_url is required")); + }; + + let api_key = self + .api_key + .as_ref() + .map(|key| key.resolve(ctx)) + .transpose()?; + + let allowed_roles = self + .allowed_roles + .iter() + .map(|role| role.resolve(ctx)) + .collect::>>()?; + + let Some(default_role) = self.default_role.as_ref() else { + return Err(anyhow::anyhow!("default_role must be provided")); + }; + let default_role = default_role.resolve(ctx)?; + + if !allowed_roles.contains(&default_role) { + return Err(anyhow::anyhow!( + "default_role must be in allowed_roles: {} not in {:?}", + default_role, + allowed_roles + )); + } + + let headers = self + .headers + .iter() + .map(|(k, v)| Ok((k.clone(), v.resolve(ctx)?))) + .collect::>>()?; + + let properties = { + let mut properties = self + .properties + .iter() + .map(|(k, (_, v))| Ok((k.clone(), v.resolve_serde::(ctx)?))) + .collect::>>()?; + + // TODO(vbv): Only do this for azure + if matches!(provider, crate::ClientProvider::OpenAI(crate::OpenAIClientProviderVariant::Azure)) { + properties + .entry("max_tokens".into()) + .or_insert(serde_json::json!(4096)); + } + properties + }; + + let query_params = self + .query_params + .iter() + .map(|(k, v)| Ok((k.clone(), v.resolve(ctx)?))) + .collect::>>()?; + + Ok(ResolvedOpenAI { + base_url, + api_key, + allowed_roles, + default_role, + allowed_metadata: self.allowed_role_metadata.resolve(ctx)?, + supported_request_modes: self.supported_request_modes.clone(), + headers, + properties, + query_params, + proxy_url: super::helpers::get_proxy_url(ctx), + }) + } + + pub fn create_standard( + mut properties: PropertyHandler, + ) -> Result>> { + let base_url = properties.ensure_base_url_with_default(UnresolvedUrl::new_static("https://api.openai.com/v1")); + + let api_key = Some( + properties + .ensure_api_key() + .map(|v| v.clone()) + .unwrap_or_else(|| StringOr::EnvVar("OPENAI_API_KEY".to_string())), + ); + + Self::create_common(properties, Some(either::Either::Left(base_url)), api_key) + } + + pub fn create_azure(mut properties: PropertyHandler) -> Result>> { + let base_url = { + let base_url = properties.ensure_base_url(false); + let resource_name = properties + .ensure_string("resource_name", false) + .map(|(key_span, v, _)| (key_span, v.clone())); + let deployment_id = properties + .ensure_string("deployment_id", false) + .map(|(key_span, v, _)| (key_span, v.clone())); + + match (base_url, resource_name, deployment_id) { + (Some(url), None, None) => Some(either::Either::Left(url.1)), + (None, Some(name), Some(id)) => Some(either::Either::Right((name.1, id.1))), + (_, None, Some((key_span, _))) => { + properties.push_error( + "resource_name must be provided when deployment_id is provided", + key_span, + ); + None + } + (_, Some((key_span, _)), None) => { + properties.push_error( + "deployment_id must be provided when resource_name is provided", + key_span, + ); + None + } + (Some((key_1_span, ..)), Some((key_2_span, _)), Some((key_3_span, _))) => { + for key in [key_1_span, key_2_span, key_3_span] { + properties.push_error( + "Only one of base_url or both (resource_name, deployment_id) must be provided", + key + ); + } + None + } + (None, None, None) => { + properties.push_option_error( + "Missing either base_url or both (resource_name, deployment_id)", + ); + None + } + } + }; + + let api_key = Some( + properties + .ensure_api_key() + .map(|v| v.clone()) + .unwrap_or_else(|| StringOr::EnvVar("AZURE_OPENAI_API_KEY".to_string())), + ); + + let mut query_params = IndexMap::new(); + if let Some((_, v, _)) = properties.ensure_string("api_version", false) { + query_params.insert("api-version".to_string(), v.clone()); + } + + let mut instance = Self::create_common(properties, base_url, api_key)?; + instance.query_params = query_params; + + Ok(instance) + } + + pub fn create_generic(mut properties: PropertyHandler) -> Result>> { + let base_url = properties.ensure_base_url(true); + + let api_key = properties.ensure_api_key().map(|v| v.clone()); + + Self::create_common( + properties, + base_url.map(|url| either::Either::Left(url.1)), + api_key, + ) + } + + pub fn create_ollama(mut properties: PropertyHandler) -> Result>> { + let base_url = properties.ensure_base_url_with_default(UnresolvedUrl::new_static("http://localhost:11434/v1")); + + let api_key = properties.ensure_api_key().map(|v| v.clone()); + + Self::create_common(properties, Some(either::Either::Left(base_url)), api_key) + } + + fn create_common( + mut properties: PropertyHandler, + base_url: Option>, + api_key: Option, + ) -> Result>> { + let allowed_roles = properties.ensure_allowed_roles().unwrap_or(vec![ + StringOr::Value("system".to_string()), + StringOr::Value("user".to_string()), + StringOr::Value("assistant".to_string()), + ]); + + let default_role = properties.ensure_default_role(allowed_roles.as_slice(), 1); + let allowed_metadata = properties.ensure_allowed_metadata(); + let supported_request_modes = properties.ensure_supported_request_modes(); + let headers = properties.ensure_headers().unwrap_or_default(); + let (properties, errors) = properties.finalize(); + + if !errors.is_empty() { + return Err(errors); + } + + Ok(Self { + base_url, + api_key, + allowed_roles, + default_role, + allowed_role_metadata: allowed_metadata, + supported_request_modes, + headers, + properties, + query_params: IndexMap::new(), + }) + } +} diff --git a/engine/baml-lib/llm-client/src/clients/round_robin.rs b/engine/baml-lib/llm-client/src/clients/round_robin.rs new file mode 100644 index 000000000..444030115 --- /dev/null +++ b/engine/baml-lib/llm-client/src/clients/round_robin.rs @@ -0,0 +1,69 @@ +use std::collections::HashSet; + +use anyhow::Result; +use baml_types::{EvaluationContext, StringOr}; + +use crate::ClientSpec; + +use super::helpers::{Error, PropertyHandler}; + +#[derive(Debug)] +pub struct UnresolvedRoundRobin { + pub strategy: Vec<(either::Either, Meta)>, + start_index: Option, +} + +pub struct ResolvedRoundRobin { + pub strategy: Vec, + pub start_index: Option, +} + +impl UnresolvedRoundRobin { + pub fn without_meta(&self) -> UnresolvedRoundRobin<()> { + UnresolvedRoundRobin { + strategy: self.strategy.iter().map(|(s, _)| (s.clone(), ())).collect(), + start_index: self.start_index, + } + } + + pub fn required_env_vars(&self) -> HashSet { + self.strategy.iter().map(|(s, _)| { + match s { + either::Either::Left(s) => s.required_env_vars(), + either::Either::Right(_) => Default::default(), + } + }).flatten().collect() + } + + pub fn resolve(&self, ctx: &EvaluationContext<'_>) -> Result { + let strategy = self.strategy.iter().map(|(s, _)| match s { + either::Either::Left(s) => ClientSpec::new_from_id(s.resolve(ctx)?.as_str()), + either::Either::Right(s) => Ok(s.clone()), + }).collect::>>()?; + + Ok(ResolvedRoundRobin { + strategy, + start_index: self.start_index, + }) + } + + pub fn create_from(mut properties: PropertyHandler) -> Result>> { + let strategy = properties.ensure_strategy(); + let start_index = properties.ensure_int("start", false).map(|(_, v, _)| v); + let errors = properties.finalize_empty(); + + if !errors.is_empty() { + return Err(errors); + } + + let strategy = strategy.expect("strategy is required"); + + Ok(Self { strategy, start_index }) + } +} + +impl super::StrategyClientProperty for UnresolvedRoundRobin { + fn strategy(&self) -> &Vec<(either::Either, Meta)> { + &self.strategy + } +} diff --git a/engine/baml-lib/llm-client/src/clients/vertex.rs b/engine/baml-lib/llm-client/src/clients/vertex.rs new file mode 100644 index 000000000..dc5eaf7e3 --- /dev/null +++ b/engine/baml-lib/llm-client/src/clients/vertex.rs @@ -0,0 +1,393 @@ +use std::collections::HashSet; + +use crate::{AllowedRoleMetadata, SupportedRequestModes, UnresolvedAllowedRoleMetadata}; +use anyhow::{Context, Result}; + +use baml_types::{GetEnvVar, StringOr, UnresolvedValue}; +use indexmap::IndexMap; +use serde::Deserialize; + +use super::helpers::{Error, PropertyHandler, UnresolvedUrl}; + +#[derive(Debug)] +enum UnresolvedServiceAccountDetails { + RawAuthorizationHeader(StringOr), + MaybeFilePathOrContent(StringOr), + Object(IndexMap)>), + Json(StringOr), +} + +#[derive(Debug, Deserialize)] +pub struct ServiceAccount { + pub token_uri: String, + pub project_id: String, + pub client_email: String, + pub private_key: String, +} + +pub enum ResolvedServiceAccountDetails { + RawAuthorizationHeader(String), + Json(ServiceAccount), +} + +impl UnresolvedServiceAccountDetails { + fn without_meta(&self) -> UnresolvedServiceAccountDetails<()> { + match self { + UnresolvedServiceAccountDetails::RawAuthorizationHeader(s) => { + UnresolvedServiceAccountDetails::RawAuthorizationHeader(s.clone()) + } + UnresolvedServiceAccountDetails::MaybeFilePathOrContent(s) => { + UnresolvedServiceAccountDetails::MaybeFilePathOrContent(s.clone()) + } + UnresolvedServiceAccountDetails::Object(s) => UnresolvedServiceAccountDetails::Object( + s.iter() + .map(|(k, v)| (k.clone(), ((), v.1.without_meta()))) + .collect(), + ), + UnresolvedServiceAccountDetails::Json(s) => { + UnresolvedServiceAccountDetails::Json(s.clone()) + } + } + } + + fn required_env_vars(&self) -> HashSet { + match self { + UnresolvedServiceAccountDetails::RawAuthorizationHeader(s) => s.required_env_vars(), + UnresolvedServiceAccountDetails::MaybeFilePathOrContent(s) => s.required_env_vars(), + UnresolvedServiceAccountDetails::Object(s) => s + .values() + .map(|(_, v)| v.required_env_vars()) + .flatten() + .collect(), + UnresolvedServiceAccountDetails::Json(s) => s.required_env_vars(), + } + } + + fn resolve(&self, ctx: &impl GetEnvVar) -> Result { + match self { + UnresolvedServiceAccountDetails::RawAuthorizationHeader(s) => Ok( + ResolvedServiceAccountDetails::RawAuthorizationHeader(s.resolve(ctx)?), + ), + UnresolvedServiceAccountDetails::MaybeFilePathOrContent(s) => { + let value = s.resolve(ctx)?; + match serde_json::from_str(&value) { + Ok(json) => Ok(ResolvedServiceAccountDetails::Json(json)), + Err(_) => { + #[cfg(not(target_arch = "wasm32"))] + { + // Not a valid JSON, so we assume it's a file path + // Load the file and parse it as JSON + let file = std::fs::read_to_string(&value).context(format!( + "Failed to read service account file: {}", + value + ))?; + let json = serde_json::from_str(&file) + .context(format!("Failed to parse service account file as JSON"))?; + Ok(ResolvedServiceAccountDetails::Json(json)) + } + #[cfg(target_arch = "wasm32")] + { + anyhow::bail!( + format!("Reading from files not supported in BAML playground. For the playground, pass in the contents of your credentials file as a string to the same environment variable you used for 'credentials'.\nFile: {}", value) + ); + } + } + } + } + UnresolvedServiceAccountDetails::Object(s) => { + let raw = s + .iter() + .map(|(k, v)| Ok((k, v.1.resolve_serde::(ctx)?))) + .collect::>>()?; + Ok(ResolvedServiceAccountDetails::Json( + serde_json::from_value(serde_json::json!(raw)) + .context(format!("Failed to parse service account JSON"))?, + )) + } + UnresolvedServiceAccountDetails::Json(s) => { + let raw = s.resolve(ctx)?; + Ok(ResolvedServiceAccountDetails::Json( + serde_json::from_str(&raw) + .context(format!("Failed to parse service account JSON"))?, + )) + } + } + } +} + +#[derive(Debug)] +pub struct UnresolvedVertex { + // Either base_url or location + base_url: either::Either, + project_id: Option, + authorization: UnresolvedServiceAccountDetails, + model: StringOr, + headers: IndexMap, + allowed_roles: Vec, + default_role: Option, + allowed_role_metadata: UnresolvedAllowedRoleMetadata, + supported_request_modes: SupportedRequestModes, + properties: IndexMap)>, +} + +pub struct ResolvedVertex { + pub base_url: String, + pub authorization: ResolvedServiceAccountDetails, + pub model: String, + pub headers: IndexMap, + pub allowed_roles: Vec, + pub default_role: String, + pub allowed_metadata: AllowedRoleMetadata, + pub supported_request_modes: SupportedRequestModes, + pub properties: IndexMap, + pub proxy_url: Option, +} + +impl UnresolvedVertex { + pub fn required_env_vars(&self) -> HashSet { + let mut env_vars = HashSet::new(); + match self.base_url { + either::Either::Left(ref base_url) => env_vars.extend(base_url.required_env_vars()), + either::Either::Right(ref location) => env_vars.extend(location.required_env_vars()), + } + if let Some(ref project_id) = self.project_id { + env_vars.extend(project_id.required_env_vars()); + } + env_vars.extend(self.authorization.required_env_vars()); + env_vars.extend(self.model.required_env_vars()); + env_vars.extend( + self.headers + .values() + .map(|v| v.required_env_vars()) + .flatten(), + ); + env_vars.extend( + self.allowed_roles + .iter() + .map(|r| r.required_env_vars()) + .flatten(), + ); + self.default_role + .as_ref() + .map(|r| env_vars.extend(r.required_env_vars())); + env_vars.extend(self.allowed_role_metadata.required_env_vars()); + env_vars.extend(self.supported_request_modes.required_env_vars()); + env_vars.extend( + self.properties + .values() + .map(|(_, v)| v.required_env_vars()) + .flatten(), + ); + + env_vars + } + + pub fn without_meta(&self) -> UnresolvedVertex<()> { + UnresolvedVertex { + base_url: self.base_url.clone(), + project_id: self.project_id.clone(), + authorization: self.authorization.without_meta(), + model: self.model.clone(), + headers: self.headers.clone(), + allowed_roles: self.allowed_roles.clone(), + default_role: self.default_role.clone(), + allowed_role_metadata: self.allowed_role_metadata.clone(), + supported_request_modes: self.supported_request_modes.clone(), + properties: self + .properties + .iter() + .map(|(k, (_, v))| (k.clone(), ((), v.without_meta()))) + .collect(), + } + } + + pub fn resolve(&self, ctx: &impl GetEnvVar) -> Result { + // Validate auth options - only one should be provided + let authorization = self.authorization.resolve(ctx)?; + + let base_url = match self.base_url.as_ref() { + either::Either::Left(url) => url.resolve(ctx), + either::Either::Right(location) => { + let project_id = match self.project_id.as_ref() { + Some(project_id) => project_id.resolve(ctx)?, + None => match &authorization { + ResolvedServiceAccountDetails::Json(service_account) => { + service_account.project_id.clone() + } + ResolvedServiceAccountDetails::RawAuthorizationHeader(_) => { + return Err(anyhow::anyhow!( + "project_id is required when using location + authorization" + )) + } + }, + }; + + let location = location.resolve(ctx)?; + Ok(format!( + "https://{location}-aiplatform.googleapis.com/v1/projects/{project_id}/locations/{location}/publishers/google/models" + )) + } + }?; + + let model = self.model.resolve(ctx)?; + + let allowed_roles = self + .allowed_roles + .iter() + .map(|role| role.resolve(ctx)) + .collect::>>()?; + + let Some(default_role) = self.default_role.as_ref() else { + return Err(anyhow::anyhow!("default_role must be provided")); + }; + let default_role = default_role.resolve(ctx)?; + + if !allowed_roles.contains(&default_role) { + return Err(anyhow::anyhow!( + "default_role must be in allowed_roles: {} not in {:?}", + default_role, + allowed_roles + )); + } + + let headers = self + .headers + .iter() + .map(|(k, v)| Ok((k.clone(), v.resolve(ctx)?))) + .collect::>>()?; + + Ok(ResolvedVertex { + base_url, + authorization, + model, + headers, + allowed_roles, + default_role, + allowed_metadata: self.allowed_role_metadata.resolve(ctx)?, + supported_request_modes: self.supported_request_modes.clone(), + properties: self + .properties + .iter() + .map(|(k, (_, v))| Ok((k.clone(), v.resolve_serde::(ctx)?))) + .collect::>>()?, + proxy_url: super::helpers::get_proxy_url(ctx), + }) + } + + pub fn create_from(mut properties: PropertyHandler) -> Result>> { + let authorization = { + let credentials = properties + .ensure_any("credentials") + .map(|(_, v)| v) + .and_then(|v| match v { + UnresolvedValue::String(s, ..) => { + Some(UnresolvedServiceAccountDetails::MaybeFilePathOrContent(s)) + } + UnresolvedValue::Map(m, ..) => Some(UnresolvedServiceAccountDetails::Object(m)), + other => { + properties.push_error( + format!( + "credentials must be a string or an object. Got: {}", + other.r#type() + ), + other.meta().clone(), + ); + None + } + }); + + let credentials_content = properties + .ensure_string("credentials_content", false) + .map(|(_, v, _)| UnresolvedServiceAccountDetails::Json(v)); + + let authz = properties + .ensure_string("authorization", false) + .map(|(_, v, _)| UnresolvedServiceAccountDetails::RawAuthorizationHeader(v)); + + match (authz, credentials, credentials_content) { + (Some(authz), _, _) => Some(authz), + (None, Some(credentials), Some(credentials_content)) => { + if cfg!(target_arch = "wasm32") { + Some(credentials_content) + } else { + Some(credentials) + } + } + (None, Some(credentials), None) => Some(credentials), + (None, None, Some(credentials_content)) => Some(credentials_content), + (None, None, None) => { + if cfg!(target_arch = "wasm32") { + Some(UnresolvedServiceAccountDetails::Json(StringOr::EnvVar( + "GOOGLE_APPLICATION_CREDENTIALS_CONTENT".to_string(), + ))) + } else { + Some(UnresolvedServiceAccountDetails::MaybeFilePathOrContent( + StringOr::EnvVar("GOOGLE_APPLICATION_CREDENTIALS".to_string()), + )) + } + } + } + }; + let model = properties.ensure_string("model", true).map(|(_, v, _)| v); + + let base_url = { + let base_url = properties.ensure_base_url(false); + let location = properties + .ensure_string("location", false) + .map(|(key_span, v, _)| (key_span, v.clone())); + + match (base_url, location) { + (Some(url), None) => Some(either::Either::Left(url.1)), + (None, Some(name)) => Some(either::Either::Right(name.1)), + (Some((key_1_span, ..)), Some((key_2_span, _))) => { + for key in [key_1_span, key_2_span] { + properties + .push_error("Only one of base_url or location must be provided", key); + } + None + } + (None, None) => { + // Its possible this will come in from credentials later + properties.push_option_error("Missing either base_url or location"); + None + } + } + }; + + let project_id = properties + .ensure_string("project_id", false) + .map(|(_, v, _)| v); + + let allowed_roles = properties.ensure_allowed_roles().unwrap_or(vec![ + StringOr::Value("system".to_string()), + StringOr::Value("user".to_string()), + StringOr::Value("assistant".to_string()), + ]); + + let default_role = properties.ensure_default_role(allowed_roles.as_slice(), 1); + let allowed_metadata = properties.ensure_allowed_metadata(); + let supported_request_modes = properties.ensure_supported_request_modes(); + let headers = properties.ensure_headers().unwrap_or_default(); + let (properties, errors) = properties.finalize(); + if !errors.is_empty() { + return Err(errors); + } + + let model = model.expect("model is required"); + let base_url = base_url.expect("base_url is required"); + let authorization = authorization.expect("authorization is required"); + + Ok(Self { + base_url, + project_id, + authorization, + model, + headers, + allowed_roles, + default_role, + allowed_role_metadata: allowed_metadata, + supported_request_modes, + properties, + }) + } +} diff --git a/engine/baml-lib/llm-client/src/clientspec.rs b/engine/baml-lib/llm-client/src/clientspec.rs new file mode 100644 index 000000000..862c6b2cc --- /dev/null +++ b/engine/baml-lib/llm-client/src/clientspec.rs @@ -0,0 +1,258 @@ +use anyhow::Result; +use std::collections::HashSet; + +use baml_types::{GetEnvVar, StringOr}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Serialize)] +pub enum ClientSpec { + Named(String), + /// Shorthand for "/" + Shorthand(ClientProvider, String), +} + +impl ClientSpec { + pub fn as_str(&self) -> String { + match self { + ClientSpec::Named(n) => n.clone(), + ClientSpec::Shorthand(provider, model) => format!("{provider}/{model}"), + } + } + + pub fn new_from_id(arg: &str) -> Result { + if arg.contains("/") { + let (provider, model) = arg.split_once("/").unwrap(); + Ok(ClientSpec::Shorthand(provider.parse()?, model.to_string())) + } else { + Ok(ClientSpec::Named(arg.into())) + } + } +} + +/// The provider for the client, e.g. baml-openai-chat +#[derive(Clone, Debug, Deserialize, Serialize)] +pub enum ClientProvider { + /// The OpenAI client provider variant + OpenAI(OpenAIClientProviderVariant), + /// The Anthropic client provider variant + Anthropic, + /// The AWS Bedrock client provider variant + AwsBedrock, + /// The Google AI client provider variant + GoogleAi, + /// The Vertex client provider variant + Vertex, + /// The strategy client provider variant + Strategy(StrategyClientProvider), +} + +/// The OpenAI client provider variant +#[derive(Clone, Debug, Deserialize, Serialize)] +pub enum OpenAIClientProviderVariant { + /// The base OpenAI client provider variant + Base, + /// The Ollama client provider variant + Ollama, + /// The Azure client provider variant + Azure, + /// The generic client provider variant + Generic, +} + +/// The strategy client provider variant +#[derive(Clone, Debug, Deserialize, Serialize)] +pub enum StrategyClientProvider { + /// The round-robin strategy client provider variant + RoundRobin, + /// The fallback strategy client provider variant + Fallback, +} + +impl std::fmt::Display for ClientProvider { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ClientProvider::OpenAI(variant) => write!(f, "{}", variant), + ClientProvider::Anthropic => write!(f, "anthropic"), + ClientProvider::AwsBedrock => write!(f, "aws-bedrock"), + ClientProvider::GoogleAi => write!(f, "google-ai"), + ClientProvider::Vertex => write!(f, "vertex-ai"), + ClientProvider::Strategy(variant) => write!(f, "{}", variant), + } + } +} + +impl std::fmt::Display for OpenAIClientProviderVariant { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + OpenAIClientProviderVariant::Base => write!(f, "openai"), + OpenAIClientProviderVariant::Ollama => write!(f, "ollama"), + OpenAIClientProviderVariant::Azure => write!(f, "azure-openai"), + OpenAIClientProviderVariant::Generic => write!(f, "openai-generic"), + } + } +} + +impl std::fmt::Display for StrategyClientProvider { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + StrategyClientProvider::RoundRobin => write!(f, "round-robin"), + StrategyClientProvider::Fallback => write!(f, "fallback"), + } + } +} + +impl std::str::FromStr for ClientProvider { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + match s { + "openai" => Ok(ClientProvider::OpenAI(OpenAIClientProviderVariant::Base)), + "baml-openai-chat" => Ok(ClientProvider::OpenAI(OpenAIClientProviderVariant::Base)), + "openai-generic" => Ok(ClientProvider::OpenAI(OpenAIClientProviderVariant::Generic)), + "azure-openai" => Ok(ClientProvider::OpenAI(OpenAIClientProviderVariant::Azure)), + "baml-azure-chat" => Ok(ClientProvider::OpenAI(OpenAIClientProviderVariant::Azure)), + "baml-ollama-chat" => Ok(ClientProvider::OpenAI(OpenAIClientProviderVariant::Ollama)), + "ollama" => Ok(ClientProvider::OpenAI(OpenAIClientProviderVariant::Ollama)), + "anthropic" => Ok(ClientProvider::Anthropic), + "baml-anthropic-chat" => Ok(ClientProvider::Anthropic), + "aws-bedrock" => Ok(ClientProvider::AwsBedrock), + "google-ai" => Ok(ClientProvider::GoogleAi), + "vertex-ai" => Ok(ClientProvider::Vertex), + "fallback" => Ok(ClientProvider::Strategy(StrategyClientProvider::Fallback)), + "baml-fallback" => Ok(ClientProvider::Strategy(StrategyClientProvider::Fallback)), + "round-robin" => Ok(ClientProvider::Strategy(StrategyClientProvider::RoundRobin)), + "baml-round-robin" => Ok(ClientProvider::Strategy(StrategyClientProvider::RoundRobin)), + _ => Err(anyhow::anyhow!("Invalid client provider: {}", s)), + } + } +} + +impl std::str::FromStr for OpenAIClientProviderVariant { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + match s { + "openai" => Ok(OpenAIClientProviderVariant::Base), + "ollama" => Ok(OpenAIClientProviderVariant::Ollama), + "azure-openai" => Ok(OpenAIClientProviderVariant::Azure), + "openai-generic" => Ok(OpenAIClientProviderVariant::Generic), + _ => Err(anyhow::anyhow!( + "Invalid OpenAI client provider variant: {}", + s + )), + } + } +} + +impl std::str::FromStr for StrategyClientProvider { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + match s { + "round-robin" => Ok(StrategyClientProvider::RoundRobin), + "fallback" => Ok(StrategyClientProvider::Fallback), + _ => Err(anyhow::anyhow!( + "Invalid strategy client provider variant: {}", + s + )), + } + } +} + +impl ClientProvider { + pub fn allowed_providers() -> &'static [&'static str] { + &[ + "openai", + "openai-generic", + "azure-openai", + "anthropic", + "ollama", + "round-robin", + "fallback", + "google-ai", + "vertex-ai", + "aws-bedrock", + ] + } +} + +impl std::fmt::Display for ClientSpec { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ClientSpec::Named(n) => write!(f, "{}", n), + ClientSpec::Shorthand(provider, model) => write!(f, "{}/{}", provider, model), + } + } +} + +#[derive(Clone, Debug)] +pub struct SupportedRequestModes { + // If unset, treat as auto + pub stream: Option, +} + +impl SupportedRequestModes { + pub fn required_env_vars(&self) -> HashSet { + HashSet::new() + } +} + +#[derive(Clone, Debug)] +pub enum UnresolvedAllowedRoleMetadata { + Value(StringOr), + All, + None, + Only(HashSet), +} + +#[derive(Clone, Debug)] +pub enum AllowedRoleMetadata { + All, + None, + Only(HashSet), +} + +impl UnresolvedAllowedRoleMetadata { + pub fn required_env_vars(&self) -> HashSet { + match self { + Self::Value(role) => role.required_env_vars(), + Self::Only(roles) => roles + .iter() + .map(|role| role.required_env_vars()) + .flatten() + .collect(), + _ => HashSet::new(), + } + } + + pub fn resolve(&self, ctx: &impl GetEnvVar) -> Result { + match self { + Self::Value(role) => { + let role = role.resolve(ctx)?; + match role.as_str() { + "all" => Ok(AllowedRoleMetadata::All), + "none" => Ok(AllowedRoleMetadata::None), + _ => Err(anyhow::anyhow!("Invalid allowed role metadata: {}. Allowed values are 'all' or 'none' or an array of roles.", role)), + } + } + Self::All => Ok(AllowedRoleMetadata::All), + Self::None => Ok(AllowedRoleMetadata::None), + Self::Only(roles) => Ok(AllowedRoleMetadata::Only( + roles + .iter() + .map(|role| role.resolve(ctx)) + .collect::>>()?, + )), + } + } +} + +impl AllowedRoleMetadata { + pub fn is_allowed(&self, key: &str) -> bool { + match self { + Self::All => true, + Self::None => false, + Self::Only(allowed) => allowed.contains(&key.to_string()), + } + } +} diff --git a/engine/baml-lib/llm-client/src/lib.rs b/engine/baml-lib/llm-client/src/lib.rs new file mode 100644 index 000000000..70a43b16e --- /dev/null +++ b/engine/baml-lib/llm-client/src/lib.rs @@ -0,0 +1,5 @@ +mod clientspec; +mod clients; + +pub use clientspec::*; +pub use clients::*; diff --git a/engine/baml-lib/parser-database/Cargo.toml b/engine/baml-lib/parser-database/Cargo.toml index 54b447258..669d39976 100644 --- a/engine/baml-lib/parser-database/Cargo.toml +++ b/engine/baml-lib/parser-database/Cargo.toml @@ -18,6 +18,7 @@ internal-baml-jinja-types = { path = "../jinja" } internal-baml-diagnostics = { path = "../diagnostics" } internal-baml-schema-ast = { path = "../schema-ast" } internal-baml-prompt-parser = { path = "../prompt-parser" } +internal-llm-client = { path = "../llm-client" } either = "1.6.1" enumflags2 = "0.7" @@ -31,6 +32,7 @@ serde_json.workspace = true regex = "1.10.2" anyhow.workspace = true itertools = "0.13.0" +ouroboros = "*" [features] default = [] diff --git a/engine/baml-lib/parser-database/src/attributes/alias.rs b/engine/baml-lib/parser-database/src/attributes/alias.rs index 87c46bee1..282c3f7fe 100644 --- a/engine/baml-lib/parser-database/src/attributes/alias.rs +++ b/engine/baml-lib/parser-database/src/attributes/alias.rs @@ -1,12 +1,22 @@ +use internal_baml_diagnostics::DatamodelError; + use crate::{coerce, context::Context, types::Attributes}; pub(super) fn visit_alias_attribute(attributes: &mut Attributes, ctx: &mut Context<'_>) { match ctx .visit_default_arg_with_idx("alias") - .map(|(_, value)| coerce::string(value, ctx.diagnostics)) { - Ok(Some(name)) => attributes.add_alias(ctx.interner.intern(name)), + Ok((_, name)) => { + if attributes.alias().is_some() { + ctx.push_attribute_validation_error("cannot be specified more than once", false); + } else if let Some(result) = name.to_unresolved_value(ctx.diagnostics) { + if result.as_str().is_some() { + attributes.add_alias(result); + } else { + ctx.push_error(DatamodelError::new_validation_error("must be a string.", result.meta().clone())); + } + } + } Err(err) => ctx.push_error(err), // not flattened for error handing legacy reasons - Ok(None) => (), }; } diff --git a/engine/baml-lib/parser-database/src/attributes/description.rs b/engine/baml-lib/parser-database/src/attributes/description.rs index 2344aa35e..798fbce35 100644 --- a/engine/baml-lib/parser-database/src/attributes/description.rs +++ b/engine/baml-lib/parser-database/src/attributes/description.rs @@ -1,15 +1,20 @@ -use crate::{context::Context, types::Attributes}; +use internal_baml_diagnostics::DatamodelError; + +use crate::{coerce, context::Context, types::Attributes}; pub(super) fn visit_description_attribute(attributes: &mut Attributes, ctx: &mut Context<'_>) { match ctx .visit_default_arg_with_idx("description") - .map(|(_, value)| value) { - Ok(description) => { + Ok((_, name)) => { if attributes.description().is_some() { - ctx.push_attribute_validation_error("Duplicate description attribute.", true); - } else { - attributes.add_description(description.clone()) + ctx.push_attribute_validation_error("cannot be specified more than once", false); + } else if let Some(result) = name.to_unresolved_value(ctx.diagnostics) { + if result.as_str().is_some() { + attributes.add_description(result); + } else { + ctx.push_error(DatamodelError::new_validation_error("must be a string.", result.meta().clone())); + } } } Err(err) => ctx.push_error(err), // not flattened for error handing legacy reasons diff --git a/engine/baml-lib/parser-database/src/attributes/mod.rs b/engine/baml-lib/parser-database/src/attributes/mod.rs index 7d27531fc..959f09853 100644 --- a/engine/baml-lib/parser-database/src/attributes/mod.rs +++ b/engine/baml-lib/parser-database/src/attributes/mod.rs @@ -1,3 +1,4 @@ +use internal_baml_diagnostics::Span; use internal_baml_schema_ast::ast::{Top, TopId, TypeExpId, TypeExpressionBlock}; mod alias; @@ -6,17 +7,17 @@ mod description; mod to_string_attribute; use crate::interner::StringId; use crate::{context::Context, types::ClassAttributes, types::EnumAttributes}; -use baml_types::Constraint; +use baml_types::{Constraint, UnresolvedValue}; use internal_baml_schema_ast::ast::{Expression, SubType}; /// #[derive(Debug, Default)] pub struct Attributes { /// Description of the node, used in describing the node to the LLM. - pub description: Option, + pub description: Option>, /// Alias for the node used when communicating with the LLM. - pub alias: Option, + pub alias: Option>, /// Whether the node is a dynamic type. pub dynamic_type: Option, @@ -30,22 +31,22 @@ pub struct Attributes { impl Attributes { /// Set a description. - pub fn add_description(&mut self, description: Expression) { + pub fn add_description(&mut self, description: UnresolvedValue) { self.description.replace(description); } /// Get the description. - pub fn description(&self) -> &Option { + pub fn description(&self) -> &Option> { &self.description } /// Set an alias. - pub fn add_alias(&mut self, alias: StringId) { + pub fn add_alias(&mut self, alias: UnresolvedValue) { self.alias.replace(alias); } /// Get the alias. - pub fn alias(&self) -> &Option { + pub fn alias(&self) -> &Option> { &self.alias } diff --git a/engine/baml-lib/parser-database/src/attributes/to_string_attribute.rs b/engine/baml-lib/parser-database/src/attributes/to_string_attribute.rs index 61524d2f5..2c094542b 100644 --- a/engine/baml-lib/parser-database/src/attributes/to_string_attribute.rs +++ b/engine/baml-lib/parser-database/src/attributes/to_string_attribute.rs @@ -8,7 +8,6 @@ use crate::{context::Context, types::Attributes}; use super::alias::visit_alias_attribute; use super::constraint::visit_constraint_attributes; use super::description::visit_description_attribute; - pub(super) fn visit(ctx: &mut Context<'_>, span: &Span, as_block: bool) -> Option { let mut modified = false; diff --git a/engine/baml-lib/parser-database/src/lib.rs b/engine/baml-lib/parser-database/src/lib.rs index bb3998589..68187f75a 100644 --- a/engine/baml-lib/parser-database/src/lib.rs +++ b/engine/baml-lib/parser-database/src/lib.rs @@ -45,7 +45,7 @@ use internal_baml_schema_ast::ast::SchemaAst; pub use tarjan::Tarjan; pub use types::{ Attributes, ContantDelayStrategy, ExponentialBackoffStrategy, PrinterType, PromptAst, - PromptVariable, RetryPolicy, RetryPolicyStrategy, StaticType, + PromptVariable, RetryPolicy, RetryPolicyStrategy, StaticType, ClientProperties }; use self::{context::Context, interner::StringId, types::Types}; diff --git a/engine/baml-lib/parser-database/src/types/configurations.rs b/engine/baml-lib/parser-database/src/types/configurations.rs index 6b09b7b4e..af6f09055 100644 --- a/engine/baml-lib/parser-database/src/types/configurations.rs +++ b/engine/baml-lib/parser-database/src/types/configurations.rs @@ -1,10 +1,11 @@ use baml_types::Constraint; +use baml_types::UnresolvedValue; use internal_baml_diagnostics::{DatamodelError, DatamodelWarning, Span}; use internal_baml_schema_ast::ast::{ Attribute, ValExpId, ValueExprBlock, WithIdentifier, WithName, WithSpan, }; use regex::Regex; -use std::collections::HashSet; +use std::{collections::HashSet, ops::Deref}; use crate::attributes::constraint::attribute_as_constraint; use crate::{coerce, coerce_array, coerce_expression::coerce_map, context::Context}; @@ -72,13 +73,13 @@ pub(crate) fn visit_retry_policy<'db>( } } ("options", Some(val)) => { - match coerce_map(val, &coerce::string_with_span, ctx.diagnostics) { - Some(val) => { - options = Some( - val.iter() - .map(|(k, v)| ((k.0.to_string(), k.1.clone()), (*v).clone())) - .collect::>(), - ); + match val.to_unresolved_value(ctx.diagnostics) { + Some(UnresolvedValue::::Map(kv, _)) => options = Some(kv), + Some(other) => { + ctx.push_error(DatamodelError::new_validation_error( + "`options` must be a map", + other.meta().clone() + )); } None => {} } @@ -251,38 +252,14 @@ pub(crate) fn visit_test_case<'db>( } } } - ("input", Some(val)) => { - if !val.is_map() { - ctx.diagnostics.push_warning(DatamodelWarning::new( - "Direct values are not supported. Please pass in parameters by name".into(), - val.span().clone(), - )); - args = Some((f.span(), Default::default())); - } else { - match coerce_map(val, &coerce::string_with_span, ctx.diagnostics) { - Some(val) => { - let params = val - .iter() - .map(|(k, v)| ((k.0.to_string(), (k.1.clone(), (*v).clone())))) - .collect(); - args = Some((f.span(), params)); - } - None => ctx.push_error(DatamodelError::new_property_not_known_error( - "input", - f.identifier().span().clone(), - ["functions", "args"].to_vec(), - )), - } - } - } ("args", Some(val)) => { - match coerce_map(val, &coerce::string_with_span, ctx.diagnostics) { - Some(val) => { - let params = val - .iter() - .map(|(k, v)| ((k.0.to_string(), (k.1.clone(), (*v).clone())))) - .collect(); - args = Some((f.span(), params)); + match val.to_unresolved_value(ctx.diagnostics) { + Some(UnresolvedValue::::Map(kv, span)) => args = Some((span, kv)), + Some(other) => { + ctx.push_error(DatamodelError::new_validation_error( + "`args` must be a map", + other.meta().clone() + )); } None => {} } diff --git a/engine/baml-lib/parser-database/src/types/mod.rs b/engine/baml-lib/parser-database/src/types/mod.rs index 9cfb09664..3a57d1598 100644 --- a/engine/baml-lib/parser-database/src/types/mod.rs +++ b/engine/baml-lib/parser-database/src/types/mod.rs @@ -1,17 +1,21 @@ +use ouroboros::self_referencing; use std::collections::{HashMap, HashSet}; use std::hash::Hash; +use std::ops::Deref; use crate::coerce; use crate::types::configurations::visit_test_case; use crate::{context::Context, DatamodelError}; use baml_types::Constraint; +use baml_types::{StringOr, UnresolvedValue}; use indexmap::IndexMap; -use internal_baml_diagnostics::Span; +use internal_baml_diagnostics::{Diagnostics, Span}; use internal_baml_prompt_parser::ast::{ChatBlock, PrinterBlock, Variable}; use internal_baml_schema_ast::ast::{ self, Expression, FieldId, RawString, ValExpId, WithIdentifier, WithName, WithSpan, }; +use internal_llm_client::{ClientProvider, PropertyHandler, UnresolvedClientProperty}; mod configurations; mod prompt; @@ -126,18 +130,22 @@ pub enum PromptAst<'a> { Chat(Vec<(Option<&'a ChatBlock>, String)>, Vec<(String, String)>), } -#[derive(Debug, Clone)] +/// The properties of the client. +/// This is highly dangerous, but i did this to only copy the options once. pub struct ClientProperties { - pub provider: (String, Span), + /// The provider for the client, e.g. baml-openai-chat + pub provider: (ClientProvider, Span), + /// The retry policy for the client pub retry_policy: Option<(String, Span)>, - pub options: Vec<(String, Expression)>, + /// The options for the client + pub options: UnresolvedClientProperty, } -#[derive(Debug, Clone)] +#[derive(Debug)] pub struct TestCase { pub functions: Vec<(String, Span)>, // The span is the span of the argument (the expression has its own span) - pub args: IndexMap, + pub args: IndexMap)>, pub args_field_span: Span, pub constraints: Vec<(Constraint, Span, Span)>, } @@ -167,14 +175,14 @@ impl PrinterType { } /// How to retry a request. -#[derive(Debug, Clone)] +#[derive(Debug)] pub struct RetryPolicy { /// The maximum number of retries. pub max_retries: u32, /// The strategy to use. pub strategy: RetryPolicyStrategy, /// Any additional options. - pub options: Option>, + pub options: Option)>>, } #[derive(Debug, Clone, Copy, serde::Serialize)] @@ -220,7 +228,7 @@ pub struct TemplateStringProperties { pub template: String, } -#[derive(Debug, Default)] +#[derive(Default)] pub(super) struct Types { pub(super) enum_attributes: HashMap, pub(super) class_attributes: HashMap, @@ -431,34 +439,61 @@ fn visit_function<'db>(idx: ValExpId, function: &'db ast::ValueExprBlock, ctx: & fn visit_client<'db>(idx: ValExpId, client: &'db ast::ValueExprBlock, ctx: &mut Context<'db>) { let mut provider = None; let mut retry_policy = None; - let mut options: Vec<(String, Expression)> = Vec::new(); + let mut options = None; client .iter_fields() .for_each(|(_idx, field)| match field.name() { - "provider" => provider = field.expr.as_ref(), - "retry_policy" => retry_policy = field.expr.as_ref(), - "options" => { - match field.expr.as_ref() { - Some(ast::Expression::Map(map, span)) => { - map.iter().for_each(|(key, value)| { - if let Some(key) = coerce::string(key, ctx.diagnostics) { - options.push((key.to_string(), value.clone())); - } else { - ctx.push_error(DatamodelError::new_validation_error( - "Expected a string key.", - span.clone(), + "provider" => { + match field + .expr + .as_ref() + .and_then(|e| e.to_unresolved_value(ctx.diagnostics)) + { + Some(e) => match e.as_static_str() { + Ok(s) => match s.parse::() { + Ok(p) => provider = Some((p, e.meta().clone())), + Err(err) => { + ctx.push_error(DatamodelError::not_found_error( + "client provider", + s, + e.meta().clone(), + ClientProvider::allowed_providers() + .iter() + .map(|v| v.to_string()) + .collect(), + false, )); } - }); + }, + Err(err) => ctx.push_error(DatamodelError::new_validation_error( + &format!("`provider` value error: {err}"), + e.meta().clone(), + )), + }, + None => ctx.push_error(DatamodelError::new_validation_error( + "Missing `provider` field in client. e.g. `provider \"openai\"`", + field.span().clone(), + )), + } + } + "retry_policy" => retry_policy = field.expr.as_ref(), + "options" => { + match field + .expr + .as_ref() + .and_then(|e| e.to_unresolved_value(ctx.diagnostics)) + { + Some(UnresolvedValue::Map(kv, _)) => { + options = Some((kv, field.identifier().span().clone())); } - Some(_) => { + Some(v) => { ctx.push_error(DatamodelError::new_validation_error( - "Expected a map.", - field.span().clone(), + &format!("Expected a key-value pair, but got a: {}", v.r#type()), + v.meta().clone(), )); } - _ => {} - }; + None => {} + } } config => ctx.push_error(DatamodelError::new_validation_error( &format!("Unknown field `{}` in client", config), @@ -477,25 +512,37 @@ fn visit_client<'db>(idx: ValExpId, client: &'db ast::ValueExprBlock, ctx: &mut None => None, }; - match (provider, options) { - (Some(provider), options) => { - match (coerce::string_with_span(provider, ctx.diagnostics), options) { - (Some(provider), options) => { + match provider { + Some(provider) => { + let (options_kv, options_span) = match options { + Some((kv, span)) => (kv, span), + None => (Default::default(), client.span().clone()), + }; + + let properties = PropertyHandler::new(options_kv, options_span); + // Parse and cache the result + match provider.0.parse_client_property(properties) { + Ok(options) => { ctx.types.client_properties.insert( idx, ClientProperties { - provider: (provider.0.to_string(), provider.1.clone()), + provider, retry_policy, options, }, ); - } - _ => { - // Errors are handled by coerce. + }, + Err(errors) => { + for error in errors { + ctx.push_error(DatamodelError::new_client_error( + error.message, + error.span, + )); + } } } } - (None, _) => ctx.push_error(DatamodelError::new_validation_error( + None => ctx.push_error(DatamodelError::new_validation_error( "Missing `provider` field in client. e.g. `provider openai`", client.span().clone(), )), diff --git a/engine/baml-lib/parser-database/src/walkers/client.rs b/engine/baml-lib/parser-database/src/walkers/client.rs index c977b8232..14f431223 100644 --- a/engine/baml-lib/parser-database/src/walkers/client.rs +++ b/engine/baml-lib/parser-database/src/walkers/client.rs @@ -1,4 +1,5 @@ use internal_baml_schema_ast::ast::{Expression, WithSpan}; +use internal_llm_client::ClientProvider; use crate::{ ast::{self, WithIdentifier}, @@ -25,67 +26,67 @@ impl<'db> ClientWalker<'db> { } /// The provider for the client, e.g. baml-openai-chat - pub fn provider(self) -> &'db str { - self.properties().provider.0.as_str() + pub fn provider(self) -> &'db ClientProvider { + &self.properties().provider.0 } - /// The model specified for the client, e.g. "gpt-3.5-turbo" - pub fn model(self) -> Option<&'db str> { - let Some((_, model)) = self.properties().options.iter().find(|(k, _)| k == "model") else { - return None; - }; - let Some((model, _)) = model.as_string_value() else { - return None; - }; - Some(model) - } + // The model specified for the client, e.g. "gpt-3.5-turbo" + // pub fn model(self) -> Option<&'db str> { + // let Some((_, model)) = self.properties().options.iter().find(|(k, _)| k == "model") else { + // return None; + // }; + // let Some((model, _)) = model.as_string_value() else { + // return None; + // }; + // Some(model) + // } - /// Returns the list of all non-strategy clients (i.e. flattens fallback/round-robin clients to their constituent clients) - pub fn flat_clients(self) -> Vec> { - // TODO(sam): how are fallback/round-robin clients represented here? - let provider = self.properties().provider.0.as_str(); + // / Returns the list of all non-strategy clients (i.e. flattens fallback/round-robin clients to their constituent clients) + // pub fn flat_clients(self) -> Vec> { + // // TODO(sam): how are fallback/round-robin clients represented here? + // let provider = self.properties().provider.0.as_str(); - if provider == "baml-fallback" || provider == "baml-round-robin" { - let Some((_, strategy)) = self - .properties() - .options - .iter() - .find(|(k, _)| k == "strategy") - else { - return vec![]; - }; - let Expression::Array(strategy, _span) = strategy else { - return vec![]; - }; + // if provider == "baml-fallback" || provider == "baml-round-robin" { + // let Some((_, strategy)) = self + // .properties() + // .options + // .iter() + // .find(|(k, _)| k == "strategy") + // else { + // return vec![]; + // }; + // let Expression::Array(strategy, _span) = strategy else { + // return vec![]; + // }; - let mut clients = vec![]; - for entry in strategy { - if let Some((s, _)) = entry.as_string_value() { - clients.push(s); - } - if let Some((m, _)) = entry.as_map() { - if let Some((_, client_name)) = m - .iter() - .filter(|(k, _)| k.as_string_value().map_or(false, |(s, _)| s == "client")) - .nth(0) - { - if let Some((client_name, _)) = client_name.as_string_value() { - clients.push(client_name); - }; - }; - } - } - let clients = clients - .into_iter() - .filter_map(|client_name| self.db.find_client(client_name)) - .flat_map(|client| client.flat_clients().into_iter()) - .collect::>(); + // let mut clients = vec![]; + // for entry in strategy { + // if let Some((s, _)) = entry.as_string_value() { + // clients.push(s); + // } + // if let Some((m, _)) = entry.as_map() { + // if let Some((_, client_name)) = m + // .iter() + // .filter(|(k, _)| k.as_string_value().map_or(false, |(s, _)| s == "client")) + // .nth(0) + // { + // if let Some((client_name, _)) = client_name.as_string_value() { + // clients.push(client_name); + // }; + // }; + // } + // } + // let clients = clients + // .into_iter() + // .filter_map(|client_name| self.db.find_client(client_name)) + // .flat_map(|client| client.flat_clients().into_iter()) + // .collect::>(); - return clients; - } + // return clients; + // } - vec![self] - } + // vec![self] + // } } // with identifier diff --git a/engine/baml-lib/parser-database/src/walkers/function.rs b/engine/baml-lib/parser-database/src/walkers/function.rs index 1627cfc81..3b5900b78 100644 --- a/engine/baml-lib/parser-database/src/walkers/function.rs +++ b/engine/baml-lib/parser-database/src/walkers/function.rs @@ -1,6 +1,7 @@ use either::Either; use internal_baml_diagnostics::DatamodelError; use internal_baml_schema_ast::ast::{ArgumentId, Identifier, WithIdentifier, WithSpan}; +use internal_llm_client::ClientSpec; use crate::{ ast::{self, WithName}, @@ -120,14 +121,14 @@ impl<'db> FunctionWalker<'db> { } } -/// Reference to a client -pub enum ClientSpec { - /// References a client by name - Named(String), - - /// Defined inline using shorthand "/" syntax - Shorthand(String, String), -} +// impl AstClientSpec { +// pub fn required_env_vars(&self) -> HashSet { +// match self { +// ClientSpec::Named(n) => HashSet::new(), +// ClientSpec::Shorthand(_, _) => HashSet::new(), +// } +// } +// } impl<'db> FunctionWalker<'db> { /// Returns the client spec for the function, if it is well-formed @@ -139,14 +140,11 @@ impl<'db> FunctionWalker<'db> { self.span().clone(), )); }; - - match client.0.split_once("/") { - // TODO: do this in a more robust way - // actually validate which clients are and aren't allowed - Some((provider, model)) => Ok(ClientSpec::Shorthand(provider.to_string(), model.to_string())), - None => match self.db.find_client(client.0.as_str()) { - Some(client) => Ok(ClientSpec::Named(client.name().to_string())), - None => { + match ClientSpec::new_from_id(client.0.as_str()) { + Ok(ClientSpec::Named(name)) => { + if let Some(client) = self.db.find_client(&name) { + Ok(ClientSpec::Named(name)) + } else { let clients = self .db .walk_clients() @@ -157,9 +155,17 @@ impl<'db> FunctionWalker<'db> { &client.0, client.1.clone(), clients.clone(), + false, )) } - }, + } + Ok(spec) => Ok(spec), + Err(e) => { + return Err(DatamodelError::new_validation_error( + &e.to_string(), + client.1.clone(), + )); + } } } } diff --git a/engine/baml-lib/parser-database/src/walkers/mod.rs b/engine/baml-lib/parser-database/src/walkers/mod.rs index 9d1fd2a92..3d03b8ed8 100644 --- a/engine/baml-lib/parser-database/src/walkers/mod.rs +++ b/engine/baml-lib/parser-database/src/walkers/mod.rs @@ -19,7 +19,7 @@ pub use client::*; pub use configuration::*; use either::Either; pub use field::*; -pub use function::{FunctionWalker, ClientSpec}; +pub use function::FunctionWalker; pub use template_string::TemplateStringWalker; use internal_baml_schema_ast::ast::{FieldType, Identifier, TopId, TypeExpId, WithName}; pub use r#class::*; diff --git a/engine/baml-lib/schema-ast/src/ast/expression.rs b/engine/baml-lib/schema-ast/src/ast/expression.rs index e4d327bf7..ec6bb431a 100644 --- a/engine/baml-lib/schema-ast/src/ast/expression.rs +++ b/engine/baml-lib/schema-ast/src/ast/expression.rs @@ -1,4 +1,7 @@ -use baml_types::TypeValue; +use baml_types::{TypeValue, UnresolvedValue as UnresolvedValueBase}; +use internal_baml_diagnostics::Diagnostics; + +type UnresolvedValue = UnresolvedValueBase; use crate::ast::Span; use bstd::dedent; @@ -324,4 +327,58 @@ impl Expression { (Map(_, _), _) => panic!("Types do not match: {:?} and {:?}", self, other), } } + + pub fn to_unresolved_value(&self, diagnostics: &mut internal_baml_diagnostics::Diagnostics,) -> Option { + use baml_types::StringOr; + + match self { + Expression::BoolValue(val, span) => Some(UnresolvedValue::Bool(*val, span.clone())), + Expression::NumericValue(val, span) => Some(UnresolvedValue::Numeric(val.clone(), span.clone())), + Expression::Identifier(identifier) => match identifier { + Identifier::ENV(val, span) => Some(UnresolvedValue::String(StringOr::EnvVar(val.to_string()), span.clone())), + Identifier::Ref(ref_identifier, span) => { + Some(UnresolvedValue::String(StringOr::Value(ref_identifier.full_name.as_str().to_string()), span.clone())) + } + Identifier::Invalid(val, span) + | Identifier::String(val, span) + | Identifier::Local(val, span) => { + match val.as_str() { + "null" => Some(UnresolvedValue::Null(span.clone())), + "true" => Some(UnresolvedValue::Bool(true, span.clone())), + "false" => Some(UnresolvedValue::Bool(false, span.clone())), + _ => Some(UnresolvedValue::String(StringOr::Value(val.to_string()), span.clone())), + } + }, + }, + Expression::StringValue(val, span) => Some(UnresolvedValue::String(StringOr::Value(val.to_string()), span.clone())), + Expression::RawStringValue(raw_string) => { + // Do standard dedenting / trimming. + let val = raw_string.value(); + Some(UnresolvedValue::String(StringOr::Value(val.to_string()), raw_string.span().clone())) + } + Expression::Array(vec, span) => { + let values = vec + .iter() + .filter_map(|e| e.to_unresolved_value(diagnostics)) + .collect::>(); + Some(UnresolvedValue::Array(values, span.clone())) + } + Expression::Map(map, span) => { + let values = map + .iter() + .filter_map(|(k, v)| { + let key = k.to_unresolved_value(diagnostics); + if let Some(UnresolvedValue::String(StringOr::Value(key), key_span)) = key { + if let Some(value) = v.to_unresolved_value(diagnostics) { + return Some((key, (key_span, value))); + } + } + None + }) + .collect::<_>(); + Some(UnresolvedValue::Map(values, span.clone())) + } + Expression::JinjaExpressionValue(jinja_expression, span) => Some(UnresolvedValue::String(StringOr::JinjaExpression(jinja_expression.clone()), span.clone())), + } + } } diff --git a/engine/baml-runtime/Cargo.toml b/engine/baml-runtime/Cargo.toml index 798c8070e..5db9034e6 100644 --- a/engine/baml-runtime/Cargo.toml +++ b/engine/baml-runtime/Cargo.toml @@ -40,6 +40,7 @@ json_comments = "0.2.2" jsonish = { path = "../baml-lib/jsonish" } internal-baml-codegen.workspace = true baml-types = { path = "../baml-lib/baml-types" } +internal-llm-client = { path = "../baml-lib/llm-client" } internal-baml-core = { path = "../baml-lib/baml-core" } internal-baml-jinja = { path = "../baml-lib/jinja-runtime" } log.workspace = true diff --git a/engine/baml-runtime/src/cli/serve/mod.rs b/engine/baml-runtime/src/cli/serve/mod.rs index 0781e27a0..92b63e659 100644 --- a/engine/baml-runtime/src/cli/serve/mod.rs +++ b/engine/baml-runtime/src/cli/serve/mod.rs @@ -56,7 +56,7 @@ pub struct ServeArgs { no_version_check: bool, } -#[derive(Serialize, Deserialize, Clone, Debug)] +#[derive(Deserialize, Clone, Debug)] pub struct BamlOptions { pub client_registry: Option, } diff --git a/engine/baml-runtime/src/client_registry/mod.rs b/engine/baml-runtime/src/client_registry/mod.rs index d65cae1c9..d36374099 100644 --- a/engine/baml-runtime/src/client_registry/mod.rs +++ b/engine/baml-runtime/src/client_registry/mod.rs @@ -1,5 +1,7 @@ // This is designed to build any type of client, not just primitives use anyhow::{Context, Result}; +pub use internal_llm_client::ClientProvider; +use internal_llm_client::{ClientSpec, PropertyHandler, UnresolvedClientProperty}; use std::collections::HashMap; use std::sync::Arc; @@ -16,15 +18,57 @@ pub enum PrimitiveClient { Vertex, } -#[derive(Serialize, Clone, Deserialize, Debug)] +#[derive(Clone, Deserialize, Debug)] pub struct ClientProperty { pub name: String, - pub provider: String, + pub provider: ClientProvider, pub retry_policy: Option, - pub options: BamlMap, + options: BamlMap, } -#[derive(Clone, Serialize, Deserialize, Debug)] +impl ClientProperty { + pub fn new(name: String, provider: ClientProvider, retry_policy: Option, options: BamlMap) -> Self { + Self { + name, + provider, + retry_policy, + options, + } + } + + pub fn from_shorthand(provider: &ClientProvider, model: &str) -> Self { + Self { + name: format!("{}/{}", provider, model), + provider: provider.clone(), + retry_policy: None, + options: vec![("model".to_string(), BamlValue::String(model.to_string()))] + .into_iter() + .collect(), + } + } + + pub fn unresolved_options(&self) -> Result> { + let property = PropertyHandler::new( + self.options + .iter() + .map(|(k, v)| Ok((k.clone(), ((), v.to_resolvable()?)))) + .collect::>()?, + (), + ); + self.provider.parse_client_property(property).map_err(|e| { + anyhow::anyhow!( + "Failed to parse client options for {}:\n{}", + self.name, + e.into_iter() + .map(|e| e.message) + .collect::>() + .join("\n") + ) + }) + } +} + +#[derive(Clone, Deserialize, Debug)] pub struct ClientRegistry { #[serde(deserialize_with = "deserialize_clients")] clients: HashMap, diff --git a/engine/baml-runtime/src/constraints.rs b/engine/baml-runtime/src/constraints.rs index d28a86092..9465417ed 100644 --- a/engine/baml-runtime/src/constraints.rs +++ b/engine/baml-runtime/src/constraints.rs @@ -329,7 +329,7 @@ mod tests { client: "test_client".to_string(), model: "test_model".to_string(), prompt: RenderedPrompt::Completion(String::new()), - request_options: HashMap::new(), + request_options: Default::default(), content: String::new(), start_time: web_time::SystemTime::UNIX_EPOCH, latency: web_time::Duration::from_millis(500), diff --git a/engine/baml-runtime/src/internal/llm_client/llm_provider.rs b/engine/baml-runtime/src/internal/llm_client/llm_provider.rs index 225118cff..3e4289723 100644 --- a/engine/baml-runtime/src/internal/llm_client/llm_provider.rs +++ b/engine/baml-runtime/src/internal/llm_client/llm_provider.rs @@ -44,13 +44,11 @@ impl TryFrom<(&ClientWalker<'_>, &RuntimeContext)> for LLMProvider { type Error = anyhow::Error; fn try_from((client, ctx): (&ClientWalker, &RuntimeContext)) -> Result { - match client.elem().provider.as_str() { - "baml-fallback" | "fallback" | "baml-round-robin" | "round-robin" => { - LLMStrategyProvider::try_from((client, ctx)).map(LLMProvider::Strategy) - } + match &client.elem().provider { + internal_llm_client::ClientProvider::Strategy(_) => LLMStrategyProvider::try_from((client, ctx)).map(LLMProvider::Strategy), _ => LLMPrimitiveProvider::try_from((client, ctx)) .map(Arc::new) - .map(LLMProvider::Primitive), + .map(LLMProvider::Primitive) } } } @@ -59,10 +57,8 @@ impl TryFrom<(&ClientProperty, &RuntimeContext)> for LLMProvider { type Error = anyhow::Error; fn try_from(value: (&ClientProperty, &RuntimeContext)) -> Result { - match value.0.provider.as_str() { - "baml-fallback" | "fallback" | "baml-round-robin" | "round-robin" => { - LLMStrategyProvider::try_from(value).map(LLMProvider::Strategy) - } + match &value.0.provider { + internal_llm_client::ClientProvider::Strategy(_) => LLMStrategyProvider::try_from(value).map(LLMProvider::Strategy), _ => LLMPrimitiveProvider::try_from(value) .map(Arc::new) .map(LLMProvider::Primitive), diff --git a/engine/baml-runtime/src/internal/llm_client/mod.rs b/engine/baml-runtime/src/internal/llm_client/mod.rs index 352d0dbee..4122fcfc5 100644 --- a/engine/baml-runtime/src/internal/llm_client/mod.rs +++ b/engine/baml-runtime/src/internal/llm_client/mod.rs @@ -5,16 +5,16 @@ pub mod llm_provider; pub mod orchestrator; pub mod primitive; -mod properties_hander; pub mod retry_policy; mod strategy; pub mod traits; use anyhow::Result; -use baml_types::{BamlValueWithMeta, JinjaExpression, ResponseCheck}; +use baml_types::{BamlMap, BamlValueWithMeta, JinjaExpression, ResponseCheck}; use internal_baml_core::ir::ClientWalker; use internal_baml_jinja::RenderedPrompt; +use internal_llm_client::AllowedRoleMetadata; use jsonish::BamlValueWithFlags; use serde::{Deserialize, Serialize}; use std::error::Error; @@ -75,34 +75,9 @@ pub struct ModelFeatures { pub chat: bool, pub anthropic_system_constraints: bool, pub resolve_media_urls: ResolveMediaUrls, - pub allowed_metadata: AllowedMetadata, + pub allowed_metadata: AllowedRoleMetadata, } -#[derive(Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum AllowedMetadata { - #[serde(rename = "all")] - All, - #[serde(rename = "none")] - None, - Only(HashSet), -} - -#[derive(Clone, Serialize, Deserialize)] -pub struct SupportedRequestModes { - // If unset, treat as auto - pub stream: Option, -} - -impl AllowedMetadata { - pub fn is_allowed(&self, key: &str) -> bool { - match self { - Self::All => true, - Self::None => false, - Self::Only(allowed) => allowed.contains(&key.to_string()), - } - } -} #[derive(Debug)] pub struct RetryLLMResponse { @@ -180,7 +155,7 @@ pub struct LLMErrorResponse { pub client: String, pub model: Option, pub prompt: RenderedPrompt, - pub request_options: HashMap, + pub request_options: BamlMap, #[cfg_attr(target_arch = "wasm32", serde(skip_serializing))] pub start_time: web_time::SystemTime, pub latency: web_time::Duration, @@ -258,7 +233,7 @@ pub struct LLMCompleteResponse { pub client: String, pub model: String, pub prompt: RenderedPrompt, - pub request_options: HashMap, + pub request_options: BamlMap, pub content: String, #[cfg_attr(target_arch = "wasm32", serde(skip_serializing))] pub start_time: web_time::SystemTime, @@ -373,27 +348,3 @@ impl crate::tracing::Visualize for LLMErrorResponse { s.join("\n") } } - -// For parsing args -fn resolve_properties_walker( - client: &ClientWalker, - ctx: &crate::RuntimeContext, -) -> Result { - use anyhow::Context; - let result = (&client.item.elem.options) - .iter() - .map(|(k, v)| { - Ok(( - k.into(), - ctx.resolve_expression::(v) - .context(format!( - "client {} could not resolve options.{}", - client.name(), - k - ))?, - )) - }) - .collect::>>()?; - - Ok(properties_hander::PropertiesHandler::new(result)) -} diff --git a/engine/baml-runtime/src/internal/llm_client/orchestrator/mod.rs b/engine/baml-runtime/src/internal/llm_client/orchestrator/mod.rs index 57b8f61ce..dc079b7fc 100644 --- a/engine/baml-runtime/src/internal/llm_client/orchestrator/mod.rs +++ b/engine/baml-runtime/src/internal/llm_client/orchestrator/mod.rs @@ -84,7 +84,7 @@ impl OrchestratorNode { } } -#[derive(Debug, Default, Clone, Serialize)] +#[derive(Debug, Default, Clone)] pub struct OrchestrationScope { pub scope: Vec, } diff --git a/engine/baml-runtime/src/internal/llm_client/primitive/anthropic/anthropic_client.rs b/engine/baml-runtime/src/internal/llm_client/primitive/anthropic/anthropic_client.rs index 58523b37b..837470a5a 100644 --- a/engine/baml-runtime/src/internal/llm_client/primitive/anthropic/anthropic_client.rs +++ b/engine/baml-runtime/src/internal/llm_client/primitive/anthropic/anthropic_client.rs @@ -1,18 +1,18 @@ use crate::internal::llm_client::{ - properties_hander::PropertiesHandler, traits::{ToProviderMessage, ToProviderMessageExt, WithClientProperties}, - AllowedMetadata, ResolveMediaUrls, SupportedRequestModes, + ResolveMediaUrls, }; use std::collections::HashMap; use anyhow::{Context, Result}; -use baml_types::{BamlMedia, BamlMediaContent}; +use baml_types::{BamlMap, BamlMedia, BamlMediaContent}; use eventsource_stream::Eventsource; use futures::StreamExt; use internal_baml_core::ir::ClientWalker; use internal_baml_jinja::{ ChatMessagePart, RenderContext_Client, RenderedChatMessage, RenderedPrompt, }; +use internal_llm_client::{anthropic::ResolvedAnthropic, AllowedRoleMetadata, ClientProvider, ResolvedClientProperty, UnresolvedClientProperty}; use crate::{ client_registry::ClientProperty, @@ -36,26 +36,13 @@ use crate::RuntimeContext; use super::types::MessageChunk; -// stores properties required for making a post request to the API -struct PostRequestProperities { - default_role: String, - base_url: String, - api_key: Option, - headers: HashMap, - proxy_url: Option, - allowed_metadata: AllowedMetadata, - // These are passed directly to the Anthropic API. - properties: HashMap, - supported_request_modes: SupportedRequestModes, -} - // represents client that interacts with the Anthropic API pub struct AnthropicClient { pub name: String, retry_policy: Option, context: RenderContext_Client, features: ModelFeatures, - properties: PostRequestProperities, + properties: ResolvedAnthropic, // clients client: reqwest::Client, @@ -64,47 +51,21 @@ pub struct AnthropicClient { // resolves/constructs PostRequestProperties from the client's options and runtime context, fleshing out the needed headers and parameters // basically just reads the client's options and matches them to needed properties or defaults them fn resolve_properties( - mut properties: PropertiesHandler, + provider: &ClientProvider, + properties: &UnresolvedClientProperty<()>, ctx: &RuntimeContext, -) -> Result { - // this is a required field - - let default_role = properties.pull_default_role("system")?; - let base_url = properties - .pull_base_url()? - .unwrap_or_else(|| "https://api.anthropic.com".into()); - let api_key = properties - .pull_api_key()? - .or_else(|| ctx.env.get("ANTHROPIC_API_KEY").map(|s| s.to_string())); - - let allowed_metadata = properties.pull_allowed_role_metadata()?; - let mut headers = properties.pull_headers()?; - headers - .entry("anthropic-version".to_string()) - .or_insert("2023-06-01".to_string()); - - let supported_request_modes = properties.pull_supported_request_modes()?; - - let mut properties = properties.finalize(); - // Anthropic has a very low max_tokens by default, so we increase it to 4096. - properties - .entry("max_tokens".into()) - .or_insert_with(|| 4096.into()); - let properties = properties; - - - Ok(PostRequestProperities { - default_role, - base_url, - api_key, - headers, - allowed_metadata, - properties, - proxy_url: ctx.env.get("BOUNDARY_PROXY_URL").map(|s| s.to_string()), - supported_request_modes, - }) -} +) -> Result { + let properties = properties.resolve(provider, &ctx.eval_ctx(false))?; + + let ResolvedClientProperty::Anthropic(props) = properties else { + anyhow::bail!( + "Invalid client property. Should have been a anthropic property but got: {}", + properties.name() + ); + }; + Ok(props) +} // getters for client info impl WithRetryPolicy for AnthropicClient { fn retry_policy_name(&self) -> Option<&str> { @@ -113,12 +74,9 @@ impl WithRetryPolicy for AnthropicClient { } impl WithClientProperties for AnthropicClient { - fn allowed_metadata(&self) -> &AllowedMetadata { + fn allowed_metadata(&self) -> &AllowedRoleMetadata { &self.properties.allowed_metadata } - fn client_properties(&self) -> &HashMap { - &self.properties.properties - } fn supports_streaming(&self) -> bool { self.properties.supported_request_modes.stream.unwrap_or(true) } @@ -292,13 +250,13 @@ impl WithStreamChat for AnthropicClient { // constructs base client and resolves properties based on context impl AnthropicClient { pub fn dynamic_new(client: &ClientProperty, ctx: &RuntimeContext) -> Result { - let properties = resolve_properties(client.property_handler()?, ctx)?; + let properties = resolve_properties(&client.provider, &client.unresolved_options()?, ctx)?; let default_role = properties.default_role.clone(); Ok(Self { name: client.name.clone(), context: RenderContext_Client { name: client.name.clone(), - provider: client.provider.clone(), + provider: client.provider.to_string(), default_role, }, features: ModelFeatures { @@ -315,14 +273,13 @@ impl AnthropicClient { } pub fn new(client: &ClientWalker, ctx: &RuntimeContext) -> Result { - let properties = super::super::resolve_properties_walker(client, ctx)?; - let properties = resolve_properties(properties, ctx)?; + let properties = resolve_properties(&client.elem().provider, &client.options(), ctx)?; let default_role = properties.default_role.clone(); Ok(Self { name: client.name().into(), context: RenderContext_Client { name: client.name().into(), - provider: client.elem().provider.clone(), + provider: client.elem().provider.to_string(), default_role, }, features: ModelFeatures { @@ -373,9 +330,7 @@ impl RequestBuilder for AnthropicClient { for (key, value) in &self.properties.headers { req = req.header(key, value); } - if let Some(key) = &self.properties.api_key { - req = req.header("x-api-key", key); - } + req = req.header("x-api-key", self.properties.api_key.clone()); if allow_proxy { req = req.header("baml-original-url", self.properties.base_url.as_str()); @@ -398,7 +353,7 @@ impl RequestBuilder for AnthropicClient { Ok(req.json(&body)) } - fn request_options(&self) -> &HashMap { + fn request_options(&self) -> &BamlMap { &self.properties.properties } } diff --git a/engine/baml-runtime/src/internal/llm_client/primitive/aws/aws_client.rs b/engine/baml-runtime/src/internal/llm_client/primitive/aws/aws_client.rs index 02e20b4a2..bb6ffd7d0 100644 --- a/engine/baml-runtime/src/internal/llm_client/primitive/aws/aws_client.rs +++ b/engine/baml-runtime/src/internal/llm_client/primitive/aws/aws_client.rs @@ -9,18 +9,20 @@ use anyhow::{Context, Result}; use aws_smithy_json::serialize::JsonObjectWriter; use aws_smithy_runtime_api::client::result::SdkError; use aws_smithy_types::Blob; -use baml_types::BamlMediaContent; +use baml_types::{BamlMap, BamlMediaContent}; use baml_types::{BamlMedia, BamlMediaType}; use futures::stream; use internal_baml_core::ir::ClientWalker; use internal_baml_jinja::{ChatMessagePart, RenderContext_Client, RenderedChatMessage}; +use internal_llm_client::aws_bedrock::ResolvedAwsBedrock; +use internal_llm_client::{AllowedRoleMetadata, ClientProvider, ResolvedClientProperty, UnresolvedClientProperty}; use serde::Deserialize; use serde_json::Map; use web_time::Instant; use web_time::SystemTime; +use crate::client_registry::ClientProperty; use crate::internal::llm_client::traits::{ToProviderMessageExt, WithClientProperties}; -use crate::internal::llm_client::{AllowedMetadata, SupportedRequestModes}; use crate::internal::llm_client::{ primitive::request::RequestBuilder, traits::{ @@ -33,96 +35,66 @@ use crate::internal::llm_client::{ use crate::{RenderCurlSettings, RuntimeContext}; -// stores properties required for making a post request to the API -struct RequestProperties { - model_id: String, - - - // (region, access_key_id, secret_access_key) - aws_access: (Option, Option, Option), - - default_role: String, - inference_config: Option, - allowed_metadata: AllowedMetadata, - - request_options: HashMap, - ctx_env: HashMap, - supported_request_modes: SupportedRequestModes, -} - // represents client that interacts with the Anthropic API pub struct AwsClient { pub name: String, retry_policy: Option, context: RenderContext_Client, features: ModelFeatures, - properties: RequestProperties, + properties: ResolvedAwsBedrock, } -fn resolve_properties(client: &ClientWalker, ctx: &RuntimeContext) -> Result { - let mut properties = super::super::resolve_properties_walker(client, ctx)?; - - let model_id = { - // We allow `provider aws-bedrock` to specify the model using either `model_id` or `model`: - // - // - the Bedrock API itself only accepts `model_id` - // - but all other providers specify the model using `model`, so for someone used to using - // "openai/gpt-4o", they'll expect to be able to use `model gpt-4o` - // - if I were on the bedrock team, I would be _very_ hesitant to add a new request field - // `model` if I already have `model_id`, so I think using `model` isn't too risky - let maybe_model_id = properties.remove_str("model_id")?; - let maybe_model = properties.remove_str("model")?; - - match (maybe_model_id, maybe_model) { - (Some(model_id), Some(model)) => anyhow::bail!("model_id and model cannot both be provided"), - (Some(model_id), None) => model_id, - (None, Some(model)) => model, - _ => anyhow::bail!("model_id or model is required"), - } - }; - - let default_role = properties.pull_default_role("user")?; - let allowed_metadata = properties.pull_allowed_role_metadata()?; +fn resolve_properties( + provider: &ClientProvider, + properties: &UnresolvedClientProperty<()>, + ctx: &RuntimeContext, +) -> Result { - let inference_config = properties - .remove_serde::("inference_configuration")? - .map(|c| c.into()); - - let aws_region = properties - .remove_str("region") - .unwrap_or_else(|_| ctx.env.get("AWS_REGION").map(|s| s.to_string())); + let properties = properties.resolve(provider, &ctx.eval_ctx(false))?; + let ResolvedClientProperty::AWSBedrock(props) = properties else { + anyhow::bail!("Invalid client property. Should have been a aws-bedrock property but got: {}", properties.name()); + }; - let aws_access_key_id = properties.remove_str("aws_access_key_id") - .unwrap_or_else(|_| ctx.env.get("AWS_ACCESS_KEY_ID").map(|s| s.to_string())); - let aws_secret_access_key = properties.remove_str("aws_secret_access_key") - .unwrap_or_else(|_| ctx.env.get("AWS_SECRET_ACCESS_KEY").map(|s| s.to_string())); + Ok(props) +} - let supported_request_modes = properties.pull_supported_request_modes()?; +impl AwsClient { - let properties = properties.finalize(); + pub fn dynamic_new(client: &ClientProperty, ctx: &RuntimeContext) -> Result { + let properties = resolve_properties(&client.provider, &client.unresolved_options()?, ctx)?; + let default_role = properties.default_role.clone(); - Ok(RequestProperties { - model_id, - aws_access: (aws_region, aws_access_key_id, aws_secret_access_key), - default_role, - inference_config, - allowed_metadata, - request_options: properties, - ctx_env: ctx.env.clone(), - supported_request_modes, - }) -} + Ok(Self { + name: client.name.clone(), + context: RenderContext_Client { + name: client.name.clone(), + provider: client.provider.to_string(), + default_role, + }, + features: ModelFeatures { + chat: true, + completion: false, + anthropic_system_constraints: true, + resolve_media_urls: ResolveMediaUrls::Always, + allowed_metadata: properties.allowed_role_metadata.clone(), + }, + retry_policy: client + .retry_policy + .as_ref() + .map(|s| s.to_string()), + properties, + }) + } -impl AwsClient { pub fn new(client: &ClientWalker, ctx: &RuntimeContext) -> Result { - let post_properties = resolve_properties(client, ctx)?; - let default_role = post_properties.default_role.clone(); // clone before moving + let properties = resolve_properties(&client.elem().provider, &client.options(), ctx)?; + let default_role = properties.default_role.clone(); // clone before moving Ok(Self { name: client.name().into(), context: RenderContext_Client { name: client.name().into(), - provider: client.elem().provider.clone(), + provider: client.elem().provider.to_string(), default_role, }, features: ModelFeatures { @@ -130,85 +102,45 @@ impl AwsClient { completion: false, anthropic_system_constraints: true, resolve_media_urls: ResolveMediaUrls::Always, - allowed_metadata: post_properties.allowed_metadata.clone(), + allowed_metadata: properties.allowed_role_metadata.clone(), }, retry_policy: client .elem() .retry_policy_id .as_ref() .map(|s| s.to_string()), - properties: post_properties, + properties, }) } - pub fn request_options(&self) -> &std::collections::HashMap { - &self.properties.request_options + pub fn request_options(&self) -> &BamlMap { + // TODO:(vbv) - use inference config for this. + static DEFAULT_REQUEST_OPTIONS: std::sync::OnceLock> = std::sync::OnceLock::new(); + DEFAULT_REQUEST_OPTIONS.get_or_init(|| Default::default()) } // TODO: this should be memoized on client construction, but because config loading is async, // we can't do this in AwsClient::new (which is called from LLMPRimitiveProvider::try_from) async fn client_anyhow(&self) -> Result { - let (aws_region, aws_access_key_id, aws_secret_access_key) = &self.properties.aws_access; - - #[cfg(not(target_arch = "wasm32"))] - let loader: ConfigLoader = aws_config::defaults(BehaviorVersion::latest()); - #[cfg(target_arch = "wasm32")] - let loader: ConfigLoader = { - use aws_config::Region; - use aws_credential_types::Credentials; - - let (aws_region, aws_access_key_id, aws_secret_access_key) = match ( - aws_region, - aws_access_key_id, - aws_secret_access_key, - ) { - (Some(aws_region), Some(aws_access_key_id), Some(aws_secret_access_key)) => { - (aws_region, aws_access_key_id, aws_secret_access_key) - } - _ => { - anyhow::bail!( - "AWS_REGION, AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY must be configured for AWS Bedrock for Web Assembly" - ) - } - }; - - let loader = super::wasm::load_aws_config() - .region(Region::new(aws_region.clone())) - .credentials_provider(Credentials::new( - aws_access_key_id.clone(), - aws_secret_access_key.clone(), - None, - None, - "baml-runtime/wasm", - )); - - loader - }; - - let loader = if let Some(aws_region) = aws_region { - loader.region(Region::new(aws_region.clone())) - } else { - loader - }; + let mut loader = super::wasm::load_aws_config(); + #[cfg(not(target_arch = "wasm32"))] + let mut loader = aws_config::defaults(BehaviorVersion::latest()); + if let Some(aws_region) = self.properties.region.as_ref() { + loader = loader.region(Region::new(aws_region.clone())); + } - let loader = match (aws_access_key_id, aws_secret_access_key) { - (Some(aws_access_key_id), Some(aws_secret_access_key)) => { - loader.credentials_provider(Credentials::new( + if let (Some(aws_access_key_id), Some(aws_secret_access_key)) = (self.properties.access_key_id.as_ref(), self.properties.secret_access_key.as_ref()) { + loader = loader.credentials_provider(Credentials::new( aws_access_key_id.clone(), aws_secret_access_key.clone(), None, None, - "baml-runtime/wasm", - )) - } - (Some(_), None) | (None, Some(_)) => { - anyhow::bail!("AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY must be configured or not configured together for AWS Bedrock") - } - _ => loader, - }; + "baml-runtime", + )); + } let config = loader .retry_config(RetryConfig::disabled()) @@ -273,9 +205,14 @@ impl AwsClient { .map(|m| self.role_to_message(m)) .collect::>>()?; + let inference_config = match self.properties.inference_config.as_ref() { + Some(curr) => Some(aws_sdk_bedrockruntime::types::InferenceConfiguration::builder().set_max_tokens(curr.max_tokens).set_temperature(curr.temperature).set_top_p(curr.top_p).set_stop_sequences(curr.stop_sequences.clone()).build()), + None => None, + }; + bedrock::operation::converse::ConverseInput::builder() - .set_inference_config(self.properties.inference_config.clone()) - .set_model_id(Some(self.properties.model_id.clone())) + .set_inference_config(inference_config) + .set_model_id(Some(self.properties.model.clone())) .set_system(system_message) .set_messages(Some(converse_messages)) .build() @@ -329,11 +266,8 @@ impl WithRetryPolicy for AwsClient { } impl WithClientProperties for AwsClient { - fn client_properties(&self) -> &HashMap { - &self.properties.request_options - } - fn allowed_metadata(&self) -> &crate::internal::llm_client::AllowedMetadata { - &self.properties.allowed_metadata + fn allowed_metadata(&self) -> &AllowedRoleMetadata { + &self.properties.allowed_role_metadata } fn supports_streaming(&self) -> bool { self.properties.supported_request_modes.stream.unwrap_or(true) @@ -359,8 +293,9 @@ impl WithStreamChat for AwsClient { chat_messages: &Vec, ) -> StreamResponse { let client = self.context.name.to_string(); - let model = Some(self.properties.model_id.clone()); - let request_options = self.properties.request_options.clone(); + let model = Some(self.properties.model.clone()); + // TODO:(vbv) - use inference config for this. + let request_options = Default::default(); let prompt = internal_baml_jinja::RenderedPrompt::Chat(chat_messages.clone()); let aws_client = match self.client_anyhow().await { @@ -454,7 +389,7 @@ impl WithStreamChat for AwsClient { content: "".to_string(), start_time: system_start, latency: instant_start.elapsed(), - model: self.properties.model_id.clone(), + model: self.properties.model.clone(), request_options, metadata: LLMCompleteResponseMetadata { baml_is_complete: false, @@ -659,8 +594,9 @@ impl WithChat for AwsClient { chat_messages: &Vec, ) -> LLMResponse { let client = self.context.name.to_string(); - let model = Some(self.properties.model_id.clone()); - let request_options = self.properties.request_options.clone(); + let model = Some(self.properties.model.clone()); + // TODO:(vbv) - use inference config for this. + let request_options = Default::default(); let prompt = internal_baml_jinja::RenderedPrompt::Chat(chat_messages.clone()); let aws_client = match self.client_anyhow().await { @@ -729,7 +665,7 @@ impl WithChat for AwsClient { start_time: system_start.clone(), latency: instant_start.elapsed(), request_options, - model: self.properties.model_id.clone(), + model: self.properties.model.clone(), metadata: LLMCompleteResponseMetadata { baml_is_complete: match response.stop_reason { bedrock::types::StopReason::StopSequence diff --git a/engine/baml-runtime/src/internal/llm_client/primitive/google/googleai_client.rs b/engine/baml-runtime/src/internal/llm_client/primitive/google/googleai_client.rs index 1dc5a929f..7f2d78c3d 100644 --- a/engine/baml-runtime/src/internal/llm_client/primitive/google/googleai_client.rs +++ b/engine/baml-runtime/src/internal/llm_client/primitive/google/googleai_client.rs @@ -1,9 +1,8 @@ use crate::client_registry::ClientProperty; -use crate::internal::llm_client::properties_hander::{PropertiesHandler}; use crate::internal::llm_client::traits::{ ToProviderMessage, ToProviderMessageExt, WithClientProperties, }; -use crate::internal::llm_client::{AllowedMetadata, ResolveMediaUrls, SupportedRequestModes}; +use crate::internal::llm_client::ResolveMediaUrls; use crate::RuntimeContext; use crate::{ internal::llm_client::{ @@ -21,25 +20,18 @@ use crate::{ request::create_client, }; use anyhow::{Context, Result}; -use baml_types::{BamlMedia, BamlMediaContent}; +use baml_types::{BamlMap, BamlMedia, BamlMediaContent}; use eventsource_stream::Eventsource; use futures::StreamExt; use http::header; use internal_baml_core::ir::ClientWalker; use internal_baml_jinja::{ChatMessagePart, RenderContext_Client, RenderedChatMessage}; +use internal_llm_client::google_ai::ResolvedGoogleAI; +use internal_llm_client::{ + AllowedRoleMetadata, ClientProvider, ResolvedClientProperty, UnresolvedClientProperty, +}; use serde_json::json; use std::collections::HashMap; -struct PostRequestProperities { - default_role: String, - api_key: Option, - headers: HashMap, - base_url: String, - proxy_url: Option, - model_id: Option, - properties: HashMap, - allowed_metadata: AllowedMetadata, - supported_request_modes: SupportedRequestModes, -} pub struct GoogleAIClient { pub name: String, @@ -47,42 +39,24 @@ pub struct GoogleAIClient { pub retry_policy: Option, pub context: RenderContext_Client, pub features: ModelFeatures, - properties: PostRequestProperities, + properties: ResolvedGoogleAI, } fn resolve_properties( - mut properties: PropertiesHandler, + provider: &ClientProvider, + properties: &UnresolvedClientProperty<()>, ctx: &RuntimeContext, -) -> Result { - let default_role = properties.pull_default_role("user")?; - let api_key = properties - .pull_api_key()? - .or_else(|| ctx.env.get("GOOGLE_API_KEY").map(|s| s.to_string())); - - let model_id = properties - .remove_str("model")? - .unwrap_or_else(|| "gemini-1.5-flash".to_string()); - - let base_url = properties - .pull_base_url()? - .unwrap_or_else(|| "https://generativelanguage.googleapis.com/v1beta".to_string()); - - let allowed_metadata = properties.pull_allowed_role_metadata()?; - let headers = properties.pull_headers()?; - - let supported_request_modes = properties.pull_supported_request_modes()?; - - Ok(PostRequestProperities { - default_role, - api_key, - headers, - base_url, - proxy_url: ctx.env.get("BOUNDARY_PROXY_URL").map(|s| s.to_string()), - model_id: Some(model_id), - properties: properties.finalize(), - allowed_metadata, - supported_request_modes, - }) +) -> Result { + let properties = properties.resolve(provider, &ctx.eval_ctx(false))?; + + let ResolvedClientProperty::GoogleAI(props) = properties else { + anyhow::bail!( + "Invalid client property. Should have been a google-ai property but got: {}", + properties.name() + ); + }; + + Ok(props) } impl WithRetryPolicy for GoogleAIClient { @@ -92,14 +66,14 @@ impl WithRetryPolicy for GoogleAIClient { } impl WithClientProperties for GoogleAIClient { - fn client_properties(&self) -> &HashMap { - &self.properties.properties - } - fn allowed_metadata(&self) -> &crate::internal::llm_client::AllowedMetadata { + fn allowed_metadata(&self) -> &AllowedRoleMetadata { &self.properties.allowed_metadata } fn supports_streaming(&self) -> bool { - self.properties.supported_request_modes.stream.unwrap_or(true) + self.properties + .supported_request_modes + .stream + .unwrap_or(true) } } @@ -125,7 +99,7 @@ impl SseResponseTrait for GoogleAIClient { ) -> StreamResponse { let prompt = prompt.clone(); let client_name = self.context.name.clone(); - let model_id = self.properties.model_id.clone().unwrap_or_default(); + let model_id = self.properties.model.clone(); let params = self.properties.properties.clone(); Ok(Box::pin( resp.bytes_stream() @@ -223,14 +197,13 @@ impl WithStreamChat for GoogleAIClient { impl GoogleAIClient { pub fn new(client: &ClientWalker, ctx: &RuntimeContext) -> Result { - let properties = super::super::resolve_properties_walker(client, ctx)?; - let properties = resolve_properties(properties, ctx)?; + let properties = resolve_properties(&client.elem().provider, &client.options(), ctx)?; let default_role = properties.default_role.clone(); Ok(Self { name: client.name().into(), context: RenderContext_Client { name: client.name().into(), - provider: client.elem().provider.clone(), + provider: client.elem().provider.to_string(), default_role, }, features: ModelFeatures { @@ -251,14 +224,14 @@ impl GoogleAIClient { } pub fn dynamic_new(client: &ClientProperty, ctx: &RuntimeContext) -> Result { - let properties = resolve_properties(client.property_handler()?, ctx)?; + let properties = resolve_properties(&client.provider, &client.unresolved_options()?, ctx)?; let default_role = properties.default_role.clone(); Ok(Self { name: client.name.clone(), context: RenderContext_Client { name: client.name.clone(), - provider: client.provider.clone(), + provider: client.provider.to_string(), default_role, }, features: ModelFeatures { @@ -294,7 +267,7 @@ impl RequestBuilder for GoogleAIClient { let baml_original_url = format!( "{}/models/{}:{}", self.properties.base_url, - self.properties.model_id.as_ref().unwrap_or(&"".to_string()), + self.properties.model.clone(), should_stream ); @@ -310,13 +283,7 @@ impl RequestBuilder for GoogleAIClient { req = req.header(key, value); } - req = req.header( - "x-goog-api-key", - self.properties - .api_key - .clone() - .unwrap_or_else(|| "".to_string()), - ); + req = req.header("x-goog-api-key", self.properties.api_key.clone()); let mut body = json!(self.properties.properties); let body_obj = body.as_object_mut().unwrap(); @@ -332,7 +299,7 @@ impl RequestBuilder for GoogleAIClient { Ok(req.json(&body)) } - fn request_options(&self) -> &HashMap { + fn request_options(&self) -> &BamlMap { &self.properties.properties } } @@ -378,13 +345,7 @@ impl WithChat for GoogleAIClient { start_time: system_now, latency: instant_now.elapsed(), request_options: self.properties.properties.clone(), - model: self - .properties - .properties - .get("model") - .and_then(|v| v.as_str().map(|s| s.to_string())) - .or_else(|| _ctx.env.get("default model").map(|s| s.to_string())) - .unwrap_or_else(|| "".to_string()), + model: self.properties.model.clone(), metadata: LLMCompleteResponseMetadata { baml_is_complete: match response.candidates[0].finish_reason { Some(FinishReason::Stop) => true, diff --git a/engine/baml-runtime/src/internal/llm_client/primitive/mod.rs b/engine/baml-runtime/src/internal/llm_client/primitive/mod.rs index 63c5ad6fd..a49bbe4b1 100644 --- a/engine/baml-runtime/src/internal/llm_client/primitive/mod.rs +++ b/engine/baml-runtime/src/internal/llm_client/primitive/mod.rs @@ -1,8 +1,9 @@ use std::sync::Arc; use anyhow::Result; -use baml_types::BamlValue; +use baml_types::{BamlMap, BamlValue}; use internal_baml_core::ir::{repr::IntermediateRepr, ClientWalker}; +use internal_llm_client::{AllowedRoleMetadata, ClientProvider, OpenAIClientProviderVariant}; use crate::{ client_registry::ClientProperty, internal::prompt_renderer::PromptRenderer, @@ -87,10 +88,7 @@ impl WithRetryPolicy for LLMPrimitiveProvider { } impl WithClientProperties for LLMPrimitiveProvider { - fn client_properties(&self) -> &std::collections::HashMap { - match_llm_provider!(self, client_properties) - } - fn allowed_metadata(&self) -> &super::AllowedMetadata { + fn allowed_metadata(&self) -> &AllowedRoleMetadata { match_llm_provider!(self, allowed_metadata) } fn supports_streaming(&self) -> bool { @@ -102,32 +100,48 @@ impl TryFrom<(&ClientProperty, &RuntimeContext)> for LLMPrimitiveProvider { type Error = anyhow::Error; fn try_from((value, ctx): (&ClientProperty, &RuntimeContext)) -> Result { - match value.provider.as_str() { - "openai" => OpenAIClient::dynamic_new(value, ctx).map(Into::into), - "openai-generic" => OpenAIClient::dynamic_new_generic(value, ctx).map(Into::into), - "azure-openai" => OpenAIClient::dynamic_new_azure(value, ctx).map(Into::into), - "ollama" => OpenAIClient::dynamic_new_ollama(value, ctx).map(Into::into), - "anthropic" => AnthropicClient::dynamic_new(value, ctx).map(Into::into), - "google-ai" => GoogleAIClient::dynamic_new(value, ctx).map(Into::into), - "vertex-ai" => VertexClient::dynamic_new(value, ctx).map(Into::into), - // dynamic_new is not implemented for aws::AwsClient - other => { - let options = [ - "anthropic", - "azure-openai", - "google-ai", - "openai", - "openai-generic", - "vertex-ai", - "fallback", - "round-robin", - ]; - anyhow::bail!( - "Unsupported provider: {}. Available ones are: {}", - other, - options.join(", ") - ) + match &value.provider { + ClientProvider::OpenAI(open_aiclient_provider_variant) => { + match open_aiclient_provider_variant { + OpenAIClientProviderVariant::Base => OpenAIClient::dynamic_new(value, ctx).map(Into::into), + OpenAIClientProviderVariant::Ollama => OpenAIClient::dynamic_new_ollama(value, ctx).map(Into::into), + OpenAIClientProviderVariant::Azure => OpenAIClient::dynamic_new_azure(value, ctx).map(Into::into), + OpenAIClientProviderVariant::Generic => OpenAIClient::dynamic_new_generic(value, ctx).map(Into::into), + } + }, + ClientProvider::Anthropic => AnthropicClient::dynamic_new(value, ctx).map(Into::into), + ClientProvider::AwsBedrock => AwsClient::dynamic_new(value, ctx).map(Into::into), + ClientProvider::GoogleAi => GoogleAIClient::dynamic_new(value, ctx).map(Into::into), + ClientProvider::Vertex => VertexClient::dynamic_new(value, ctx).map(Into::into), + ClientProvider::Strategy(strategy_client_provider) => { + unimplemented!("Strategy client providers are not supported yet in LLMPrimitiveProvider") } + + // "openai" => OpenAIClient::dynamic_new(value, ctx).map(Into::into), + // "openai-generic" => OpenAIClient::dynamic_new_generic(value, ctx).map(Into::into), + // "azure-openai" => OpenAIClient::dynamic_new_azure(value, ctx).map(Into::into), + // "ollama" => OpenAIClient::dynamic_new_ollama(value, ctx).map(Into::into), + // "anthropic" => AnthropicClient::dynamic_new(value, ctx).map(Into::into), + // "google-ai" => GoogleAIClient::dynamic_new(value, ctx).map(Into::into), + // "vertex-ai" => VertexClient::dynamic_new(value, ctx).map(Into::into), + // // dynamic_new is not implemented for aws::AwsClient + // other => { + // let options = [ + // "anthropic", + // "azure-openai", + // "google-ai", + // "openai", + // "openai-generic", + // "vertex-ai", + // "fallback", + // "round-robin", + // ]; + // anyhow::bail!( + // "Unsupported provider: {}. Available ones are: {}", + // other, + // options.join(", ") + // ) + // } } } } @@ -136,36 +150,21 @@ impl TryFrom<(&ClientWalker<'_>, &RuntimeContext)> for LLMPrimitiveProvider { type Error = anyhow::Error; fn try_from((client, ctx): (&ClientWalker, &RuntimeContext)) -> Result { - match client.elem().provider.as_str() { - "baml-openai-chat" | "openai" => OpenAIClient::new(client, ctx).map(Into::into), - "openai-generic" => OpenAIClient::new_generic(client, ctx).map(Into::into), - "baml-azure-chat" | "azure-openai" => { - OpenAIClient::new_azure(client, ctx).map(Into::into) - } - "baml-anthropic-chat" | "anthropic" => { - AnthropicClient::new(client, ctx).map(Into::into) + match &client.elem().provider { + ClientProvider::OpenAI(open_aiclient_provider_variant) => { + match open_aiclient_provider_variant { + OpenAIClientProviderVariant::Base => OpenAIClient::new(client, ctx).map(Into::into), + OpenAIClientProviderVariant::Ollama => OpenAIClient::new_ollama(client, ctx).map(Into::into), + OpenAIClientProviderVariant::Azure => OpenAIClient::new_azure(client, ctx).map(Into::into), + OpenAIClientProviderVariant::Generic => OpenAIClient::new_generic(client, ctx).map(Into::into), + } } - "baml-ollama-chat" | "ollama" => OpenAIClient::new_ollama(client, ctx).map(Into::into), - "google-ai" => GoogleAIClient::new(client, ctx).map(Into::into), - "aws-bedrock" => aws::AwsClient::new(client, ctx).map(Into::into), - "vertex-ai" => VertexClient::new(client, ctx).map(Into::into), - other => { - let options = [ - "anthropic", - "aws-bedrock", - "azure-openai", - "google-ai", - "openai", - "openai-generic", - "vertex-ai", - "fallback", - "round-robin", - ]; - anyhow::bail!( - "Unsupported provider: {}. Available ones are: {}", - other, - options.join(", ") - ) + ClientProvider::Anthropic => AnthropicClient::new(client, ctx).map(Into::into), + ClientProvider::AwsBedrock => AwsClient::new(client, ctx).map(Into::into), + ClientProvider::GoogleAi => GoogleAIClient::new(client, ctx).map(Into::into), + ClientProvider::Vertex => VertexClient::new(client, ctx).map(Into::into), + ClientProvider::Strategy(strategy_client_provider) => { + unimplemented!("Strategy client providers are not supported yet in LLMPrimitiveProvider") } } .into() @@ -247,9 +246,7 @@ impl LLMPrimitiveProvider { &match_llm_provider!(self, context).name } - pub fn request_options(&self) -> &std::collections::HashMap { + pub fn request_options(&self) -> &BamlMap { match_llm_provider!(self, request_options) } } - -use super::resolve_properties_walker; diff --git a/engine/baml-runtime/src/internal/llm_client/primitive/openai/openai_client.rs b/engine/baml-runtime/src/internal/llm_client/primitive/openai/openai_client.rs index fb66fff46..6736413b5 100644 --- a/engine/baml-runtime/src/internal/llm_client/primitive/openai/openai_client.rs +++ b/engine/baml-runtime/src/internal/llm_client/primitive/openai/openai_client.rs @@ -2,16 +2,18 @@ use std::collections::HashMap; use crate::internal::llm_client::ResolveMediaUrls; use anyhow::Result; -use baml_types::{BamlMedia, BamlMediaContent, BamlMediaType}; +use baml_types::{BamlMap, BamlMedia, BamlMediaContent, BamlMediaType}; use internal_baml_core::ir::ClientWalker; use internal_baml_jinja::{ChatMessagePart, RenderContext_Client, RenderedChatMessage}; +use internal_llm_client::openai::ResolvedOpenAI; +use internal_llm_client::AllowedRoleMetadata; use serde_json::json; use crate::internal::llm_client::{ ErrorCode, LLMCompleteResponse, LLMCompleteResponseMetadata, LLMErrorResponse, }; -use super::properties::{self, PostRequestProperties}; +use super::properties; use super::types::{ChatCompletionResponse, ChatCompletionResponseDelta, FinishReason}; use crate::client_registry::ClientProperty; @@ -39,7 +41,7 @@ pub struct OpenAIClient { retry_policy: Option, context: RenderContext_Client, features: ModelFeatures, - properties: PostRequestProperties, + properties: ResolvedOpenAI, // clients client: reqwest::Client, } @@ -51,10 +53,7 @@ impl WithRetryPolicy for OpenAIClient { } impl WithClientProperties for OpenAIClient { - fn client_properties(&self) -> &HashMap { - &self.properties.properties - } - fn allowed_metadata(&self) -> &crate::internal::llm_client::AllowedMetadata { + fn allowed_metadata(&self) -> &AllowedRoleMetadata { &self.properties.allowed_metadata } fn supports_streaming(&self) -> bool { @@ -300,7 +299,7 @@ impl RequestBuilder for OpenAIClient { Ok(req.json(&body)) } - fn request_options(&self) -> &HashMap { + fn request_options(&self) -> &BamlMap { &self.properties.properties } } @@ -427,7 +426,7 @@ macro_rules! make_openai_client { provider: $provider.into(), context: RenderContext_Client { name: $client.name.clone(), - provider: $client.provider.clone(), + provider: $client.provider.to_string(), default_role: $properties.default_role.clone(), }, features: ModelFeatures { @@ -448,7 +447,7 @@ macro_rules! make_openai_client { provider: $provider.into(), context: RenderContext_Client { name: $client.name().into(), - provider: $client.elem().provider.clone(), + provider: $client.elem().provider.to_string(), default_role: $properties.default_role.clone(), }, features: ModelFeatures { @@ -471,31 +470,27 @@ macro_rules! make_openai_client { impl OpenAIClient { pub fn new(client: &ClientWalker, ctx: &RuntimeContext) -> Result { - let properties = super::super::resolve_properties_walker(client, ctx)?; - let properties = properties::openai::resolve_properties(properties, ctx)?; + let properties = properties::resolve_properties(&client.elem().provider, &client.options(), ctx)?; make_openai_client!(client, properties, "openai") } pub fn new_generic(client: &ClientWalker, ctx: &RuntimeContext) -> Result { - let properties = super::super::resolve_properties_walker(client, ctx)?; - let properties = properties::generic::resolve_properties(properties, ctx)?; + let properties = properties::resolve_properties(&client.elem().provider, &client.options(), ctx)?; make_openai_client!(client, properties, "openai-generic") } pub fn new_ollama(client: &ClientWalker, ctx: &RuntimeContext) -> Result { - let properties = super::super::resolve_properties_walker(client, ctx)?; - let properties = properties::ollama::resolve_properties(properties, ctx)?; + let properties = properties::resolve_properties(&client.elem().provider, &client.options(), ctx)?; make_openai_client!(client, properties, "ollama") } pub fn new_azure(client: &ClientWalker, ctx: &RuntimeContext) -> Result { - let properties = super::super::resolve_properties_walker(client, ctx)?; - let properties = properties::azure::resolve_properties(properties, ctx)?; + let properties = properties::resolve_properties(&client.elem().provider, &client.options(), ctx)?; make_openai_client!(client, properties, "azure") } pub fn dynamic_new(client: &ClientProperty, ctx: &RuntimeContext) -> Result { - let properties = properties::openai::resolve_properties(client.property_handler()?, &ctx)?; + let properties = properties::resolve_properties(&client.provider, &client.unresolved_options()?, ctx)?; make_openai_client!(client, properties, "openai", dynamic) } @@ -503,7 +498,7 @@ impl OpenAIClient { client: &ClientProperty, ctx: &RuntimeContext, ) -> Result { - let properties = properties::generic::resolve_properties(client.property_handler()?, ctx)?; + let properties = properties::resolve_properties(&client.provider, &client.unresolved_options()?, ctx)?; make_openai_client!(client, properties, "openai-generic", dynamic) } @@ -511,7 +506,7 @@ impl OpenAIClient { client: &ClientProperty, ctx: &RuntimeContext, ) -> Result { - let properties = properties::ollama::resolve_properties(client.property_handler()?, ctx)?; + let properties = properties::resolve_properties(&client.provider, &client.unresolved_options()?, ctx)?; make_openai_client!(client, properties, "ollama", dynamic) } @@ -519,7 +514,7 @@ impl OpenAIClient { client: &ClientProperty, ctx: &RuntimeContext, ) -> Result { - let properties = properties::azure::resolve_properties(client.property_handler()?, ctx)?; + let properties = properties::resolve_properties(&client.provider, &client.unresolved_options()?, ctx)?; make_openai_client!(client, properties, "azure", dynamic) } } diff --git a/engine/baml-runtime/src/internal/llm_client/primitive/openai/properties/mod.rs b/engine/baml-runtime/src/internal/llm_client/primitive/openai/properties/mod.rs index d23a0363d..b690ca8f6 100644 --- a/engine/baml-runtime/src/internal/llm_client/primitive/openai/properties/mod.rs +++ b/engine/baml-runtime/src/internal/llm_client/primitive/openai/properties/mod.rs @@ -1,20 +1,21 @@ -pub(crate) mod azure; -pub(crate) mod generic; -pub(crate) mod ollama; -pub(crate) mod openai; +use internal_llm_client::{openai::ResolvedOpenAI, ClientProvider, ResolvedClientProperty, UnresolvedClientProperty}; -use crate::internal::llm_client::{AllowedMetadata, SupportedRequestModes}; -use std::collections::HashMap; +use crate::RuntimeContext; -pub struct PostRequestProperties { - pub default_role: String, - pub base_url: String, - pub api_key: Option, - pub headers: HashMap, - pub query_params: HashMap, - pub proxy_url: Option, - // These are passed directly to the OpenAI API. - pub properties: HashMap, - pub allowed_metadata: AllowedMetadata, - pub supported_request_modes: SupportedRequestModes, + +pub fn resolve_properties( + provider: &ClientProvider, + properties: &UnresolvedClientProperty<()>, + ctx: &RuntimeContext, +) -> anyhow::Result { + let properties = properties.resolve(provider, &ctx.eval_ctx(false))?; + + let ResolvedClientProperty::OpenAI(props) = properties else { + anyhow::bail!( + "Invalid client property. Should have been a openai property but got: {}", + properties.name() + ); + }; + + Ok(props) } diff --git a/engine/baml-runtime/src/internal/llm_client/primitive/openai/properties/openai.rs b/engine/baml-runtime/src/internal/llm_client/primitive/openai/properties/openai.rs index cbf5fc49e..a18d70d92 100644 --- a/engine/baml-runtime/src/internal/llm_client/primitive/openai/properties/openai.rs +++ b/engine/baml-runtime/src/internal/llm_client/primitive/openai/properties/openai.rs @@ -1,45 +1,25 @@ use std::collections::HashMap; use anyhow::{Context, Result}; +use internal_llm_client::{openai::ResolvedOpenAI, ClientProvider, ResolvedClientProperty, UnresolvedClientProperty}; -use crate::{ - internal::llm_client::{properties_hander::PropertiesHandler, AllowedMetadata}, - RuntimeContext, -}; +use crate::RuntimeContext; use super::PostRequestProperties; pub fn resolve_properties( - mut properties: PropertiesHandler, + provider: &ClientProvider, + properties: &UnresolvedClientProperty<()>, ctx: &RuntimeContext, -) -> Result { - let default_role = properties.pull_default_role("system")?; - let base_url = properties - .pull_base_url()? - .unwrap_or_else(|| "https://api.openai.com/v1".to_string()); +) -> Result { + let properties = properties.resolve(provider, &ctx.eval_ctx(false))?; - let api_key = properties - .pull_api_key()? - .or_else(|| ctx.env.get("OPENAI_API_KEY").map(|s| s.to_string())); + let ResolvedClientProperty::OpenAI(props) = properties else { + anyhow::bail!( + "Invalid client property. Should have been a openai property but got: {}", + properties.name() + ); + }; - let allowed_metadata = properties.pull_allowed_role_metadata()?; - let headers = properties.pull_headers()?; - - let supported_request_modes = properties.pull_supported_request_modes()?; - - Ok(PostRequestProperties { - default_role, - base_url, - api_key, - headers, - properties: properties.finalize(), - allowed_metadata, - proxy_url: ctx - .env - .get("BOUNDARY_PROXY_URL") - .map(|s| Some(s.to_string())) - .unwrap_or(None), - query_params: Default::default(), - supported_request_modes, - }) + Ok(props) } diff --git a/engine/baml-runtime/src/internal/llm_client/primitive/request.rs b/engine/baml-runtime/src/internal/llm_client/primitive/request.rs index 7188050f8..5fe90c3cb 100644 --- a/engine/baml-runtime/src/internal/llm_client/primitive/request.rs +++ b/engine/baml-runtime/src/internal/llm_client/primitive/request.rs @@ -1,6 +1,7 @@ use std::collections::HashMap; use anyhow::{Context, Result}; +use baml_types::BamlMap; use internal_baml_jinja::RenderedChatMessage; use reqwest::Response; use serde::de::DeserializeOwned; @@ -16,7 +17,7 @@ pub trait RequestBuilder { stream: bool, ) -> Result; - fn request_options(&self) -> &HashMap; + fn request_options(&self) -> &BamlMap; fn http_client(&self) -> &reqwest::Client; } @@ -91,6 +92,17 @@ pub async fn make_request( let status = response.status(); if !status.is_success() { + let url = response.url().to_string(); + let text = response.text().await.map_or_else( + |_| "".to_string(), + |text| { + if text.is_empty() { + "".to_string() + } else { + text + } + }, + ); return Err(LLMResponse::LLMFailure(LLMErrorResponse { client: client.context().name.to_string(), model: None, @@ -98,10 +110,7 @@ pub async fn make_request( start_time: system_now, request_options: client.request_options().clone(), latency: instant_now.elapsed(), - message: format!( - "Request failed: {}", - response.text().await.unwrap_or("".into()) - ), + message: format!("Request failed: {}\n{}", url, text), code: ErrorCode::from_status(status), })); } diff --git a/engine/baml-runtime/src/internal/llm_client/primitive/vertex/types.rs b/engine/baml-runtime/src/internal/llm_client/primitive/vertex/types.rs index 26a230540..af3e11e9f 100644 --- a/engine/baml-runtime/src/internal/llm_client/primitive/vertex/types.rs +++ b/engine/baml-runtime/src/internal/llm_client/primitive/vertex/types.rs @@ -232,7 +232,7 @@ pub enum HarmSeverity { #[serde(rename_all = "camelCase")] pub struct Candidate { pub index: Option, - pub content: Content, + pub content: Option, pub finish_reason: Option, pub safety_ratings: Option>, pub citation_metadata: Option, diff --git a/engine/baml-runtime/src/internal/llm_client/primitive/vertex/vertex_client.rs b/engine/baml-runtime/src/internal/llm_client/primitive/vertex/vertex_client.rs index f0c3d4bd8..be64f13d7 100644 --- a/engine/baml-runtime/src/internal/llm_client/primitive/vertex/vertex_client.rs +++ b/engine/baml-runtime/src/internal/llm_client/primitive/vertex/vertex_client.rs @@ -1,9 +1,8 @@ use crate::client_registry::ClientProperty; -use crate::internal::llm_client::properties_hander::{ PropertiesHandler}; use crate::internal::llm_client::traits::{ ToProviderMessage, ToProviderMessageExt, WithClientProperties, }; -use crate::internal::llm_client::{AllowedMetadata, ResolveMediaUrls, SupportedRequestModes}; +use crate::internal::llm_client::ResolveMediaUrls; #[cfg(target_arch = "wasm32")] use crate::internal::wasm_jwt::{encode_jwt, JwtError}; use crate::RuntimeContext; @@ -25,6 +24,10 @@ use crate::{ use anyhow::{Context, Result}; use chrono::{Duration, Utc}; use futures::StreamExt; +use internal_llm_client::vertex::{ResolvedServiceAccountDetails, ResolvedVertex, ServiceAccount}; +use internal_llm_client::{ + AllowedRoleMetadata, ClientProvider, ResolvedClientProperty, UnresolvedClientProperty, +}; #[cfg(not(target_arch = "wasm32"))] use jsonwebtoken::{encode, Algorithm, EncodingKey, Header}; use serde::{Deserialize, Serialize}; @@ -42,40 +45,13 @@ use internal_baml_jinja::{RenderContext_Client, RenderedChatMessage}; use serde_json::json; use std::collections::HashMap; -enum ServiceAccountDetails { - None, - RawAuthorizationHeader(String), - FilePath(String), - Json(serde_json::Map), -} - -impl Default for ServiceAccountDetails { - fn default() -> Self { - ServiceAccountDetails::None - } -} - -struct PostRequestProperties { - default_role: String, - base_url: Option, - service_account_details: ServiceAccountDetails, - headers: HashMap, - proxy_url: Option, - properties: HashMap, - project_id: Option, - model_id: Option, - location: Option, - allowed_metadata: AllowedMetadata, - supported_request_modes: SupportedRequestModes, -} - pub struct VertexClient { pub name: String, pub client: reqwest::Client, pub retry_policy: Option, pub context: RenderContext_Client, pub features: ModelFeatures, - properties: PostRequestProperties, + properties: ResolvedVertex, } #[derive(Debug, Serialize, Deserialize)] @@ -87,100 +63,38 @@ struct Claims { iat: i64, } -#[derive(Debug, Deserialize)] -struct ServiceAccount { - client_email: String, - private_key: String, +// This is currently hardcoded, but we could make it a property if we wanted +// https://developers.google.com/identity/protocols/oauth2/scopes +const DEFAULT_SCOPE: &str = "https://www.googleapis.com/auth/cloud-platform"; + +impl Claims { + fn from_service_account(service_account: &ServiceAccount) -> Claims { + let now = Utc::now(); + Claims { + iss: service_account.client_email.clone(), + scope: DEFAULT_SCOPE.to_string(), + aud: service_account.token_uri.clone(), + exp: (now + Duration::hours(1)).timestamp(), + iat: now.timestamp(), + } + } } fn resolve_properties( - mut properties: PropertiesHandler, + provider: &ClientProvider, + properties: &UnresolvedClientProperty<()>, ctx: &RuntimeContext, -) -> Result { - let default_role = properties.pull_default_role("user")?; - - let base_url = properties.pull_base_url()?; - let allowed_metadata = properties.pull_allowed_role_metadata()?; - - let service_account_details = { - let authz = properties.remove_str("authorization")?; - let creds = properties.remove_serde::("credentials")?; - let creds_content = properties.remove_str("credentials_content")?; - - // Ensure that at most one of authz, creds, and creds_content is provided - if [authz.is_some(), creds.is_some(), creds_content.is_some()] - .iter() - .filter(|&&b| b) - .count() - > 1 - { - anyhow::bail!( - "Only one of authorization, credentials, and credentials_content can be provided" - ); - } +) -> Result { + let properties = properties.resolve(provider, &ctx.eval_ctx(false))?; - if let Some(authz) = authz { - ServiceAccountDetails::RawAuthorizationHeader(authz) - } else if let Some(creds) = creds { - match creds { - serde_json::Value::String(s) => match serde_json::from_str(&s) { - Ok(service_account) => ServiceAccountDetails::Json(service_account), - Err(_) => ServiceAccountDetails::FilePath(s), - }, - serde_json::Value::Object(o) => ServiceAccountDetails::Json(o), - _ => anyhow::bail!("credentials must be a string or JSON object"), - } - } else if let Some(creds_content) = creds_content { - ServiceAccountDetails::Json( - serde_json::from_str(&creds_content) - .context("Failed to parse credentials_content as a JSON object")?, - ) - } else if let Some(path) = ctx.env.get("GOOGLE_APPLICATION_CREDENTIALS") { - ServiceAccountDetails::FilePath(path.to_string()) - } else if let Some(creds_content) = ctx.env.get("GOOGLE_APPLICATION_CREDENTIALS_CONTENT") { - ServiceAccountDetails::Json( - serde_json::from_str(&creds_content) - .context("Failed to parse credentials_content as a JSON object")?, - ) - } else { - ServiceAccountDetails::None - } - }; - let headers = properties.pull_headers()?; - - let project_id = properties.remove_str("project_id")?; - let model_id = properties.remove_str("model")?; - let location = properties.remove_str("location")?; - - // Ensure that project_id, model_id, and location are provided - let project_id = match project_id { - Some(project_id) => project_id, - None => anyhow::bail!("project_id must be provided"), - }; - let model_id = match model_id { - Some(model_id) => model_id, - None => anyhow::bail!("model must be provided"), - }; - let location = match location { - Some(location) => location, - None => anyhow::bail!("location must be provided"), + let ResolvedClientProperty::Vertex(props) = properties else { + anyhow::bail!( + "Invalid client property. Should have been a vertex property but got: {}", + properties.name() + ); }; - let supported_request_modes = properties.pull_supported_request_modes()?; - - Ok(PostRequestProperties { - default_role, - base_url, - service_account_details, - headers, - properties: properties.finalize(), - project_id: Some(project_id), - model_id: Some(model_id), - location: Some(location), - proxy_url: ctx.env.get("BOUNDARY_PROXY_URL").map(|s| s.to_string()), - allowed_metadata, - supported_request_modes, - }) + Ok(props) } impl WithRetryPolicy for VertexClient { @@ -190,14 +104,14 @@ impl WithRetryPolicy for VertexClient { } impl WithClientProperties for VertexClient { - fn client_properties(&self) -> &HashMap { - &self.properties.properties - } - fn allowed_metadata(&self) -> &crate::internal::llm_client::AllowedMetadata { + fn allowed_metadata(&self) -> &AllowedRoleMetadata { &self.properties.allowed_metadata } fn supports_streaming(&self) -> bool { - self.properties.supported_request_modes.stream.unwrap_or(true) + self.properties + .supported_request_modes + .stream + .unwrap_or(true) } } @@ -223,7 +137,7 @@ impl SseResponseTrait for VertexClient { ) -> StreamResponse { let prompt = prompt.clone(); let client_name = self.context.name.clone(); - let model_id = self.properties.model_id.clone().unwrap_or_default(); + let model_id = self.properties.model.clone(); let params = self.properties.properties.clone(); Ok(Box::pin( resp.bytes_stream() @@ -280,10 +194,13 @@ impl SseResponseTrait for VertexClient { ))); } }; - if let Some(choice) = event.candidates.get(0) { - if let Some(content) = choice.content.parts.get(0) { - inner.content += &content.text; + if let Some(content) = choice + .content + .as_ref() + .and_then(|c| c.parts.get(0).map(|p| p.text.as_ref())) + { + inner.content += content; } match choice.finish_reason.as_ref() { Some(FinishReason::Stop) => { @@ -294,6 +211,7 @@ impl SseResponseTrait for VertexClient { _ => (), } } + inner.latency = instant_start.elapsed(); std::future::ready(Some(LLMResponse::Success(inner.clone()))) @@ -321,14 +239,13 @@ impl WithStreamChat for VertexClient { impl VertexClient { pub fn new(client: &ClientWalker, ctx: &RuntimeContext) -> Result { - let properties = super::super::resolve_properties_walker(client, ctx)?; - let properties = resolve_properties(properties, ctx)?; + let properties = resolve_properties(&client.elem().provider, client.options(), ctx)?; let default_role = properties.default_role.clone(); Ok(Self { name: client.name().into(), context: RenderContext_Client { name: client.name().into(), - provider: client.elem().provider.clone(), + provider: client.elem().provider.to_string(), default_role, }, features: ModelFeatures { @@ -349,14 +266,14 @@ impl VertexClient { } pub fn dynamic_new(client: &ClientProperty, ctx: &RuntimeContext) -> Result { - let properties = resolve_properties(client.property_handler()?, ctx)?; + let properties = resolve_properties(&client.provider, &client.unresolved_options()?, ctx)?; let default_role = properties.default_role.clone(); Ok(Self { name: client.name.clone(), context: RenderContext_Client { name: client.name.clone(), - provider: client.provider.clone(), + provider: client.provider.to_string(), default_role, }, features: ModelFeatures { @@ -374,16 +291,8 @@ impl VertexClient { } async fn get_access_token(service_account: &ServiceAccount) -> Result { - let now = Utc::now(); - let claims = Claims { - iss: service_account.client_email.clone(), - scope: "https://www.googleapis.com/auth/cloud-platform".to_string(), - aud: "https://oauth2.googleapis.com/token".to_string(), - exp: (now + Duration::hours(1)).timestamp(), - iat: now.timestamp(), - }; - // Create the JWT + let claims = Claims::from_service_account(service_account); #[cfg(not(target_arch = "wasm32"))] let jwt = encode( @@ -404,7 +313,7 @@ async fn get_access_token(service_account: &ServiceAccount) -> Result { ("assertion", &jwt), ]; let res: Value = client - .post("https://oauth2.googleapis.com/token") + .post(&service_account.token_uri) .form(¶ms) .send() .await? @@ -439,41 +348,10 @@ impl RequestBuilder for VertexClient { should_stream = "streamGenerateContent?alt=sse"; } - let location = self - .properties - .location - .clone() - .unwrap_or_else(|| "".to_string()); - let project_id = self - .properties - .project_id - .clone() - .unwrap_or_else(|| "".to_string()); - - let model_id = self - .properties - .model_id - .clone() - .unwrap_or_else(|| "".to_string()); - - let base_url = self - .properties - .base_url - .clone() - .unwrap_or_else(|| "".to_string()); - - let baml_original_url = if base_url != "" { - format!("{}{}:{}", base_url, model_id, should_stream) - } else { - format!( - "https://{}-aiplatform.googleapis.com/v1/projects/{}/locations/{}/publishers/google/models/{}:{}", - location, - project_id, - location, - model_id, - should_stream - ) - }; + let base_url = self.properties.base_url.clone(); + let model = self.properties.model.clone(); + let baml_original_url = format!("{}/{}:{}", base_url, model, should_stream); + let mut req = match (&self.properties.proxy_url, allow_proxy) { (Some(proxy_url), true) => { let req = self.client.post(proxy_url.clone()); @@ -482,32 +360,11 @@ impl RequestBuilder for VertexClient { _ => self.client.post(baml_original_url), }; - let access_token = match &self.properties.service_account_details { - ServiceAccountDetails::None => { - anyhow::bail!("No service account was specified."); - } - ServiceAccountDetails::RawAuthorizationHeader(token) => token.to_string(), - ServiceAccountDetails::FilePath(path) => { - #[cfg(not(target_arch = "wasm32"))] - { - let file = File::open(path)?; - let reader = BufReader::new(file); - let service_account: ServiceAccount = serde_json::from_reader(reader)?; - - get_access_token(&service_account).await? - } - #[cfg(target_arch = "wasm32")] - { - anyhow::bail!( - "Reading from files not supported in BAML playground. Pass in your credentials file as a string to the 'GOOGLE_APPLICATION_CREDENTIALS_CONTENT' environment variable." - ); - } - } - ServiceAccountDetails::Json(token) => { - let service_account: ServiceAccount = - serde_json::from_value(serde_json::Value::Object(token.clone()))?; - get_access_token(&service_account).await? - } + let access_token = match &self.properties.authorization { + ResolvedServiceAccountDetails::RawAuthorizationHeader(token) => token.to_string(), + ResolvedServiceAccountDetails::Json(token) => get_access_token(&token) + .await + .context("Failed to get access token")?, }; req = req.header("Authorization", format!("Bearer {}", access_token)); @@ -526,9 +383,12 @@ impl RequestBuilder for VertexClient { either::Either::Right(messages) => body_obj.extend(self.chat_to_message(messages)?), } - Ok(req.json(&body)) + let req = req.json(&body); + + Ok(req) } - fn request_options(&self) -> &HashMap { + + fn request_options(&self) -> &indexmap::IndexMap { &self.properties.properties } } @@ -566,12 +426,32 @@ impl WithChat for VertexClient { code: ErrorCode::Other(200), }); } + + let content = if let Some(content) = response.candidates.get(0).and_then(|c| { + c.content + .as_ref() + .and_then(|c| c.parts.get(0).map(|p| p.text.clone())) + }) { + content + } else { + return LLMResponse::LLMFailure(LLMErrorResponse { + client: self.context.name.to_string(), + model: None, + prompt: internal_baml_jinja::RenderedPrompt::Chat(prompt.clone()), + start_time: system_now, + request_options: self.properties.properties.clone(), + latency: instant_now.elapsed(), + message: "No content".to_string(), + code: ErrorCode::Other(200), + }); + }; + let usage_metadata = response.usage_metadata.clone().unwrap(); LLMResponse::Success(LLMCompleteResponse { client: self.context.name.to_string(), prompt: internal_baml_jinja::RenderedPrompt::Chat(prompt.clone()), - content: response.candidates[0].content.parts[0].text.clone(), + content, start_time: system_now, latency: instant_now.elapsed(), request_options: self.properties.properties.clone(), diff --git a/engine/baml-runtime/src/internal/llm_client/properties_hander.rs b/engine/baml-runtime/src/internal/llm_client/properties_hander.rs deleted file mode 100644 index 76e5ad3ae..000000000 --- a/engine/baml-runtime/src/internal/llm_client/properties_hander.rs +++ /dev/null @@ -1,165 +0,0 @@ -use anyhow::{Context, Result}; -use std::collections::HashMap; - -use super::{AllowedMetadata, SupportedRequestModes}; - -pub(super) struct PropertiesHandler { - properties: HashMap, -} - -impl PropertiesHandler { - pub fn new(properties: HashMap) -> Self { - Self { properties } - } - - pub fn finalize(self) -> HashMap { - self.properties - } - - fn get(&mut self, key: &str) -> Option { - self.properties.remove(key) - } - - fn remove(&mut self, key: &str) -> Option { - // Ban certain keys - match key { - "allowed_role_metadata" - | "supports_streaming" - | "base_url" - | "api_key" - | "headers" - | "default_role" => { - unreachable!("{} is a reserved key in options", key) - } - _ => self.properties.remove(key), - } - } - - pub fn remove_serde(&mut self, key: &str) -> Result> { - match self.remove(key) { - Some(value) => Ok(Some( - serde_json::from_value(value).context(format!("Failed to parse: {key}"))?, - )), - None => Ok(None), - } - } - - pub fn remove_str(&mut self, key: &str) -> Result> { - match self.remove(key) { - Some(value) => match value.as_str() { - Some(s) => Ok(Some(s.to_string())), - None => anyhow::bail!("{} must be a string", key), - }, - None => Ok(None), - } - } - - pub fn pull_headers(&mut self) -> Result> { - let headers = self.get("headers").map(|v| { - if let Some(v) = v.as_object() { - v.iter() - .map(|(k, v)| { - Ok(( - k.to_string(), - match v { - serde_json::Value::String(s) => s.to_string(), - _ => anyhow::bail!("Header '{k}' must be a string"), - }, - )) - }) - .collect::>>() - } else { - Ok(Default::default()) - } - }); - let headers = match headers { - Some(h) => h?, - None => Default::default(), - }; - - Ok(headers) - } - - pub fn pull_allowed_role_metadata(&mut self) -> Result { - let allowed_metadata = match self.get("allowed_role_metadata") { - Some(allowed_metadata) => serde_json::from_value(allowed_metadata).context( - "allowed_role_metadata must be an array of keys. For example: ['key1', 'key2']", - )?, - None => AllowedMetadata::None, - }; - - Ok(allowed_metadata) - } - - pub fn pull_base_url(&mut self) -> Result> { - self.get("base_url").map_or(Ok(None), |v| { - match v - .as_str() - .map(|s| Some(s)) - .ok_or_else(|| anyhow::anyhow!("base_url must be a string"))? - { - Some(s) if s.is_empty() => { - anyhow::bail!("base_url must be a non-empty string") - } - Some(s) => Ok(Some(s.to_string())), - None => Ok(None), - } - }) - } - - pub fn pull_default_role(&mut self, default: &str) -> Result { - let default_role = self.get("default_role").map(|v| { - v.as_str() - .map(|s| s.to_string()) - .ok_or_else(|| anyhow::anyhow!("default_role must be a string")) - }); - match default_role { - Some(Ok(role)) => Ok(role), - Some(Err(e)) => Err(e), - None => Ok(default.to_string()), - } - } - - pub fn pull_api_key(&mut self) -> Result> { - let api_key = self.get("api_key").map(|v| { - v.as_str() - .map(|s| s.to_string()) - .ok_or_else(|| anyhow::anyhow!("api_key must be a string")) - }); - match api_key { - Some(Ok(key)) => Ok(Some(key)), - Some(Err(e)) => Err(e), - None => Ok(None), - } - } - - pub fn pull_supported_request_modes(&mut self) -> Result { - let supports_streaming = match self.get("supports_streaming") { - Some(v) => match v { - serde_json::Value::Bool(s) => Some(s), - _ => { - return Err(anyhow::anyhow!( - "supports_streaming must be a bool: Got {:?}", - v - )) - } - }, - None => None, - }; - - Ok(SupportedRequestModes { - stream: supports_streaming, - }) - } -} - -impl crate::client_registry::ClientProperty { - pub(super) fn property_handler(&self) -> Result { - Ok(PropertiesHandler::new( - self.options - .iter() - .map(|(k, v)| Ok((k.clone(), serde_json::json!(v)))) - .collect::>>()?, - )) - } -} diff --git a/engine/baml-runtime/src/internal/llm_client/strategy/fallback.rs b/engine/baml-runtime/src/internal/llm_client/strategy/fallback.rs index 25c5756d5..4b9a952e9 100644 --- a/engine/baml-runtime/src/internal/llm_client/strategy/fallback.rs +++ b/engine/baml-runtime/src/internal/llm_client/strategy/fallback.rs @@ -2,14 +2,12 @@ use std::collections::HashMap; use anyhow::{Context, Result}; -use internal_baml_core::ir::{repr::ClientSpec, ClientWalker}; +use internal_baml_core::ir::ClientWalker; +use internal_llm_client::{ClientProvider, ClientSpec, ResolvedClientProperty, UnresolvedClientProperty}; use crate::{ client_registry::ClientProperty, - internal::llm_client::{ - orchestrator::{ExecutionScope, IterOrchestrator, OrchestrationScope, OrchestrationState}, - properties_hander::PropertiesHandler, - }, + internal::llm_client::orchestrator::{ExecutionScope, IterOrchestrator, OrchestrationScope, OrchestrationState}, runtime_interface::InternalClientLookup, RuntimeContext, }; @@ -22,34 +20,18 @@ pub struct FallbackStrategy { } fn resolve_strategy( - mut properties: PropertiesHandler, - _ctx: &RuntimeContext, + provider: &ClientProvider, + properties: &UnresolvedClientProperty<()>, + ctx: &RuntimeContext, ) -> Result> { - let strategy = properties - .remove_serde::>("strategy") - .context("Failed to resolve strategy into string[]")?; - - let strategy = if let Some(strategy) = strategy { - if strategy.is_empty() { - anyhow::bail!("Empty strategy array, at least one client is required"); - } - strategy - } else { - anyhow::bail!("Missing a strategy field"); - }; - - let properties = properties.finalize(); - if !properties.is_empty() { - let supported_keys = ["strategy"]; - let unknown_keys = properties.keys().map(String::from).collect::>(); + let properties = properties.resolve(provider, &ctx.eval_ctx(false))?; + let ResolvedClientProperty::Fallback(props) = properties else { anyhow::bail!( - "Unknown keys: {}. Supported keys are: {}", - unknown_keys.join(", "), - supported_keys.join(", ") + "Invalid client property. Should have been a fallback property but got: {}", + properties.name() ); - } - - Ok(strategy.into_iter().map(ClientSpec::new_from_id).collect()) + }; + Ok(props.strategy) } impl TryFrom<(&ClientProperty, &RuntimeContext)> for FallbackStrategy { @@ -58,7 +40,7 @@ impl TryFrom<(&ClientProperty, &RuntimeContext)> for FallbackStrategy { fn try_from( (client, ctx): (&ClientProperty, &RuntimeContext), ) -> std::result::Result { - let strategy = resolve_strategy(client.property_handler()?, ctx)?; + let strategy = resolve_strategy(&client.provider, &client.unresolved_options()?, ctx)?; Ok(Self { name: client.name.clone(), retry_policy: client.retry_policy.clone(), @@ -71,8 +53,7 @@ impl TryFrom<(&ClientWalker<'_>, &RuntimeContext)> for FallbackStrategy { type Error = anyhow::Error; fn try_from((client, ctx): (&ClientWalker, &RuntimeContext)) -> Result { - let properties = super::super::resolve_properties_walker(client, ctx)?; - let strategy = resolve_strategy(properties, ctx)?; + let strategy = resolve_strategy(&client.elem().provider, client.options(), ctx)?; Ok(Self { name: client.item.elem.name.clone(), retry_policy: client.retry_policy().as_ref().map(String::from), diff --git a/engine/baml-runtime/src/internal/llm_client/strategy/mod.rs b/engine/baml-runtime/src/internal/llm_client/strategy/mod.rs index 12910d99a..79c54c804 100644 --- a/engine/baml-runtime/src/internal/llm_client/strategy/mod.rs +++ b/engine/baml-runtime/src/internal/llm_client/strategy/mod.rs @@ -5,6 +5,7 @@ mod fallback; pub mod roundrobin; use internal_baml_core::ir::ClientWalker; +use internal_llm_client::{ClientProvider, StrategyClientProvider}; use crate::{ client_registry::ClientProperty, runtime_interface::InternalClientLookup, RuntimeContext, @@ -41,19 +42,23 @@ impl TryFrom<(&ClientWalker<'_>, &RuntimeContext)> for LLMStrategyProvider { type Error = anyhow::Error; fn try_from((client, ctx): (&ClientWalker, &RuntimeContext)) -> Result { - match client.elem().provider.as_str() { - "baml-round-robin" | "round-robin" => RoundRobinStrategy::try_from((client, ctx)) - .map(Arc::new) - .map(LLMStrategyProvider::RoundRobin), - "baml-fallback" | "fallback" => { - FallbackStrategy::try_from((client, ctx)).map(LLMStrategyProvider::Fallback) + match &client.elem().provider { + ClientProvider::Strategy(strategy_client_provider) => { + match strategy_client_provider { + StrategyClientProvider::RoundRobin => { + RoundRobinStrategy::try_from((client, ctx)) + .map(Arc::new) + .map(LLMStrategyProvider::RoundRobin) + } + StrategyClientProvider::Fallback => { + FallbackStrategy::try_from((client, ctx)).map(LLMStrategyProvider::Fallback) + } + } } - other => { - let options = ["round-robin", "fallback"]; - anyhow::bail!( - "Unsupported strategy provider: {}. Available ones are: {}", - other, - options.join(", ") + _ => { + anyhow::bail!( + "Unsupported strategy provider: {}", + client.elem().provider, ) } } @@ -64,12 +69,16 @@ impl TryFrom<(&ClientProperty, &RuntimeContext)> for LLMStrategyProvider { type Error = anyhow::Error; fn try_from((client, ctx): (&ClientProperty, &RuntimeContext)) -> Result { - match client.provider.as_str() { - "baml-round-robin" | "round-robin" => RoundRobinStrategy::try_from((client, ctx)) - .map(Arc::new) - .map(LLMStrategyProvider::RoundRobin), - "baml-fallback" | "fallback" => { - FallbackStrategy::try_from((client, ctx)).map(LLMStrategyProvider::Fallback) + match &client.provider { + ClientProvider::Strategy(strategy) => match strategy { + StrategyClientProvider::RoundRobin => { + RoundRobinStrategy::try_from((client, ctx)) + .map(Arc::new) + .map(LLMStrategyProvider::RoundRobin) + } + StrategyClientProvider::Fallback => { + FallbackStrategy::try_from((client, ctx)).map(LLMStrategyProvider::Fallback) + } } other => { let options = ["round-robin", "fallback"]; diff --git a/engine/baml-runtime/src/internal/llm_client/strategy/roundrobin.rs b/engine/baml-runtime/src/internal/llm_client/strategy/roundrobin.rs index 2af6eac7c..1e6dd3c60 100644 --- a/engine/baml-runtime/src/internal/llm_client/strategy/roundrobin.rs +++ b/engine/baml-runtime/src/internal/llm_client/strategy/roundrobin.rs @@ -7,16 +7,16 @@ use std::{ }, }; -use internal_baml_core::ir::{repr::ClientSpec, ClientWalker}; +use internal_baml_core::ir::ClientWalker; +use internal_llm_client::{ + ClientProvider, ClientSpec, ResolvedClientProperty, UnresolvedClientProperty, +}; use crate::{ client_registry::ClientProperty, - internal::llm_client::{ - orchestrator::{ - ExecutionScope, IterOrchestrator, OrchestrationScope, OrchestrationState, - OrchestratorNodeIterator, - }, - properties_hander::PropertiesHandler, + internal::llm_client::orchestrator::{ + ExecutionScope, IterOrchestrator, OrchestrationScope, OrchestrationState, + OrchestratorNodeIterator, }, runtime_interface::InternalClientLookup, RuntimeContext, @@ -24,13 +24,12 @@ use crate::{ use serde::Serialize; use serde::Serializer; -#[derive(Serialize, Debug)] +#[derive(Debug, Serialize)] pub struct RoundRobinStrategy { pub name: String, pub(super) retry_policy: Option, // TODO: We can add conditions to each client client_specs: Vec, - #[serde(serialize_with = "serialize_atomic")] current_index: AtomicUsize, } @@ -54,58 +53,30 @@ impl RoundRobinStrategy { } fn resolve_strategy( - mut properties: PropertiesHandler, - _ctx: &RuntimeContext, + provider: &ClientProvider, + properties: &UnresolvedClientProperty<()>, + ctx: &RuntimeContext, ) -> Result<(Vec, usize)> { - let strategy = properties - .remove_serde::>("strategy") - .context("Failed to parse strategy into string[]")?; - - let strategy = if let Some(strategy) = strategy { - if strategy.is_empty() { - anyhow::bail!("Empty strategy array, at least one client is required"); - } - strategy - } else { - anyhow::bail!("Missing a strategy field"); - }; - - let start = properties - .remove_serde::("start") - .context("Failed to parse start: not a number")?; - - let properties = properties.finalize(); - if !properties.is_empty() { - let supported_keys = ["strategy", "start"]; - let unknown_keys = properties.keys().map(String::from).collect::>(); + let properties = properties.resolve(provider, &ctx.eval_ctx(false))?; + let ResolvedClientProperty::RoundRobin(props) = properties else { anyhow::bail!( - "Unknown keys: {}. Supported keys are: {}", - unknown_keys.join(", "), - supported_keys.join(", ") + "Invalid client property. Should have been a round-robin property but got: {}", + properties.name() ); - } - - let start = match start { - Some(start) => start % strategy.len(), + }; + let start = match props.start_index { + Some(start) => (start as usize) % props.strategy.len(), None => { - #[cfg(not(target_arch = "wasm32"))] - { - fastrand::usize(..strategy.len()) - } - - // For VSCode, we don't want a random start point, - // as it can make rendering inconsistent - #[cfg(target_arch = "wasm32")] - { + if cfg!(target_arch = "wasm32") { + // For VSCode, we don't want a random start point, + // as it can make rendering inconsistent 0 + } else { + fastrand::usize(..props.strategy.len()) } } }; - - Ok(( - strategy.into_iter().map(ClientSpec::new_from_id).collect(), - start, - )) + Ok((props.strategy, start)) } impl TryFrom<(&ClientProperty, &RuntimeContext)> for RoundRobinStrategy { @@ -114,7 +85,8 @@ impl TryFrom<(&ClientProperty, &RuntimeContext)> for RoundRobinStrategy { fn try_from( (client, ctx): (&ClientProperty, &RuntimeContext), ) -> std::result::Result { - let (strategy, start) = resolve_strategy(client.property_handler()?, ctx)?; + let (strategy, start) = + resolve_strategy(&client.provider, &client.unresolved_options()?, ctx)?; Ok(RoundRobinStrategy { name: client.name.clone(), @@ -129,8 +101,7 @@ impl TryFrom<(&ClientWalker<'_>, &RuntimeContext)> for RoundRobinStrategy { type Error = anyhow::Error; fn try_from((client, ctx): (&ClientWalker, &RuntimeContext)) -> Result { - let properties = super::super::resolve_properties_walker(client, ctx)?; - let (strategy, start) = resolve_strategy(properties, ctx)?; + let (strategy, start) = resolve_strategy(&client.elem().provider, client.options(), ctx)?; Ok(Self { name: client.item.elem.name.clone(), retry_policy: client.retry_policy().as_ref().map(String::from), diff --git a/engine/baml-runtime/src/internal/llm_client/traits/mod.rs b/engine/baml-runtime/src/internal/llm_client/traits/mod.rs index 62379ed2d..7bc160d44 100644 --- a/engine/baml-runtime/src/internal/llm_client/traits/mod.rs +++ b/engine/baml-runtime/src/internal/llm_client/traits/mod.rs @@ -2,6 +2,7 @@ use std::{collections::HashMap, path::PathBuf, pin::Pin}; use anyhow::{Context, Result}; use aws_smithy_types::byte_stream::error::Error; +use internal_llm_client::AllowedRoleMetadata; use serde_json::{json, Map}; mod chat; @@ -33,8 +34,7 @@ pub trait WithRetryPolicy { } pub trait WithClientProperties { - fn client_properties(&self) -> &HashMap; - fn allowed_metadata(&self) -> &super::AllowedMetadata; + fn allowed_metadata(&self) -> &AllowedRoleMetadata; fn supports_streaming(&self) -> bool; } @@ -283,7 +283,11 @@ where .await?; let request_builder = self - .build_request(either::Right(&chat_messages), false, render_settings.stream && self.supports_streaming()) + .build_request( + either::Right(&chat_messages), + false, + render_settings.stream && self.supports_streaming(), + ) .await?; let mut request = request_builder.build()?; let url_header_value = { @@ -334,7 +338,12 @@ pub trait WithStreamable { impl WithStreamable for T where - T: WithClient + WithStreamChat + WithStreamCompletion + WithClientProperties + WithChat + WithCompletion, + T: WithClient + + WithStreamChat + + WithStreamCompletion + + WithClientProperties + + WithChat + + WithCompletion, { #[allow(async_fn_in_trait)] async fn stream(&self, ctx: &RuntimeContext, prompt: &RenderedPrompt) -> StreamResponse { @@ -368,19 +377,15 @@ where self.stream_chat(ctx, p).await } else { let res = self.chat(ctx, p).await; - Ok(Box::pin(futures::stream::once(async move { - res - }))) + Ok(Box::pin(futures::stream::once(async move { res }))) } - }, + } RenderedPrompt::Completion(p) => { if self.supports_streaming() { self.stream_completion(ctx, p).await } else { let res = self.completion(ctx, p).await; - Ok(Box::pin(futures::stream::once(async move { - res - }))) + Ok(Box::pin(futures::stream::once(async move { res }))) } } } @@ -615,15 +620,7 @@ async fn to_base64_with_inferred_mime_type( if let Some((mime_type, base64)) = as_base64(&media_url.url.as_str()) { return Ok((base64.to_string(), mime_type.to_string())); } - let response = match fetch_with_proxy( - &media_url.url, - ctx.env - .get("BOUNDARY_PROXY_URL") - .as_deref() - .map(|s| s.as_str()), - ) - .await - { + let response = match fetch_with_proxy(&media_url.url, ctx.proxy_url()).await { Ok(response) => response, Err(e) => return Err(anyhow::anyhow!("Failed to fetch media: {e:?}")), }; diff --git a/engine/baml-runtime/src/internal/prompt_renderer/mod.rs b/engine/baml-runtime/src/internal/prompt_renderer/mod.rs index 24e855c84..2dbbfa65a 100644 --- a/engine/baml-runtime/src/internal/prompt_renderer/mod.rs +++ b/engine/baml-runtime/src/internal/prompt_renderer/mod.rs @@ -1,4 +1,5 @@ mod render_output_format; +use internal_llm_client::ClientSpec; use jsonish::BamlValueWithFlags; use render_output_format::render_output_format; @@ -6,10 +7,7 @@ use anyhow::Result; use baml_types::{BamlValue, FieldType}; use internal_baml_core::{ error_unsupported, - ir::{ - repr::{ClientSpec, IntermediateRepr}, - FunctionWalker, IRHelper, - }, + ir::{repr::IntermediateRepr, FunctionWalker, IRHelper}, }; use internal_baml_jinja::{ types::OutputFormatContent, RenderContext, RenderContext_Client, RenderedPrompt, @@ -95,7 +93,7 @@ impl PromptRenderer { }) .collect::>(), ir, - &ctx.env, + ctx.env_vars(), ) } } diff --git a/engine/baml-runtime/src/internal/prompt_renderer/render_output_format.rs b/engine/baml-runtime/src/internal/prompt_renderer/render_output_format.rs index 4d93fa0af..d33a78041 100644 --- a/engine/baml-runtime/src/internal/prompt_renderer/render_output_format.rs +++ b/engine/baml-runtime/src/internal/prompt_renderer/render_output_format.rs @@ -122,14 +122,16 @@ fn find_existing_class_field<'a>( desc = OverridableValue::::from(attrs.meta.get("description")); } + let eval_ctx = ctx.eval_ctx(false); + if matches!(alias, OverridableValue::Unset) { - if let Some(_alias) = field_walker.alias(&ctx.env)? { + if let Some(_alias) = field_walker.alias(&eval_ctx)? { alias = OverridableValue::Set(_alias); } } if matches!(desc, OverridableValue::Unset) { - if let Some(_desc) = field_walker.description(&ctx.env)? { + if let Some(_desc) = field_walker.description(&eval_ctx)? { desc = OverridableValue::Set(_desc); } } @@ -176,18 +178,19 @@ fn find_enum_value( desc = OverridableValue::::from(attrs.meta.get("description")); } + let eval_ctx = ctx.eval_ctx(false); if let Some(value) = value_walker { - if value.skip(&ctx.env)? && !matches!(skip, OverridableValue::Set(false)) { + if value.skip(&eval_ctx)? && !matches!(skip, OverridableValue::Set(false)) { return Ok(None); } if matches!(alias, OverridableValue::Unset) { - if let Some(_alias) = value.alias(&ctx.env)? { + if let Some(_alias) = value.alias(&eval_ctx)? { alias = OverridableValue::Set(_alias); } } if matches!(desc, OverridableValue::Unset) { - if let Some(_desc) = value.description(&ctx.env)? { + if let Some(_desc) = value.description(&eval_ctx)? { desc = OverridableValue::Set(_desc); } } @@ -214,6 +217,8 @@ fn relevant_data_models<'a>( let mut recursive_classes = IndexSet::new(); let mut start: Vec = vec![output.clone()]; + let eval_ctx = ctx.eval_ctx(false); + while let Some(output) = start.pop() { match ir.distribute_constraints(&output) { (FieldType::Enum(enm), constraints) => { @@ -247,7 +252,7 @@ fn relevant_data_models<'a>( if matches!(alias, OverridableValue::Unset) { if let Ok(walker) = walker { - if let Some(a) = walker.alias(&ctx.env)? { + if let Some(a) = walker.alias(&eval_ctx)? { alias = OverridableValue::Set(a); } } @@ -324,7 +329,7 @@ fn relevant_data_models<'a>( if matches!(alias, OverridableValue::Unset) { if let Ok(walker) = walker { - if let Some(a) = walker.alias(&ctx.env)? { + if let Some(a) = walker.alias(&eval_ctx)? { alias = OverridableValue::Set(a); } } diff --git a/engine/baml-runtime/src/lib.rs b/engine/baml-runtime/src/lib.rs index 5308ed79c..c33718a28 100644 --- a/engine/baml-runtime/src/lib.rs +++ b/engine/baml-runtime/src/lib.rs @@ -189,8 +189,9 @@ impl BamlRuntime { function_name: &str, test_name: &str, ctx: &RuntimeContext, + strict: bool, ) -> Result<(BamlMap, Vec)> { - let params = self.inner.get_test_params(function_name, test_name, ctx)?; + let params = self.inner.get_test_params(function_name, test_name, ctx, strict)?; let constraints = self .inner .get_test_constraints(function_name, test_name, &ctx)?; @@ -202,8 +203,9 @@ impl BamlRuntime { function_name: &str, test_name: &str, ctx: &RuntimeContext, + strict: bool, ) -> Result> { - let (params, _) = self.get_test_params_and_constraints(function_name, test_name, ctx)?; + let (params, _) = self.get_test_params_and_constraints(function_name, test_name, ctx, strict)?; Ok(params) } @@ -222,7 +224,7 @@ impl BamlRuntime { let run_to_response = || async { let rctx = ctx.create_ctx(None, None)?; let (params, constraints) = - self.get_test_params_and_constraints(function_name, test_name, &rctx)?; + self.get_test_params_and_constraints(function_name, test_name, &rctx, true)?; let rctx_stream = ctx.create_ctx(None, None)?; let mut stream = self.inner.stream_function_impl( function_name.into(), diff --git a/engine/baml-runtime/src/runtime/runtime_interface.rs b/engine/baml-runtime/src/runtime/runtime_interface.rs index db850ad73..e93a32742 100644 --- a/engine/baml-runtime/src/runtime/runtime_interface.rs +++ b/engine/baml-runtime/src/runtime/runtime_interface.rs @@ -2,7 +2,7 @@ use std::{collections::HashMap, path::PathBuf, sync::Arc}; use super::InternalBamlRuntime; use crate::internal::llm_client::traits::WithClientProperties; -use crate::internal::llm_client::{AllowedMetadata, LLMResponse}; +use crate::internal::llm_client::LLMResponse; use crate::{ client_registry::ClientProperty, internal::{ @@ -24,16 +24,14 @@ use crate::{ RuntimeContext, RuntimeInterface, }; use anyhow::{Context, Result}; -use baml_types::{BamlMap, BamlValue, Constraint}; +use baml_types::{BamlMap, BamlValue, Constraint, EvaluationContext}; use internal_baml_core::{ internal_baml_diagnostics::SourceFile, - ir::{ - repr::{ClientSpec, IntermediateRepr}, - ArgCoercer, FunctionWalker, IRHelper, - }, + ir::{repr::IntermediateRepr, ArgCoercer, FunctionWalker, IRHelper}, validate, }; use internal_baml_jinja::RenderedPrompt; +use internal_llm_client::{AllowedRoleMetadata, ClientSpec}; impl<'a> InternalClientLookup<'a> for InternalBamlRuntime { // Gets a top-level client/strategy by name @@ -44,14 +42,7 @@ impl<'a> InternalClientLookup<'a> for InternalBamlRuntime { ) -> Result> { match client_spec { ClientSpec::Shorthand(provider, model) => { - let client_property = ClientProperty { - name: format!("{}/{}", provider, model), - provider: provider.into(), - retry_policy: None, - options: vec![("model".to_string(), BamlValue::String(model.to_string()))] - .into_iter() - .collect(), - }; + let client_property = ClientProperty::from_shorthand(provider, model); // TODO: allow other providers let llm_primitive_provider = LLMPrimitiveProvider::try_from((&client_property, ctx)) @@ -150,7 +141,7 @@ impl InternalRuntimeInterface for InternalBamlRuntime { ctx: &RuntimeContext, params: &BamlMap, node_index: Option, - ) -> Result<(RenderedPrompt, OrchestrationScope, AllowedMetadata)> { + ) -> Result<(RenderedPrompt, OrchestrationScope, AllowedRoleMetadata)> { let func = self.get_function(function_name, ctx)?; let baml_args = self.ir().check_function_params( &func, @@ -240,11 +231,14 @@ impl InternalRuntimeInterface for InternalBamlRuntime { function_name: &str, test_name: &str, ctx: &RuntimeContext, + strict: bool, ) -> Result> { let func = self.get_function(function_name, ctx)?; let test = self.ir().find_test(&func, test_name)?; - match test.test_case_params(&ctx.env) { + let eval_ctx = ctx.eval_ctx(strict); + + match test.test_case_params(&eval_ctx) { Ok(params) => { // Collect all errors and return them as a single error. let mut errors = Vec::new(); @@ -283,7 +277,10 @@ impl InternalRuntimeInterface for InternalBamlRuntime { } fn get_test_constraints( - &self, function_name: &str, test_name: &str, ctx: &RuntimeContext + &self, + function_name: &str, + test_name: &str, + ctx: &RuntimeContext, ) -> Result> { let func = self.get_function(function_name, ctx)?; let walker = self.ir().find_test(&func, test_name)?; diff --git a/engine/baml-runtime/src/runtime_interface.rs b/engine/baml-runtime/src/runtime_interface.rs index fd1149007..2659636e8 100644 --- a/engine/baml-runtime/src/runtime_interface.rs +++ b/engine/baml-runtime/src/runtime_interface.rs @@ -1,14 +1,13 @@ use anyhow::Result; use baml_types::{BamlMap, BamlValue, Constraint}; use internal_baml_core::internal_baml_diagnostics::Diagnostics; -use internal_baml_core::ir::repr::ClientSpec; use internal_baml_core::ir::{repr::IntermediateRepr, FunctionWalker}; use internal_baml_jinja::RenderedPrompt; +use internal_llm_client::{AllowedRoleMetadata, ClientSpec}; use std::{collections::HashMap, sync::Arc}; use crate::internal::llm_client::llm_provider::LLMProvider; use crate::internal::llm_client::orchestrator::{OrchestrationScope, OrchestratorNode}; -use crate::internal::llm_client::AllowedMetadata; use crate::tracing::{BamlTracer, TracingSpan}; use crate::types::on_log_event::LogEventCallbackSync; use crate::{ @@ -139,7 +138,7 @@ pub trait InternalRuntimeInterface { ctx: &RuntimeContext, params: &BamlMap, node_index: Option, - ) -> Result<(RenderedPrompt, OrchestrationScope, AllowedMetadata)>; + ) -> Result<(RenderedPrompt, OrchestrationScope, AllowedRoleMetadata)>; #[allow(async_fn_in_trait)] async fn render_raw_curl( @@ -158,6 +157,7 @@ pub trait InternalRuntimeInterface { function_name: &str, test_name: &str, ctx: &RuntimeContext, + strict: bool, ) -> Result>; fn get_test_constraints( diff --git a/engine/baml-runtime/src/tracing/api_wrapper/core_types.rs b/engine/baml-runtime/src/tracing/api_wrapper/core_types.rs index 05dd30a39..1b96e14d9 100644 --- a/engine/baml-runtime/src/tracing/api_wrapper/core_types.rs +++ b/engine/baml-runtime/src/tracing/api_wrapper/core_types.rs @@ -1,5 +1,6 @@ use std::{collections::HashMap, path::PathBuf}; +use baml_types::BamlMap; use indexmap::IndexMap; use serde::{Deserialize, Serialize}; use serde_json::Value; @@ -181,7 +182,7 @@ pub enum Role { pub(crate) struct LLMEventInput { pub prompt: LLMEventInputPrompt, #[serde(rename = "invocation_params")] - pub request_options: HashMap, + pub request_options: BamlMap, } #[derive(Serialize, Debug, Clone)] diff --git a/engine/baml-runtime/src/types/context_manager.rs b/engine/baml-runtime/src/types/context_manager.rs index 23b0429cf..56370196d 100644 --- a/engine/baml-runtime/src/types/context_manager.rs +++ b/engine/baml-runtime/src/types/context_manager.rs @@ -16,7 +16,6 @@ type BamlContext = (uuid::Uuid, String, HashMap); #[derive(Clone)] pub struct RuntimeContextManager { baml_src_reader: Arc, - context: Arc>>, env_vars: HashMap, global_tags: Arc>>, @@ -129,14 +128,14 @@ impl RuntimeContextManager { let (cls, enm) = tb.map(|tb| tb.to_overrides()).unwrap_or_default(); - let mut ctx = RuntimeContext { - baml_src: self.baml_src_reader.clone(), - env: self.env_vars.clone(), + let mut ctx = RuntimeContext::new( + self.baml_src_reader.clone(), + self.env_vars.clone(), tags, - client_overrides: Default::default(), - class_override: cls, - enum_overrides: enm, - }; + Default::default(), + cls, + enm, + ); let client_overrides = match cb { Some(cb) => Some( @@ -151,24 +150,17 @@ impl RuntimeContextManager { Ok(ctx) } - pub fn create_ctx_with_default>( - &self, - env_vars: impl Iterator, - ) -> RuntimeContext { + pub fn create_ctx_with_default(&self) -> RuntimeContext { let ctx = self.context.lock().unwrap(); - let env_vars = env_vars - .map(|x| (x.as_ref().to_string(), format!("${{{}}}", x.as_ref()))) - .chain(self.env_vars.iter().map(|(k, v)| (k.clone(), v.clone()))); - - RuntimeContext { - baml_src: self.baml_src_reader.clone(), - env: env_vars.collect(), - tags: ctx.last().map(|(.., x)| x).cloned().unwrap_or_default(), - client_overrides: Default::default(), - class_override: Default::default(), - enum_overrides: Default::default(), - } + RuntimeContext::new( + self.baml_src_reader.clone(), + self.env_vars.clone(), + ctx.last().map(|(.., x)| x).cloned().unwrap_or_default(), + Default::default(), + Default::default(), + Default::default(), + ) } pub fn context_depth(&self) -> usize { diff --git a/engine/baml-runtime/src/types/expression_helper.rs b/engine/baml-runtime/src/types/expression_helper.rs deleted file mode 100644 index 16df949e5..000000000 --- a/engine/baml-runtime/src/types/expression_helper.rs +++ /dev/null @@ -1,57 +0,0 @@ -use anyhow::Result; -use std::collections::HashMap; - -use internal_baml_core::ir::repr::Expression; -use serde_json::json; - -use crate::RuntimeContext; - -pub fn to_value(ctx: &RuntimeContext, expr: &Expression) -> Result { - // serde_json::Value::serialize(&self.into(), serializer) - Ok(match expr { - Expression::Identifier(idn) => match idn { - internal_baml_core::ir::repr::Identifier::ENV(key) => match ctx.env.get(key) { - None => anyhow::bail!("unset env variable '{}'", key), - Some(val) => serde_json::Value::String(val.to_string()), - }, - _ => serde_json::Value::String(idn.name()), - }, - Expression::Bool(b) => serde_json::Value::Bool(*b), - Expression::Numeric(n) => serde_json::Value::Number(n.parse().unwrap()), - Expression::String(s) => serde_json::Value::String(s.clone()), - Expression::RawString(s) => serde_json::Value::String(s.to_string()), - Expression::List(items) => serde_json::Value::Array( - items - .iter() - .map(|item| to_value(ctx, item)) - .collect::>>()?, - ), - Expression::Map(kv) => { - let res = kv - .iter() - .map(|(k, v)| { - let k = match k { - Expression::String(s) => s.clone(), - Expression::Identifier(internal_baml_core::ir::repr::Identifier::ENV( - key, - )) => match ctx.env.get(key) { - None => anyhow::bail!("unset env variable '{}'", key), - Some(val) => val.to_string(), - }, - Expression::Identifier( - internal_baml_core::ir::repr::Identifier::Local(key), - ) => key.clone(), - Expression::Identifier(internal_baml_core::ir::repr::Identifier::Ref( - key, - )) => key.join("."), - _ => anyhow::bail!("invalid key {:#?}", k), - }; - let v = to_value(ctx, v)?; - Ok((k, v)) - }) - .collect::>>()?; - json!(res) - }, - Expression::JinjaExpression(_) => anyhow::bail!("Unable to normalize jinja expression to a value without a context."), - }) -} diff --git a/engine/baml-runtime/src/types/mod.rs b/engine/baml-runtime/src/types/mod.rs index cda1bf1ff..1fabe04c4 100644 --- a/engine/baml-runtime/src/types/mod.rs +++ b/engine/baml-runtime/src/types/mod.rs @@ -1,5 +1,5 @@ mod context_manager; -mod expression_helper; +// mod expression_helper; pub mod on_log_event; mod response; pub(crate) mod runtime_context; diff --git a/engine/baml-runtime/src/types/runtime_context.rs b/engine/baml-runtime/src/types/runtime_context.rs index 393524061..8bb9d5592 100644 --- a/engine/baml-runtime/src/types/runtime_context.rs +++ b/engine/baml-runtime/src/types/runtime_context.rs @@ -1,7 +1,7 @@ use anyhow::Result; -use baml_types::BamlValue; +use baml_types::{BamlValue, EvaluationContext, UnresolvedValue}; use indexmap::IndexMap; -use internal_baml_core::ir::{repr::Expression, FieldType}; +use internal_baml_core::ir::FieldType; use serde; use serde_json; use std::{collections::HashMap, sync::Arc}; @@ -53,7 +53,7 @@ cfg_if::cfg_if!( pub struct RuntimeContext { // path to baml_src in the local filesystem pub baml_src: Arc, - pub env: HashMap, + env: HashMap, pub tags: HashMap, pub client_overrides: Option<(Option, HashMap>)>, pub class_override: IndexMap, @@ -61,20 +61,51 @@ pub struct RuntimeContext { } impl RuntimeContext { + pub fn eval_ctx<'a>(&'a self, strict: bool) -> EvaluationContext<'a> { + EvaluationContext::new(&self.env, !strict) + } + + pub fn env_vars(&self) -> &HashMap { + &self.env + } + + pub fn proxy_url(&self) -> Option<&str> { + self.env.get("BOUNDARY_PROXY_URL").map(|s| s.as_str()) + } + + pub fn new( + baml_src: Arc, + env: HashMap, + tags: HashMap, + client_overrides: Option<(Option, HashMap>)>, + class_override: IndexMap, + enum_overrides: IndexMap, + ) -> RuntimeContext { + RuntimeContext { + baml_src, + env, + tags, + client_overrides, + class_override, + enum_overrides, + } + } + pub fn resolve_expression( &self, - expr: &Expression, + expr: &UnresolvedValue<()>, + // If true, will return an error if any environment variables are not set + // otherwise, will return a value with the missing environment variables replaced with the string "${key}" + strict: bool, ) -> Result { - match super::expression_helper::to_value(self, expr) { - Ok(v) => serde_json::from_value(v).map_err(|e| e.into()), - Err(e) => Err(e), - } - .map_err(|e| { - anyhow::anyhow!( + let ctx = EvaluationContext::new(&self.env, strict); + match expr.resolve_serde::(&ctx) { + Ok(v) => Ok(v), + Err(e) => anyhow::bail!( "Failed to resolve expression {:?} with error: {:?}", expr, e - ) - }) + ), + } } } diff --git a/engine/baml-runtime/tests/test_runtime.rs b/engine/baml-runtime/tests/test_runtime.rs index c83c61cba..c3323def9 100644 --- a/engine/baml-runtime/tests/test_runtime.rs +++ b/engine/baml-runtime/tests/test_runtime.rs @@ -182,7 +182,7 @@ mod internal_tests { .create_ctx_manager(BamlValue::String("test".to_string()), None) .create_ctx_with_default(missing_env_vars.iter()); - let params = runtime.get_test_params(function_name, test_name, &ctx)?; + let params = runtime.get_test_params(function_name, test_name, &ctx, true)?; let render_prompt_future = runtime @@ -262,7 +262,7 @@ mod internal_tests { .create_ctx_manager(BamlValue::String("test".to_string()), None) .create_ctx_with_default(missing_env_vars.iter()); - let params = runtime.get_test_params(function_name, test_name, &ctx)?; + let params = runtime.get_test_params(function_name, test_name, &ctx, true)?; let render_prompt_future = runtime @@ -345,7 +345,7 @@ test ImageReceiptTest { let function_name = "ExtractReceipt"; let test_name = "ImageReceiptTest"; - let params = runtime.get_test_params(function_name, test_name, &ctx)?; + let params = runtime.get_test_params(function_name, test_name, &ctx, true)?; let render_prompt_future = runtime .internal() @@ -428,7 +428,7 @@ test TestName { let function_name = "Bot"; let test_name = "TestName"; - let params = runtime.get_test_params(function_name, test_name, &ctx)?; + let params = runtime.get_test_params(function_name, test_name, &ctx, true)?; let render_prompt_future = runtime .internal() @@ -498,7 +498,7 @@ test TestTree { let function_name = "BuildTree"; let test_name = "TestTree"; - let params = runtime.get_test_params(function_name, test_name, &ctx)?; + let params = runtime.get_test_params(function_name, test_name, &ctx, true)?; let render_prompt_future = runtime .internal() diff --git a/engine/baml-schema-wasm/Cargo.toml b/engine/baml-schema-wasm/Cargo.toml index d83bcede2..2d07a93b8 100644 --- a/engine/baml-schema-wasm/Cargo.toml +++ b/engine/baml-schema-wasm/Cargo.toml @@ -23,6 +23,7 @@ indexmap.workspace = true internal-baml-codegen.workspace = true internal-baml-core.workspace = true jsonish = { path = "../baml-lib/jsonish" } +internal-llm-client = { path = "../baml-lib/llm-client" } js-sys = "=0.3.69" log.workspace = true serde.workspace = true diff --git a/engine/baml-schema-wasm/src/runtime_wasm/mod.rs b/engine/baml-schema-wasm/src/runtime_wasm/mod.rs index 62557b82e..3553f296a 100644 --- a/engine/baml-schema-wasm/src/runtime_wasm/mod.rs +++ b/engine/baml-schema-wasm/src/runtime_wasm/mod.rs @@ -4,7 +4,6 @@ use crate::runtime_wasm::runtime_prompt::WasmPrompt; use anyhow::Context; use baml_runtime::internal::llm_client::orchestrator::OrchestrationScope; use baml_runtime::internal::llm_client::orchestrator::OrchestratorNode; -use baml_runtime::internal::llm_client::AllowedMetadata; use baml_runtime::internal::prompt_renderer::PromptRenderer; use baml_runtime::BamlSrcReader; use baml_runtime::InternalRuntimeInterface; @@ -16,6 +15,7 @@ use baml_types::{BamlMediaType, BamlValue, GeneratorOutputType, TypeValue}; use indexmap::IndexMap; use internal_baml_codegen::version_check::GeneratorType; use internal_baml_codegen::version_check::{check_version, VersionCheckMode}; +use internal_llm_client::AllowedRoleMetadata; use jsonish::deserializer::deserialize_flags::Flag; use jsonish::BamlValueWithFlags; @@ -459,9 +459,9 @@ impl WasmLLMFailure { self.scope.name() } pub fn prompt(&self) -> WasmPrompt { - // TODO: This is a hack. We shouldn't hardcode AllowedMetadata::All + // TODO: This is a hack. We shouldn't hardcode AllowedRoleMetadata::All // here, but instead plumb it through the LLMErrors - (&self.prompt, &self.scope, &AllowedMetadata::All).into() + (&self.prompt, &self.scope, &AllowedRoleMetadata::All).into() } } @@ -473,9 +473,9 @@ impl WasmLLMResponse { } pub fn prompt(&self) -> WasmPrompt { - // TODO: This is a hack. We shouldn't hardcode AllowedMetadata::All + // TODO: This is a hack. We shouldn't hardcode AllowedRoleMetadata::All // here, but instead plumb it through the LLMErrors - (&self.prompt, &self.scope, &AllowedMetadata::All).into() + (&self.prompt, &self.scope, &AllowedRoleMetadata::All).into() } } @@ -962,6 +962,12 @@ impl WasmRuntime { #[wasm_bindgen] pub fn list_functions(&self) -> Vec { + let ctx = &self + .runtime + .create_ctx_manager(BamlValue::String("wasm".to_string()), None); + let ctx = ctx.create_ctx_with_default(); + let ctx = ctx.eval_ctx(false); + self.runtime .internal() .ir() @@ -1008,7 +1014,7 @@ impl WasmRuntime { test_cases: f .walk_tests() .map(|tc| { - let params = match tc.test_case_params(&self.runtime.env_vars()) { + let params = match tc.test_case_params(&ctx) { Ok(params) => Ok(params .iter() .map(|(k, v)| { @@ -1311,12 +1317,19 @@ impl WasmRuntime { #[wasm_bindgen] pub fn list_testcases(&self) -> Vec { + let ctx = self + .runtime + .create_ctx_manager(BamlValue::String("wasm".to_string()), None); + + let ctx = ctx.create_ctx_with_default(); + let ctx = ctx.eval_ctx(true); + self.runtime .internal() .ir() .walk_tests() .map(|tc| { - let params = match tc.test_case_params(&self.runtime.env_vars()) { + let params = match tc.test_case_params(&ctx) { Ok(params) => Ok(params .iter() .map(|(k, v)| { @@ -1513,18 +1526,17 @@ impl WasmFunction { wasm_call_context: &WasmCallContext, get_baml_src_cb: js_sys::Function, ) -> JsResult { - let missing_env_vars = rt.runtime.internal().ir().required_env_vars(); let ctx = rt .runtime .create_ctx_manager( BamlValue::String("wasm".to_string()), js_fn_to_baml_src_reader(get_baml_src_cb), ) - .create_ctx_with_default(missing_env_vars.iter()); + .create_ctx_with_default(); let params = rt .runtime - .get_test_params(&self.name, &test_name, &ctx) + .get_test_params(&self.name, &test_name, &ctx, false) .map_err(|e| JsError::new(format!("{e:?}").as_str()))?; rt.runtime @@ -1540,7 +1552,7 @@ impl WasmFunction { pub fn client_name(&self, rt: &WasmRuntime) -> Result { let rt: &BamlRuntime = &rt.runtime; let ctx_manager = rt.create_ctx_manager(BamlValue::String("wasm".to_string()), None); - let ctx = ctx_manager.create_ctx_with_default(rt.env_vars().keys().map(|k| k.as_str())); + let ctx = ctx_manager.create_ctx_with_default(); let ir = rt.internal().ir(); let walker = ir .find_function(&self.name) @@ -1560,19 +1572,17 @@ impl WasmFunction { expand_images: bool, get_baml_src_cb: js_sys::Function, ) -> Result { - let missing_env_vars = rt.runtime.internal().ir().required_env_vars(); - let ctx = rt .runtime .create_ctx_manager( BamlValue::String("wasm".to_string()), js_fn_to_baml_src_reader(get_baml_src_cb), ) - .create_ctx_with_default(missing_env_vars.iter()); + .create_ctx_with_default(); let params = rt .runtime - .get_test_params(&self.name, &test_name, &ctx) + .get_test_params(&self.name, &test_name, &ctx, false) .map_err(|e| JsError::new(format!("{e:?}").as_str()))?; let result = rt @@ -1644,10 +1654,9 @@ impl WasmFunction { pub fn orchestration_graph(&self, rt: &WasmRuntime) -> Result, JsValue> { let rt: &BamlRuntime = &rt.runtime; - let missing_env_vars = rt.internal().ir().required_env_vars(); let ctx = rt .create_ctx_manager(BamlValue::String("wasm".to_string()), None) - .create_ctx_with_default(missing_env_vars.iter()); + .create_ctx_with_default(); let ir = rt.internal().ir(); let walker = ir diff --git a/engine/baml-schema-wasm/src/runtime_wasm/runtime_prompt.rs b/engine/baml-schema-wasm/src/runtime_wasm/runtime_prompt.rs index c193ac890..e7f7f0e8d 100644 --- a/engine/baml-schema-wasm/src/runtime_wasm/runtime_prompt.rs +++ b/engine/baml-schema-wasm/src/runtime_wasm/runtime_prompt.rs @@ -1,12 +1,10 @@ use std::collections::HashMap; use baml_runtime::{ - internal::llm_client::{ - orchestrator::{ExecutionScope, OrchestrationScope}, - AllowedMetadata, - }, + internal::llm_client::orchestrator::{ExecutionScope, OrchestrationScope}, ChatMessagePart, RenderedPrompt, }; +use internal_llm_client::AllowedRoleMetadata; use serde_json::json; use crate::runtime_wasm::ToJsValue; @@ -23,7 +21,7 @@ pub struct WasmScope { pub struct WasmPrompt { prompt: RenderedPrompt, pub client_name: String, - allowed: AllowedMetadata, + allowed: AllowedRoleMetadata, } impl From for WasmScope { @@ -32,9 +30,13 @@ impl From for WasmScope { } } -impl From<(&RenderedPrompt, &OrchestrationScope, &AllowedMetadata)> for WasmPrompt { +impl From<(&RenderedPrompt, &OrchestrationScope, &AllowedRoleMetadata)> for WasmPrompt { fn from( - (prompt, client_name, allowed): (&RenderedPrompt, &OrchestrationScope, &AllowedMetadata), + (prompt, client_name, allowed): ( + &RenderedPrompt, + &OrchestrationScope, + &AllowedRoleMetadata, + ), ) -> Self { WasmPrompt { prompt: prompt.clone(), diff --git a/engine/language_client_codegen/src/ruby/expression.rs b/engine/language_client_codegen/src/ruby/expression.rs index 567e58381..94a57dd82 100644 --- a/engine/language_client_codegen/src/ruby/expression.rs +++ b/engine/language_client_codegen/src/ruby/expression.rs @@ -1,45 +1,46 @@ -use baml_types::{BamlMediaType, TypeValue}; -use internal_baml_core::ir::{Expression, Identifier}; +use baml_types::{BamlMediaType, StringOr, TypeValue, UnresolvedValue}; use super::ruby_language_features::ToRuby; -impl ToRuby for Expression { - fn to_ruby(&self) -> String { - match self { - Expression::List(values) => { - format!( - "[{}]", - values - .iter() - .map(|v| v.to_ruby()) - .collect::>() - .join(", ") - ) - } - Expression::Map(values) => { - format!( - "{{ {} }}", - values - .iter() - .map(|(k, v)| format!("{}: {}", k.to_ruby(), v.to_ruby())) - .collect::>() - .join(", ") - ) - } - Expression::Identifier(idn) => match idn { - Identifier::ENV(idn) => format!("process.env.{}", idn), - Identifier::Local(k) => format!("\"{}\"", k.replace('"', "\\\"")), - Identifier::Ref(r) => format!("\"{}\"", r.join(".")), - Identifier::Primitive(p) => p.to_ruby(), - }, - Expression::String(val) => format!("\"{}\"", val.escape_default()), - Expression::RawString(val) => format!("`{}`", val.replace('`', "\\`")), - Expression::Numeric(val) => val.clone(), - Expression::Bool(val) => val.to_string(), - Expression::JinjaExpression(val) => val.to_string(), - } - } -} +// impl ToRuby for UnresolvedValue { +// fn to_ruby(&self) -> String { +// match self { +// UnresolvedValue::Array(values) => { +// format!( +// "[{}]", +// values +// .iter() +// .map(|v| v.to_ruby()) +// .collect::>() +// .join(", ") +// ) +// } +// UnresolvedValue::Map(values) => { +// format!( +// "{{ {} }}", +// values +// .iter() +// .map(|(k, v)| format!("{}: {}", k.to_ruby(), v.to_ruby())) +// .collect::>() +// .join(", ") +// ) +// } +// UnresolvedValue::Identifier(idn) => match idn { +// Identifier::ENV(idn) => format!("process.env.{}", idn), +// Identifier::Local(k) => format!("\"{}\"", k.replace('"', "\\\"")), +// Identifier::Ref(r) => format!("\"{}\"", r.join(".")), +// Identifier::Primitive(p) => p.to_ruby(), +// }, +// UnresolvedValue::String(val) => match val { +// StringOr::EnvVar(s) => format!("process.env.{}", s), +// StringOr::Value(v) => format!("\"{}\"", v.replace('"', "\\\"")), +// StringOr::JinjaExpression(jinja_expression) => , +// }, +// UnresolvedValue::Numeric(val) => val.clone(), +// UnresolvedValue::Bool(val) => val.to_string(), +// } +// } +// } impl ToRuby for TypeValue { fn to_ruby(&self) -> String { diff --git a/engine/language_client_python/src/errors.rs b/engine/language_client_python/src/errors.rs index dd5f0de93..b07a26f7b 100644 --- a/engine/language_client_python/src/errors.rs +++ b/engine/language_client_python/src/errors.rs @@ -75,7 +75,8 @@ impl BamlError { LLMResponse::LLMFailure(failed) => match &failed.code { baml_runtime::internal::llm_client::ErrorCode::Other(2) => { PyErr::new::(format!( - "Something went wrong with the LLM client: {}", + "Something went wrong with the LLM client {}: {}", + failed.client, failed.message )) } diff --git a/engine/language_client_python/src/types/client_registry.rs b/engine/language_client_python/src/types/client_registry.rs index ec611cfc7..d7498d957 100644 --- a/engine/language_client_python/src/types/client_registry.rs +++ b/engine/language_client_python/src/types/client_registry.rs @@ -1,9 +1,12 @@ +use std::str::FromStr; + use baml_runtime::client_registry; use pyo3::prelude::{pymethods, PyResult}; use pyo3::{PyObject, Python, ToPyObject}; use crate::errors::BamlInvalidArgumentError; use crate::parse_py_type::parse_py_type; +use client_registry::ClientProvider; crate::lang_wrapper!(ClientRegistry, client_registry::ClientRegistry); @@ -36,13 +39,15 @@ impl ClientRegistry { )); }; - let client_property = baml_runtime::client_registry::ClientProperty { - name, - provider, - retry_policy, - options: args_map, + let provider = match ClientProvider::from_str(&provider) { + Ok(provider) => provider, + Err(e) => { + return Err(BamlInvalidArgumentError::new_err(format!("Invalid provider: {:?}", e))); + } }; + let client_property = client_registry::ClientProperty::new(name, provider, retry_policy, args_map); + self.inner.add_client(client_property); Ok(()) } diff --git a/engine/language_client_ruby/ext/ruby_ffi/src/types/client_registry.rs b/engine/language_client_ruby/ext/ruby_ffi/src/types/client_registry.rs index 80312af1f..65a681bfe 100644 --- a/engine/language_client_ruby/ext/ruby_ffi/src/types/client_registry.rs +++ b/engine/language_client_ruby/ext/ruby_ffi/src/types/client_registry.rs @@ -3,6 +3,7 @@ use magnus::{ class, function, method, scan_args::scan_args, Error, Module, Object, RHash, Ruby, Value, }; use std::cell::RefCell; +use std::str::FromStr; use crate::ruby_to_json; use crate::Result; @@ -35,13 +36,19 @@ impl ClientRegistry { } }; - let client_property = baml_runtime::client_registry::ClientProperty { - name, - provider, - retry_policy, - options, + + let provider = match client_registry::ClientProvider::from_str(&provider) { + Ok(provider) => provider, + Err(e) => { + return Err(Error::new( + ruby.exception_syntax_error(), + format!("Invalid provider: {:?}", e), + )); + } }; + let client_property = client_registry::ClientProperty::new(name, provider, retry_policy, options); + rb_self.inner.borrow_mut().add_client(client_property); Ok(()) } diff --git a/engine/language_client_typescript/src/types/client_registry.rs b/engine/language_client_typescript/src/types/client_registry.rs index ef088d96d..4cb4975d8 100644 --- a/engine/language_client_typescript/src/types/client_registry.rs +++ b/engine/language_client_typescript/src/types/client_registry.rs @@ -1,7 +1,10 @@ +use std::str::FromStr; + use baml_runtime::client_registry; use napi::Env; use napi::JsObject; use napi_derive::napi; +use client_registry::ClientProvider; use crate::errors::invalid_argument_error; use crate::parse_ts_types; @@ -35,13 +38,16 @@ impl ClientRegistry { } let args_map = args.as_map_owned().unwrap(); - let client_property = baml_runtime::client_registry::ClientProperty { - name, - provider, - retry_policy, - options: args_map, + + let provider = match ClientProvider::from_str(&provider) { + Ok(provider) => provider, + Err(e) => { + return Err(invalid_argument_error(&format!("Invalid provider: {:?}", e))); + } }; + let client_property = client_registry::ClientProperty::new(name, provider, retry_policy, args_map); + self.inner.add_client(client_property); Ok(()) } diff --git a/integ-tests/baml_src/clients.baml b/integ-tests/baml_src/clients.baml index fa601112a..94daa276e 100644 --- a/integ-tests/baml_src/clients.baml +++ b/integ-tests/baml_src/clients.baml @@ -83,7 +83,6 @@ client Gemini { safetySettings { category HARM_CATEGORY_HATE_SPEECH threshold BLOCK_LOW_AND_ABOVE - } } } @@ -92,10 +91,8 @@ client Vertex { provider vertex-ai options { model gemini-1.5-pro - project_id sam-project-vertex-1 location us-central1 credentials env.INTEG_TESTS_GOOGLE_APPLICATION_CREDENTIALS - } } @@ -110,7 +107,6 @@ client AwsBedrock { // model_id "anthropic.claude-3-haiku-20240307-v1:0" model_id "meta.llama3-8b-instruct-v1:0" // model_id "mistral.mistral-7b-instruct-v0:2" - api_key "" } } @@ -125,7 +121,6 @@ client AwsBedrockInvalidRegion { // model_id "anthropic.claude-3-haiku-20240307-v1:0" model_id "meta.llama3-8b-instruct-v1:0" // model_id "mistral.mistral-7b-instruct-v0:2" - api_key "" } } @@ -151,7 +146,7 @@ client ClaudeWithCaching { options { model claude-3-haiku-20240307 api_key env.ANTHROPIC_API_KEY - max_tokens 1000 + max_tokens 500 allowed_role_metadata ["cache_control"] headers { "anthropic-beta" "prompt-caching-2024-07-31" diff --git a/integ-tests/baml_src/test-files/providers/providers.baml b/integ-tests/baml_src/test-files/providers/providers.baml index 4c013b9c4..d8225b21d 100644 --- a/integ-tests/baml_src/test-files/providers/providers.baml +++ b/integ-tests/baml_src/test-files/providers/providers.baml @@ -93,8 +93,39 @@ function TestCaching(input: string, not_cached: string) -> string { client ClaudeWithCaching prompt #" {{ _.role('system', cache_control={"type": "ephemeral"}) }} - Describe this in 5 words: {{ input }} + Generate the following story + {{ input }} + + {# Haiku require 2048 tokens to cache -#} + {{ input }} + {{ _.role('user') }} {{ not_cached }} "# } + +test TestName { + functions [TestCaching] + args { + input #" +In a near-future society where dreams have become a tradable commodity and shared experience, a lonely and socially awkward teenager named Alex discovers they possess a rare and powerful ability to not only view but also manipulate the dreams of others. Initially thrilled by this newfound power, Alex begins subtly altering the dreams of classmates and family members, helping them overcome fears, boost confidence, or experience fantastical adventures. As Alex's skills grow, so does their influence. They start selling premium dream experiences on the black market, crafting intricate and addictive dreamscapes for wealthy clients. However, the line between dream and reality begins to blur for those exposed to Alex's creations. Some clients struggle to differentiate between their true memories and the artificial ones implanted by Alex's dream manipulation. + +Complications arise when a mysterious government agency takes notice of Alex's unique abilities. They offer Alex a chance to use their gift for "the greater good," hinting at applications in therapy, criminal rehabilitation, and even national security. Simultaneously, an underground resistance movement reaches out, warning Alex about the dangers of dream manipulation and the potential for mass control and exploitation. Caught between these opposing forces, Alex must navigate a complex web of ethical dilemmas. They grapple with questions of free will, the nature of consciousness, and the responsibility that comes with having power over people's minds. As the consequences of their actions spiral outward, affecting the lives of loved ones and strangers alike, Alex is forced to confront the true nature of their ability and decide how—or if—it should be used. + +The story explores themes of identity, the subconscious mind, the ethics of technology, and the power of imagination. It delves into the potential consequences of a world where our most private thoughts and experiences are no longer truly our own, and examines the fine line between helping others and manipulating them for personal gain or a perceived greater good. The narrative further expands on the societal implications of such abilities, questioning the moral boundaries of altering consciousness and the potential for abuse in a world where dreams can be commodified. It challenges the reader to consider the impact of technology on personal autonomy and the ethical responsibilities of those who wield such power. + +As Alex's journey unfolds, they encounter various individuals whose lives have been touched by their dream manipulations, each presenting a unique perspective on the ethical quandaries at hand. From a classmate who gains newfound confidence to a wealthy client who becomes addicted to the dreamscapes, the ripple effects of Alex's actions are profound and far-reaching. The government agency's interest in Alex's abilities raises questions about the potential for state control and surveillance, while the resistance movement highlights the dangers of unchecked power and the importance of safeguarding individual freedoms. + +Ultimately, Alex's story is one of self-discovery and moral reckoning, as they must decide whether to embrace their abilities for personal gain, align with the government's vision of a controlled utopia, or join the resistance in their fight for freedom and autonomy. The narrative invites readers to reflect on the nature of reality, the boundaries of human experience, and the ethical implications of a world where dreams are no longer private sanctuaries but shared and manipulated commodities. It also explores the psychological impact on Alex, who must deal with the burden of knowing the intimate fears and desires of others, and the isolation that comes from being unable to share their own dreams without altering them. + +The story further examines the technological advancements that have made dream manipulation possible, questioning the role of innovation in society and the potential for both progress and peril. It considers the societal divide between those who can afford to buy enhanced dream experiences and those who cannot, highlighting issues of inequality and access. As Alex becomes more entangled in the web of their own making, they must confront the possibility that their actions could lead to unintended consequences, not just for themselves but for the fabric of society as a whole. + +In the end, Alex's journey is a cautionary tale about the power of dreams and the responsibilities that come with wielding such influence. It serves as a reminder of the importance of ethical considerations in the face of technological advancement and the need to balance innovation with humanity. The story leaves readers pondering the true cost of a world where dreams are no longer sacred, and the potential for both wonder and danger in the uncharted territories of the mind. But it's also a story about the power of imagination and the potential for change, even in a world where our deepest thoughts are no longer our own. And it's a story about the power of choice, and the importance of fighting for the freedom to dream. + +In conclusion, this story is a reflection on the power of dreams and the responsibilities that come with wielding such influence. It serves as a reminder of the importance of ethical considerations in the face of technological advancement and the need to balance innovation with humanity. The story leaves readers pondering the true cost of a world where dreams are no longer sacred, and the potential for both wonder and danger in the uncharted territories of the mind. But it's also a story about the power of imagination and the potential for change, even in a world where our deepest thoughts are no longer our own. And it's a story about the power of choice, and the importance of fighting for the freedom to dream. + "# + not_cached #" + hello world + "# + } +} diff --git a/integ-tests/baml_src/test-files/strategies/fallback.baml b/integ-tests/baml_src/test-files/strategies/fallback.baml index 048226463..00eed8147 100644 --- a/integ-tests/baml_src/test-files/strategies/fallback.baml +++ b/integ-tests/baml_src/test-files/strategies/fallback.baml @@ -35,7 +35,8 @@ client FaultyAzureClient { provider azure-openai options { model unknown-model - api_key env.OPENAI_API_KEY + resource_name "unknown-resource-id" + deployment_id "unknown-deployment-id" } } diff --git a/integ-tests/baml_src/test-files/testing_pipeline/resume.baml b/integ-tests/baml_src/test-files/testing_pipeline/resume.baml index 1a4852dbc..773155850 100644 --- a/integ-tests/baml_src/test-files/testing_pipeline/resume.baml +++ b/integ-tests/baml_src/test-files/testing_pipeline/resume.baml @@ -65,7 +65,7 @@ function ExtractResume(resume: string, img: image?) -> Resume { test sam_resume { functions [ExtractResume] - input { + args { img { url "https://avatars.githubusercontent.com/u/1016595?v=4" } @@ -122,7 +122,7 @@ test sam_resume { test vaibhav_resume { functions [ExtractResume] - input { + args { resume #" Vaibhav Gupta linkedin/vaigup diff --git a/integ-tests/python/baml_client/inlinedbaml.py b/integ-tests/python/baml_client/inlinedbaml.py index 5b61d4c61..c0fb4f46e 100644 --- a/integ-tests/python/baml_client/inlinedbaml.py +++ b/integ-tests/python/baml_client/inlinedbaml.py @@ -16,7 +16,7 @@ file_map = { - "clients.baml": "retry_policy Bar {\n max_retries 3\n strategy {\n type exponential_backoff\n }\n}\n\nretry_policy Foo {\n max_retries 3\n strategy {\n type constant_delay\n delay_ms 100\n }\n}\n\nclient GPT4 {\n provider openai\n options {\n model gpt-4o\n api_key env.OPENAI_API_KEY\n }\n} \n\n\nclient GPT4o {\n provider openai\n options {\n model gpt-4o\n api_key env.OPENAI_API_KEY\n }\n} \n\n\nclient GPT4Turbo {\n retry_policy Bar\n provider openai\n options {\n model gpt-4-turbo\n api_key env.OPENAI_API_KEY\n }\n} \n\nclient GPT35 {\n provider openai\n options {\n model \"gpt-3.5-turbo\"\n api_key env.OPENAI_API_KEY\n }\n}\n\nclient GPT35LegacyProvider {\n provider openai\n options {\n model \"gpt-3.5-turbo\"\n api_key env.OPENAI_API_KEY\n }\n}\n\n\nclient Ollama {\n provider ollama\n options {\n model llama2\n }\n}\n\nclient GPT35Azure {\n provider azure-openai\n options {\n resource_name \"west-us-azure-baml\"\n deployment_id \"gpt-35-turbo-default\"\n // base_url \"https://west-us-azure-baml.openai.azure.com/openai/deployments/gpt-35-turbo-default\"\n api_version \"2024-02-01\"\n api_key env.AZURE_OPENAI_API_KEY\n }\n}\n\nclient Gemini {\n provider google-ai\n options {\n model gemini-1.5-pro-001\n api_key env.GOOGLE_API_KEY\n safetySettings {\n category HARM_CATEGORY_HATE_SPEECH\n threshold BLOCK_LOW_AND_ABOVE\n\n }\n }\n}\n\nclient Vertex {\n provider vertex-ai \n options {\n model gemini-1.5-pro\n project_id sam-project-vertex-1\n location us-central1\n credentials env.INTEG_TESTS_GOOGLE_APPLICATION_CREDENTIALS\n\n }\n}\n\n\nclient AwsBedrock {\n provider aws-bedrock\n options {\n inference_configuration {\n max_tokens 100\n }\n // model \"anthropic.claude-3-5-sonnet-20240620-v1:0\"\n // model_id \"anthropic.claude-3-haiku-20240307-v1:0\"\n model_id \"meta.llama3-8b-instruct-v1:0\"\n // model_id \"mistral.mistral-7b-instruct-v0:2\"\n api_key \"\"\n }\n}\n\nclient AwsBedrockInvalidRegion {\n provider aws-bedrock\n options {\n region \"us-invalid-7\"\n inference_configuration {\n max_tokens 100\n }\n // model \"anthropic.claude-3-5-sonnet-20240620-v1:0\"\n // model_id \"anthropic.claude-3-haiku-20240307-v1:0\"\n model_id \"meta.llama3-8b-instruct-v1:0\"\n // model_id \"mistral.mistral-7b-instruct-v0:2\"\n api_key \"\"\n }\n}\n\nclient Sonnet {\n provider anthropic\n options {\n model claude-3-5-sonnet-20241022\n api_key env.ANTHROPIC_API_KEY\n }\n}\n\nclient Claude {\n provider anthropic\n options {\n model claude-3-haiku-20240307\n api_key env.ANTHROPIC_API_KEY\n max_tokens 1000\n }\n}\n\nclient ClaudeWithCaching {\n provider anthropic\n options {\n model claude-3-haiku-20240307\n api_key env.ANTHROPIC_API_KEY\n max_tokens 1000\n allowed_role_metadata [\"cache_control\"]\n headers {\n \"anthropic-beta\" \"prompt-caching-2024-07-31\"\n }\n }\n}\n\nclient Resilient_SimpleSyntax {\n retry_policy Foo\n provider baml-fallback\n options {\n strategy [\n GPT4Turbo\n GPT35\n Lottery_SimpleSyntax\n ]\n }\n} \n \nclient Lottery_SimpleSyntax {\n provider baml-round-robin\n options {\n start 0\n strategy [\n GPT35\n Claude\n ]\n }\n}\n\nclient TogetherAi {\n provider \"openai-generic\"\n options {\n base_url \"https://api.together.ai/v1\"\n api_key env.TOGETHER_API_KEY\n model \"meta-llama/Llama-3-70b-chat-hf\"\n }\n}\n\n", + "clients.baml": "retry_policy Bar {\n max_retries 3\n strategy {\n type exponential_backoff\n }\n}\n\nretry_policy Foo {\n max_retries 3\n strategy {\n type constant_delay\n delay_ms 100\n }\n}\n\nclient GPT4 {\n provider openai\n options {\n model gpt-4o\n api_key env.OPENAI_API_KEY\n }\n} \n\n\nclient GPT4o {\n provider openai\n options {\n model gpt-4o\n api_key env.OPENAI_API_KEY\n }\n} \n\n\nclient GPT4Turbo {\n retry_policy Bar\n provider openai\n options {\n model gpt-4-turbo\n api_key env.OPENAI_API_KEY\n }\n} \n\nclient GPT35 {\n provider openai\n options {\n model \"gpt-3.5-turbo\"\n api_key env.OPENAI_API_KEY\n }\n}\n\nclient GPT35LegacyProvider {\n provider openai\n options {\n model \"gpt-3.5-turbo\"\n api_key env.OPENAI_API_KEY\n }\n}\n\n\nclient Ollama {\n provider ollama\n options {\n model llama2\n }\n}\n\nclient GPT35Azure {\n provider azure-openai\n options {\n resource_name \"west-us-azure-baml\"\n deployment_id \"gpt-35-turbo-default\"\n // base_url \"https://west-us-azure-baml.openai.azure.com/openai/deployments/gpt-35-turbo-default\"\n api_version \"2024-02-01\"\n api_key env.AZURE_OPENAI_API_KEY\n }\n}\n\nclient Gemini {\n provider google-ai\n options {\n model gemini-1.5-pro-001\n api_key env.GOOGLE_API_KEY\n safetySettings {\n category HARM_CATEGORY_HATE_SPEECH\n threshold BLOCK_LOW_AND_ABOVE\n }\n }\n}\n\nclient Vertex {\n provider vertex-ai \n options {\n model gemini-1.5-pro\n location us-central1\n credentials env.INTEG_TESTS_GOOGLE_APPLICATION_CREDENTIALS\n }\n}\n\n\nclient AwsBedrock {\n provider aws-bedrock\n options {\n inference_configuration {\n max_tokens 100\n }\n // model \"anthropic.claude-3-5-sonnet-20240620-v1:0\"\n // model_id \"anthropic.claude-3-haiku-20240307-v1:0\"\n model_id \"meta.llama3-8b-instruct-v1:0\"\n // model_id \"mistral.mistral-7b-instruct-v0:2\"\n }\n}\n\nclient AwsBedrockInvalidRegion {\n provider aws-bedrock\n options {\n region \"us-invalid-7\"\n inference_configuration {\n max_tokens 100\n }\n // model \"anthropic.claude-3-5-sonnet-20240620-v1:0\"\n // model_id \"anthropic.claude-3-haiku-20240307-v1:0\"\n model_id \"meta.llama3-8b-instruct-v1:0\"\n // model_id \"mistral.mistral-7b-instruct-v0:2\"\n }\n}\n\nclient Sonnet {\n provider anthropic\n options {\n model claude-3-5-sonnet-20241022\n api_key env.ANTHROPIC_API_KEY\n }\n}\n\nclient Claude {\n provider anthropic\n options {\n model claude-3-haiku-20240307\n api_key env.ANTHROPIC_API_KEY\n max_tokens 1000\n }\n}\n\nclient ClaudeWithCaching {\n provider anthropic\n options {\n model claude-3-haiku-20240307\n api_key env.ANTHROPIC_API_KEY\n max_tokens 500\n allowed_role_metadata [\"cache_control\"]\n headers {\n \"anthropic-beta\" \"prompt-caching-2024-07-31\"\n }\n }\n}\n\nclient Resilient_SimpleSyntax {\n retry_policy Foo\n provider baml-fallback\n options {\n strategy [\n GPT4Turbo\n GPT35\n Lottery_SimpleSyntax\n ]\n }\n} \n \nclient Lottery_SimpleSyntax {\n provider baml-round-robin\n options {\n start 0\n strategy [\n GPT35\n Claude\n ]\n }\n}\n\nclient TogetherAi {\n provider \"openai-generic\"\n options {\n base_url \"https://api.together.ai/v1\"\n api_key env.TOGETHER_API_KEY\n model \"meta-llama/Llama-3-70b-chat-hf\"\n }\n}\n\n", "custom-task.baml": "class BookOrder {\n orderId string @description(#\"\n The ID of the book order\n \"#)\n title string @description(#\"\n The title of the ordered book\n \"#)\n quantity int @description(#\"\n The quantity of books ordered\n \"#)\n price float @description(#\"\n The price of the book\n \"#)\n}\n\nclass FlightConfirmation {\n confirmationNumber string @description(#\"\n The flight confirmation number\n \"#)\n flightNumber string @description(#\"\n The flight number\n \"#)\n departureTime string @description(#\"\n The scheduled departure time of the flight\n \"#)\n arrivalTime string @description(#\"\n The scheduled arrival time of the flight\n \"#)\n seatNumber string @description(#\"\n The seat number assigned on the flight\n \"#)\n}\n\nclass GroceryReceipt {\n receiptId string @description(#\"\n The ID of the grocery receipt\n \"#)\n storeName string @description(#\"\n The name of the grocery store\n \"#)\n items (string | int | float)[] @description(#\"\n A list of items purchased. Each item consists of a name, quantity, and price.\n \"#)\n totalAmount float @description(#\"\n The total amount spent on groceries\n \"#)\n}\n\nclass CustomTaskResult {\n bookOrder BookOrder | null\n flightConfirmation FlightConfirmation | null\n groceryReceipt GroceryReceipt | null\n}\n\nfunction CustomTask(input: string) -> BookOrder | FlightConfirmation | GroceryReceipt {\n client \"openai/gpt-4o-mini\"\n prompt #\"\n Given the input string, extract either an order for a book, a flight confirmation, or a grocery receipt.\n\n {{ ctx.output_format }}\n\n Input:\n \n {{ input}}\n \"#\n}\n\ntest CustomTask {\n functions [CustomTask]\n args {\n input #\"\nDear [Your Name],\n\nThank you for booking with [Airline Name]! We are pleased to confirm your upcoming flight.\n\nFlight Confirmation Details:\n\nBooking Reference: ABC123\nPassenger Name: [Your Name]\nFlight Number: XY789\nDeparture Date: September 15, 2024\nDeparture Time: 10:30 AM\nArrival Time: 1:45 PM\nDeparture Airport: John F. Kennedy International Airport (JFK), New York, NY\nArrival Airport: Los Angeles International Airport (LAX), Los Angeles, CA\nSeat Number: 12A\nClass: Economy\nBaggage Allowance:\n\nChecked Baggage: 1 piece, up to 23 kg\nCarry-On Baggage: 1 piece, up to 7 kg\nImportant Information:\n\nPlease arrive at the airport at least 2 hours before your scheduled departure.\nCheck-in online via our website or mobile app to save time at the airport.\nEnsure that your identification documents are up to date and match the name on your booking.\nContact Us:\n\nIf you have any questions or need to make changes to your booking, please contact our customer service team at 1-800-123-4567 or email us at support@[airline].com.\n\nWe wish you a pleasant journey and thank you for choosing [Airline Name].\n\nBest regards,\n\n[Airline Name] Customer Service\n \"#\n }\n}", "fiddle-examples/chain-of-thought.baml": "class Email {\n subject string\n body string\n from_address string\n}\n\nenum OrderStatus {\n ORDERED\n SHIPPED\n DELIVERED\n CANCELLED\n}\n\nclass OrderInfo {\n order_status OrderStatus\n tracking_number string?\n estimated_arrival_date string?\n}\n\nfunction GetOrderInfo(email: Email) -> OrderInfo {\n client GPT4\n prompt #\"\n Given the email below:\n\n ```\n from: {{email.from_address}}\n Email Subject: {{email.subject}}\n Email Body: {{email.body}}\n ```\n\n Extract this info from the email in JSON format:\n {{ ctx.output_format }}\n\n Before you output the JSON, please explain your\n reasoning step-by-step. Here is an example on how to do this:\n 'If we think step by step we can see that ...\n therefore the output JSON is:\n {\n ... the json schema ...\n }'\n \"#\n}", "fiddle-examples/chat-roles.baml": "// This will be available as an enum in your Python and Typescript code.\nenum Category2 {\n Refund\n CancelOrder\n TechnicalSupport\n AccountIssue\n Question\n}\n\nfunction ClassifyMessage2(input: string) -> Category {\n client GPT4\n\n prompt #\"\n {{ _.role(\"system\") }}\n // You can use _.role(\"system\") to indicate that this text should be a system message\n\n Classify the following INPUT into ONE\n of the following categories:\n\n {{ ctx.output_format }}\n\n {{ _.role(\"user\") }}\n // And _.role(\"user\") to indicate that this text should be a user message\n\n INPUT: {{ input }}\n\n Response:\n \"#\n}", @@ -86,14 +86,14 @@ "test-files/functions/prompts/no-chat-messages.baml": "\n\nfunction PromptTestClaude(input: string) -> string {\n client Sonnet\n prompt #\"\n Tell me a haiku about {{ input }}\n \"#\n}\n\n\nfunction PromptTestStreaming(input: string) -> string {\n client GPT35\n prompt #\"\n Tell me a short story about {{ input }}\n \"#\n}\n\ntest TestName {\n functions [PromptTestStreaming]\n args {\n input #\"\n hello world\n \"#\n }\n}\n", "test-files/functions/prompts/with-chat-messages.baml": "\nfunction PromptTestOpenAIChat(input: string) -> string {\n client GPT35\n prompt #\"\n {{ _.role(\"system\") }}\n You are an assistant that always responds in a very excited way with emojis and also outputs this word 4 times after giving a response: {{ input }}\n \n {{ _.role(\"user\") }}\n Tell me a haiku about {{ input }}\n \"#\n}\n\nfunction PromptTestOpenAIChatNoSystem(input: string) -> string {\n client GPT35\n prompt #\"\n You are an assistant that always responds in a very excited way with emojis and also outputs this word 4 times after giving a response: {{ input }}\n \n {{ _.role(\"user\") }}\n Tell me a haiku about {{ input }}\n \"#\n}\n\nfunction PromptTestClaudeChat(input: string) -> string {\n client Claude\n prompt #\"\n {{ _.role(\"system\") }}\n You are an assistant that always responds in a very excited way with emojis and also outputs this word 4 times after giving a response: {{ input }}\n \n {{ _.role(\"user\") }}\n Tell me a haiku about {{ input }}\n \"#\n}\n\nfunction PromptTestClaudeChatNoSystem(input: string) -> string {\n client Claude\n prompt #\"\n You are an assistant that always responds in a very excited way with emojis and also outputs this word 4 times after giving a response: {{ input }}\n \n {{ _.role(\"user\") }}\n Tell me a haiku about {{ input }}\n \"#\n}\n\ntest TestSystemAndNonSystemChat1 {\n functions [PromptTestClaude, PromptTestOpenAI, PromptTestOpenAIChat, PromptTestOpenAIChatNoSystem, PromptTestClaudeChat, PromptTestClaudeChatNoSystem]\n args {\n input \"cats\"\n }\n}\n\ntest TestSystemAndNonSystemChat2 {\n functions [PromptTestClaude, PromptTestOpenAI, PromptTestOpenAIChat, PromptTestOpenAIChatNoSystem, PromptTestClaudeChat, PromptTestClaudeChatNoSystem]\n args {\n input \"lion\"\n }\n}", "test-files/functions/v2/basic.baml": "\n\nfunction ExtractResume2(resume: string) -> Resume {\n client GPT4\n prompt #\"\n {{ _.role('system') }}\n\n Extract the following information from the resume:\n\n Resume:\n <<<<\n {{ resume }}\n <<<<\n\n Output JSON schema:\n {{ ctx.output_format }}\n\n JSON:\n \"#\n}\n\n\nclass WithReasoning {\n value string\n reasoning string @description(#\"\n Why the value is a good fit.\n \"#)\n}\n\n\nclass SearchParams {\n dateRange int? @description(#\"\n In ISO duration format, e.g. P1Y2M10D.\n \"#)\n location string[]\n jobTitle WithReasoning? @description(#\"\n An exact job title, not a general category.\n \"#)\n company WithReasoning? @description(#\"\n The exact name of the company, not a product or service.\n \"#)\n description WithReasoning[] @description(#\"\n Any specific projects or features the user is looking for.\n \"#)\n tags (Tag | string)[]\n}\n\nenum Tag {\n Security\n AI\n Blockchain\n}\n\nfunction GetQuery(query: string) -> SearchParams {\n client GPT4\n prompt #\"\n Extract the following information from the query:\n\n Query:\n <<<<\n {{ query }}\n <<<<\n\n OUTPUT_JSON_SCHEMA:\n {{ ctx.output_format }}\n\n Before OUTPUT_JSON_SCHEMA, list 5 intentions the user may have.\n --- EXAMPLES ---\n 1. \n 2. \n 3. \n 4. \n 5. \n\n {\n ... // OUTPUT_JSON_SCHEMA\n }\n \"#\n}\n\nclass RaysData {\n dataType DataType\n value Resume | Event\n}\n\nenum DataType {\n Resume\n Event\n}\n\nclass Event {\n title string\n date string\n location string\n description string\n}\n\nfunction GetDataType(text: string) -> RaysData {\n client GPT4\n prompt #\"\n Extract the relevant info.\n\n Text:\n <<<<\n {{ text }}\n <<<<\n\n Output JSON schema:\n {{ ctx.output_format }}\n\n JSON:\n \"#\n}", - "test-files/providers/providers.baml": "function TestAnthropic(input: string) -> string {\n client Claude\n prompt #\"\n Write a nice haiku about {{ input }}\n \"#\n}\n\nfunction PromptTestOpenAI(input: string) -> string {\n client GPT35\n prompt #\"\n Write a nice haiku about {{ input }}\n \"#\n}\n\nfunction TestOpenAILegacyProvider(input: string) -> string {\n client GPT35LegacyProvider\n prompt #\"\n Write a nice haiku about {{ input }}\n \"#\n}\n\nfunction TestAzure(input: string) -> string {\n client GPT35Azure\n prompt #\"\n Write a nice haiku about {{ input }}\n \"#\n}\n\nfunction TestOllama(input: string) -> string {\n client Ollama\n prompt #\"\n Write a nice haiku about {{ input }}\n \"#\n}\n\nfunction TestGemini(input: string) -> string {\n client Gemini\n prompt #\"\n Write a nice short story about {{ input }}\n \"#\n}\n\nfunction TestVertex(input: string) -> string {\n client Vertex\n prompt #\"\n Write a nice short story about {{ input }}\n \"#\n\n}\n\nfunction TestAws(input: string) -> string {\n client AwsBedrock\n prompt #\"\n Write a nice short story about {{ input }}\n \"#\n}\n\nfunction TestAwsInvalidRegion(input: string) -> string {\n client AwsBedrockInvalidRegion\n prompt #\"\n Write a nice short story about {{ input }}\n \"#\n}\n\nfunction TestOpenAIShorthand(input: string) -> string {\n client \"openai/gpt-4o-mini\"\n prompt #\"\n Write a nice short story about {{ input }}\n \"#\n}\n\nfunction TestAnthropicShorthand(input: string) -> string {\n client \"anthropic/claude-3-haiku-20240307\"\n prompt #\"\n Write a nice short story about {{ input }}\n \"#\n}\n\ntest TestProvider {\n functions [\n TestAnthropic, TestVertex, PromptTestOpenAI, TestAzure, TestOllama, TestGemini, TestAws,\n TestAwsInvalidRegion,\n TestOpenAIShorthand,\n TestAnthropicShorthand\n ]\n args {\n input \"Donkey kong and peanut butter\"\n }\n}\n\n\nfunction TestCaching(input: string, not_cached: string) -> string {\n client ClaudeWithCaching\n prompt #\"\n {{ _.role('system', cache_control={\"type\": \"ephemeral\"}) }}\n Describe this in 5 words: {{ input }}\n {{ _.role('user') }}\n {{ not_cached }}\n \"#\n}\n", + "test-files/providers/providers.baml": "function TestAnthropic(input: string) -> string {\n client Claude\n prompt #\"\n Write a nice haiku about {{ input }}\n \"#\n}\n\nfunction PromptTestOpenAI(input: string) -> string {\n client GPT35\n prompt #\"\n Write a nice haiku about {{ input }}\n \"#\n}\n\nfunction TestOpenAILegacyProvider(input: string) -> string {\n client GPT35LegacyProvider\n prompt #\"\n Write a nice haiku about {{ input }}\n \"#\n}\n\nfunction TestAzure(input: string) -> string {\n client GPT35Azure\n prompt #\"\n Write a nice haiku about {{ input }}\n \"#\n}\n\nfunction TestOllama(input: string) -> string {\n client Ollama\n prompt #\"\n Write a nice haiku about {{ input }}\n \"#\n}\n\nfunction TestGemini(input: string) -> string {\n client Gemini\n prompt #\"\n Write a nice short story about {{ input }}\n \"#\n}\n\nfunction TestVertex(input: string) -> string {\n client Vertex\n prompt #\"\n Write a nice short story about {{ input }}\n \"#\n\n}\n\nfunction TestAws(input: string) -> string {\n client AwsBedrock\n prompt #\"\n Write a nice short story about {{ input }}\n \"#\n}\n\nfunction TestAwsInvalidRegion(input: string) -> string {\n client AwsBedrockInvalidRegion\n prompt #\"\n Write a nice short story about {{ input }}\n \"#\n}\n\nfunction TestOpenAIShorthand(input: string) -> string {\n client \"openai/gpt-4o-mini\"\n prompt #\"\n Write a nice short story about {{ input }}\n \"#\n}\n\nfunction TestAnthropicShorthand(input: string) -> string {\n client \"anthropic/claude-3-haiku-20240307\"\n prompt #\"\n Write a nice short story about {{ input }}\n \"#\n}\n\ntest TestProvider {\n functions [\n TestAnthropic, TestVertex, PromptTestOpenAI, TestAzure, TestOllama, TestGemini, TestAws,\n TestAwsInvalidRegion,\n TestOpenAIShorthand,\n TestAnthropicShorthand\n ]\n args {\n input \"Donkey kong and peanut butter\"\n }\n}\n\n\nfunction TestCaching(input: string, not_cached: string) -> string {\n client ClaudeWithCaching\n prompt #\"\n {{ _.role('system', cache_control={\"type\": \"ephemeral\"}) }}\n Generate the following story\n {{ input }}\n\n {# Haiku require 2048 tokens to cache -#}\n {{ input }}\n\n {{ _.role('user') }}\n {{ not_cached }}\n \"#\n}\n\ntest TestName {\n functions [TestCaching]\n args {\n input #\"\nIn a near-future society where dreams have become a tradable commodity and shared experience, a lonely and socially awkward teenager named Alex discovers they possess a rare and powerful ability to not only view but also manipulate the dreams of others. Initially thrilled by this newfound power, Alex begins subtly altering the dreams of classmates and family members, helping them overcome fears, boost confidence, or experience fantastical adventures. As Alex's skills grow, so does their influence. They start selling premium dream experiences on the black market, crafting intricate and addictive dreamscapes for wealthy clients. However, the line between dream and reality begins to blur for those exposed to Alex's creations. Some clients struggle to differentiate between their true memories and the artificial ones implanted by Alex's dream manipulation.\n\nComplications arise when a mysterious government agency takes notice of Alex's unique abilities. They offer Alex a chance to use their gift for \"the greater good,\" hinting at applications in therapy, criminal rehabilitation, and even national security. Simultaneously, an underground resistance movement reaches out, warning Alex about the dangers of dream manipulation and the potential for mass control and exploitation. Caught between these opposing forces, Alex must navigate a complex web of ethical dilemmas. They grapple with questions of free will, the nature of consciousness, and the responsibility that comes with having power over people's minds. As the consequences of their actions spiral outward, affecting the lives of loved ones and strangers alike, Alex is forced to confront the true nature of their ability and decide how—or if—it should be used.\n\nThe story explores themes of identity, the subconscious mind, the ethics of technology, and the power of imagination. It delves into the potential consequences of a world where our most private thoughts and experiences are no longer truly our own, and examines the fine line between helping others and manipulating them for personal gain or a perceived greater good. The narrative further expands on the societal implications of such abilities, questioning the moral boundaries of altering consciousness and the potential for abuse in a world where dreams can be commodified. It challenges the reader to consider the impact of technology on personal autonomy and the ethical responsibilities of those who wield such power.\n\nAs Alex's journey unfolds, they encounter various individuals whose lives have been touched by their dream manipulations, each presenting a unique perspective on the ethical quandaries at hand. From a classmate who gains newfound confidence to a wealthy client who becomes addicted to the dreamscapes, the ripple effects of Alex's actions are profound and far-reaching. The government agency's interest in Alex's abilities raises questions about the potential for state control and surveillance, while the resistance movement highlights the dangers of unchecked power and the importance of safeguarding individual freedoms.\n\nUltimately, Alex's story is one of self-discovery and moral reckoning, as they must decide whether to embrace their abilities for personal gain, align with the government's vision of a controlled utopia, or join the resistance in their fight for freedom and autonomy. The narrative invites readers to reflect on the nature of reality, the boundaries of human experience, and the ethical implications of a world where dreams are no longer private sanctuaries but shared and manipulated commodities. It also explores the psychological impact on Alex, who must deal with the burden of knowing the intimate fears and desires of others, and the isolation that comes from being unable to share their own dreams without altering them.\n\nThe story further examines the technological advancements that have made dream manipulation possible, questioning the role of innovation in society and the potential for both progress and peril. It considers the societal divide between those who can afford to buy enhanced dream experiences and those who cannot, highlighting issues of inequality and access. As Alex becomes more entangled in the web of their own making, they must confront the possibility that their actions could lead to unintended consequences, not just for themselves but for the fabric of society as a whole.\n\nIn the end, Alex's journey is a cautionary tale about the power of dreams and the responsibilities that come with wielding such influence. It serves as a reminder of the importance of ethical considerations in the face of technological advancement and the need to balance innovation with humanity. The story leaves readers pondering the true cost of a world where dreams are no longer sacred, and the potential for both wonder and danger in the uncharted territories of the mind. But it's also a story about the power of imagination and the potential for change, even in a world where our deepest thoughts are no longer our own. And it's a story about the power of choice, and the importance of fighting for the freedom to dream.\n\nIn conclusion, this story is a reflection on the power of dreams and the responsibilities that come with wielding such influence. It serves as a reminder of the importance of ethical considerations in the face of technological advancement and the need to balance innovation with humanity. The story leaves readers pondering the true cost of a world where dreams are no longer sacred, and the potential for both wonder and danger in the uncharted territories of the mind. But it's also a story about the power of imagination and the potential for change, even in a world where our deepest thoughts are no longer our own. And it's a story about the power of choice, and the importance of fighting for the freedom to dream.\n \"#\n not_cached #\"\n hello world\n \"#\n }\n}\n", "test-files/strategies/fallback-shorthand.baml": "\nclient FallbackToShorthand {\n provider fallback\n options {\n strategy [\n \"openai/does-not-exist\",\n \"openai/gpt-4o-mini\"\n ]\n }\n}\n\n\nfunction TestFallbackToShorthand(input: string) -> string {\n client FallbackToShorthand\n // TODO make it return the client name instead\n prompt #\"\n Say a haiku about {{input}}.\n \"#\n}\n\ntest TestProvider_FallbackToShorthand {\n functions [\n TestFallbackToShorthand\n ]\n args {\n input \"Donkey kong and peanut butter\"\n }\n}\n", - "test-files/strategies/fallback.baml": "// Happy path fallbacks.\nclient FaultyClient {\n provider openai\n options {\n model unknown-model\n api_key env.OPENAI_API_KEY\n }\n}\n\n\nclient FallbackClient {\n provider fallback\n options {\n // first 2 clients are expected to fail.\n strategy [\n FaultyClient,\n RetryClientConstant,\n GPT35\n Gemini\n\n ]\n }\n}\n\nfunction TestFallbackClient() -> string {\n client FallbackClient\n // TODO make it return the client name instead\n prompt #\"\n Say a haiku about mexico.\n \"#\n}\n\n// Fallbacks should fail gracefully.\nclient FaultyAzureClient {\n provider azure-openai\n options {\n model unknown-model\n api_key env.OPENAI_API_KEY\n }\n}\n\nclient SingleFallbackClient {\n provider fallback\n options {\n // first 2 clients are expected to fail.\n strategy [\n FaultyAzureClient\n ]\n }\n}\n\nfunction TestSingleFallbackClient() -> string {\n client SingleFallbackClient\n // TODO make it return the client name instead\n prompt #\"\n Say a haiku about mexico.\n \"#\n}\n", + "test-files/strategies/fallback.baml": "// Happy path fallbacks.\nclient FaultyClient {\n provider openai\n options {\n model unknown-model\n api_key env.OPENAI_API_KEY\n }\n}\n\n\nclient FallbackClient {\n provider fallback\n options {\n // first 2 clients are expected to fail.\n strategy [\n FaultyClient,\n RetryClientConstant,\n GPT35\n Gemini\n\n ]\n }\n}\n\nfunction TestFallbackClient() -> string {\n client FallbackClient\n // TODO make it return the client name instead\n prompt #\"\n Say a haiku about mexico.\n \"#\n}\n\n// Fallbacks should fail gracefully.\nclient FaultyAzureClient {\n provider azure-openai\n options {\n model unknown-model\n resource_name \"unknown-resource-id\"\n deployment_id \"unknown-deployment-id\"\n }\n}\n\nclient SingleFallbackClient {\n provider fallback\n options {\n // first 2 clients are expected to fail.\n strategy [\n FaultyAzureClient\n ]\n }\n}\n\nfunction TestSingleFallbackClient() -> string {\n client SingleFallbackClient\n // TODO make it return the client name instead\n prompt #\"\n Say a haiku about mexico.\n \"#\n}\n", "test-files/strategies/retry.baml": "\nretry_policy Exponential {\n max_retries 3\n strategy {\n type exponential_backoff\n }\n}\n\nretry_policy Constant {\n max_retries 3\n strategy {\n type constant_delay\n delay_ms 100\n }\n}\n\nclient RetryClientConstant {\n provider openai\n retry_policy Constant\n options {\n model \"gpt-3.5-turbo\"\n api_key \"blah\"\n }\n}\n\nclient RetryClientExponential {\n provider openai\n retry_policy Exponential\n options {\n model \"gpt-3.5-turbo\"\n api_key \"blahh\"\n }\n}\n\nfunction TestRetryConstant() -> string {\n client RetryClientConstant\n prompt #\"\n Say a haiku\n \"#\n}\n\nfunction TestRetryExponential() -> string {\n client RetryClientExponential\n prompt #\"\n Say a haiku\n \"#\n}\n", "test-files/strategies/roundrobin.baml": "", "test-files/template_string/template_string.baml": "\nfunction Completion(prefix: string, suffix: string, language: string) -> string {\n client \"openai/gpt-4o\"\n prompt ##\"\n {{ _.role(\"system\", cache_control={\"type\": \"ephemeral\"}) }}\n\n You are a programmer that suggests code completions in the %INSERT-HERE% part below with {{ language }} code. Only output the code that replaces %INSERT-HERE% part, NOT THE SUFFIX OR PREFIX. Respond only with code, and with no markdown formatting.\n\n Try to complete a whole section inside curlies when you can.\n\n {% if language == \"baml\" %}\n {{ BAMLBackground2()}}\n\n Examples:\n INPUT:\n ---\n class MyObject {{\"{\"}}%INSERT-HERE%\n }\n ---\n OUTPUT:\n ---\n property string\n ---\n In this example, we just inserted one line, with tabs for a fake property to aid the user.\n\n INPUT:\n ---\n function foo(input: string) -> string {{\"{\"}} %INSERT-HERE%\n prompt #\"\n {{ \"{{ input }}\" }}\n \"#\n }\n ---\n OUTPUT:\n ---\n client \"openai/gpt-4o\"\n ---\n In this example, no need to add the prompt because it was part of the suffix after INSERT-HERE\n\n INPUT:\n OUTPUT: N/A\n In this example there was nothing to complete, so we returned N/A.\n\n Ignore the \"---\" in your outputs.\n {% endif %}\n\n\n {{ _.role(\"user\") }}\n INPUT:\n ---\n {{ prefix }}%INSERT-HERE%{{ suffix }}\n ---\n \"##\n}\n\ntest CompletionTest3 {\n functions [Completion]\n args {\n prefix ##\"function foo(input: string) -> string {\n client \"openai/gpt-4o\"\n prompt #\"\n \"##\n suffix \"\"\n language \"baml\"\n }\n}\n\ntest CompletionTest2 {\n functions [Completion]\n args {\n prefix \"function foo(input: string) -> string {\\n\"\n suffix \"\\n prompt #\\n\\\"\"\n language \"baml\"\n }\n}\n \ntemplate_string Hi(\n hello: string,\n world: string,\n) ##\"\n {{ hello }} {{ world }}\n\"##\n\ntemplate_string Hi3(\n hello: string,\n world: string,\n) #\"\n {{ hello }} {{ world }}\n\"#\n\ntemplate_string BAMLBackground2() ##\"\n \n BAML is a domain-specific language for building LLM prompts as functions.\n client \"openai/gpt-4o\"\n // prompt with jinja syntax inside here. with double curly braces for variables.\n // make sure to include: {{ \"{{ ctx.output_format }}\"}} in the prompt, which prints the output schema instructions so the LLM returns the output in the correct format (json or string, etc.). DO NOT write the output schema manually.\n prompt #\"\n \n \"#\n }\n\n 3. You do not need to specify to \"answer in JSON format\". Only write in the prompt brief instruction, and any other task-specific things to keep in mind for the task.\n 4. Write a {{ \"{{ _.role(\\\"user\\\") }}\" }} tag to indicate where the user's inputs start. So if there's a convo you can write\n #\"{{ \"{{ _.role(\\\"user\\\") }}\" }} {{ \"{{ some-variable }}\" }}#\n \n \n\n The @asserts only go in the \"output\" types. Don't use them in inputs.\n Do NOT use numbers as confidence intervals if you need to use them. Prefer an enum with descriptions or literals like \"high\", \"medium\", \"low\".\n\n Dedent all declarations.\n\"##\n\ntemplate_string BamlTests() ##\"\n // For image inputs:\n test ImageTest {\n functions [MyFunction]\n args {\n imageArg {\n file \"../images/test.png\"\n // Optional: media_type \"image/png\"\n }\n // Or using URL:\n // imageArg {\n // url \"https://example.com/image.png\"\n // }\n }\n }\n\n // For array/object inputs:\n test ComplexTest {\n functions [MyFunction]\n args {\n input {\n name \"Complex Object\"\n tags [\n \"tag1\",\n #\"\n Multi-line\n tag here\n \"#\n ]\n status PENDING\n type \"error\"\n count 100\n enabled false\n score 7.8\n }\n }\n }\n\"##\n", "test-files/testing_pipeline/output-format.baml": "class Recipe {\n ingredients map\n recipe_type \"breakfast\" | \"dinner\"\n}\n\nclass Quantity {\n amount int | float\n unit string?\n}\n\nfunction AaaSamOutputFormat(recipe: string) -> Recipe {\n client GPT35\n prompt #\"\n Return this value back to me: {{recipe}}\n\n {{ctx.output_format(map_style='angle')}}\n \"#\n}\n\ntest MyOutput {\n functions [AaaSamOutputFormat]\n args {\n recipe #\"\n Here's a simple recipe for beef stew:\nIngredients:\n\n2 lbs beef chuck, cut into 1-inch cubes\n2 tbsp vegetable oil\n1 onion, diced\n3 carrots, sliced\n2 celery stalks, chopped\n2 potatoes, cubed\n3 cloves garlic, minced\n4 cups beef broth\n1 can (14.5 oz) diced tomatoes\n1 tbsp Worcestershire sauce\n1 tsp dried thyme\n1 bay leaf\nSalt and pepper to taste\n\nInstructions:\n\nSeason beef with salt and pepper. Heat oil in a large pot over medium-high heat. Brown the beef in batches, then set aside.\nIn the same pot, sauté onion, carrots, and celery until softened, about 5 minutes.\nAdd garlic and cook for another minute.\nReturn beef to the pot. Add broth, tomatoes, Worcestershire sauce, thyme, and bay leaf.\nBring to a boil, then reduce heat and simmer covered for 1 hour.\nAdd potatoes and continue simmering for 30-45 minutes, until beef and potatoes are tender.\nRemove bay leaf, adjust seasoning if needed, and serve hot.\n\nWould you like any additional information or variations on this recipe?\n \"#\n }\n}\n", - "test-files/testing_pipeline/resume.baml": "class Resume {\n name string\n email string\n phone string\n experience Education[]\n education string[]\n skills string[]\n}\n\nclass Education {\n institution string\n location string\n degree string\n major string[]\n graduation_date string?\n}\n\ntemplate_string AddRole(foo: string) #\"\n {{ _.role('system')}}\n You are a {{ foo }}. be nice\n\n {{ _.role('user') }}\n\"#\n\nclient TestClient {\n provider fallback\n retry_policy Constant\n options {\n strategy [\n Claude\n GPT35\n AwsBedrock\n ]\n }\n}\n\nclient Claude2 {\n provider anthropic\n options {\n model claude-3-haiku-20240307\n api_key env.FOOBAR3\n max_tokens 1000\n }\n}\n\nfunction ExtractResume(resume: string, img: image?) -> Resume {\n client Claude2\n prompt #\"\n {{ AddRole(\"Software Engineer\") }}\n\n Extract data:\n \n\n <<<<\n {{ resume }}\n <<<<\n\n {% if img %}\n {{img}}\n {% endif %}\n\n {{ ctx.output_format }}\n \"#\n}\n\ntest sam_resume {\n functions [ExtractResume]\n input {\n img {\n url \"https://avatars.githubusercontent.com/u/1016595?v=4\"\n }\n resume #\"\n Sam Lijin\n he/him | jobs@sxlijin.com | sxlijin.github.io | 111-222-3333 | sxlijin | sxlijin\n\n Experience\n Trunk\n | July 2021 - current\n Trunk Check | Senior Software Engineer | Services TL, Mar 2023 - current | IC, July 2021 - Feb 2023\n Proposed, designed, and led a team of 3 to build a web experience for Check (both a web-only onboarding flow and SaaS offerings)\n Proposed and built vulnerability scanning into Check, enabling it to compete with security products such as Snyk\n Helped grow Check from <1K users to 90K+ users by focusing on product-led growth\n Google | Sept 2017 - June 2021\n User Identity SRE | Senior Software Engineer | IC, Mar 2021 - June 2021\n Designed an incremental key rotation system to limit the global outage risk to Google SSO\n Discovered and severed an undocumented Gmail serving dependency on Identity-internal systems\n Cloud Firestore | Senior Software Engineer | EngProd TL, Aug 2019 - Feb 2021 | IC, Sept 2017 - July 2019\n Metadata TTL system: backlog of XX trillion records, sustained 1M ops/sec, peaking at 3M ops/sec\n\n Designed and implemented a logging system with novel observability and privacy requirements\n Designed and implemented Jepsen-style testing to validate correctness guarantees\n Datastore Migration: zero downtime, xM RPS and xxPB of data over xM customers and 36 datacenters\n\n Designed composite index migration, queue processing migration, progressive rollout, fast rollback, and disk stockout mitigations; implemented transaction log replay, state transitions, and dark launch process\n Designed and implemented end-to-end correctness and performance testing\n Velocity improvements for 60-eng org\n\n Proposed and implemented automated rollbacks: got us out of a 3-month release freeze and prevented 5 outages over the next 6 months\n Proposed and implemented new development and release environments spanning 30+ microservices\n Incident response for API proxy rollback affecting every Google Cloud service\n\n Google App Engine Memcache | Software Engineer | EngProd TL, Apr 2019 - July 2019\n Proposed and led execution of test coverage improvement strategy for a new control plane: reduced rollbacks and ensured strong consistency of a distributed cache serving xxM QPS\n Designed and implemented automated performance regression testing for two critical serving paths\n Used to validate Google-wide rollout of AMD CPUs, by proving a 50p latency delta of <10µs\n Implemented on shared Borg (i.e. vulnerable to noisy neighbors) with <12% variance\n Miscellaneous | Sept 2017 - June 2021\n Redesigned the Noogler training on Google-internal storage technologies & trained 2500+ Nooglers\n Landed multiple google3-wide refactorings, each spanning xxK files (e.g. SWIG to CLIF)\n Education\n Vanderbilt University (Nashville, TN) | May 2017 | B.S. in Computer Science, Mathematics, and Political Science\n\n Stuyvesant HS (New York, NY) | 2013\n\n Skills\n C++, Java, Typescript, Javascript, Python, Bash; light experience with Rust, Golang, Scheme\n gRPC, Bazel, React, Linux\n Hobbies: climbing, skiing, photography\n \"#\n }\n}\n\ntest vaibhav_resume {\n functions [ExtractResume]\n input {\n resume #\"\n Vaibhav Gupta\n linkedin/vaigup\n (972) 400-5279\n vaibhavtheory@gmail.com\n EXPERIENCE\n Google,\n Software Engineer\n Dec 2018-Present\n Seattle, WA\n •\n Augmented Reality,\n Depth Team\n •\n Technical Lead for on-device optimizations\n •\n Optimized and designed front\n facing depth algorithm\n on Pixel 4\n •\n Focus: C++ and SIMD on custom silicon\n \n \n EDUCATION\n University of Texas at Austin\n Aug 2012-May 2015\n Bachelors of Engineering, Integrated Circuits\n Bachelors of Computer Science\n \"#\n }\n}", + "test-files/testing_pipeline/resume.baml": "class Resume {\n name string\n email string\n phone string\n experience Education[]\n education string[]\n skills string[]\n}\n\nclass Education {\n institution string\n location string\n degree string\n major string[]\n graduation_date string?\n}\n\ntemplate_string AddRole(foo: string) #\"\n {{ _.role('system')}}\n You are a {{ foo }}. be nice\n\n {{ _.role('user') }}\n\"#\n\nclient TestClient {\n provider fallback\n retry_policy Constant\n options {\n strategy [\n Claude\n GPT35\n AwsBedrock\n ]\n }\n}\n\nclient Claude2 {\n provider anthropic\n options {\n model claude-3-haiku-20240307\n api_key env.FOOBAR3\n max_tokens 1000\n }\n}\n\nfunction ExtractResume(resume: string, img: image?) -> Resume {\n client Claude2\n prompt #\"\n {{ AddRole(\"Software Engineer\") }}\n\n Extract data:\n \n\n <<<<\n {{ resume }}\n <<<<\n\n {% if img %}\n {{img}}\n {% endif %}\n\n {{ ctx.output_format }}\n \"#\n}\n\ntest sam_resume {\n functions [ExtractResume]\n args {\n img {\n url \"https://avatars.githubusercontent.com/u/1016595?v=4\"\n }\n resume #\"\n Sam Lijin\n he/him | jobs@sxlijin.com | sxlijin.github.io | 111-222-3333 | sxlijin | sxlijin\n\n Experience\n Trunk\n | July 2021 - current\n Trunk Check | Senior Software Engineer | Services TL, Mar 2023 - current | IC, July 2021 - Feb 2023\n Proposed, designed, and led a team of 3 to build a web experience for Check (both a web-only onboarding flow and SaaS offerings)\n Proposed and built vulnerability scanning into Check, enabling it to compete with security products such as Snyk\n Helped grow Check from <1K users to 90K+ users by focusing on product-led growth\n Google | Sept 2017 - June 2021\n User Identity SRE | Senior Software Engineer | IC, Mar 2021 - June 2021\n Designed an incremental key rotation system to limit the global outage risk to Google SSO\n Discovered and severed an undocumented Gmail serving dependency on Identity-internal systems\n Cloud Firestore | Senior Software Engineer | EngProd TL, Aug 2019 - Feb 2021 | IC, Sept 2017 - July 2019\n Metadata TTL system: backlog of XX trillion records, sustained 1M ops/sec, peaking at 3M ops/sec\n\n Designed and implemented a logging system with novel observability and privacy requirements\n Designed and implemented Jepsen-style testing to validate correctness guarantees\n Datastore Migration: zero downtime, xM RPS and xxPB of data over xM customers and 36 datacenters\n\n Designed composite index migration, queue processing migration, progressive rollout, fast rollback, and disk stockout mitigations; implemented transaction log replay, state transitions, and dark launch process\n Designed and implemented end-to-end correctness and performance testing\n Velocity improvements for 60-eng org\n\n Proposed and implemented automated rollbacks: got us out of a 3-month release freeze and prevented 5 outages over the next 6 months\n Proposed and implemented new development and release environments spanning 30+ microservices\n Incident response for API proxy rollback affecting every Google Cloud service\n\n Google App Engine Memcache | Software Engineer | EngProd TL, Apr 2019 - July 2019\n Proposed and led execution of test coverage improvement strategy for a new control plane: reduced rollbacks and ensured strong consistency of a distributed cache serving xxM QPS\n Designed and implemented automated performance regression testing for two critical serving paths\n Used to validate Google-wide rollout of AMD CPUs, by proving a 50p latency delta of <10µs\n Implemented on shared Borg (i.e. vulnerable to noisy neighbors) with <12% variance\n Miscellaneous | Sept 2017 - June 2021\n Redesigned the Noogler training on Google-internal storage technologies & trained 2500+ Nooglers\n Landed multiple google3-wide refactorings, each spanning xxK files (e.g. SWIG to CLIF)\n Education\n Vanderbilt University (Nashville, TN) | May 2017 | B.S. in Computer Science, Mathematics, and Political Science\n\n Stuyvesant HS (New York, NY) | 2013\n\n Skills\n C++, Java, Typescript, Javascript, Python, Bash; light experience with Rust, Golang, Scheme\n gRPC, Bazel, React, Linux\n Hobbies: climbing, skiing, photography\n \"#\n }\n}\n\ntest vaibhav_resume {\n functions [ExtractResume]\n args {\n resume #\"\n Vaibhav Gupta\n linkedin/vaigup\n (972) 400-5279\n vaibhavtheory@gmail.com\n EXPERIENCE\n Google,\n Software Engineer\n Dec 2018-Present\n Seattle, WA\n •\n Augmented Reality,\n Depth Team\n •\n Technical Lead for on-device optimizations\n •\n Optimized and designed front\n facing depth algorithm\n on Pixel 4\n •\n Focus: C++ and SIMD on custom silicon\n \n \n EDUCATION\n University of Texas at Austin\n Aug 2012-May 2015\n Bachelors of Engineering, Integrated Circuits\n Bachelors of Computer Science\n \"#\n }\n}", } def get_baml_files(): diff --git a/integ-tests/python/report.html b/integ-tests/python/report.html index 0bafb9d9c..61d9ab562 100644 --- a/integ-tests/python/report.html +++ b/integ-tests/python/report.html @@ -3,11 +3,11 @@
Test Report

Summary

Started 2024-11-11 13:26:51
Ended 2024-11-11 13:30:07
Duration 0:03:16.206012
Total run time 0:03:14.943801
97
3 failed 93 passed 1 skipped

Warnings

Warnings 1

  • PytestUnhandledCoroutineWarning async def functions are not natively supported and have been skipped. You need to install a suitable plugin for your async framework, for example: - anyio - pytest-asyncio - pytest-tornasync - pytest-trio - pytest-twisted
    /Users/vbv/Library/Caches/pypoetry/virtualenvs/python-integ-tests-lSPb1NE9-py3.12/lib/python3.12/site-packages/_pytest/python.py:148

Tests

tests/test_functions.py 3901 0:03:14.941336

PASSED test_env_vars_reset 0:00:03.317093

Started 2024-11-11 13:26:52
Ended 2024-11-11 13:26:55
Duration 0:00:03.317093
Markers
asyncio
Fixtures event_loop_policy cleanup event_loop

Setup

Call

Captured stdout call
Context depth is greater than 0!
+    
 
 Test Report 
 
 
 
 
 
Summary
 
 
 
 Started 2024-11-25 16:34:22 
 
 Ended  2024-11-25 16:37:37  
 
 Duration 0:03:15.093315 
 
 Total run time 0:03:14.063044 
 
 
 
      100  
  2  failed  98  passed 
 
 
 
 
 
 
Tests
 
  
  tests/test_functions.py   295   0:03:14.061376  
  
 
  
 PASSED  test_env_vars_reset  0:00:01.749110 
  
 
 
 Started 2024-11-25 16:34:22 
 
 Ended 2024-11-25 16:34:24 
 
 Duration 0:00:01.749110 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
Context depth is greater than 0!
 Except but ending trace!
 Context depth is greater than 0!
-
 
 
 
Captured stderr call
 
[2024-11-11T19:26:53Z WARN  baml_events] Function ExtractPeople:
-    Client: GPT4 (<unknown>) - 234ms
+
 
 
 
Captured stderr call
 
[2024-11-26T00:34:23Z WARN  baml_events] Function ExtractPeople:
+    Client: GPT4 (<unknown>) - 431ms
     ---PROMPT---
     [chat] system: You are an expert extraction algorithm. Only extract relevant information from the text. If you do not know the value of an attribute asked to extract, return null for the attribute's value.
     
@@ -23,7 +23,8 @@
     ---REQUEST OPTIONS---
     model: "gpt-4o"
     ---ERROR (InvalidAuthentication (401))---
-    Request failed: {
+    Request failed: https://api.openai.com/v1/chat/completions
+    {
         "error": {
             "message": "Incorrect API key provided: sk-12345*7890. You can find your API key at https://platform.openai.com/account/api-keys.",
             "type": "invalid_request_error",
@@ -32,8 +33,8 @@
         }
     }
     
-[2024-11-11T19:26:55Z INFO  baml_events] Function ExtractPeople:
-    Client: GPT4 (gpt-4o-2024-08-06) - 2696ms. StopReason: stop. Tokens(in/out): 124/22
+[2024-11-26T00:34:24Z INFO  baml_events] Function ExtractPeople:
+    Client: GPT4 (gpt-4o-2024-08-06) - 783ms. StopReason: stop. Tokens(in/out): 124/22
     ---PROMPT---
     [chat] system: You are an expert extraction algorithm. Only extract relevant information from the text. If you do not know the value of an attribute asked to extract, return null for the attribute's value.
     
@@ -60,13 +61,13 @@
         "hair_color": "BLACK"
       }
     ]
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_sync  0:00:00.420527 
  
 
 
 Started 2024-11-11 13:26:55 
 
 Ended 2024-11-11 13:26:56 
 
 Duration 0:00:00.420527 
 
 Markers   
 
 Fixtures   event_loop_policy   cleanup   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
got response key
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_sync  0:00:00.366913 
  
 
 
 Started 2024-11-25 16:34:24 
 
 Ended 2024-11-25 16:34:25 
 
 Duration 0:00:00.366913 
 
 Markers   
 
 Fixtures   event_loop_policy   cleanup   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
got response key
 true
 52
-
 
 
 
Captured stderr call
 
[2024-11-11T19:26:56Z INFO  baml_events] Function TestFnNamedArgsSingleClass:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 417ms. StopReason: stop. Tokens(in/out): 19/5
+
 
 
 
Captured stderr call
 
[2024-11-26T00:34:25Z INFO  baml_events] Function TestFnNamedArgsSingleClass:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 363ms. StopReason: stop. Tokens(in/out): 19/5
     ---PROMPT---
-    [chat] system: Print these values back to me:
+    [chat] user: Print these values back to me:
     key
     true
     52
@@ -77,39 +78,94 @@
     52
     ---Parsed Response (string)---
     "key\ntrue\n52"
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  TestAllInputs::test_single_bool  0:00:00.392597 
  
 
 
 Started 2024-11-11 13:26:56 
 
 Ended 2024-11-11 13:26:56 
 
 Duration 0:00:00.392597 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:26:56Z INFO  baml_events] Function TestFnNamedArgsSingleBool:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 388ms. StopReason: stop. Tokens(in/out): 15/1
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  TestAllInputs::test_single_bool  0:00:00.406893 
  
 
 
 Started 2024-11-25 16:34:25 
 
 Ended 2024-11-25 16:34:25 
 
 Duration 0:00:00.406893 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:34:25Z INFO  baml_events] Function TestFnNamedArgsSingleBool:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 404ms. StopReason: stop. Tokens(in/out): 15/1
     ---PROMPT---
-    [chat] system: Return this value back to me: true
+    [chat] user: Return this value back to me: true
     
     ---LLM REPLY---
     true
     ---Parsed Response (string)---
     "true"
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  TestAllInputs::test_single_string_list  0:00:00.517886 
  
 
 
 Started 2024-11-11 13:26:56 
 
 Ended 2024-11-11 13:26:57 
 
 Duration 0:00:00.517886 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:26:57Z INFO  baml_events] Function TestFnNamedArgsSingleStringList:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 511ms. StopReason: stop. Tokens(in/out): 23/9
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  TestAllInputs::test_single_string_list  0:00:00.357174 
  
 
 
 Started 2024-11-25 16:34:25 
 
 Ended 2024-11-25 16:34:25 
 
 Duration 0:00:00.357174 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:34:25Z INFO  baml_events] Function TestFnNamedArgsSingleStringList:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 354ms. StopReason: stop. Tokens(in/out): 23/9
     ---PROMPT---
-    [chat] system: Return this value back to me: ["a", "b", "c"]
+    [chat] user: Return this value back to me: ["a", "b", "c"]
     
     ---LLM REPLY---
     ["a", "b", "c"]
     ---Parsed Response (string)---
     "[\"a\", \"b\", \"c\"]"
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  TestAllInputs::test_return_literal_union  0:00:00.462986 
  
 
 
 Started 2024-11-11 13:26:57 
 
 Ended 2024-11-11 13:26:57 
 
 Duration 0:00:00.462986 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:26:57Z INFO  baml_events] Function LiteralUnionsTest:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 453ms. StopReason: stop. Tokens(in/out): 31/1
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 FAILED  TestAllInputs::test_return_literal_union  0:00:00.428855 
  
 
 
 Started 2024-11-25 16:34:25 
 
 Ended 2024-11-25 16:34:26 
 
 Duration 0:00:00.428855 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
baml_py.internal_monkeypatch.BamlValidationError: BamlValidationError(message=Failed to parse LLM response: Failed to coerce value: : Failed to find any (1 | true | "string output") in 3 items
+  - : Expected 1, got Object([("Answer", Number(Number(1)))]).
+  - : Expected true, got Object([("Answer", Number(Number(1)))]).
+  - : Expected "string output", got Object([("Answer", Number(Number(1)))])., raw_output={
+  "Answer": 1
+}, prompt=[chat] user: Return one of these values: 
+Answer in JSON using any of these schemas:
+1 or true or "string output"
+)
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
self = 
+
+>   ???
+
+tests/test_functions.py:115: 
+_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
+
+self = , input = 'a'
+baml_options = {}
+
+    async def LiteralUnionsTest(
+        self,
+        input: str,
+        baml_options: BamlCallOptions = {},
+    ) -> Union[Literal[1], Literal[True], Literal["string output"]]:
+      __tb__ = baml_options.get("tb", None)
+      if __tb__ is not None:
+        tb = __tb__._tb # type: ignore (we know how to use this private attribute)
+      else:
+        tb = None
+      __cr__ = baml_options.get("client_registry", None)
+    
+      raw = await self.__runtime.call_function(
+        "LiteralUnionsTest",
+        {
+          "input": input,
+        },
+        self.__ctx_manager.get(),
+        tb,
+        __cr__,
+      )
+>     return cast(Union[Literal[1], Literal[True], Literal["string output"]], raw.cast_to(types, types))
+E     baml_py.internal_monkeypatch.BamlValidationError: BamlValidationError(message=Failed to parse LLM response: Failed to coerce value: : Failed to find any (1 | true | "string output") in 3 items
+E       - : Expected 1, got Object([("Answer", Number(Number(1)))]).
+E       - : Expected true, got Object([("Answer", Number(Number(1)))]).
+E       - : Expected "string output", got Object([("Answer", Number(Number(1)))])., raw_output={
+E       "Answer": 1
+E     }, prompt=[chat] user: Return one of these values: 
+E     Answer in JSON using any of these schemas:
+E     1 or true or "string output"
+E     )
+
+baml_client/async_client.py:1408: BamlValidationError
 
 
 
Captured stderr call
 
[2024-11-26T00:34:26Z WARN  baml_events] Function LiteralUnionsTest:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 426ms. StopReason: stop. Tokens(in/out): 31/9
     ---PROMPT---
-    [chat] system: Return one of these values: 
+    [chat] user: Return one of these values: 
     Answer in JSON using any of these schemas:
     1 or true or "string output"
     
     ---LLM REPLY---
-    1
-    ---Parsed Response (int)---
-    1
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  TestAllInputs::test_constraints  0:00:00.725449 
  
 
 
 Started 2024-11-11 13:26:57 
 
 Ended 2024-11-11 13:26:58 
 
 Duration 0:00:00.725449 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:26:58Z INFO  baml_events] Function PredictAge:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 710ms. StopReason: stop. Tokens(in/out): 116/36
-    ---PROMPT---
-    [chat] system: Using your understanding of the historical popularity
+    {
+      "Answer": 1
+    }
+    ---Parsed Response (Error)---
+    Failed to coerce value: <root>: Failed to find any (1 | true | "string output") in 3 items
+      - <root>: Expected 1, got Object([("Answer", Number(Number(1)))]).
+      - <root>: Expected true, got Object([("Answer", Number(Number(1)))]).
+      - <root>: Expected "string output", got Object([("Answer", Number(Number(1)))]).
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  TestAllInputs::test_constraints  0:00:00.838754 
  
 
 
 Started 2024-11-25 16:34:26 
 
 Ended 2024-11-25 16:34:27 
 
 Duration 0:00:00.838754 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:34:27Z INFO  baml_events] Function PredictAge:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 827ms. StopReason: stop. Tokens(in/out): 116/36
+    ---PROMPT---
+    [chat] user: Using your understanding of the historical popularity
     of names, predict the age of a person with the name
     Greg in years. Also predict their genus and
     species. It's Homo sapiens (with exactly that spelling
@@ -132,7 +188,7 @@
       "planetary_age": {
         "age": 41
       },
-      "certainty": 90,
+      "certainty": 100,
       "species": "Homo sapiens"
     }
     ---Parsed Response (class FooAny)---
@@ -150,7 +206,7 @@
         }
       },
       "certainty": {
-        "value": 90,
+        "value": 100,
         "checks": {
           "unreasonably_certain": {
             "name": "unreasonably_certain",
@@ -162,6 +218,11 @@
       "species": {
         "value": "Homo sapiens",
         "checks": {
+          "trivial": {
+            "name": "trivial",
+            "expression": "this == \"Homo sapiens\"",
+            "status": "succeeded"
+          },
           "regex_good": {
             "name": "regex_good",
             "expression": "this|regex_match(\"Homo\")",
@@ -171,19 +232,14 @@
             "name": "regex_bad",
             "expression": "this|regex_match(\"neanderthalensis\")",
             "status": "failed"
-          },
-          "trivial": {
-            "name": "trivial",
-            "expression": "this == \"Homo sapiens\"",
-            "status": "succeeded"
           }
         }
       }
     }
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  TestAllInputs::test_constraint_union_variant_checking  0:00:00.917260 
  
 
 
 Started 2024-11-11 13:26:58 
 
 Ended 2024-11-11 13:26:59 
 
 Duration 0:00:00.917260 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:26:59Z INFO  baml_events] Function ExtractContactInfo:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 894ms. StopReason: stop. Tokens(in/out): 98/39
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  TestAllInputs::test_constraint_union_variant_checking  0:00:00.721323 
  
 
 
 Started 2024-11-25 16:34:27 
 
 Ended 2024-11-25 16:34:27 
 
 Duration 0:00:00.721323 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:34:27Z INFO  baml_events] Function ExtractContactInfo:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 708ms. StopReason: stop. Tokens(in/out): 98/39
     ---PROMPT---
-    [chat] system: Extract a primary contact info, and if possible a secondary contact
+    [chat] user: Extract a primary contact info, and if possible a secondary contact
     info, from this document:
     
     Reach me at help@boundaryml.com, or 111-222-3333 if needed.
@@ -220,10 +276,10 @@
         "value": "111-222-3333"
       }
     }
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  TestAllInputs::test_return_malformed_constraint  0:00:01.874296 
  
 
 
 Started 2024-11-11 13:26:59 
 
 Ended 2024-11-11 13:27:01 
 
 Duration 0:00:01.874296 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:27:01Z WARN  baml_events] Function ReturnMalformedConstraints:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 1867ms. StopReason: stop. Tokens(in/out): 28/9
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  TestAllInputs::test_return_malformed_constraint  0:00:00.529349 
  
 
 
 Started 2024-11-25 16:34:27 
 
 Ended 2024-11-25 16:34:28 
 
 Duration 0:00:00.529349 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:34:28Z WARN  baml_events] Function ReturnMalformedConstraints:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 527ms. StopReason: stop. Tokens(in/out): 28/9
     ---PROMPT---
-    [chat] system: Return the integer after 1
+    [chat] user: Return the integer after 1
     
     Answer in JSON using this schema:
     {
@@ -238,12 +294,12 @@
     Failed to coerce value: <root>: Failed while parsing required fields: missing=0, unparsed=1
       - <root>: Failed to parse field foo: foo: Failed to evaluate constraints: unknown method: object has no method named length (in <string>:1)
         - foo: Failed to evaluate constraints: unknown method: object has no method named length (in <string>:1)
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  TestAllInputs::test_use_malformed_constraint  0:00:00.003463 
  
 
 
 Started 2024-11-11 13:27:01 
 
 Ended 2024-11-11 13:27:01 
 
 Duration 0:00:00.003463 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:27:01Z ERROR baml_runtime::tracing]   Error: a: Failed to evaluate assert: Error evaluating constraint: unknown method: object has no method named length (in <string>:1)
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  TestAllInputs::test_use_malformed_constraint  0:00:00.001317 
  
 
 
 Started 2024-11-25 16:34:28 
 
 Ended 2024-11-25 16:34:28 
 
 Duration 0:00:00.001317 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:34:28Z ERROR baml_runtime::tracing]   Error: a: Failed to evaluate assert: Error evaluating constraint: unknown method: object has no method named length (in <string>:1)
     
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  TestAllInputs::test_single_class  0:00:00.948883 
  
 
 
 Started 2024-11-11 13:27:01 
 
 Ended 2024-11-11 13:27:02 
 
 Duration 0:00:00.948883 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:27:02Z INFO  baml_events] Function TestFnNamedArgsSingleClass:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 942ms. StopReason: stop. Tokens(in/out): 19/5
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  TestAllInputs::test_single_class  0:00:00.453934 
  
 
 
 Started 2024-11-25 16:34:28 
 
 Ended 2024-11-25 16:34:28 
 
 Duration 0:00:00.453934 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:34:28Z INFO  baml_events] Function TestFnNamedArgsSingleClass:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 452ms. StopReason: stop. Tokens(in/out): 19/5
     ---PROMPT---
-    [chat] system: Print these values back to me:
+    [chat] user: Print these values back to me:
     key
     true
     52
@@ -254,10 +310,10 @@
     52
     ---Parsed Response (string)---
     "key\ntrue\n52"
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  TestAllInputs::test_multiple_args  0:00:00.460532 
  
 
 
 Started 2024-11-11 13:27:02 
 
 Ended 2024-11-11 13:27:02 
 
 Duration 0:00:00.460532 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:27:02Z INFO  baml_events] Function TestMulticlassNamedArgs:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 452ms. StopReason: stop. Tokens(in/out): 25/11
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  TestAllInputs::test_multiple_args  0:00:00.525176 
  
 
 
 Started 2024-11-25 16:34:28 
 
 Ended 2024-11-25 16:34:29 
 
 Duration 0:00:00.525176 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:34:29Z INFO  baml_events] Function TestMulticlassNamedArgs:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 522ms. StopReason: stop. Tokens(in/out): 25/11
     ---PROMPT---
-    [chat] system: Print these values back to me:
+    [chat] user: Print these values back to me:
     key
     true
     52
@@ -266,73 +322,73 @@
     64
     
     ---LLM REPLY---
-    key  
-    true  
-    52  
-    key  
-    true  
+    key
+    true
+    52
+    key
+    true
     64
     ---Parsed Response (string)---
-    "key  \ntrue  \n52  \nkey  \ntrue  \n64"
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  TestAllInputs::test_single_enum_list  0:00:00.405180 
  
 
 
 Started 2024-11-11 13:27:02 
 
 Ended 2024-11-11 13:27:03 
 
 Duration 0:00:00.405180 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:27:03Z INFO  baml_events] Function TestFnNamedArgsSingleEnumList:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 397ms. StopReason: stop. Tokens(in/out): 18/4
+    "key\ntrue\n52\nkey\ntrue\n64"
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  TestAllInputs::test_single_enum_list  0:00:00.347136 
  
 
 
 Started 2024-11-25 16:34:29 
 
 Ended 2024-11-25 16:34:29 
 
 Duration 0:00:00.347136 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:34:29Z INFO  baml_events] Function TestFnNamedArgsSingleEnumList:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 344ms. StopReason: stop. Tokens(in/out): 18/4
     ---PROMPT---
-    [chat] system: Print these values back to me:
+    [chat] user: Print these values back to me:
     ["TWO"]
     
     ---LLM REPLY---
     ["TWO"]
     ---Parsed Response (string)---
     "[\"TWO\"]"
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  TestAllInputs::test_single_float  0:00:00.515763 
  
 
 
 Started 2024-11-11 13:27:03 
 
 Ended 2024-11-11 13:27:03 
 
 Duration 0:00:00.515763 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:27:03Z INFO  baml_events] Function TestFnNamedArgsSingleFloat:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 509ms. StopReason: stop. Tokens(in/out): 18/3
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  TestAllInputs::test_single_float  0:00:00.369850 
  
 
 
 Started 2024-11-25 16:34:29 
 
 Ended 2024-11-25 16:34:30 
 
 Duration 0:00:00.369850 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:34:30Z INFO  baml_events] Function TestFnNamedArgsSingleFloat:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 367ms. StopReason: stop. Tokens(in/out): 18/3
     ---PROMPT---
-    [chat] system: Return this value back to me: 3.12
+    [chat] user: Return this value back to me: 3.12
     
     ---LLM REPLY---
     3.12
     ---Parsed Response (string)---
     "3.12"
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  TestAllInputs::test_single_int  0:00:00.424194 
  
 
 
 Started 2024-11-11 13:27:03 
 
 Ended 2024-11-11 13:27:04 
 
 Duration 0:00:00.424194 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:27:04Z INFO  baml_events] Function TestFnNamedArgsSingleInt:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 419ms. StopReason: stop. Tokens(in/out): 17/2
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  TestAllInputs::test_single_int  0:00:00.327321 
  
 
 
 Started 2024-11-25 16:34:30 
 
 Ended 2024-11-25 16:34:30 
 
 Duration 0:00:00.327321 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:34:30Z INFO  baml_events] Function TestFnNamedArgsSingleInt:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 325ms. StopReason: stop. Tokens(in/out): 17/2
     ---PROMPT---
-    [chat] system: Return this value back to me: 3566
+    [chat] user: Return this value back to me: 3566
     
     ---LLM REPLY---
     3566
     ---Parsed Response (string)---
     "3566"
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  TestAllInputs::test_single_literal_int  0:00:00.356345 
  
 
 
 Started 2024-11-11 13:27:04 
 
 Ended 2024-11-11 13:27:04 
 
 Duration 0:00:00.356345 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:27:04Z INFO  baml_events] Function TestNamedArgsLiteralInt:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 350ms. StopReason: stop. Tokens(in/out): 16/1
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  TestAllInputs::test_single_literal_int  0:00:00.299279 
  
 
 
 Started 2024-11-25 16:34:30 
 
 Ended 2024-11-25 16:34:30 
 
 Duration 0:00:00.299279 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:34:30Z INFO  baml_events] Function TestNamedArgsLiteralInt:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 297ms. StopReason: stop. Tokens(in/out): 16/1
     ---PROMPT---
-    [chat] system: Return this value back to me: 1
+    [chat] user: Return this value back to me: 1
     
     ---LLM REPLY---
     1
     ---Parsed Response (string)---
     "1"
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  TestAllInputs::test_single_literal_bool  0:00:00.376583 
  
 
 
 Started 2024-11-11 13:27:04 
 
 Ended 2024-11-11 13:27:04 
 
 Duration 0:00:00.376583 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:27:04Z INFO  baml_events] Function TestNamedArgsLiteralBool:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 371ms. StopReason: stop. Tokens(in/out): 15/1
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  TestAllInputs::test_single_literal_bool  0:00:00.882065 
  
 
 
 Started 2024-11-25 16:34:30 
 
 Ended 2024-11-25 16:34:31 
 
 Duration 0:00:00.882065 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:34:31Z INFO  baml_events] Function TestNamedArgsLiteralBool:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 880ms. StopReason: stop. Tokens(in/out): 15/1
     ---PROMPT---
-    [chat] system: Return this value back to me: true
+    [chat] user: Return this value back to me: true
     
     ---LLM REPLY---
     true
     ---Parsed Response (string)---
     "true"
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  TestAllInputs::test_single_literal_string  0:00:00.433627 
  
 
 
 Started 2024-11-11 13:27:04 
 
 Ended 2024-11-11 13:27:05 
 
 Duration 0:00:00.433627 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:27:05Z INFO  baml_events] Function TestNamedArgsLiteralString:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 427ms. StopReason: stop. Tokens(in/out): 16/8
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  TestAllInputs::test_single_literal_string  0:00:00.435087 
  
 
 
 Started 2024-11-25 16:34:31 
 
 Ended 2024-11-25 16:34:32 
 
 Duration 0:00:00.435087 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:34:32Z INFO  baml_events] Function TestNamedArgsLiteralString:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 433ms. StopReason: stop. Tokens(in/out): 16/2
     ---PROMPT---
-    [chat] system: Return this value back to me: My String
+    [chat] user: Return this value back to me: My String
     
     ---LLM REPLY---
-    The value you entered is: My String
+    My String
     ---Parsed Response (string)---
-    "The value you entered is: My String"
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  TestAllInputs::test_class_with_literal_prop  0:00:00.841751 
  
 
 
 Started 2024-11-11 13:27:05 
 
 Ended 2024-11-11 13:27:06 
 
 Duration 0:00:00.841751 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:27:06Z INFO  baml_events] Function FnLiteralClassInputOutput:
-    Client: GPT4 (gpt-4o-2024-08-06) - 832ms. StopReason: stop. Tokens(in/out): 30/13
+    "My String"
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  TestAllInputs::test_class_with_literal_prop  0:00:01.559398 
  
 
 
 Started 2024-11-25 16:34:32 
 
 Ended 2024-11-25 16:34:33 
 
 Duration 0:00:01.559398 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:34:33Z INFO  baml_events] Function FnLiteralClassInputOutput:
+    Client: GPT4 (gpt-4o-2024-08-06) - 1555ms. StopReason: stop. Tokens(in/out): 30/13
     ---PROMPT---
-    [chat] system: Return the same object you were given.
+    [chat] user: Return the same object you were given.
     Answer in JSON using this schema:
     {
       prop: "hello",
@@ -348,17 +404,16 @@
     {
       "prop": "hello"
     }
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  TestAllInputs::test_literal_classs_with_literal_union_prop  0:00:00.662402 
  
 
 
 Started 2024-11-11 13:27:06 
 
 Ended 2024-11-11 13:27:06 
 
 Duration 0:00:00.662402 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:27:06Z INFO  baml_events] Function FnLiteralUnionClassInputOutput:
-    Client: GPT4 (gpt-4o-2024-08-06) - 654ms. StopReason: stop. Tokens(in/out): 54/13
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  TestAllInputs::test_literal_classs_with_literal_union_prop  0:00:00.642134 
  
 
 
 Started 2024-11-25 16:34:33 
 
 Ended 2024-11-25 16:34:34 
 
 Duration 0:00:00.642134 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:34:34Z INFO  baml_events] Function FnLiteralUnionClassInputOutput:
+    Client: GPT4 (gpt-4o-2024-08-06) - 638ms. StopReason: stop. Tokens(in/out): 49/13
     ---PROMPT---
-    [chat] system: Return the same object you were given.
+    [chat] user: Return the same object you were given.
     Answer in JSON using any of these schemas:
     {
       prop: "one",
     } or {
       prop: "two",
-    }
-    user: {
+    }{
         "prop": "one",
     }
     
@@ -372,10 +427,10 @@
     {
       "prop": "one"
     }
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  TestAllInputs::test_single_map_string_to_string  0:00:00.545213 
  
 
 
 Started 2024-11-11 13:27:06 
 
 Ended 2024-11-11 13:27:07 
 
 Duration 0:00:00.545213 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:27:07Z INFO  baml_events] Function TestFnNamedArgsSingleMapStringToString:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 538ms. StopReason: stop. Tokens(in/out): 29/15
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  TestAllInputs::test_single_map_string_to_string  0:00:00.504573 
  
 
 
 Started 2024-11-25 16:34:34 
 
 Ended 2024-11-25 16:34:34 
 
 Duration 0:00:00.504573 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:34:34Z INFO  baml_events] Function TestFnNamedArgsSingleMapStringToString:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 502ms. StopReason: stop. Tokens(in/out): 29/15
     ---PROMPT---
-    [chat] system: Return this value back to me: {"lorem": "ipsum", "dolor": "sit"}
+    [chat] user: Return this value back to me: {"lorem": "ipsum", "dolor": "sit"}
     
     ---LLM REPLY---
     {"lorem": "ipsum", "dolor": "sit"}
@@ -384,29 +439,27 @@
       "lorem": "ipsum",
       "dolor": "sit"
     }
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  TestAllInputs::test_single_map_string_to_class  0:00:00.629675 
  
 
 
 Started 2024-11-11 13:27:07 
 
 Ended 2024-11-11 13:27:07 
 
 Duration 0:00:00.629675 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:27:07Z INFO  baml_events] Function TestFnNamedArgsSingleMapStringToClass:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 622ms. StopReason: stop. Tokens(in/out): 28/18
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  TestAllInputs::test_single_map_string_to_class  0:00:00.583404 
  
 
 
 Started 2024-11-25 16:34:34 
 
 Ended 2024-11-25 16:34:35 
 
 Duration 0:00:00.583404 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:34:35Z INFO  baml_events] Function TestFnNamedArgsSingleMapStringToClass:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 579ms. StopReason: stop. Tokens(in/out): 28/14
     ---PROMPT---
-    [chat] system: Return this value back to me: {"lorem": {
+    [chat] user: Return this value back to me: {"lorem": {
         "word": "ipsum",
     }}
     
     ---LLM REPLY---
-    {
-        "lorem": {
-            "word": "ipsum"
-        }
-    }
+    {"lorem": {
+        "word": "ipsum",
+    }}
     ---Parsed Response (map<string, class StringToClassEntry>)---
     {
       "lorem": {
         "word": "ipsum"
       }
     }
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  TestAllInputs::test_single_map_string_to_map  0:00:00.440707 
  
 
 
 Started 2024-11-11 13:27:07 
 
 Ended 2024-11-11 13:27:08 
 
 Duration 0:00:00.440707 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:27:08Z INFO  baml_events] Function TestFnNamedArgsSingleMapStringToMap:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 434ms. StopReason: stop. Tokens(in/out): 25/11
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  TestAllInputs::test_single_map_string_to_map  0:00:00.468936 
  
 
 
 Started 2024-11-25 16:34:35 
 
 Ended 2024-11-25 16:34:35 
 
 Duration 0:00:00.468936 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:34:35Z INFO  baml_events] Function TestFnNamedArgsSingleMapStringToMap:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 466ms. StopReason: stop. Tokens(in/out): 25/11
     ---PROMPT---
-    [chat] system: Return this value back to me: {"lorem": {"word": "ipsum"}}
+    [chat] user: Return this value back to me: {"lorem": {"word": "ipsum"}}
     
     ---LLM REPLY---
     {"lorem": {"word": "ipsum"}}
@@ -416,58 +469,122 @@
         "word": "ipsum"
       }
     }
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_should_work_for_all_outputs  0:00:05.795874 
  
 
 
 Started 2024-11-11 13:27:08 
 
 Ended 2024-11-11 13:27:14 
 
 Duration 0:00:05.795874 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:27:08Z INFO  baml_events] Function FnOutputBool:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 370ms. StopReason: stop. Tokens(in/out): 15/1
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  TestAllInputs::test_enum_key_in_map  0:00:00.770730 
  
 
 
 Started 2024-11-25 16:34:35 
 
 Ended 2024-11-25 16:34:36 
 
 Duration 0:00:00.770730 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:34:36Z INFO  baml_events] Function InOutEnumMapKey:
+    Client: openai/gpt-4o (gpt-4o-2024-08-06) - 763ms. StopReason: stop. Tokens(in/out): 43/20
+    ---PROMPT---
+    [chat] user: Merge these: {"A": "A"} {"B": "B"}
+    
+    Answer in JSON using this schema:
+    map<'A' or 'B' or 'C', string>
+    
+    ---LLM REPLY---
+    ```json
+    {
+      "A": "A",
+      "B": "B"
+    }
+    ```
+    ---Parsed Response (map<string, string>)---
+    {
+      "A": "A",
+      "B": "B"
+    }
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  TestAllInputs::test_literal_string_union_key_in_map  0:00:00.853689 
  
 
 
 Started 2024-11-25 16:34:36 
 
 Ended 2024-11-25 16:34:37 
 
 Duration 0:00:00.853689 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:34:37Z INFO  baml_events] Function InOutLiteralStringUnionMapKey:
+    Client: openai/gpt-4o (gpt-4o-2024-08-06) - 850ms. StopReason: stop. Tokens(in/out): 48/20
+    ---PROMPT---
+    [chat] user: Merge these:
+    
+    {"one": "1"}
+    
+    {"two": "2"}
+    
+    Answer in JSON using this schema:
+    map<"one" or "two" or "three" or "four", string>
+    
+    ---LLM REPLY---
+    ```json
+    {
+      "one": "1",
+      "two": "2"
+    }
+    ```
+    ---Parsed Response (map<string, string>)---
+    {
+      "one": "1",
+      "two": "2"
+    }
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  TestAllInputs::test_single_literal_string_key_in_map  0:00:00.907595 
  
 
 
 Started 2024-11-25 16:34:37 
 
 Ended 2024-11-25 16:34:38 
 
 Duration 0:00:00.907595 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:34:38Z INFO  baml_events] Function InOutSingleLiteralStringMapKey:
+    Client: openai/gpt-4o (gpt-4o-2024-08-06) - 904ms. StopReason: stop. Tokens(in/out): 35/13
+    ---PROMPT---
+    [chat] user: Return the same map you were given:
+    
+    {"key": "1"}
+    
+    Answer in JSON using this schema:
+    map<"key", string>
+    
+    ---LLM REPLY---
+    ```json
+    {
+      "key": "1"
+    }
+    ```
+    ---Parsed Response (map<string, string>)---
+    {
+      "key": "1"
+    }
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_should_work_for_all_outputs  0:00:04.834969 
  
 
 
 Started 2024-11-25 16:34:38 
 
 Ended 2024-11-25 16:34:43 
 
 Duration 0:00:04.834969 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:34:38Z INFO  baml_events] Function FnOutputBool:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 332ms. StopReason: stop. Tokens(in/out): 15/1
     ---PROMPT---
-    [chat] system: Return a true: Answer as a bool
+    [chat] user: Return a true: Answer as a bool
     
     ---LLM REPLY---
     true
     ---Parsed Response (bool)---
     true
-[2024-11-11T19:27:09Z INFO  baml_events] Function FnOutputInt:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 370ms. StopReason: stop. Tokens(in/out): 17/1
+[2024-11-26T00:34:39Z INFO  baml_events] Function FnOutputInt:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 365ms. StopReason: stop. Tokens(in/out): 17/1
     ---PROMPT---
-    [chat] system: Return the integer 5 with no additional context.
+    [chat] user: Return the integer 5 with no additional context.
     
     ---LLM REPLY---
     5
     ---Parsed Response (int)---
     5
-[2024-11-11T19:27:09Z INFO  baml_events] Function FnOutputLiteralInt:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 403ms. StopReason: stop. Tokens(in/out): 18/1
+[2024-11-26T00:34:39Z INFO  baml_events] Function FnOutputLiteralInt:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 398ms. StopReason: stop. Tokens(in/out): 18/7
     ---PROMPT---
-    [chat] system: Return an integer: Answer using this specific value:
+    [chat] user: Return an integer: Answer using this specific value:
     5
     
     ---LLM REPLY---
-    5
+    The integer value is 5.
     ---Parsed Response (int)---
     5
-[2024-11-11T19:27:09Z INFO  baml_events] Function FnOutputLiteralBool:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 382ms. StopReason: stop. Tokens(in/out): 18/1
+[2024-11-26T00:34:39Z INFO  baml_events] Function FnOutputLiteralBool:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 308ms. StopReason: stop. Tokens(in/out): 18/1
     ---PROMPT---
-    [chat] system: Return a false: Answer using this specific value:
+    [chat] user: Return a false: Answer using this specific value:
     false
     
     ---LLM REPLY---
     false
     ---Parsed Response (bool)---
     false
-[2024-11-11T19:27:10Z INFO  baml_events] Function FnOutputLiteralString:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 669ms. StopReason: stop. Tokens(in/out): 21/2
+[2024-11-26T00:34:40Z INFO  baml_events] Function FnOutputLiteralString:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 716ms. StopReason: stop. Tokens(in/out): 21/8
     ---PROMPT---
-    [chat] system: Return a string: Answer using this specific value:
+    [chat] user: Return a string: Answer using this specific value:
     "example output"
     
     ---LLM REPLY---
-    example output
+    The answer is: "example output"
     ---Parsed Response (string)---
     "example output"
-[2024-11-11T19:27:11Z INFO  baml_events] Function FnOutputClassList:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 1222ms. StopReason: stop. Tokens(in/out): 46/63
+[2024-11-26T00:34:41Z INFO  baml_events] Function FnOutputClassList:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 719ms. StopReason: stop. Tokens(in/out): 46/62
     ---PROMPT---
-    [chat] system: Return a JSON array that follows this schema: 
+    [chat] user: Return a JSON array that follows this schema: 
     Answer with a JSON Array using this schema:
     [
       {
@@ -480,38 +597,38 @@
     
     ---LLM REPLY---
     [
-        {
-          "prop1": "apple",
-          "prop2": 5
-        },
-        {
-          "prop1": "banana",
-          "prop2": 3
-        },
-        {
-          "prop1": "orange",
-          "prop2": 8
-        }
-      ]
+      {
+        "prop1": "apple",
+        "prop2": 10
+      },
+      {
+        "prop1": "banana",
+        "prop2": 20
+      },
+      {
+        "prop1": "orange",
+        "prop2": 15
+      }
+    ]
     ---Parsed Response (list<class TestOutputClass>)---
     [
       {
         "prop1": "apple",
-        "prop2": 5
+        "prop2": 10
       },
       {
         "prop1": "banana",
-        "prop2": 3
+        "prop2": 20
       },
       {
         "prop1": "orange",
-        "prop2": 8
+        "prop2": 15
       }
     ]
-[2024-11-11T19:27:12Z INFO  baml_events] Function FnOutputClassWithEnum:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 619ms. StopReason: stop. Tokens(in/out): 48/21
+[2024-11-26T00:34:41Z INFO  baml_events] Function FnOutputClassWithEnum:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 487ms. StopReason: stop. Tokens(in/out): 48/20
     ---PROMPT---
-    [chat] system: Return a made up json blob that matches this schema:
+    [chat] user: Return a made up json blob that matches this schema:
     Answer in JSON using this schema:
     {
       prop1: string,
@@ -523,18 +640,18 @@
     
     ---LLM REPLY---
     {
-      "prop1": "Hello, World!",
+      "prop1": "Hello World",
       "prop2": "TWO"
     }
     ---Parsed Response (class TestClassWithEnum)---
     {
-      "prop1": "Hello, World!",
+      "prop1": "Hello World",
       "prop2": "TWO"
     }
-[2024-11-11T19:27:12Z INFO  baml_events] Function FnOutputClass:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 596ms. StopReason: stop. Tokens(in/out): 50/18
+[2024-11-26T00:34:42Z INFO  baml_events] Function FnOutputClass:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 510ms. StopReason: stop. Tokens(in/out): 50/20
     ---PROMPT---
-    [chat] system: Return a JSON blob with this schema: 
+    [chat] user: Return a JSON blob with this schema: 
     Answer in JSON using this schema:
     {
       prop1: string,
@@ -547,18 +664,18 @@
     
     ---LLM REPLY---
     {
-      "prop1": "example",
+      "prop1": "Hello, World!",
       "prop2": 540
     }
     ---Parsed Response (class TestOutputClass)---
     {
-      "prop1": "example",
+      "prop1": "Hello, World!",
       "prop2": 540
     }
-[2024-11-11T19:27:13Z INFO  baml_events] Function FnEnumListOutput:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 434ms. StopReason: stop. Tokens(in/out): 51/11
+[2024-11-26T00:34:42Z INFO  baml_events] Function FnEnumListOutput:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 380ms. StopReason: stop. Tokens(in/out): 51/12
     ---PROMPT---
-    [chat] system: Print out two of these values randomly selected from the list below in a json array.
+    [chat] user: Print out two of these values randomly selected from the list below in a json array.
     
     Answer with a JSON Array using this schema:
     [
@@ -569,18 +686,18 @@
     
     ---LLM REPLY---
     [
-      "ONE",
-      "TWO"
+      'TWO',
+      'THREE'
     ]
     ---Parsed Response (list<enum EnumOutput>)---
     [
-      "ONE",
-      "TWO"
+      "TWO",
+      "THREE"
     ]
-[2024-11-11T19:27:14Z INFO  baml_events] Function FnEnumOutput:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 672ms. StopReason: stop. Tokens(in/out): 42/20
+[2024-11-26T00:34:43Z INFO  baml_events] Function FnEnumOutput:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 593ms. StopReason: stop. Tokens(in/out): 42/26
     ---PROMPT---
-    [chat] system: Choose one of these values randomly. Before you give the answer, write out an unrelated haiku about the ocean.
+    [chat] user: Choose one of these values randomly. Before you give the answer, write out an unrelated haiku about the ocean.
     
     VALUE_ENUM
     ----
@@ -589,68 +706,72 @@
     - THREE
     
     ---LLM REPLY---
-    In the vast ocean
-    Waves whisper ancient secrets
-    Majestic beauty
+    Waves crash on the shore
+    Whispers of the deep blue sea
+    Nature's symphony
     
-    VALUE_ENUM: TWO
+    Randomly selected value: TWO
     ---Parsed Response (enum EnumOutput)---
     "TWO"
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_should_work_with_image_url  0:00:01.069187 
  
 
 
 Started 2024-11-11 13:27:14 
 
 Ended 2024-11-11 13:27:15 
 
 Duration 0:00:01.069187 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:27:15Z INFO  baml_events] Function TestImageInput:
-    Client: GPT4o (gpt-4o-2024-08-06) - 1061ms. StopReason: stop. Tokens(in/out): 275/6
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_should_work_with_image_url  0:00:01.997576 
  
 
 
 Started 2024-11-25 16:34:43 
 
 Ended 2024-11-25 16:34:45 
 
 Duration 0:00:01.997576 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:34:45Z INFO  baml_events] Function TestImageInput:
+    Client: GPT4o (gpt-4o-2024-08-06) - 1994ms. StopReason: stop. Tokens(in/out): 275/7
     ---PROMPT---
     [chat] user: Describe this in 4 words. One word must be the color<image_placeholder: https://upload.wikimedia.org/wikipedia/en/4/4d/Shrek_%28character%29.png>
     
     ---LLM REPLY---
-    Green ogre in clothes.
+    Green ogre, brown vest.
     ---Parsed Response (string)---
-    "Green ogre in clothes."
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_should_work_with_image_list  0:00:01.501662 
  
 
 
 Started 2024-11-11 13:27:15 
 
 Ended 2024-11-11 13:27:16 
 
 Duration 0:00:01.501662 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:27:16Z INFO  baml_events] Function TestImageListInput:
-    Client: GPT4o (gpt-4o-2024-08-06) - 1494ms. StopReason: stop. Tokens(in/out): 528/9
+    "Green ogre, brown vest."
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_should_work_with_image_list  0:00:03.173481 
  
 
 
 Started 2024-11-25 16:34:45 
 
 Ended 2024-11-25 16:34:48 
 
 Duration 0:00:03.173481 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:34:48Z INFO  baml_events] Function TestImageListInput:
+    Client: GPT4o (gpt-4o-2024-08-06) - 3169ms. StopReason: stop. Tokens(in/out): 528/9
     ---PROMPT---
     [chat] user: What colors do these have in common? [<image_placeholder: https://upload.wikimedia.org/wikipedia/en/4/4d/Shrek_%28character%29.png>,<image_placeholder: https://www.google.com/images/branding/googlelogo/2x/googlelogo_color_92x30dp.png>]
     
     ---LLM REPLY---
-    Both images have the color green in common.
+    The images have the color green in common.
     ---Parsed Response (string)---
-    "Both images have the color green in common."
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_should_work_with_vertex  0:00:09.695080 
  
 
 
 Started 2024-11-11 13:27:16 
 
 Ended 2024-11-11 13:27:26 
 
 Duration 0:00:09.695080 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:27:26Z INFO  baml_events] Function TestVertex:
-    Client: Vertex () - 9686ms. StopReason: "STOP". Tokens(in/out): 8/481
+    "The images have the color green in common."
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_should_work_with_vertex  0:00:11.122248 
  
 
 
 Started 2024-11-25 16:34:48 
 
 Ended 2024-11-25 16:34:59 
 
 Duration 0:00:11.122248 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:34:59Z INFO  baml_events] Function TestVertex:
+    Client: Vertex () - 11117ms. StopReason: "STOP". Tokens(in/out): 8/563
     ---PROMPT---
     [chat] user: Write a nice short story about donkey kong
     
     ---LLM REPLY---
-    The humid jungle air hung heavy as Donkey Kong ambled through his banana grove, the morning sun dappling through the leaves onto his fur. He hummed a happy, off-key tune, gently patting his belly full of freshly picked bananas. Life was good. He had a mountain of bananas, a beautiful sunset every evening, and not a single pesky plumber in sight. 
+    The morning sun, dappled and green through the jungle canopy, woke Donkey Kong with a yawn. He scratched his furry belly, remembering the dream he'd had – a mountain of perfectly ripe bananas, each one singing a merry tune. He glanced over at the pile of bananas beside him, already half gone from his midnight snack. 
+    
+    His stomach rumbled, reminding him of more important matters. Food. He grabbed a plump banana and peeled it with practiced ease. Just as he was about to take a bite, a tiny voice squeaked, "Daddy, I want banana!"
+    
+    Donkey Kong smiled. His little buddy, Diddy Kong, was awake. Diddy, full of energy even before breakfast, was already bouncing around like a furry pinball. Donkey Kong chuckled and broke the banana in half, offering one to Diddy. 
     
-    He stopped by a particularly plump bunch, its golden fruit almost glowing in the morning light.  He reached out, eager to pluck the prize, when a flash of pink caught his eye.
+    "Easy now, little buddy," he rumbled, his voice deep and booming. "We need to go find more bananas after breakfast."
     
-    There, nestled amongst the leaves, was a tiny, whimpering creature. Pink and hairless, it barely resembled the jungle animals Donkey Kong knew. He leaned closer, his fur brushing against the little thing. It let out a tiny squeak and instinctively clung to his fur.
+    Diddy, mouth full of banana, nodded vigorously. After a quick breakfast, they set off into the jungle. Donkey Kong moved with surprising grace for his size, swinging from vines and leaping over streams. Diddy followed close behind, mimicking his every move with his own brand of chaotic energy.
     
-    Donkey Kong was confused. This creature was clearly lost and scared, yet...he felt a strange protectiveness towards it.  He couldn't leave it alone. With a grunt, he scooped the tiny creature into his large hand. It was soft and warm, and it looked at him with wide, trusting eyes. 
+    They searched high and low, but the banana trees seemed to be strangely empty.  Discouraged, they came to a stop by a rushing waterfall. Just as Donkey Kong was about to suggest they head back, Diddy let out a joyous whoop. 
     
-    He decided to call it "Pinky."
+    "Look, Daddy!" He pointed towards a small cave hidden behind the cascading water.  Donkey Kong squinted. He could just make out a faint golden glow emanating from within. Could it be…?
     
-    For weeks, Donkey Kong cared for Pinky, feeding it mashed bananas and keeping it safe from curious monkeys and mischievous snakes. He even let Pinky sleep in a nest of soft leaves in his treetop hammock. Pinky, in turn, filled Donkey Kong’s days with unexpected joy. 
+    With a roar of determination, Donkey Kong plunged through the waterfall.  Diddy, a little worried, followed close behind. The cave was small, but what it lacked in size, it made up for in sheer, glorious abundance. 
     
-    One evening, as the sun dipped below the horizon, painting the sky in fiery hues, Pinky started making strange noises. Donkey Kong watched, puzzled, as it pointed towards the sky and let out a high-pitched squeal. He tilted his head, trying to understand. Was it…was Pinky trying to sing?
+    Bananas.
     
-    Donkey Kong let out a rumbling laugh, and soon, the jungle echoed with a strange but harmonious duet – Donkey Kong’s booming voice and Pinky’s squeaky trills, a melody of friendship echoing in the twilight.
+    Mountains of them, piled high to the ceiling.  Each one was perfectly ripe, glowing with an ethereal light. Donkey Kong and Diddy stared, speechless for a moment, before erupting in joyous yells.  Diddy jumped up and down, his signature red cap almost flying off his head. Donkey Kong let out a triumphant roar that echoed through the jungle.
     
-    Even though they were different, Donkey Kong knew one thing for sure - he'd found a friend in the most unexpected place. And as he watched Pinky snuggle into his fur, he realized that sometimes, the best things in life come in the smallest packages. 
+    They feasted until they could eat no more, filling their tummies and their hearts with happiness. It was the best banana discovery ever, and as Donkey Kong settled down for a nap, his belly full and Diddy snuggled sleepily by his side, he knew that no matter what tomorrow brought, they would face it together, fueled by friendship and the memory of the secret cave of glowing bananas. 
     
     ---Parsed Response (string)---
-    "The humid jungle air hung heavy as Donkey Kong ambled through his banana grove, the morning sun dappling through the leaves onto his fur. He hummed a happy, off-key tune, gently patting his belly full of freshly picked bananas. Life was good. He had a mountain of bananas, a beautiful sunset every evening, and not a single pesky plumber in sight. \n\nHe stopped by a particularly plump bunch, its golden fruit almost glowing in the morning light.  He reached out, eager to pluck the prize, when a flash of pink caught his eye.\n\nThere, nestled amongst the leaves, was a tiny, whimpering creature. Pink and hairless, it barely resembled the jungle animals Donkey Kong knew. He leaned closer, his fur brushing against the little thing. It let out a tiny squeak and instinctively clung to his fur.\n\nDonkey Kong was confused. This creature was clearly lost and scared, yet...he felt a strange protectiveness towards it.  He couldn't leave it alone. With a grunt, he scooped the tiny creature into his large hand. It was soft and warm, and it looked at him with wide, trusting eyes. \n\nHe decided to call it \"Pinky.\"\n\nFor weeks, Donkey Kong cared for Pinky, feeding it mashed bananas and keeping it safe from curious monkeys and mischievous snakes. He even let Pinky sleep in a nest of soft leaves in his treetop hammock. Pinky, in turn, filled Donkey Kong’s days with unexpected joy. \n\nOne evening, as the sun dipped below the horizon, painting the sky in fiery hues, Pinky started making strange noises. Donkey Kong watched, puzzled, as it pointed towards the sky and let out a high-pitched squeal. He tilted his head, trying to understand. Was it…was Pinky trying to sing?\n\nDonkey Kong let out a rumbling laugh, and soon, the jungle echoed with a strange but harmonious duet – Donkey Kong’s booming voice and Pinky’s squeaky trills, a melody of friendship echoing in the twilight.\n\nEven though they were different, Donkey Kong knew one thing for sure - he'd found a friend in the most unexpected place. And as he watched Pinky snuggle into his fur, he realized that sometimes, the best things in life come in the smallest packages. \n"
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_should_work_with_image_base64  0:00:01.879766 
  
 
 
 Started 2024-11-11 13:27:26 
 
 Ended 2024-11-11 13:27:28 
 
 Duration 0:00:01.879766 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:27:28Z INFO  baml_events] Function TestImageInput:
-    Client: GPT4o (gpt-4o-2024-08-06) - 1851ms. StopReason: stop. Tokens(in/out): 275/6
+    "The morning sun, dappled and green through the jungle canopy, woke Donkey Kong with a yawn. He scratched his furry belly, remembering the dream he'd had – a mountain of perfectly ripe bananas, each one singing a merry tune. He glanced over at the pile of bananas beside him, already half gone from his midnight snack. \n\nHis stomach rumbled, reminding him of more important matters. Food. He grabbed a plump banana and peeled it with practiced ease. Just as he was about to take a bite, a tiny voice squeaked, \"Daddy, I want banana!\"\n\nDonkey Kong smiled. His little buddy, Diddy Kong, was awake. Diddy, full of energy even before breakfast, was already bouncing around like a furry pinball. Donkey Kong chuckled and broke the banana in half, offering one to Diddy. \n\n\"Easy now, little buddy,\" he rumbled, his voice deep and booming. \"We need to go find more bananas after breakfast.\"\n\nDiddy, mouth full of banana, nodded vigorously. After a quick breakfast, they set off into the jungle. Donkey Kong moved with surprising grace for his size, swinging from vines and leaping over streams. Diddy followed close behind, mimicking his every move with his own brand of chaotic energy.\n\nThey searched high and low, but the banana trees seemed to be strangely empty.  Discouraged, they came to a stop by a rushing waterfall. Just as Donkey Kong was about to suggest they head back, Diddy let out a joyous whoop. \n\n\"Look, Daddy!\" He pointed towards a small cave hidden behind the cascading water.  Donkey Kong squinted. He could just make out a faint golden glow emanating from within. Could it be…?\n\nWith a roar of determination, Donkey Kong plunged through the waterfall.  Diddy, a little worried, followed close behind. The cave was small, but what it lacked in size, it made up for in sheer, glorious abundance. \n\nBananas.\n\nMountains of them, piled high to the ceiling.  Each one was perfectly ripe, glowing with an ethereal light. Donkey Kong and Diddy stared, speechless for a moment, before erupting in joyous yells.  Diddy jumped up and down, his signature red cap almost flying off his head. Donkey Kong let out a triumphant roar that echoed through the jungle.\n\nThey feasted until they could eat no more, filling their tummies and their hearts with happiness. It was the best banana discovery ever, and as Donkey Kong settled down for a nap, his belly full and Diddy snuggled sleepily by his side, he knew that no matter what tomorrow brought, they would face it together, fueled by friendship and the memory of the secret cave of glowing bananas. \n"
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_should_work_with_image_base64  0:00:01.577244 
  
 
 
 Started 2024-11-25 16:34:59 
 
 Ended 2024-11-25 16:35:01 
 
 Duration 0:00:01.577244 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:35:00Z INFO  baml_events] Function TestImageInput:
+    Client: GPT4o (gpt-4o-2024-08-06) - 1548ms. StopReason: stop. Tokens(in/out): 275/8
     ---PROMPT---
     [chat] user: Describe this in 4 words. One word must be the color<image_placeholder base64>
     
     ---LLM REPLY---
-    Green animated cartoon ogre.
+    Green, smiling, animated ogre.
     ---Parsed Response (string)---
-    "Green animated cartoon ogre."
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_should_work_with_audio_base64  0:00:00.810606 
  
 
 
 Started 2024-11-11 13:27:28 
 
 Ended 2024-11-11 13:27:29 
 
 Duration 0:00:00.810606 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:27:29Z INFO  baml_events] Function AudioInput:
-    Client: Gemini () - 795ms. StopReason: "STOP". Tokens(in/out): 114/1
+    "Green, smiling, animated ogre."
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_should_work_with_audio_base64  0:00:01.141043 
  
 
 
 Started 2024-11-25 16:35:01 
 
 Ended 2024-11-25 16:35:02 
 
 Duration 0:00:01.141043 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:35:02Z INFO  baml_events] Function AudioInput:
+    Client: Gemini () - 1125ms. StopReason: "STOP". Tokens(in/out): 114/1
     ---PROMPT---
     [chat] user: Does this sound like a roar? Yes or no? One word no other characters.<audio_placeholder base64>
     
@@ -659,8 +780,8 @@
     
     ---Parsed Response (string)---
     "Yes \n"
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_should_work_with_audio_url  0:00:01.018003 
  
 
 
 Started 2024-11-11 13:27:29 
 
 Ended 2024-11-11 13:27:30 
 
 Duration 0:00:01.018003 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:27:30Z INFO  baml_events] Function AudioInput:
-    Client: Gemini () - 884ms. StopReason: "STOP". Tokens(in/out): 178/1
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_should_work_with_audio_url  0:00:01.322203 
  
 
 
 Started 2024-11-25 16:35:02 
 
 Ended 2024-11-25 16:35:03 
 
 Duration 0:00:01.322203 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:35:03Z INFO  baml_events] Function AudioInput:
+    Client: Gemini () - 1054ms. StopReason: "STOP". Tokens(in/out): 178/1
     ---PROMPT---
     [chat] user: Does this sound like a roar? Yes or no? One word no other characters.<audio_placeholder base64>
     
@@ -669,17 +790,18 @@
     
     ---Parsed Response (string)---
     "No \n"
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_works_with_retries2  0:00:02.030863 
  
 
 
 Started 2024-11-11 13:27:30 
 
 Ended 2024-11-11 13:27:32 
 
 Duration 0:00:02.030863 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
Expected error LLM call failed: LLMErrorResponse { client: "RetryClientExponential", model: None, prompt: Chat([RenderedChatMessage { role: "system", allow_duplicate_role: false, parts: [Text("Say a haiku")] }]), request_options: {"model": String("gpt-3.5-turbo")}, start_time: SystemTime { tv_sec: 1731353251, tv_nsec: 940087000 }, latency: 161.383042ms, message: "Request failed: {\n    \"error\": {\n        \"message\": \"Incorrect API key provided: blahh. You can find your API key at https://platform.openai.com/account/api-keys.\",\n        \"type\": \"invalid_request_error\",\n        \"param\": null,\n        \"code\": \"invalid_api_key\"\n    }\n}\n", code: InvalidAuthentication }
-
 
 
 
Captured stderr call
 
[2024-11-11T19:27:32Z WARN  baml_events] Function TestRetryExponential:
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_works_with_retries2  0:00:02.572222 
  
 
 
 Started 2024-11-25 16:35:03 
 
 Ended 2024-11-25 16:35:06 
 
 Duration 0:00:02.572222 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
Expected error LLM call failed: LLMErrorResponse { client: "RetryClientExponential", model: None, prompt: Chat([RenderedChatMessage { role: "user", allow_duplicate_role: false, parts: [Text("Say a haiku")] }]), request_options: {"model": String("gpt-3.5-turbo")}, start_time: SystemTime { tv_sec: 1732581305, tv_nsec: 706095000 }, latency: 336.116875ms, message: "Request failed: https://api.openai.com/v1/chat/completions\n{\n    \"error\": {\n        \"message\": \"Incorrect API key provided: blahh. You can find your API key at https://platform.openai.com/account/api-keys.\",\n        \"type\": \"invalid_request_error\",\n        \"param\": null,\n        \"code\": \"invalid_api_key\"\n    }\n}\n", code: InvalidAuthentication }
+
 
 
 
Captured stderr call
 
[2024-11-26T00:35:06Z WARN  baml_events] Function TestRetryExponential:
     (3 other previous tries)
-    Client: RetryClientExponential (<unknown>) - 161ms
+    Client: RetryClientExponential (<unknown>) - 336ms
     ---PROMPT---
-    [chat] system: Say a haiku
+    [chat] user: Say a haiku
     
     ---REQUEST OPTIONS---
     model: "gpt-3.5-turbo"
     ---ERROR (InvalidAuthentication (401))---
-    Request failed: {
+    Request failed: https://api.openai.com/v1/chat/completions
+    {
         "error": {
             "message": "Incorrect API key provided: blahh. You can find your API key at https://platform.openai.com/account/api-keys.",
             "type": "invalid_request_error",
@@ -688,462 +810,423 @@
         }
     }
     
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_works_with_fallbacks  0:00:01.676532 
  
 
 
 Started 2024-11-11 13:27:32 
 
 Ended 2024-11-11 13:27:33 
 
 Duration 0:00:01.676532 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:27:33Z INFO  baml_events] Function TestFallbackClient:
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_works_with_fallbacks  0:00:02.786931 
  
 
 
 Started 2024-11-25 16:35:06 
 
 Ended 2024-11-25 16:35:08 
 
 Duration 0:00:02.786931 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:35:08Z INFO  baml_events] Function TestFallbackClient:
     (5 other previous tries)
-    Client: GPT35 (gpt-3.5-turbo-0125) - 532ms. StopReason: stop. Tokens(in/out): 14/17
+    Client: GPT35 (gpt-3.5-turbo-0125) - 688ms. StopReason: stop. Tokens(in/out): 14/16
     ---PROMPT---
-    [chat] system: Say a haiku about mexico.
+    [chat] user: Say a haiku about mexico.
     
     ---LLM REPLY---
-    Desert meets ocean,
-    Vibrant culture, spicy cuisine,
-    Mexico's beauty.
+    Colorful fiestas
+    Mariachis playing tunes
+    Mexico's charm shines
     ---Parsed Response (string)---
-    "Desert meets ocean,\nVibrant culture, spicy cuisine,\nMexico's beauty."
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_works_with_failing_azure_fallback  0:00:00.003238 
  
 
 
 Started 2024-11-11 13:27:33 
 
 Ended 2024-11-11 13:27:33 
 
 Duration 0:00:00.003238 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:27:33Z ERROR baml_runtime::tracing] Either base_url or both (resource_name, deployment_id) must be provided
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_claude  0:00:00.907101 
  
 
 
 Started 2024-11-11 13:27:33 
 
 Ended 2024-11-11 13:27:34 
 
 Duration 0:00:00.907101 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:27:34Z INFO  baml_events] Function PromptTestClaude:
-    Client: Sonnet (claude-3-5-sonnet-20241022) - 900ms. StopReason: "end_turn". Tokens(in/out): 19/34
+    "Colorful fiestas\nMariachis playing tunes\nMexico's charm shines"
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_works_with_failing_azure_fallback  0:00:00.028621 
  
 
 
 Started 2024-11-25 16:35:08 
 
 Ended 2024-11-25 16:35:08 
 
 Duration 0:00:00.028621 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:35:08Z WARN  baml_events] Function TestSingleFallbackClient:
+    Client: FaultyAzureClient (<unknown>) - 25ms
+    ---PROMPT---
+    [chat] user: Say a haiku about mexico.
+    
+    ---REQUEST OPTIONS---
+    model: "unknown-model"
+    max_tokens: 4096
+    ---ERROR (Unspecified error code: 2)---
+    reqwest::Error { kind: Request, url: Url { scheme: "https", cannot_be_a_base: false, username: "", password: None, host: Some(Domain("unknown-resource-id.openai.azure.com")), port: None, path: "/openai/deployments/unknown-deployment-id/chat/completions", query: None, fragment: None }, source: hyper_util::client::legacy::Error(Connect, ConnectError("dns error", Custom { kind: Uncategorized, error: "failed to lookup address information: nodename nor servname provided, or not known" })) }
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_claude  0:00:01.943309 
  
 
 
 Started 2024-11-25 16:35:08 
 
 Ended 2024-11-25 16:35:10 
 
 Duration 0:00:01.943309 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:35:10Z INFO  baml_events] Function PromptTestClaude:
+    Client: Sonnet (claude-3-5-sonnet-20241022) - 1939ms. StopReason: "end_turn". Tokens(in/out): 19/35
     ---PROMPT---
     [chat] user: Tell me a haiku about Mt Rainier is tall
     
     ---LLM REPLY---
     Here's a haiku about Mt. Rainier:
     
-    Rainier stands supreme
-    Piercing clouds with snowy peak
-    Guardian of time
+    Rainier stands proud, bold
+    Fourteen thousand feet of snow
+    Pierce the summer clouds
     ---Parsed Response (string)---
-    "Here's a haiku about Mt. Rainier:\n\nRainier stands supreme\nPiercing clouds with snowy peak\nGuardian of time"
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_gemini  0:00:08.387621 
  
 
 
 Started 2024-11-11 13:27:34 
 
 Ended 2024-11-11 13:27:43 
 
 Duration 0:00:08.387621 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
LLM output from Gemini: Dr. Pepper wasn't a doctor, not really. He didn't have a practice, or a degree hanging on the wall, but everyone in town called him "Doc" anyway. Maybe it was the twinkle in his eye, the way he listened with a tilted head and a soft "hmm" as you poured your heart out, or perhaps it was the steaming mugs of his famous concoction he offered, always brewed to perfection. 
+    "Here's a haiku about Mt. Rainier:\n\nRainier stands proud, bold\nFourteen thousand feet of snow\nPierce the summer clouds"
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_gemini  0:00:07.602795 
  
 
 
 Started 2024-11-25 16:35:10 
 
 Ended 2024-11-25 16:35:18 
 
 Duration 0:00:07.602795 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
LLM output from Gemini: Dr. Pete Pepper wasn't a real doctor, but his name always elicited a chuckle. He owned a small, dusty bookstore tucked between a bakery and a vintage clothing shop. It was the kind of place where sunlight snuck through the cracks in the blinds, illuminating motes of dust dancing above stacks of forgotten stories. 
+
+One rainy afternoon, a young girl named Lily shuffled in, her frown deeper than the puddles gathering on the sidewalk outside. "My dad says reading is boring," she mumbled, kicking at a loose floorboard. 
+
+Dr. Pete, perched on a rolling ladder behind the counter, raised an eyebrow. "Did he now?" He hopped down, his knees protesting with a loud creak. "And what does your dad find exciting?"
 
-Doc Pepper's "cure-all," as he playfully called it, wasn't found in any medical journal. It was a secret blend of spices and fruits, simmered low and slow on his ancient stovetop. A sip of Doc's brew could chase away the blues, mend a broken heart, or simply warm you from the inside out on a cold day. 
+"He likes fixing cars," Lily admitted, her voice barely a whisper. "He says books don't go anywhere."
 
-One blustery afternoon, a young woman named Lily arrived at Doc's doorstep, her shoulders slumped with worry. Her dreams of becoming a writer felt as distant as the stars. Doc listened patiently, his brow furrowed in thought. Then, with a knowing smile, he handed her a steaming mug.
+Dr. Pete smiled, his eyes twinkling. "Ah, but that's where he's wrong," he said, leading Lily through a labyrinth of bookshelves. "Books take you everywhere! They can take you to the moon, to the bottom of the ocean, even inside a car engine!" 
 
-"There's magic in every story," he said, his voice a low rumble, "sometimes you just need a little something to help you taste it."
+He pulled out a book with a shiny cover depicting a sleek red race car. Lily's eyes widened. Dr. Pete opened it, and they both leaned in, the scent of old paper and ink filling the air. 
 
-Lily took a tentative sip. The flavors danced on her tongue: a hint of cinnamon, a whisper of cherry, a touch of something she couldn't quite place, yet felt strangely familiar.  As the warmth spread through her, she felt a spark ignite within.  She spent the rest of the afternoon at Doc's table, scribbling furiously in her notebook, the words flowing as effortlessly as the steam from her mug.
+From that day on, Lily became a regular, her initial frown replaced by a wide, gap-toothed grin. The bookstore, once silent, echoed with her laughter as she devoured tales of daring adventurers, fantastical creatures, and yes, even the intricate workings of car engines. 
 
-Doc Pepper's brew didn't hold any magical properties, not really. But sometimes, all it took was a listening ear, a warm smile, and a unique blend of flavors to remind people of the magic they already possessed. And that, perhaps, was the best medicine of all. 
+Dr. Pete, with a twinkle in his eye and a book in his hand, had proven that sometimes, the most exciting journeys start in the quiet corners of a dusty bookstore. 
 
-
 
 
 
Captured stderr call
 
[2024-11-11T19:27:43Z INFO  baml_events] Function TestGemini:
-    Client: Gemini () - 8379ms. StopReason: "STOP". Tokens(in/out): 10/411
+
 
 
 
Captured stderr call
 
[2024-11-26T00:35:18Z INFO  baml_events] Function TestGemini:
+    Client: Gemini () - 7596ms. StopReason: "STOP". Tokens(in/out): 10/387
     ---PROMPT---
     [chat] user: Write a nice short story about Dr. Pepper
     
     ---LLM REPLY---
-    Dr. Pepper wasn't a doctor, not really. He didn't have a practice, or a degree hanging on the wall, but everyone in town called him "Doc" anyway. Maybe it was the twinkle in his eye, the way he listened with a tilted head and a soft "hmm" as you poured your heart out, or perhaps it was the steaming mugs of his famous concoction he offered, always brewed to perfection. 
+    Dr. Pete Pepper wasn't a real doctor, but his name always elicited a chuckle. He owned a small, dusty bookstore tucked between a bakery and a vintage clothing shop. It was the kind of place where sunlight snuck through the cracks in the blinds, illuminating motes of dust dancing above stacks of forgotten stories. 
     
-    Doc Pepper's "cure-all," as he playfully called it, wasn't found in any medical journal. It was a secret blend of spices and fruits, simmered low and slow on his ancient stovetop. A sip of Doc's brew could chase away the blues, mend a broken heart, or simply warm you from the inside out on a cold day. 
+    One rainy afternoon, a young girl named Lily shuffled in, her frown deeper than the puddles gathering on the sidewalk outside. "My dad says reading is boring," she mumbled, kicking at a loose floorboard. 
     
-    One blustery afternoon, a young woman named Lily arrived at Doc's doorstep, her shoulders slumped with worry. Her dreams of becoming a writer felt as distant as the stars. Doc listened patiently, his brow furrowed in thought. Then, with a knowing smile, he handed her a steaming mug.
+    Dr. Pete, perched on a rolling ladder behind the counter, raised an eyebrow. "Did he now?" He hopped down, his knees protesting with a loud creak. "And what does your dad find exciting?"
     
-    "There's magic in every story," he said, his voice a low rumble, "sometimes you just need a little something to help you taste it."
+    "He likes fixing cars," Lily admitted, her voice barely a whisper. "He says books don't go anywhere."
     
-    Lily took a tentative sip. The flavors danced on her tongue: a hint of cinnamon, a whisper of cherry, a touch of something she couldn't quite place, yet felt strangely familiar.  As the warmth spread through her, she felt a spark ignite within.  She spent the rest of the afternoon at Doc's table, scribbling furiously in her notebook, the words flowing as effortlessly as the steam from her mug.
+    Dr. Pete smiled, his eyes twinkling. "Ah, but that's where he's wrong," he said, leading Lily through a labyrinth of bookshelves. "Books take you everywhere! They can take you to the moon, to the bottom of the ocean, even inside a car engine!" 
     
-    Doc Pepper's brew didn't hold any magical properties, not really. But sometimes, all it took was a listening ear, a warm smile, and a unique blend of flavors to remind people of the magic they already possessed. And that, perhaps, was the best medicine of all. 
+    He pulled out a book with a shiny cover depicting a sleek red race car. Lily's eyes widened. Dr. Pete opened it, and they both leaned in, the scent of old paper and ink filling the air. 
+    
+    From that day on, Lily became a regular, her initial frown replaced by a wide, gap-toothed grin. The bookstore, once silent, echoed with her laughter as she devoured tales of daring adventurers, fantastical creatures, and yes, even the intricate workings of car engines. 
+    
+    Dr. Pete, with a twinkle in his eye and a book in his hand, had proven that sometimes, the most exciting journeys start in the quiet corners of a dusty bookstore. 
     
     ---Parsed Response (string)---
-    "Dr. Pepper wasn't a doctor, not really. He didn't have a practice, or a degree hanging on the wall, but everyone in town called him \"Doc\" anyway. Maybe it was the twinkle in his eye, the way he listened with a tilted head and a soft \"hmm\" as you poured your heart out, or perhaps it was the steaming mugs of his famous concoction he offered, always brewed to perfection. \n\nDoc Pepper's \"cure-all,\" as he playfully called it, wasn't found in any medical journal. It was a secret blend of spices and fruits, simmered low and slow on his ancient stovetop. A sip of Doc's brew could chase away the blues, mend a broken heart, or simply warm you from the inside out on a cold day. \n\nOne blustery afternoon, a young woman named Lily arrived at Doc's doorstep, her shoulders slumped with worry. Her dreams of becoming a writer felt as distant as the stars. Doc listened patiently, his brow furrowed in thought. Then, with a knowing smile, he handed her a steaming mug.\n\n\"There's magic in every story,\" he said, his voice a low rumble, \"sometimes you just need a little something to help you taste it.\"\n\nLily took a tentative sip. The flavors danced on her tongue: a hint of cinnamon, a whisper of cherry, a touch of something she couldn't quite place, yet felt strangely familiar.  As the warmth spread through her, she felt a spark ignite within.  She spent the rest of the afternoon at Doc's table, scribbling furiously in her notebook, the words flowing as effortlessly as the steam from her mug.\n\nDoc Pepper's brew didn't hold any magical properties, not really. But sometimes, all it took was a listening ear, a warm smile, and a unique blend of flavors to remind people of the magic they already possessed. And that, perhaps, was the best medicine of all. \n"
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_gemini_streaming  0:00:07.311081 
  
 
 
 Started 2024-11-11 13:27:43 
 
 Ended 2024-11-11 13:27:50 
 
 Duration 0:00:07.311081 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
LLM output from Gemini: Dr. Pete Pepper wasn't a doctor of medicine, but a doctor of fizz. His small, unassuming soda shop, tucked away on a side street bustling with life, was his laboratory. It was here he tirelessly toiled, not with beakers and Bunsen burners, but with bubbling fountains and exotic syrups, chasing the perfect effervescent concoction. 
+    "Dr. Pete Pepper wasn't a real doctor, but his name always elicited a chuckle. He owned a small, dusty bookstore tucked between a bakery and a vintage clothing shop. It was the kind of place where sunlight snuck through the cracks in the blinds, illuminating motes of dust dancing above stacks of forgotten stories. \n\nOne rainy afternoon, a young girl named Lily shuffled in, her frown deeper than the puddles gathering on the sidewalk outside. \"My dad says reading is boring,\" she mumbled, kicking at a loose floorboard. \n\nDr. Pete, perched on a rolling ladder behind the counter, raised an eyebrow. \"Did he now?\" He hopped down, his knees protesting with a loud creak. \"And what does your dad find exciting?\"\n\n\"He likes fixing cars,\" Lily admitted, her voice barely a whisper. \"He says books don't go anywhere.\"\n\nDr. Pete smiled, his eyes twinkling. \"Ah, but that's where he's wrong,\" he said, leading Lily through a labyrinth of bookshelves. \"Books take you everywhere! They can take you to the moon, to the bottom of the ocean, even inside a car engine!\" \n\nHe pulled out a book with a shiny cover depicting a sleek red race car. Lily's eyes widened. Dr. Pete opened it, and they both leaned in, the scent of old paper and ink filling the air. \n\nFrom that day on, Lily became a regular, her initial frown replaced by a wide, gap-toothed grin. The bookstore, once silent, echoed with her laughter as she devoured tales of daring adventurers, fantastical creatures, and yes, even the intricate workings of car engines. \n\nDr. Pete, with a twinkle in his eye and a book in his hand, had proven that sometimes, the most exciting journeys start in the quiet corners of a dusty bookstore. \n"
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_gemini_streaming  0:00:08.970334 
  
 
 
 Started 2024-11-25 16:35:18 
 
 Ended 2024-11-25 16:35:27 
 
 Duration 0:00:08.970334 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
LLM output from Gemini: The old diner was quiet, the silence broken only by the rhythmic whir of the ceiling fan and the sizzle of the grill. A lone figure sat at the counter, a Stetson pulled low over his eyes. He was nursing a glass of something dark and mysterious, the ice clinking softly against the rim. 
+
+Ellie, the diner owner, knew the drink well. "Another Dr. Pepper, Earl?" she asked, wiping down the counter with a practiced hand. 
 
-He already had his "Cherry Smile" and "Lemon Zing," but they lacked a certain... mystery. One breezy afternoon, a traveling musician with a velvet voice and a guitar case full of stories wandered in. He introduced himself as "Dusty," ordered a "Sunset Shimmer" – a drink Pete invented on the spot – and began spinning tales of faraway lands, filled with spices that danced on the tongue and fruits bursting with the sun's fire.
+Earl chuckled, a low rumble in his chest. "Ain't nothin' else like it, Ellie. Not in this town, not in this world."
 
-Inspired, Dr. Pepper combined cinnamon syrup, a dash of vanilla, and a secret ingredient whispered by the wind as Dusty strummed his guitar. He poured the concoction into an ice-filled glass, the amber liquid fizzing with an alluring aroma.  Dusty took a sip, his eyes widening. 
+Ellie smiled, knowing the truth behind his words. Dr. Pepper wasn't just a drink in this town; it was a legend. They said it tasted like the West Texas wind after a storm, like freedom and possibility all bottled up.
 
-"This, my friend," he declared, "is a symphony in a glass! What do you call it?"
+Earl had been coming to the diner for over fifty years, always ordering the same thing. He claimed he tasted something different in every glass, a new story unfolding with each sip. 
 
-Dr. Pepper, a humble smile on his face, looked at the musician, then at his creation, a new melody playing on his heartstrings. 
+One day, a young man, all city swagger and designer sunglasses, sauntered in. He ordered a cola, scoffing at Earl's Dr. Pepper. "Seriously, old timer?" he laughed. "That stuff is older than you are."
 
-"This," he said, "is Dr. Pepper."
+Earl took a long, slow sip, his eyes twinkling. "Son," he drawled, "you're right. It is older. It’s older than time itself. Each sip is a journey through history, a taste of memories yet to be made." 
 
-And so it was. The recipe, a secret whispered only to the wind, became a legend. The little soda shop, once overlooked, became a haven for dreamers and adventurers, all drawn to the magic brewed within, all thanks to Dr. Pepper, the doctor of fizz, who found his perfect note of mystery in the most unexpected of places. 
+Intrigued, the young man hesitantly took a sip from Earl's glass. His eyes widened. He tasted the dusty roads, the scent of sagebrush, the thrill of a cattle drive at dawn. It was unlike anything he had ever experienced.
 
-
 
 
 
Captured stderr call
 
[2024-11-11T19:27:50Z INFO  baml_events] Function TestGemini:
-    Client: Gemini (gemini-1.5-pro-001) - 7284ms. StopReason: Stop. Tokens(in/out): unknown/unknown
+From that day on, the young man became a regular, his love for Dr. Pepper rivaling even Earl's. The diner, once quiet, became a place where stories flowed as freely as the legendary drink. And Earl? He just smiled, knowing that the legend of Dr. Pepper, like a good story, would live on forever. 
+
+
 
 
 
Captured stderr call
 
[2024-11-26T00:35:27Z INFO  baml_events] Function TestGemini:
+    Client: Gemini (gemini-1.5-pro-001) - 8953ms. StopReason: Stop. Tokens(in/out): unknown/unknown
     ---PROMPT---
     [chat] user: Write a nice short story about Dr. Pepper
     
     ---LLM REPLY---
-    Dr. Pete Pepper wasn't a doctor of medicine, but a doctor of fizz. His small, unassuming soda shop, tucked away on a side street bustling with life, was his laboratory. It was here he tirelessly toiled, not with beakers and Bunsen burners, but with bubbling fountains and exotic syrups, chasing the perfect effervescent concoction. 
+    The old diner was quiet, the silence broken only by the rhythmic whir of the ceiling fan and the sizzle of the grill. A lone figure sat at the counter, a Stetson pulled low over his eyes. He was nursing a glass of something dark and mysterious, the ice clinking softly against the rim. 
+    
+    Ellie, the diner owner, knew the drink well. "Another Dr. Pepper, Earl?" she asked, wiping down the counter with a practiced hand. 
     
-    He already had his "Cherry Smile" and "Lemon Zing," but they lacked a certain... mystery. One breezy afternoon, a traveling musician with a velvet voice and a guitar case full of stories wandered in. He introduced himself as "Dusty," ordered a "Sunset Shimmer" – a drink Pete invented on the spot – and began spinning tales of faraway lands, filled with spices that danced on the tongue and fruits bursting with the sun's fire.
+    Earl chuckled, a low rumble in his chest. "Ain't nothin' else like it, Ellie. Not in this town, not in this world."
     
-    Inspired, Dr. Pepper combined cinnamon syrup, a dash of vanilla, and a secret ingredient whispered by the wind as Dusty strummed his guitar. He poured the concoction into an ice-filled glass, the amber liquid fizzing with an alluring aroma.  Dusty took a sip, his eyes widening. 
+    Ellie smiled, knowing the truth behind his words. Dr. Pepper wasn't just a drink in this town; it was a legend. They said it tasted like the West Texas wind after a storm, like freedom and possibility all bottled up.
     
-    "This, my friend," he declared, "is a symphony in a glass! What do you call it?"
+    Earl had been coming to the diner for over fifty years, always ordering the same thing. He claimed he tasted something different in every glass, a new story unfolding with each sip. 
     
-    Dr. Pepper, a humble smile on his face, looked at the musician, then at his creation, a new melody playing on his heartstrings. 
+    One day, a young man, all city swagger and designer sunglasses, sauntered in. He ordered a cola, scoffing at Earl's Dr. Pepper. "Seriously, old timer?" he laughed. "That stuff is older than you are."
     
-    "This," he said, "is Dr. Pepper."
+    Earl took a long, slow sip, his eyes twinkling. "Son," he drawled, "you're right. It is older. It’s older than time itself. Each sip is a journey through history, a taste of memories yet to be made." 
     
-    And so it was. The recipe, a secret whispered only to the wind, became a legend. The little soda shop, once overlooked, became a haven for dreamers and adventurers, all drawn to the magic brewed within, all thanks to Dr. Pepper, the doctor of fizz, who found his perfect note of mystery in the most unexpected of places. 
+    Intrigued, the young man hesitantly took a sip from Earl's glass. His eyes widened. He tasted the dusty roads, the scent of sagebrush, the thrill of a cattle drive at dawn. It was unlike anything he had ever experienced.
+    
+    From that day on, the young man became a regular, his love for Dr. Pepper rivaling even Earl's. The diner, once quiet, became a place where stories flowed as freely as the legendary drink. And Earl? He just smiled, knowing that the legend of Dr. Pepper, like a good story, would live on forever. 
     
     ---Parsed Response (string)---
-    "Dr. Pete Pepper wasn't a doctor of medicine, but a doctor of fizz. His small, unassuming soda shop, tucked away on a side street bustling with life, was his laboratory. It was here he tirelessly toiled, not with beakers and Bunsen burners, but with bubbling fountains and exotic syrups, chasing the perfect effervescent concoction. \n\nHe already had his \"Cherry Smile\" and \"Lemon Zing,\" but they lacked a certain... mystery. One breezy afternoon, a traveling musician with a velvet voice and a guitar case full of stories wandered in. He introduced himself as \"Dusty,\" ordered a \"Sunset Shimmer\" – a drink Pete invented on the spot – and began spinning tales of faraway lands, filled with spices that danced on the tongue and fruits bursting with the sun's fire.\n\nInspired, Dr. Pepper combined cinnamon syrup, a dash of vanilla, and a secret ingredient whispered by the wind as Dusty strummed his guitar. He poured the concoction into an ice-filled glass, the amber liquid fizzing with an alluring aroma.  Dusty took a sip, his eyes widening. \n\n\"This, my friend,\" he declared, \"is a symphony in a glass! What do you call it?\"\n\nDr. Pepper, a humble smile on his face, looked at the musician, then at his creation, a new melody playing on his heartstrings. \n\n\"This,\" he said, \"is Dr. Pepper.\"\n\nAnd so it was. The recipe, a secret whispered only to the wind, became a legend. The little soda shop, once overlooked, became a haven for dreamers and adventurers, all drawn to the magic brewed within, all thanks to Dr. Pepper, the doctor of fizz, who found his perfect note of mystery in the most unexpected of places. \n"
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_aws  0:00:03.759515 
  
 
 
 Started 2024-11-11 13:27:50 
 
 Ended 2024-11-11 13:27:54 
 
 Duration 0:00:03.759515 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:27:50Z WARN  aws_runtime::env_config::normalize] section [Connection 1] ignored; config must be in the AWS config file rather than the credentials file
-[2024-11-11T19:27:50Z INFO  aws_config::meta::region] load_region; provider=EnvironmentVariableRegionProvider { env: Env(Real) }
-[2024-11-11T19:27:54Z INFO  baml_events] Function TestAws:
-    Client: AwsBedrock (anthropic.claude-3-5-sonnet-20240620-v1:0) - 3582ms. StopReason: max_tokens. Tokens(in/out): 19/99
+    "The old diner was quiet, the silence broken only by the rhythmic whir of the ceiling fan and the sizzle of the grill. A lone figure sat at the counter, a Stetson pulled low over his eyes. He was nursing a glass of something dark and mysterious, the ice clinking softly against the rim. \n\nEllie, the diner owner, knew the drink well. \"Another Dr. Pepper, Earl?\" she asked, wiping down the counter with a practiced hand. \n\nEarl chuckled, a low rumble in his chest. \"Ain't nothin' else like it, Ellie. Not in this town, not in this world.\"\n\nEllie smiled, knowing the truth behind his words. Dr. Pepper wasn't just a drink in this town; it was a legend. They said it tasted like the West Texas wind after a storm, like freedom and possibility all bottled up.\n\nEarl had been coming to the diner for over fifty years, always ordering the same thing. He claimed he tasted something different in every glass, a new story unfolding with each sip. \n\nOne day, a young man, all city swagger and designer sunglasses, sauntered in. He ordered a cola, scoffing at Earl's Dr. Pepper. \"Seriously, old timer?\" he laughed. \"That stuff is older than you are.\"\n\nEarl took a long, slow sip, his eyes twinkling. \"Son,\" he drawled, \"you're right. It is older. It’s older than time itself. Each sip is a journey through history, a taste of memories yet to be made.\" \n\nIntrigued, the young man hesitantly took a sip from Earl's glass. His eyes widened. He tasted the dusty roads, the scent of sagebrush, the thrill of a cattle drive at dawn. It was unlike anything he had ever experienced.\n\nFrom that day on, the young man became a regular, his love for Dr. Pepper rivaling even Earl's. The diner, once quiet, became a place where stories flowed as freely as the legendary drink. And Earl? He just smiled, knowing that the legend of Dr. Pepper, like a good story, would live on forever. \n"
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_aws  0:00:01.731086 
  
 
 
 Started 2024-11-25 16:35:27 
 
 Ended 2024-11-25 16:35:29 
 
 Duration 0:00:01.731086 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:35:27Z WARN  aws_runtime::env_config::normalize] section [Connection 1] ignored; config must be in the AWS config file rather than the credentials file
+[2024-11-26T00:35:29Z INFO  baml_events] Function TestAws:
+    Client: AwsBedrock (meta.llama3-8b-instruct-v1:0) - 1559ms. StopReason: max_tokens. Tokens(in/out): 25/100
     ---PROMPT---
     [chat] user: Write a nice short story about Mt Rainier is tall
     
     ---LLM REPLY---
-    As the morning mist lifted, Sarah gazed up at the majestic peak of Mount Rainier, its snow-capped summit piercing the sky. She had lived in Washington her whole life, but the mountain's sheer size never failed to take her breath away.
     
-    Sarah remembered her grandfather's stories about climbing Rainier in his youth. "It's so tall," he'd say, "that when you're up there, you feel like you can touch the stars
+    
+    As the sun rose over the small town of Ashford, Washington, a sense of awe washed over the residents. For on this day, they would be treated to a rare sight: a clear view of Mt. Rainier's towering peak.
+    
+    The mountain, a dormant volcano, stood sentinel over the surrounding landscape, its snow-capped summit reaching for the clouds at an astonishing 14,411 feet. It was a sight that never failed to inspire, and today was no exception.
+    
+    Lena
     ---Parsed Response (string)---
-    "As the morning mist lifted, Sarah gazed up at the majestic peak of Mount Rainier, its snow-capped summit piercing the sky. She had lived in Washington her whole life, but the mountain's sheer size never failed to take her breath away.\n\nSarah remembered her grandfather's stories about climbing Rainier in his youth. \"It's so tall,\" he'd say, \"that when you're up there, you feel like you can touch the stars"
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_openai_shorthand  0:00:08.077263 
  
 
 
 Started 2024-11-11 13:27:54 
 
 Ended 2024-11-11 13:28:02 
 
 Duration 0:00:08.077263 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:28:02Z INFO  baml_events] Function TestOpenAIShorthand:
-    Client: openai/gpt-4o-mini (gpt-4o-mini-2024-07-18) - 8068ms. StopReason: stop. Tokens(in/out): 18/558
+    "\n\nAs the sun rose over the small town of Ashford, Washington, a sense of awe washed over the residents. For on this day, they would be treated to a rare sight: a clear view of Mt. Rainier's towering peak.\n\nThe mountain, a dormant volcano, stood sentinel over the surrounding landscape, its snow-capped summit reaching for the clouds at an astonishing 14,411 feet. It was a sight that never failed to inspire, and today was no exception.\n\nLena"
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_openai_shorthand  0:00:10.524816 
  
 
 
 Started 2024-11-25 16:35:29 
 
 Ended 2024-11-25 16:35:39 
 
 Duration 0:00:10.524816 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:35:39Z INFO  baml_events] Function TestOpenAIShorthand:
+    Client: openai/gpt-4o-mini (gpt-4o-mini-2024-07-18) - 10516ms. StopReason: stop. Tokens(in/out): 18/695
     ---PROMPT---
-    [chat] system: Write a nice short story about Mt Rainier is tall
+    [chat] user: Write a nice short story about Mt Rainier is tall
     
     ---LLM REPLY---
-    Once upon a time, in the heart of the Pacific Northwest, there stood a majestic giant known as Mt. Rainier. Its snow-capped peak pierced the sky, reaching heights of 14,411 feet, making it the tallest mountain in Washington. For centuries, it overlooked the lush valleys and bustling towns below, a silent sentinel that inspired stories and dreams.
+    Once upon a time, in a small town nestled at the foot of the majestic Mount Rainier, the people lived in the shadow of its towering presence. Each morning, as the sun rose, it painted the snow-capped peak with hues of gold and pink, bringing warmth to the chilly air. The mountain stood tall, a steadfast guardian watching over the town and its inhabitants.
     
-    In a quaint little village called Ashford, a young girl named Lily was fascinated by Mt. Rainier. Every morning, she would awaken to the sight of its towering presence, a constant reminder of the beauty and strength of nature. She often imagined herself climbing its rugged slopes, feeling the crisp air against her skin and gazing at the world from the peak. 
+    In the heart of the town lived a young girl named Lila. With a spirit as vibrant as the wildflowers that dotted the meadows, Lila had always dreamt of climbing to the summit of Mount Rainier. She imagined herself standing at the top, feeling the clouds at her fingertips and the world sprawling out below her.
     
-    One summer day, Lily decided it was time to embark on an adventure. With a small backpack filled with snacks and a notebook for sketches, she set off on the trail that twisted through emerald forests and blooming wildflowers. As she trekked, towering trees loomed overhead, but nothing compared to the view she longed for. 
+    One sunny day, filled with excitement, Lila shared her dream with her best friend, Sam. “We should climb it together!” she exclaimed, her eyes sparkling like the morning dew. Sam, though a bit more cautious, couldn't resist Lila's enthusiasm. They spent weeks preparing, gathering supplies, studying maps, and listening to the stories of seasoned climbers in the town.
     
-    As she hiked, she encountered hikers of all ages—each one carrying their own stories of triumphs and tribulations. An elderly couple told her of their first marriage anniversary spent at the mountain’s summit, while a young boy shared how he dreamed of standing on top one day, just like the superheroes in his comic books. Their tales only fueled Lily's desire to reach the peak.
+    Finally, the day of their adventure arrived. With backpacks packed and hearts pounding, they set off early in the morning. The path twisted and turned, winding through tall trees and over babbling brooks. As they climbed higher, the air grew cooler, and the view grew more breathtaking. Lila felt alive, her spirit soaring with the altitude.
     
-    Finally, after hours of climbing, the trees began to thin, revealing breathtaking vistas. The sun dipped low, painting the skies with hues of orange and pink, while Mt. Rainier towered majestically, its glaciers shimmering in the fading light. The mountain stood timeless, its beauty untouched by the passage of time.
+    As they neared the summit, the trail became steeper, and the snow deepened. On a particularly challenging stretch, Sam paused, his breath coming in quick bursts. “Lila, it’s so tall... What if we can’t make it?” he asked, doubt creeping into his voice.
     
-    Lily found a rocky outcropping to sit on, pulling out her notebook. She sketched the scene before her, trying to capture the enormity of the mountain in her lines. In that moment of solitude, she felt a connection to all those who had stood where she was, gazing at the world below. She understood that Mt. Rainier wasn’t just a mountain; it was a symbol of aspiration, resilience, and the wild spirit of the land.
+    Lila looked up at the mountain, its peak piercing the sky. “But that’s what makes it beautiful, Sam,” she replied, her voice firm and filled with resolve. “It’s tall, and so are we! Let’s take it one step at a time.”
     
-    As twilight descended and the stars began to twinkle, Lily made her way back down the trail, her heart full. She knew she would return to the mountain, both to conquer its peak and to continue weaving its stories into her own life.
+    With renewed determination, they pushed forward, each step fueled by their friendship and dreams. Finally, after hours of climbing, they reached a plateau that led to the summit. The moment they stepped onto the top of Mount Rainier, a feeling of pure joy washed over them. They stood together, arms raised in triumph, the world sprawling out in every direction beneath their feet.
     
-    In the years that followed, Mt. Rainier remained a constant in her journey—a reminder that greatness isn’t just about height, but about the adventures we seek and the stories we create along the way. And so, the tall mountain stood proudly, forever inspiring those who dared to dream.
+    The town looked like a miniature model, and the rivers sparkled like ribbons of light. The clouds danced around them, and Lila felt like she could touch the sky. In that moment of accomplishment, they understood that the mountain, as tall as it was, symbolized more than just a physical challenge. It represented hope, perseverance, and the strength found in friendship.
+    
+    As the sun began to set, casting a warm glow over the summit, Lila and Sam sat side by side, taking it all in. They realized that it wasn’t just the height of the mountain that mattered, but the journey they had taken to get there. 
+    
+    With a smile, Lila turned to Sam and said, “We’re so much taller than we thought we could be.” And they both knew that together, they could conquer anything, even the biggest mountains in life.
+    
+    And as they made their way back down, laughter echoed through the valleys, a sweet reminder that the tallest of peaks were meant to be climbed, and the greatest adventures were done side by side.
     ---Parsed Response (string)---
-    "Once upon a time, in the heart of the Pacific Northwest, there stood a majestic giant known as Mt. Rainier. Its snow-capped peak pierced the sky, reaching heights of 14,411 feet, making it the tallest mountain in Washington. For centuries, it overlooked the lush valleys and bustling towns below, a silent sentinel that inspired stories and dreams.\n\nIn a quaint little village called Ashford, a young girl named Lily was fascinated by Mt. Rainier. Every morning, she would awaken to the sight of its towering presence, a constant reminder of the beauty and strength of nature. She often imagined herself climbing its rugged slopes, feeling the crisp air against her skin and gazing at the world from the peak. \n\nOne summer day, Lily decided it was time to embark on an adventure. With a small backpack filled with snacks and a notebook for sketches, she set off on the trail that twisted through emerald forests and blooming wildflowers. As she trekked, towering trees loomed overhead, but nothing compared to the view she longed for. \n\nAs she hiked, she encountered hikers of all ages—each one carrying their own stories of triumphs and tribulations. An elderly couple told her of their first marriage anniversary spent at the mountain’s summit, while a young boy shared how he dreamed of standing on top one day, just like the superheroes in his comic books. Their tales only fueled Lily's desire to reach the peak.\n\nFinally, after hours of climbing, the trees began to thin, revealing breathtaking vistas. The sun dipped low, painting the skies with hues of orange and pink, while Mt. Rainier towered majestically, its glaciers shimmering in the fading light. The mountain stood timeless, its beauty untouched by the passage of time.\n\nLily found a rocky outcropping to sit on, pulling out her notebook. She sketched the scene before her, trying to capture the enormity of the mountain in her lines. In that moment of solitude, she felt a connection to all those who had stood where she was, gazing at the world below. She understood that Mt. Rainier wasn’t just a mountain; it was a symbol of aspiration, resilience, and the wild spirit of the land.\n\nAs twilight descended and the stars began to twinkle, Lily made her way back down the trail, her heart full. She knew she would return to the mountain, both to conquer its peak and to continue weaving its stories into her own life.\n\nIn the years that followed, Mt. Rainier remained a constant in her journey—a reminder that greatness isn’t just about height, but about the adventures we seek and the stories we create along the way. And so, the tall mountain stood proudly, forever inspiring those who dared to dream."
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_openai_shorthand_streaming  0:00:13.036828 
  
 
 
 Started 2024-11-11 13:28:02 
 
 Ended 2024-11-11 13:28:15 
 
 Duration 0:00:13.036828 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:28:15Z INFO  baml_events] Function TestOpenAIShorthand:
-    Client: openai/gpt-4o-mini (gpt-4o-mini-2024-07-18) - 13025ms. StopReason: stop. Tokens(in/out): 18/717
+    "Once upon a time, in a small town nestled at the foot of the majestic Mount Rainier, the people lived in the shadow of its towering presence. Each morning, as the sun rose, it painted the snow-capped peak with hues of gold and pink, bringing warmth to the chilly air. The mountain stood tall, a steadfast guardian watching over the town and its inhabitants.\n\nIn the heart of the town lived a young girl named Lila. With a spirit as vibrant as the wildflowers that dotted the meadows, Lila had always dreamt of climbing to the summit of Mount Rainier. She imagined herself standing at the top, feeling the clouds at her fingertips and the world sprawling out below her.\n\nOne sunny day, filled with excitement, Lila shared her dream with her best friend, Sam. “We should climb it together!” she exclaimed, her eyes sparkling like the morning dew. Sam, though a bit more cautious, couldn't resist Lila's enthusiasm. They spent weeks preparing, gathering supplies, studying maps, and listening to the stories of seasoned climbers in the town.\n\nFinally, the day of their adventure arrived. With backpacks packed and hearts pounding, they set off early in the morning. The path twisted and turned, winding through tall trees and over babbling brooks. As they climbed higher, the air grew cooler, and the view grew more breathtaking. Lila felt alive, her spirit soaring with the altitude.\n\nAs they neared the summit, the trail became steeper, and the snow deepened. On a particularly challenging stretch, Sam paused, his breath coming in quick bursts. “Lila, it’s so tall... What if we can’t make it?” he asked, doubt creeping into his voice.\n\nLila looked up at the mountain, its peak piercing the sky. “But that’s what makes it beautiful, Sam,” she replied, her voice firm and filled with resolve. “It’s tall, and so are we! Let’s take it one step at a time.”\n\nWith renewed determination, they pushed forward, each step fueled by their friendship and dreams. Finally, after hours of climbing, they reached a plateau that led to the summit. The moment they stepped onto the top of Mount Rainier, a feeling of pure joy washed over them. They stood together, arms raised in triumph, the world sprawling out in every direction beneath their feet.\n\nThe town looked like a miniature model, and the rivers sparkled like ribbons of light. The clouds danced around them, and Lila felt like she could touch the sky. In that moment of accomplishment, they understood that the mountain, as tall as it was, symbolized more than just a physical challenge. It represented hope, perseverance, and the strength found in friendship.\n\nAs the sun began to set, casting a warm glow over the summit, Lila and Sam sat side by side, taking it all in. They realized that it wasn’t just the height of the mountain that mattered, but the journey they had taken to get there. \n\nWith a smile, Lila turned to Sam and said, “We’re so much taller than we thought we could be.” And they both knew that together, they could conquer anything, even the biggest mountains in life.\n\nAnd as they made their way back down, laughter echoed through the valleys, a sweet reminder that the tallest of peaks were meant to be climbed, and the greatest adventures were done side by side."
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_openai_shorthand_streaming  0:00:09.234752 
  
 
 
 Started 2024-11-25 16:35:39 
 
 Ended 2024-11-25 16:35:48 
 
 Duration 0:00:09.234752 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:35:48Z INFO  baml_events] Function TestOpenAIShorthand:
+    Client: openai/gpt-4o-mini (gpt-4o-mini-2024-07-18) - 9225ms. StopReason: stop. Tokens(in/out): 18/684
     ---PROMPT---
-    [chat] system: Write a nice short story about Mt Rainier is tall
+    [chat] user: Write a nice short story about Mt Rainier is tall
     
     ---LLM REPLY---
-    Once upon a time in a small town nestled in the shadow of Mt. Rainier, there lived a young girl named Lily. Every morning, she would rise early to greet the sun and watch as its golden rays kissed the snow-capped peak of the majestic mountain. To her, it was not just a mountain; it was a giant guardian, ever watchful and protective over her town.
+    Once upon a time, in the heart of the Pacific Northwest, stood the majestic Mt. Rainier, a towering giant cloaked in snow and mystery. The mountain was not just a geological wonder but also a guardian of tales, dreams, and adventures that surrounded its rocky slopes.
     
-    Lily was filled with wonder about the great mountain. She often imagined herself climbing to its summit, dancing on its clouds, and having conversations with the birds that soared above. But despite her dreams, she also understood the mountain's towering height, which struck awe in the hearts of the townsfolk.
+    In a small town nestled at its base lived a young girl named Lila. Her days were filled with dreams of exploring the great outdoors. She often gazed up at Mt. Rainier, where the clouds kissed the peak and the sun painted its face in hues of orange and pink at dusk. To Lila, the mountain was a symbol of adventure and a challenge waiting to be conquered.
     
-    One sunny day, equipped with a backpack filled with snacks and her trusty sketchbook, Lily set out on an adventure. She had convinced her best friend, Max, to join her on a hike to a viewpoint known as "Eagle's Perch." The trail, though challenging, promised breathtaking views of Mt. Rainier.
+    One summer morning, she gathered her courage and decided to climb Mt. Rainier. With her backpack filled with snacks, a water bottle, and a trusty old map, she set off toward the trailhead, her heart bubbling with excitement. The path wound through ancient forests where tall evergreens whispered secrets to the wind, and streams sang cheerful songs as they danced over pebbles.
     
-    As they trekked through the dense forest, the air filled with the sweet scent of pine and wildflowers. They laughed, shared stories, and encouraged one another as they climbed over roots and rocks. With every step, the towering mountain loomed closer, its beauty more mesmerizing than ever. The sunlight caught the glistening snow on the peak, creating a halo that seemed to beckon them forward.
+    As she climbed higher, the air grew thinner, but Lila felt invigorated by the beauty surrounding her. Wildflowers blanketed the meadows, and the sweet scent of pine surrounded her like an embrace. With each step, she could feel Mt. Rainier’s spirit encouraging her.
     
-    After what felt like hours of hiking, they finally arrived at Eagle's Perch. Lily's breath hitched in her throat as she took in the panoramic view. The town below looked like a miniature model, and Mt. Rainier stood proudly, high above it all, a testament to nature's grandeur. 
+    Hours passed, and Lila reached a large boulder that jutted out, forming a perfect perch to catch her breath. As she paused, she noticed a group of mountain goats grazing nearby. Their nimble feet danced over the rocky terrain, reminding her of how small and insignificant she felt compared to the mountain’s grandeur. Yet, she stood tall, inspired and filled with determination.
     
-    As Max pulled out his camera to capture the moment, Lily sat down with her sketchbook, eager to translate her awe onto the page. She began to draw the mountain, its icy crown and rugged slopes, the way it dominated the skyline. Lost in her art, time seemed to stand still.
+    After a short rest, she continued her ascent. The trail became steeper, but Lila was undeterred. She recollected a saying her grandmother used to share: “The tallest mountains can be climbed one step at a time.” With renewed strength, she pressed on, focusing on the ground beneath her feet, counting each step as a small victory.
     
-    Suddenly, a soft rustling interrupted her concentration. Lily looked up to find a wise old owl perched on a nearby branch, its golden eyes observing her with curiosity. She smiled and whispered, “Hello, Mr. Owl. Isn’t Mt. Rainier incredible?”
+    As the sun began to set, Lila finally reached the mountain’s summit. She stood at the top, breathless not just from the climb but from the view that sprawled before her. The world below was a tapestry of greens and blues, the valleys cradling shimmering lakes and winding rivers. She could see the horizon stretch endlessly, meeting the sky in a brilliant palette of oranges, purples, and golds.
     
-    The owl blinked slowly, as if considering her words. “Indeed, young one. But what makes it tall is not just its height; it is the stories it holds. Every season, every storm, and every smile shared under its gaze adds to its majesty.”
+    In that moment, Lila understood; Mt. Rainier was not just a mountain. It was a teacher. A reminder that the greatest heights are conquered not just by strength but by persistence, hope, and belief in oneself. She took a deep breath, feeling the crisp, cool air fill her lungs, and a smile spread across her face as she whispered, “I did it!”
     
-    Lily’s eyes sparkled with understanding. She returned to her sketch, not just drawing the mountain, but also envisioning the laughter of children playing, the stories told around campfires, and the warmth of community gathered beneath its towers. She realized that Mt. Rainier was not just an imposing figure but a source of inspiration and connection for everyone.
+    As the stars began to twinkle above, Lila sat in serene silence, cradled by the universe and enveloped in the mountain’s ancient embrace. Mt. Rainier, tall and proud, stood watch over her – a friend who would always guide her back into the embrace of nature and adventure.
     
-    As the sun began to set, casting brilliant hues of orange and pink across the sky, Lily and Max headed back down the trail, their hearts full of adventure. They promised to return, to explore further, and to continue sharing stories beneath the mighty mountain. 
-    
-    From that day on, every time Lily gazed at Mt. Rainier, she did so with renewed respect. It was tall, yes—towering and majestic—but it was also a tale woven into her very being, reminding her that sometimes, the most significant heights are not measured in feet, but in the bonds created and the stories shared under its watchful gaze.
+    And so, Lila made her way back down the mountain, the memory of her climb forever etched in her heart, knowing that she could achieve great things, one step at a time.
     ---Parsed Response (string)---
-    "Once upon a time in a small town nestled in the shadow of Mt. Rainier, there lived a young girl named Lily. Every morning, she would rise early to greet the sun and watch as its golden rays kissed the snow-capped peak of the majestic mountain. To her, it was not just a mountain; it was a giant guardian, ever watchful and protective over her town.\n\nLily was filled with wonder about the great mountain. She often imagined herself climbing to its summit, dancing on its clouds, and having conversations with the birds that soared above. But despite her dreams, she also understood the mountain's towering height, which struck awe in the hearts of the townsfolk.\n\nOne sunny day, equipped with a backpack filled with snacks and her trusty sketchbook, Lily set out on an adventure. She had convinced her best friend, Max, to join her on a hike to a viewpoint known as \"Eagle's Perch.\" The trail, though challenging, promised breathtaking views of Mt. Rainier.\n\nAs they trekked through the dense forest, the air filled with the sweet scent of pine and wildflowers. They laughed, shared stories, and encouraged one another as they climbed over roots and rocks. With every step, the towering mountain loomed closer, its beauty more mesmerizing than ever. The sunlight caught the glistening snow on the peak, creating a halo that seemed to beckon them forward.\n\nAfter what felt like hours of hiking, they finally arrived at Eagle's Perch. Lily's breath hitched in her throat as she took in the panoramic view. The town below looked like a miniature model, and Mt. Rainier stood proudly, high above it all, a testament to nature's grandeur. \n\nAs Max pulled out his camera to capture the moment, Lily sat down with her sketchbook, eager to translate her awe onto the page. She began to draw the mountain, its icy crown and rugged slopes, the way it dominated the skyline. Lost in her art, time seemed to stand still.\n\nSuddenly, a soft rustling interrupted her concentration. Lily looked up to find a wise old owl perched on a nearby branch, its golden eyes observing her with curiosity. She smiled and whispered, “Hello, Mr. Owl. Isn’t Mt. Rainier incredible?”\n\nThe owl blinked slowly, as if considering her words. “Indeed, young one. But what makes it tall is not just its height; it is the stories it holds. Every season, every storm, and every smile shared under its gaze adds to its majesty.”\n\nLily’s eyes sparkled with understanding. She returned to her sketch, not just drawing the mountain, but also envisioning the laughter of children playing, the stories told around campfires, and the warmth of community gathered beneath its towers. She realized that Mt. Rainier was not just an imposing figure but a source of inspiration and connection for everyone.\n\nAs the sun began to set, casting brilliant hues of orange and pink across the sky, Lily and Max headed back down the trail, their hearts full of adventure. They promised to return, to explore further, and to continue sharing stories beneath the mighty mountain. \n\nFrom that day on, every time Lily gazed at Mt. Rainier, she did so with renewed respect. It was tall, yes—towering and majestic—but it was also a tale woven into her very being, reminding her that sometimes, the most significant heights are not measured in feet, but in the bonds created and the stories shared under its watchful gaze."
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_anthropic_shorthand  0:00:03.637927 
  
 
 
 Started 2024-11-11 13:28:15 
 
 Ended 2024-11-11 13:28:18 
 
 Duration 0:00:03.637927 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:28:18Z INFO  baml_events] Function TestAnthropicShorthand:
-    Client: anthropic/claude-3-haiku-20240307 (claude-3-haiku-20240307) - 3629ms. StopReason: "end_turn". Tokens(in/out): 19/380
+    "Once upon a time, in the heart of the Pacific Northwest, stood the majestic Mt. Rainier, a towering giant cloaked in snow and mystery. The mountain was not just a geological wonder but also a guardian of tales, dreams, and adventures that surrounded its rocky slopes.\n\nIn a small town nestled at its base lived a young girl named Lila. Her days were filled with dreams of exploring the great outdoors. She often gazed up at Mt. Rainier, where the clouds kissed the peak and the sun painted its face in hues of orange and pink at dusk. To Lila, the mountain was a symbol of adventure and a challenge waiting to be conquered.\n\nOne summer morning, she gathered her courage and decided to climb Mt. Rainier. With her backpack filled with snacks, a water bottle, and a trusty old map, she set off toward the trailhead, her heart bubbling with excitement. The path wound through ancient forests where tall evergreens whispered secrets to the wind, and streams sang cheerful songs as they danced over pebbles.\n\nAs she climbed higher, the air grew thinner, but Lila felt invigorated by the beauty surrounding her. Wildflowers blanketed the meadows, and the sweet scent of pine surrounded her like an embrace. With each step, she could feel Mt. Rainier’s spirit encouraging her.\n\nHours passed, and Lila reached a large boulder that jutted out, forming a perfect perch to catch her breath. As she paused, she noticed a group of mountain goats grazing nearby. Their nimble feet danced over the rocky terrain, reminding her of how small and insignificant she felt compared to the mountain’s grandeur. Yet, she stood tall, inspired and filled with determination.\n\nAfter a short rest, she continued her ascent. The trail became steeper, but Lila was undeterred. She recollected a saying her grandmother used to share: “The tallest mountains can be climbed one step at a time.” With renewed strength, she pressed on, focusing on the ground beneath her feet, counting each step as a small victory.\n\nAs the sun began to set, Lila finally reached the mountain’s summit. She stood at the top, breathless not just from the climb but from the view that sprawled before her. The world below was a tapestry of greens and blues, the valleys cradling shimmering lakes and winding rivers. She could see the horizon stretch endlessly, meeting the sky in a brilliant palette of oranges, purples, and golds.\n\nIn that moment, Lila understood; Mt. Rainier was not just a mountain. It was a teacher. A reminder that the greatest heights are conquered not just by strength but by persistence, hope, and belief in oneself. She took a deep breath, feeling the crisp, cool air fill her lungs, and a smile spread across her face as she whispered, “I did it!”\n\nAs the stars began to twinkle above, Lila sat in serene silence, cradled by the universe and enveloped in the mountain’s ancient embrace. Mt. Rainier, tall and proud, stood watch over her – a friend who would always guide her back into the embrace of nature and adventure.\n\nAnd so, Lila made her way back down the mountain, the memory of her climb forever etched in her heart, knowing that she could achieve great things, one step at a time."
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_anthropic_shorthand  0:00:02.926146 
  
 
 
 Started 2024-11-25 16:35:48 
 
 Ended 2024-11-25 16:35:51 
 
 Duration 0:00:02.926146 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:35:51Z INFO  baml_events] Function TestAnthropicShorthand:
+    Client: anthropic/claude-3-haiku-20240307 (claude-3-haiku-20240307) - 2920ms. StopReason: "end_turn". Tokens(in/out): 19/376
     ---PROMPT---
     [chat] user: Write a nice short story about Mt Rainier is tall
     
     ---LLM REPLY---
     Here is a short story about Mount Rainier:
     
-    The Towering Titan
-    
-    As the sun slowly rose over the horizon, its golden rays began to illuminate the magnificent sight before me. There, standing tall and proud, was the majestic Mount Rainier, its snow-capped peak reaching up towards the heavens.
+    The Majestic Sentinel
     
-    I had always known that Rainier was an impressive mountain, but nothing could have prepared me for the sheer scale and grandeur of this towering titan. At 14,411 feet (4,392 meters) high, it is the tallest mountain in the Cascade Range and the most prominent peak in the contiguous United States.
+    As the morning sun peeked over the horizon, its golden rays illuminated the towering presence of Mount Rainier. Standing tall and proud at 14,411 feet, the dormant volcano seemed to watch over the surrounding landscape, a silent guardian keeping vigilant watch.
     
-    As I gazed upon its rugged, glaciated slopes, I was struck by a sense of awe and wonder. The mountain seemed to command the very landscape, its presence dominating the horizon and dwarfing the surrounding peaks. It was a true testament to the raw power and beauty of nature.
+    For generations, the mountain had stood as a beacon, a landmark that guided weary travelers and inspired awe in all who gazed upon its snow-capped peak. Its sheer size and scale were humbling, a testament to the raw power and grandeur of the natural world.
     
-    I couldn't help but feel humbled by the sheer size and scale of Rainier. It was a humbling reminder of the incredible forces that have shaped our planet over millions of years, and the enduring strength and resilience of these majestic mountains.
+    To the indigenous peoples who had called this region home for centuries, Mount Rainier was more than just a mountain – it was a sacred place, a symbol of their connection to the land and the rhythms of the universe. They revered its strength and reveled in the way its presence seemed to imbue the entire region with a sense of timelessness and wonder.
     
-    As the sun continued to rise, the mountain's hues shifted, from the deep blues and purples of the early morning to the dazzling whites and grays of the snow and ice. It was a mesmerizing display, one that left me feeling both small and insignificant, and yet deeply connected to the natural world around me.
+    As the years passed, and more and more people came to experience the splendor of Mount Rainier, its legend only grew. Hikers, climbers, and nature enthusiasts from around the world flocked to its slopes, seeking to conquer its challenges and bask in the glory of its majesty.
     
-    In that moment, I knew that I had witnessed something truly special – a glimpse into the grand and timeless beauty of our world, embodied in the towering presence of Mount Rainier.
+    Yet, no matter how many times the mountain was scaled or explored, it remained a force to be reckoned with, a constant reminder of the enduring power of the natural world. And as the sun continued to rise and set, casting its ever-shifting light upon the mountain's flanks, Mount Rainier stood tall and steadfast, a majestic sentinel watching over the land it had called home for countless millennia.
     ---Parsed Response (string)---
-    "Here is a short story about Mount Rainier:\n\nThe Towering Titan\n\nAs the sun slowly rose over the horizon, its golden rays began to illuminate the magnificent sight before me. There, standing tall and proud, was the majestic Mount Rainier, its snow-capped peak reaching up towards the heavens.\n\nI had always known that Rainier was an impressive mountain, but nothing could have prepared me for the sheer scale and grandeur of this towering titan. At 14,411 feet (4,392 meters) high, it is the tallest mountain in the Cascade Range and the most prominent peak in the contiguous United States.\n\nAs I gazed upon its rugged, glaciated slopes, I was struck by a sense of awe and wonder. The mountain seemed to command the very landscape, its presence dominating the horizon and dwarfing the surrounding peaks. It was a true testament to the raw power and beauty of nature.\n\nI couldn't help but feel humbled by the sheer size and scale of Rainier. It was a humbling reminder of the incredible forces that have shaped our planet over millions of years, and the enduring strength and resilience of these majestic mountains.\n\nAs the sun continued to rise, the mountain's hues shifted, from the deep blues and purples of the early morning to the dazzling whites and grays of the snow and ice. It was a mesmerizing display, one that left me feeling both small and insignificant, and yet deeply connected to the natural world around me.\n\nIn that moment, I knew that I had witnessed something truly special – a glimpse into the grand and timeless beauty of our world, embodied in the towering presence of Mount Rainier."
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_anthropic_shorthand_streaming  0:00:03.438224 
  
 
 
 Started 2024-11-11 13:28:18 
 
 Ended 2024-11-11 13:28:22 
 
 Duration 0:00:03.438224 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:28:22Z INFO  baml_events] Function TestAnthropicShorthand:
-    Client: anthropic/claude-3-haiku-20240307 (claude-3-haiku-20240307) - 3406ms. StopReason: "end_turn". Tokens(in/out): 19/430
+    "Here is a short story about Mount Rainier:\n\nThe Majestic Sentinel\n\nAs the morning sun peeked over the horizon, its golden rays illuminated the towering presence of Mount Rainier. Standing tall and proud at 14,411 feet, the dormant volcano seemed to watch over the surrounding landscape, a silent guardian keeping vigilant watch.\n\nFor generations, the mountain had stood as a beacon, a landmark that guided weary travelers and inspired awe in all who gazed upon its snow-capped peak. Its sheer size and scale were humbling, a testament to the raw power and grandeur of the natural world.\n\nTo the indigenous peoples who had called this region home for centuries, Mount Rainier was more than just a mountain – it was a sacred place, a symbol of their connection to the land and the rhythms of the universe. They revered its strength and reveled in the way its presence seemed to imbue the entire region with a sense of timelessness and wonder.\n\nAs the years passed, and more and more people came to experience the splendor of Mount Rainier, its legend only grew. Hikers, climbers, and nature enthusiasts from around the world flocked to its slopes, seeking to conquer its challenges and bask in the glory of its majesty.\n\nYet, no matter how many times the mountain was scaled or explored, it remained a force to be reckoned with, a constant reminder of the enduring power of the natural world. And as the sun continued to rise and set, casting its ever-shifting light upon the mountain's flanks, Mount Rainier stood tall and steadfast, a majestic sentinel watching over the land it had called home for countless millennia."
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_anthropic_shorthand_streaming  0:00:02.665601 
  
 
 
 Started 2024-11-25 16:35:51 
 
 Ended 2024-11-25 16:35:54 
 
 Duration 0:00:02.665601 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:35:54Z INFO  baml_events] Function TestAnthropicShorthand:
+    Client: anthropic/claude-3-haiku-20240307 (claude-3-haiku-20240307) - 2644ms. StopReason: "end_turn". Tokens(in/out): 19/379
     ---PROMPT---
     [chat] user: Write a nice short story about Mt Rainier is tall
     
     ---LLM REPLY---
     Here is a short story about Mt. Rainier:
     
-    The Majestic Mountain
+    The Towering Titan
     
-    As the sun began to rise over the Cascade mountain range, its warm golden rays slowly crept up the towering, snow-capped peak of Mount Rainier. Standing tall at 14,411 feet above sea level, the ancient volcano dominated the horizon, its grandeur inspiring a sense of awe in all who gazed upon it.
+    Gazing up at the majestic Mt. Rainier, its snowcapped peak piercing the azure sky, Sarah felt a sense of awe wash over her. Standing at 14,411 feet, the towering volcanic mountain reigned supreme over the surrounding landscape, a testament to the raw power and grandeur of nature.
     
-    Emily had lived in the Pacific Northwest her entire life, but she never grew tired of seeing Mount Rainier loom in the distance. Each morning, as she sipped her coffee and looked out the window, she couldn't help but marvel at its sheer size and power. The mountain was a constant, steadfast presence, a reminder of the incredible forces of nature that had shaped this land over millions of years.
+    As she began her hike along the winding trails, Sarah couldn't help but be humbled by the sheer scale of the mountain before her. With each step, the summit seemed to loom ever higher, a challenge that beckoned to be conquered.
     
-    Today, Emily decided, she would venture closer to the mountain. She packed a backpack with supplies and headed out, following the winding road that led up into the national park. As she drove higher, the world around her transformed - dense forests gave way to alpine meadows dotted with wildflowers, and the air grew crisp and thin.
+    The trail grew steeper, the air thinner, but Sarah's determination only strengthened. She pressed on, her eyes fixed on the distant crest, each glimpse spurring her forward. The view became more breathtaking with every stride, the world below shrinking away as she ascended.
     
-    Finally, she reached the trailhead and set off on foot, her gaze fixed upon the majestic peak looming ever closer. The hike was challenging, but with each step, Emily felt a growing sense of connection to this ancient, awe-inspiring wonder of nature. When she finally reached a lookout point, she stood in stunned silence, taking in the breathtaking panorama before her.
+    Finally, reaching the summit, Sarah stood in reverent silence, taking in the stunning panorama that stretched out before her. The Cascades, the forests, the glistening glaciers – all seemed diminished in the shadow of this towering titan, a natural wonder that commanded respect and awe.
     
-    Mount Rainier stood tall and proud, its glaciers glittering in the sunlight like diamonds. Emily felt small and humbled in its presence, yet also filled with a deep appreciation for the power and beauty of the natural world. In that moment, she knew she would carry this experience with her forever, a reminder of the grandeur that exists when we take the time to connect with the landscapes that surround us.
+    In that moment, Sarah felt a deep connection to the mountain, a sense of being part of something greater than herself. Mt. Rainier, with its majestic presence and timeless beauty, had left an indelible mark on her soul, a reminder of the power and magnificence of the natural world.
+    
+    As she began her descent, Sarah knew that she would carry the memory of this experience with her forever, a testament to the towering splendor of Mt. Rainier.
     ---Parsed Response (string)---
-    "Here is a short story about Mt. Rainier:\n\nThe Majestic Mountain\n\nAs the sun began to rise over the Cascade mountain range, its warm golden rays slowly crept up the towering, snow-capped peak of Mount Rainier. Standing tall at 14,411 feet above sea level, the ancient volcano dominated the horizon, its grandeur inspiring a sense of awe in all who gazed upon it.\n\nEmily had lived in the Pacific Northwest her entire life, but she never grew tired of seeing Mount Rainier loom in the distance. Each morning, as she sipped her coffee and looked out the window, she couldn't help but marvel at its sheer size and power. The mountain was a constant, steadfast presence, a reminder of the incredible forces of nature that had shaped this land over millions of years.\n\nToday, Emily decided, she would venture closer to the mountain. She packed a backpack with supplies and headed out, following the winding road that led up into the national park. As she drove higher, the world around her transformed - dense forests gave way to alpine meadows dotted with wildflowers, and the air grew crisp and thin.\n\nFinally, she reached the trailhead and set off on foot, her gaze fixed upon the majestic peak looming ever closer. The hike was challenging, but with each step, Emily felt a growing sense of connection to this ancient, awe-inspiring wonder of nature. When she finally reached a lookout point, she stood in stunned silence, taking in the breathtaking panorama before her.\n\nMount Rainier stood tall and proud, its glaciers glittering in the sunlight like diamonds. Emily felt small and humbled in its presence, yet also filled with a deep appreciation for the power and beauty of the natural world. In that moment, she knew she would carry this experience with her forever, a reminder of the grandeur that exists when we take the time to connect with the landscapes that surround us."
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_fallback_to_shorthand  0:00:00.793172 
  
 
 
 Started 2024-11-11 13:28:22 
 
 Ended 2024-11-11 13:28:23 
 
 Duration 0:00:00.793172 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:28:23Z INFO  baml_events] Function TestFallbackToShorthand:
+    "Here is a short story about Mt. Rainier:\n\nThe Towering Titan\n\nGazing up at the majestic Mt. Rainier, its snowcapped peak piercing the azure sky, Sarah felt a sense of awe wash over her. Standing at 14,411 feet, the towering volcanic mountain reigned supreme over the surrounding landscape, a testament to the raw power and grandeur of nature.\n\nAs she began her hike along the winding trails, Sarah couldn't help but be humbled by the sheer scale of the mountain before her. With each step, the summit seemed to loom ever higher, a challenge that beckoned to be conquered.\n\nThe trail grew steeper, the air thinner, but Sarah's determination only strengthened. She pressed on, her eyes fixed on the distant crest, each glimpse spurring her forward. The view became more breathtaking with every stride, the world below shrinking away as she ascended.\n\nFinally, reaching the summit, Sarah stood in reverent silence, taking in the stunning panorama that stretched out before her. The Cascades, the forests, the glistening glaciers – all seemed diminished in the shadow of this towering titan, a natural wonder that commanded respect and awe.\n\nIn that moment, Sarah felt a deep connection to the mountain, a sense of being part of something greater than herself. Mt. Rainier, with its majestic presence and timeless beauty, had left an indelible mark on her soul, a reminder of the power and magnificence of the natural world.\n\nAs she began her descent, Sarah knew that she would carry the memory of this experience with her forever, a testament to the towering splendor of Mt. Rainier."
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_fallback_to_shorthand  0:00:01.082669 
  
 
 
 Started 2024-11-25 16:35:54 
 
 Ended 2024-11-25 16:35:55 
 
 Duration 0:00:01.082669 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:35:55Z INFO  baml_events] Function TestFallbackToShorthand:
     (1 other previous tries)
-    Client: openai/gpt-4o-mini (gpt-4o-mini-2024-07-18) - 561ms. StopReason: stop. Tokens(in/out): 18/21
+    Client: openai/gpt-4o-mini (gpt-4o-mini-2024-07-18) - 741ms. StopReason: stop. Tokens(in/out): 18/22
     ---PROMPT---
-    [chat] system: Say a haiku about Mt Rainier is tall.
+    [chat] user: Say a haiku about Mt Rainier is tall.
     
     ---LLM REPLY---
-    Majestic and proud,  
-    Mt. Rainier pierces sky,  
-    Whispers of the clouds.
+    Mount Rainier stands proud,  
+    Cloaked in snow and whispered clouds,  
+    Nature's towering crown.
     ---Parsed Response (string)---
-    "Majestic and proud,  \nMt. Rainier pierces sky,  \nWhispers of the clouds."
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 FAILED  test_aws_streaming  0:00:00.319182 
  
 
 
 Started 2024-11-11 13:28:23 
 
 Ended 2024-11-11 13:28:23 
 
 Duration 0:00:00.319182 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
baml_py.BamlClientHttpError: LLM call failed: LLMErrorResponse { client: "AwsBedrock", model: Some("anthropic.claude-3-5-sonnet-20240620-v1:0"), prompt: Chat([RenderedChatMessage { role: "user", allow_duplicate_role: false, parts: [Text("Write a nice short story about Mt Rainier is tall")] }]), request_options: {"api_key": String("")}, start_time: SystemTime { tv_sec: 1731353303, tv_nsec: 114527000 }, latency: 308.279084ms, message: "ServiceError(\n    ServiceError {\n        source: ThrottlingException(\n            ThrottlingException {\n                message: Some(\n                    \"Too many requests, please wait before trying again.\",\n                ),\n                meta: ErrorMetadata {\n                    code: Some(\n                        \"ThrottlingException\",\n                    ),\n                    message: Some(\n                        \"Too many requests, please wait before trying again.\",\n                    ),\n                    extras: Some(\n                        {\n                            \"aws_request_id\": \"758ee731-5022-4f89-8299-a27622e30ef0\",\n                        },\n                    ),\n                },\n            },\n        ),\n        raw: Response {\n            status: StatusCode(\n                429,\n            ),\n            headers: Headers {\n                headers: {\n                    \"date\": HeaderValue {\n                        _private: H0(\n                            \"Mon, 11 Nov 2024 19:28:23 GMT\",\n                        ),\n                    },\n                    \"content-type\": HeaderValue {\n                        _private: H0(\n                            \"application/json\",\n                        ),\n                    },\n                    \"content-length\": HeaderValue {\n                        _private: H0(\n                            \"65\",\n                        ),\n                    },\n                    \"x-amzn-requestid\": HeaderValue {\n                        _private: H0(\n                            \"758ee731-5022-4f89-8299-a27622e30ef0\",\n                        ),\n                    },\n                    \"x-amzn-errortype\": HeaderValue {\n                        _private: H0(\n                            \"ThrottlingException:http://internal.amazon.com/coral/com.amazon.bedrock/\",\n                        ),\n                    },\n                },\n            },\n            body: SdkBody {\n                inner: Once(\n                    Some(\n                        b\"{\\\"message\\\":\\\"Too many requests, please wait before trying again.\\\"}\",\n                    ),\n                ),\n                retryable: true,\n            },\n            extensions: Extensions {\n                extensions_02x: Extensions,\n                extensions_1x: Extensions,\n            },\n        },\n    },\n)", code: RateLimited }
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
>   ???
-
-tests/test_functions.py:424: 
-_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
-../../engine/language_client_python/python_src/baml_py/stream.py:81: in get_final_response
-    return self.__final_coerce((await asyncio.wrap_future(final)))
-_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
-
-x = 
-
->     lambda x: cast(str, x.cast_to(types, types)),
-      self.__ctx_manager.get(),
-    )
-E   baml_py.BamlClientHttpError: LLM call failed: LLMErrorResponse { client: "AwsBedrock", model: Some("anthropic.claude-3-5-sonnet-20240620-v1:0"), prompt: Chat([RenderedChatMessage { role: "user", allow_duplicate_role: false, parts: [Text("Write a nice short story about Mt Rainier is tall")] }]), request_options: {"api_key": String("")}, start_time: SystemTime { tv_sec: 1731353303, tv_nsec: 114527000 }, latency: 308.279084ms, message: "ServiceError(\n    ServiceError {\n        source: ThrottlingException(\n            ThrottlingException {\n                message: Some(\n                    \"Too many requests, please wait before trying again.\",\n                ),\n                meta: ErrorMetadata {\n                    code: Some(\n                        \"ThrottlingException\",\n                    ),\n                    message: Some(\n                        \"Too many requests, please wait before trying again.\",\n                    ),\n                    extras: Some(\n                        {\n                            \"aws_request_id\": \"758ee731-5022-4f89-8299-a27622e30ef0\",\n                        },\n                    ),\n                },\n            },\n        ),\n        raw: Response {\n            status: StatusCode(\n                429,\n            ),\n            headers: Headers {\n                headers: {\n                    \"date\": HeaderValue {\n                        _private: H0(\n                            \"Mon, 11 Nov 2024 19:28:23 GMT\",\n                        ),\n                    },\n                    \"content-type\": HeaderValue {\n                        _private: H0(\n                            \"application/json\",\n                        ),\n                    },\n                    \"content-length\": HeaderValue {\n                        _private: H0(\n                            \"65\",\n                        ),\n                    },\n                    \"x-amzn-requestid\": HeaderValue {\n                        _private: H0(\n                            \"758ee731-5022-4f89-8299-a27622e30ef0\",\n                        ),\n                    },\n                    \"x-amzn-errortype\": HeaderValue {\n                        _private: H0(\n                            \"ThrottlingException:http://internal.amazon.com/coral/com.amazon.bedrock/\",\n                        ),\n                    },\n                },\n            },\n            body: SdkBody {\n                inner: Once(\n                    Some(\n                        b\"{\\\"message\\\":\\\"Too many requests, please wait before trying again.\\\"}\",\n                    ),\n                ),\n                retryable: true,\n            },\n            extensions: Extensions {\n                extensions_02x: Extensions,\n                extensions_1x: Extensions,\n            },\n        },\n    },\n)", code: RateLimited }
-
-baml_client/async_client.py:5055: BamlClientHttpError
 
 
 
Captured stderr call
 
[2024-11-11T19:28:23Z WARN  aws_runtime::env_config::normalize] section [Connection 1] ignored; config must be in the AWS config file rather than the credentials file
-[2024-11-11T19:28:23Z INFO  aws_config::meta::region] load_region; provider=EnvironmentVariableRegionProvider { env: Env(Real) }
-[2024-11-11T19:28:23Z WARN  baml_events] Function TestAws:
-    Client: AwsBedrock (anthropic.claude-3-5-sonnet-20240620-v1:0) - 308ms
+    "Mount Rainier stands proud,  \nCloaked in snow and whispered clouds,  \nNature's towering crown."
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_aws_streaming  0:00:01.716384 
  
 
 
 Started 2024-11-25 16:35:55 
 
 Ended 2024-11-25 16:35:57 
 
 Duration 0:00:01.716384 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:35:55Z WARN  aws_runtime::env_config::normalize] section [Connection 1] ignored; config must be in the AWS config file rather than the credentials file
+[2024-11-26T00:35:57Z INFO  baml_events] Function TestAws:
+    Client: AwsBedrock (meta.llama3-8b-instruct-v1:0) - 1706ms. StopReason: unknown. Tokens(in/out): 25/100
     ---PROMPT---
     [chat] user: Write a nice short story about Mt Rainier is tall
     
-    ---REQUEST OPTIONS---
-    api_key: ""
-    ---ERROR (RateLimited (429))---
-    ServiceError(
-        ServiceError {
-            source: ThrottlingException(
-                ThrottlingException {
-                    message: Some(
-                        "Too many requests, please wait before trying again.",
-                    ),
-                    meta: ErrorMetadata {
-                        code: Some(
-                            "ThrottlingException",
-                        ),
-                        message: Some(
-                            "Too many requests, please wait before trying again.",
-                        ),
-                        extras: Some(
-                            {
-                                "aws_request_id": "758ee731-5022-4f89-8299-a27622e30ef0",
-                            },
-                        ),
-                    },
-                },
-            ),
-            raw: Response {
-                status: StatusCode(
-                    429,
-                ),
-                headers: Headers {
-                    headers: {
-                        "date": HeaderValue {
-                            _private: H0(
-                                "Mon, 11 Nov 2024 19:28:23 GMT",
-                            ),
-                        },
-                        "content-type": HeaderValue {
-                            _private: H0(
-                                "application/json",
-                            ),
-                        },
-                        "content-length": HeaderValue {
-                            _private: H0(
-                                "65",
-                            ),
-                        },
-                        "x-amzn-requestid": HeaderValue {
-                            _private: H0(
-                                "758ee731-5022-4f89-8299-a27622e30ef0",
-                            ),
-                        },
-                        "x-amzn-errortype": HeaderValue {
-                            _private: H0(
-                                "ThrottlingException:http://internal.amazon.com/coral/com.amazon.bedrock/",
-                            ),
-                        },
-                    },
-                },
-                body: SdkBody {
-                    inner: Once(
-                        Some(
-                            b"{\"message\":\"Too many requests, please wait before trying again.\"}",
-                        ),
-                    ),
-                    retryable: true,
-                },
-                extensions: Extensions {
-                    extensions_02x: Extensions,
-                    extensions_1x: Extensions,
-                },
-            },
-        },
-    )
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_streaming  0:00:03.571474 
  
 
 
 Started 2024-11-11 13:28:23 
 
 Ended 2024-11-11 13:28:27 
 
 Duration 0:00:03.571474 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:28:27Z INFO  baml_events] Function PromptTestStreaming:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 3562ms. StopReason: stop. Tokens(in/out): 19/305
-    ---PROMPT---
-    [chat] system: Tell me a short story about Programming languages are fun to create
-    
     ---LLM REPLY---
-    Once upon a time, in a magical land of imagination and innovation, there lived a young and curious wizard named Oliver. Oliver had always been fascinated by the art of programming and had spent many nights experimenting with different languages and algorithms.
+    .
     
-    One day, Oliver had a brilliant idea - he decided to create his very own programming language. He gathered his tools and set to work, combining the best features of his favorite languages with his own unique twist.
+    Here a few joy of the beauty of the mountains that the world is to know the heart of the world, the littleness of the littleness of the moment, the world's second. "The world is a big and the world of the moment, the moment of the moment, the world of the mind, the world of the moment, the one that was a moment of the moment, the little of the world, the little of the book of the moment, the one of the moment
+    ---Parsed Response (string)---
+    ".\n\nHere a few joy of the beauty of the mountains that the world is to know the heart of the world, the littleness of the littleness of the moment, the world's second. \"The world is a big and the world of the moment, the moment of the moment, the world of the mind, the world of the moment, the one that was a moment of the moment, the little of the world, the little of the book of the moment, the one of the moment"
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_streaming  0:00:02.264261 
  
 
 
 Started 2024-11-25 16:35:57 
 
 Ended 2024-11-25 16:35:59 
 
 Duration 0:00:02.264261 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:35:59Z INFO  baml_events] Function PromptTestStreaming:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 2254ms. StopReason: stop. Tokens(in/out): 19/240
+    ---PROMPT---
+    [chat] user: Tell me a short story about Programming languages are fun to create
     
-    As he coded away, lines of syntax and functions danced across his screen like colorful ribbons in the wind. He carefully crafted each element, giving his language a personality and charm all its own.
+    ---LLM REPLY---
+    There once was a young programmer named Lily who had a passion for creating new things. She spent hours at her computer, coding and experimenting with different programming languages. One day, she had an idea to create her own programming language from scratch.
     
-    Finally, after many hours of hard work and dedication, Oliver stood back and marveled at his creation - a sleek and powerful programming language, ready to be unleashed upon the world.
+    Lily started by researching various programming languages and studying their syntax and features. She then began to brainstorm how she could combine the best aspects of each language to create something unique and innovative.
     
-    As he shared his new language with other wizards and programmers, they were amazed by its simplicity and elegance. They quickly adopted Oliver's language, using it to bring their own ideas to life in ways they had never imagined.
+    After weeks of hard work and late nights, Lily finally unveiled her new programming language, which she called "LilyScript." It was a versatile language that allowed for easy manipulation of data and seamless integration with other programming languages.
     
-    And so, Oliver's programming language became a legend in the magical land, inspiring others to create their own languages and pushing the boundaries of what was thought possible.
+    To her delight, LilyScript quickly gained popularity among other programmers who found it easy to learn and fun to use. Lily was overjoyed to see her creation being used by others to develop amazing new software and applications.
     
-    And as for Oliver, he smiled contentedly, knowing that he had created something truly special - a language that brought joy and magic to all who used it. And so, the tale of the wizard programmer and his enchanted language lived on, reminding us all that programming languages are not just tools, but feats of creativity and wonder.
+    Programming languages are often seen as tools for solving problems, but for Lily, creating her own language was a creative and exciting endeavor. She realized that programming languages are not just functional, but can also be a form of art, allowing for boundless imagination and creativity. And for Lily, that was the most fun part of all.
     ---Parsed Response (string)---
-    "Once upon a time, in a magical land of imagination and innovation, there lived a young and curious wizard named Oliver. Oliver had always been fascinated by the art of programming and had spent many nights experimenting with different languages and algorithms.\n\nOne day, Oliver had a brilliant idea - he decided to create his very own programming language. He gathered his tools and set to work, combining the best features of his favorite languages with his own unique twist.\n\nAs he coded away, lines of syntax and functions danced across his screen like colorful ribbons in the wind. He carefully crafted each element, giving his language a personality and charm all its own.\n\nFinally, after many hours of hard work and dedication, Oliver stood back and marveled at his creation - a sleek and powerful programming language, ready to be unleashed upon the world.\n\nAs he shared his new language with other wizards and programmers, they were amazed by its simplicity and elegance. They quickly adopted Oliver's language, using it to bring their own ideas to life in ways they had never imagined.\n\nAnd so, Oliver's programming language became a legend in the magical land, inspiring others to create their own languages and pushing the boundaries of what was thought possible.\n\nAnd as for Oliver, he smiled contentedly, knowing that he had created something truly special - a language that brought joy and magic to all who used it. And so, the tale of the wizard programmer and his enchanted language lived on, reminding us all that programming languages are not just tools, but feats of creativity and wonder."
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_streaming_uniterated  0:00:03.757225 
  
 
 
 Started 2024-11-11 13:28:27 
 
 Ended 2024-11-11 13:28:30 
 
 Duration 0:00:03.757225 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:28:30Z INFO  baml_events] Function PromptTestStreaming:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 3747ms. StopReason: stop. Tokens(in/out): 19/304
+    "There once was a young programmer named Lily who had a passion for creating new things. She spent hours at her computer, coding and experimenting with different programming languages. One day, she had an idea to create her own programming language from scratch.\n\nLily started by researching various programming languages and studying their syntax and features. She then began to brainstorm how she could combine the best aspects of each language to create something unique and innovative.\n\nAfter weeks of hard work and late nights, Lily finally unveiled her new programming language, which she called \"LilyScript.\" It was a versatile language that allowed for easy manipulation of data and seamless integration with other programming languages.\n\nTo her delight, LilyScript quickly gained popularity among other programmers who found it easy to learn and fun to use. Lily was overjoyed to see her creation being used by others to develop amazing new software and applications.\n\nProgramming languages are often seen as tools for solving problems, but for Lily, creating her own language was a creative and exciting endeavor. She realized that programming languages are not just functional, but can also be a form of art, allowing for boundless imagination and creativity. And for Lily, that was the most fun part of all."
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_streaming_uniterated  0:00:01.840321 
  
 
 
 Started 2024-11-25 16:35:59 
 
 Ended 2024-11-25 16:36:01 
 
 Duration 0:00:01.840321 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:36:01Z INFO  baml_events] Function PromptTestStreaming:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 1832ms. StopReason: stop. Tokens(in/out): 19/240
     ---PROMPT---
-    [chat] system: Tell me a short story about The color blue makes me sad
+    [chat] user: Tell me a short story about The color blue makes me sad
     
     ---LLM REPLY---
-    Once there was a girl named Lily who had always felt a deep connection to the color blue. She found comfort in the cool, calming hue that surrounded her at every turn. But as she grew older, she began to notice a shift in her relationship with the color.
+    Sarah loved the color blue. It reminded her of the clear skies of summer, the depths of the ocean, and the calming presence of a gentle breeze. But one day, everything changed.
     
-    The more Lily immersed herself in the blue world, the sadder she felt. The azure skies that once brought her joy now seemed gloomy and foreboding. The gentle waves of the sea, once a source of peace, now brought a deep sense of melancholy.
+    Sarah's best friend, Jenny, was involved in a car accident. The car that hit her was blue. Ever since then, the color blue made Sarah sad. Every time she saw it, memories of that fateful day flooded her mind.
     
-    Lily couldn't understand why the color she had always loved now made her so sad. She tried to avoid anything blue, but it seemed to follow her everywhere she went. The more she pushed it away, the more it seemed to seep into every corner of her life.
+    She tried to avoid anything blue, but it was everywhere. The blue sky, blue flowers, blue clothing – it was impossible to escape. Sarah would often find herself overcome with grief whenever she caught a glimpse of the color that once brought her so much joy.
     
-    One day, as Lily sat by the ocean, watching the waves crash against the shore, she realized the truth. The color blue wasn't making her sad. It was simply reflecting the emotions she had been suppressing for so long. The sadness was already inside her, and the blue was just a mirror showing her the depths of her own sorrow.
+    But as time passed, Sarah realized that she couldn't let the color blue control her emotions forever. She started to associate the color with happier memories of her friend, remembering all the good times they shared together.
     
-    With this realization, Lily embraced the color blue once again. She allowed herself to feel the sadness and move through it, knowing that only by acknowledging her feelings could she find true healing. And as she did, the color blue transformed once more, becoming a source of strength and understanding, a reminder that even in sadness, there is beauty and growth.
+    Slowly but surely, the color blue lost its power over Sarah. It no longer made her sad, but instead served as a reminder of the love and friendship she shared with Jenny. And in that way, the color blue became a source of comfort and healing for Sarah, rather than sadness.
     ---Parsed Response (string)---
-    "Once there was a girl named Lily who had always felt a deep connection to the color blue. She found comfort in the cool, calming hue that surrounded her at every turn. But as she grew older, she began to notice a shift in her relationship with the color.\n\nThe more Lily immersed herself in the blue world, the sadder she felt. The azure skies that once brought her joy now seemed gloomy and foreboding. The gentle waves of the sea, once a source of peace, now brought a deep sense of melancholy.\n\nLily couldn't understand why the color she had always loved now made her so sad. She tried to avoid anything blue, but it seemed to follow her everywhere she went. The more she pushed it away, the more it seemed to seep into every corner of her life.\n\nOne day, as Lily sat by the ocean, watching the waves crash against the shore, she realized the truth. The color blue wasn't making her sad. It was simply reflecting the emotions she had been suppressing for so long. The sadness was already inside her, and the blue was just a mirror showing her the depths of her own sorrow.\n\nWith this realization, Lily embraced the color blue once again. She allowed herself to feel the sadness and move through it, knowing that only by acknowledging her feelings could she find true healing. And as she did, the color blue transformed once more, becoming a source of strength and understanding, a reminder that even in sadness, there is beauty and growth."
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_streaming_sync  0:00:02.798666 
  
 
 
 Started 2024-11-11 13:28:30 
 
 Ended 2024-11-11 13:28:33 
 
 Duration 0:00:02.798666 
 
 Markers   
 
 Fixtures   event_loop_policy   cleanup   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:28:33Z INFO  baml_events] Function PromptTestStreaming:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 2790ms. StopReason: stop. Tokens(in/out): 19/267
+    "Sarah loved the color blue. It reminded her of the clear skies of summer, the depths of the ocean, and the calming presence of a gentle breeze. But one day, everything changed.\n\nSarah's best friend, Jenny, was involved in a car accident. The car that hit her was blue. Ever since then, the color blue made Sarah sad. Every time she saw it, memories of that fateful day flooded her mind.\n\nShe tried to avoid anything blue, but it was everywhere. The blue sky, blue flowers, blue clothing – it was impossible to escape. Sarah would often find herself overcome with grief whenever she caught a glimpse of the color that once brought her so much joy.\n\nBut as time passed, Sarah realized that she couldn't let the color blue control her emotions forever. She started to associate the color with happier memories of her friend, remembering all the good times they shared together.\n\nSlowly but surely, the color blue lost its power over Sarah. It no longer made her sad, but instead served as a reminder of the love and friendship she shared with Jenny. And in that way, the color blue became a source of comfort and healing for Sarah, rather than sadness."
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_streaming_sync  0:00:02.413249 
  
 
 
 Started 2024-11-25 16:36:01 
 
 Ended 2024-11-25 16:36:03 
 
 Duration 0:00:02.413249 
 
 Markers   
 
 Fixtures   event_loop_policy   cleanup   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:36:03Z INFO  baml_events] Function PromptTestStreaming:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 2407ms. StopReason: stop. Tokens(in/out): 19/283
     ---PROMPT---
-    [chat] system: Tell me a short story about Programming languages are fun to create
+    [chat] user: Tell me a short story about Programming languages are fun to create
     
     ---LLM REPLY---
-    Once upon a time, in a world where creativity and innovation reigned supreme, a group of brilliant minds came together to create their own programming language. They were inspired by the endless possibilities and the beauty of code, and they wanted to develop a language that would make programming both powerful and fun.
+    Once upon a time, in a land where technology reigned supreme, a group of brilliant programmers set out to create their own programming languages. Each one had a unique vision and sought to bring their creations to life.
+    
+    The first programmer, a young and ambitious woman, crafted a language that was sleek, intuitive, and easy to learn. It was designed for beginners, making it accessible to anyone who wanted to delve into the world of coding.
     
-    After many hours of brainstorming, designing, and testing, they finally unveiled their creation to the world. Their programming language was a masterpiece, with elegant syntax, powerful features, and a user-friendly interface. Programmers from all corners of the globe were mesmerized by its capabilities and quickly embraced it as their new favorite tool.
+    The second programmer, a seasoned veteran in the field, developed a language that was powerful, efficient, and perfect for handling complex tasks. It was a tool for advanced programmers, enabling them to tackle any challenge with ease.
     
-    As more and more people started using the language, a vibrant community emerged, sharing ideas, collaborating on projects, and pushing the boundaries of what was thought possible. The programming language became a symbol of creativity and innovation, inspiring the next generation of programmers to dream big and think outside the box.
+    And lastly, the third programmer, a quirky and creative individual, came up with a language that was quirky, unconventional, and full of surprises. It was meant to inspire creativity and innovation, allowing users to think outside the box and push the boundaries of what was possible.
     
-    And so, the creators of the language watched with pride as their creation took on a life of its own, empowering people to bring their ideas to life and revolutionize the world of technology. For them, creating a programming language was not just about writing code – it was about sparking a passion for coding and unleashing the limitless potential of the human mind. And that, they knew, was the true magic of programming languages.
+    As they shared their creations with the world, programmers everywhere rejoiced at the prospect of exploring these new languages. They found joy in experimenting, problem-solving, and creating new projects that were only limited by their imaginations.
+    
+    And so, the world of programming was forever changed, as these languages brought a sense of fun, excitement, and endless possibilities to all who used them. And the programmers lived happily ever after, knowing that they had left their mark on the world of technology.
     ---Parsed Response (string)---
-    "Once upon a time, in a world where creativity and innovation reigned supreme, a group of brilliant minds came together to create their own programming language. They were inspired by the endless possibilities and the beauty of code, and they wanted to develop a language that would make programming both powerful and fun.\n\nAfter many hours of brainstorming, designing, and testing, they finally unveiled their creation to the world. Their programming language was a masterpiece, with elegant syntax, powerful features, and a user-friendly interface. Programmers from all corners of the globe were mesmerized by its capabilities and quickly embraced it as their new favorite tool.\n\nAs more and more people started using the language, a vibrant community emerged, sharing ideas, collaborating on projects, and pushing the boundaries of what was thought possible. The programming language became a symbol of creativity and innovation, inspiring the next generation of programmers to dream big and think outside the box.\n\nAnd so, the creators of the language watched with pride as their creation took on a life of its own, empowering people to bring their ideas to life and revolutionize the world of technology. For them, creating a programming language was not just about writing code – it was about sparking a passion for coding and unleashing the limitless potential of the human mind. And that, they knew, was the true magic of programming languages."
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_streaming_uniterated_sync  0:00:05.731819 
  
 
 
 Started 2024-11-11 13:28:33 
 
 Ended 2024-11-11 13:28:39 
 
 Duration 0:00:05.731819 
 
 Markers   
 
 Fixtures   event_loop_policy   cleanup   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:28:39Z INFO  baml_events] Function PromptTestStreaming:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 5724ms. StopReason: stop. Tokens(in/out): 19/425
+    "Once upon a time, in a land where technology reigned supreme, a group of brilliant programmers set out to create their own programming languages. Each one had a unique vision and sought to bring their creations to life.\n\nThe first programmer, a young and ambitious woman, crafted a language that was sleek, intuitive, and easy to learn. It was designed for beginners, making it accessible to anyone who wanted to delve into the world of coding.\n\nThe second programmer, a seasoned veteran in the field, developed a language that was powerful, efficient, and perfect for handling complex tasks. It was a tool for advanced programmers, enabling them to tackle any challenge with ease.\n\nAnd lastly, the third programmer, a quirky and creative individual, came up with a language that was quirky, unconventional, and full of surprises. It was meant to inspire creativity and innovation, allowing users to think outside the box and push the boundaries of what was possible.\n\nAs they shared their creations with the world, programmers everywhere rejoiced at the prospect of exploring these new languages. They found joy in experimenting, problem-solving, and creating new projects that were only limited by their imaginations.\n\nAnd so, the world of programming was forever changed, as these languages brought a sense of fun, excitement, and endless possibilities to all who used them. And the programmers lived happily ever after, knowing that they had left their mark on the world of technology."
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_streaming_uniterated_sync  0:00:02.961963 
  
 
 
 Started 2024-11-25 16:36:03 
 
 Ended 2024-11-25 16:36:06 
 
 Duration 0:00:02.961963 
 
 Markers   
 
 Fixtures   event_loop_policy   cleanup   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:36:06Z INFO  baml_events] Function PromptTestStreaming:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 2954ms. StopReason: stop. Tokens(in/out): 19/315
     ---PROMPT---
-    [chat] system: Tell me a short story about The color blue makes me sad
+    [chat] user: Tell me a short story about The color blue makes me sad
     
     ---LLM REPLY---
-    Once there was a girl named Lily who had always been drawn to the color blue. She loved the deep, calming shade of the sky on a clear day and the way the ocean shimmered in the sunlight. Blue had always been her favorite color, but as she grew older, she found that the color blue began to make her feel a sense of sadness.
+    Every time Lisa saw the color blue, it reminded her of the day her father passed away. He had been wearing a blue shirt, and since then, the color had taken on a heavy weight in her heart.
     
-    It all started when Lily's best friend, Sarah, moved away to a different city. Sarah had always been there for Lily, and they had shared so many happy memories together. But as Sarah's departure drew near, Lily found herself feeling increasingly sad every time she saw the color blue. It reminded her of the vastness of the ocean that separated them, and the endless sky that seemed to stretch on forever, just like the distance between them.
+    She found herself avoiding anything blue, from the sky to her favorite sweater. The mere sight of it would bring back memories of that fateful day, the sound of hospital machines beeping in the background, and the look of sorrow on her mother's face.
     
-    As the days went by, Lily found it harder and harder to shake off the feeling of sadness that the color blue brought her. She tried to avoid anything blue, but it was everywhere – in the clothes she wore, the sky above her, and the sea that beckoned her. She missed her friend more than words could say, and the color blue only served as a constant reminder of that loss.
+    No matter how hard she tried to shake off the sadness that accompanied the color blue, it seemed to linger everywhere she turned. It was as if the color had seeped into her soul and would never let her forget the pain and loss she had experienced.
     
-    But one day, as Lily was sitting by the ocean, watching the waves crash against the shore, she realized something. The color blue may have made her sad, but it also held a beauty and a serenity that she couldn't deny. She thought of all the happy memories she had shared with Sarah, and how the color blue had been a part of so many of them. And in that moment, Lily realized that while the color blue may bring sadness, it also held a deep sense of comfort and familiarity that she couldn't ignore.
+    But one day, as she sat by the ocean watching the waves crash against the shore, she realized that blue wasn't just a color of sadness. It was also a color of peace and tranquility, of vastness and wonder. And in that moment, she decided to embrace the color blue once again, not as a reminder of her father's passing, but as a symbol of the love and memories they had shared.
     
-    And so, Lily learned to embrace the color blue once again, not as a reminder of loss, but as a reminder of the love and memories she shared with her friend. The color blue would always have a bittersweet tinge for her, but it would also be a constant reminder of the beauty and joy that can be found even in moments of sadness.
+    From then on, whenever she saw the color blue, she would smile, knowing that her father was watching over her, bringing her comfort and joy in the most unexpected of places. And as she gazed at the endless expanse of blue sky above her, she felt a sense of peace and happiness wash over her, knowing that her father's spirit was always with her, guiding her through the darkest of days.
     ---Parsed Response (string)---
-    "Once there was a girl named Lily who had always been drawn to the color blue. She loved the deep, calming shade of the sky on a clear day and the way the ocean shimmered in the sunlight. Blue had always been her favorite color, but as she grew older, she found that the color blue began to make her feel a sense of sadness.\n\nIt all started when Lily's best friend, Sarah, moved away to a different city. Sarah had always been there for Lily, and they had shared so many happy memories together. But as Sarah's departure drew near, Lily found herself feeling increasingly sad every time she saw the color blue. It reminded her of the vastness of the ocean that separated them, and the endless sky that seemed to stretch on forever, just like the distance between them.\n\nAs the days went by, Lily found it harder and harder to shake off the feeling of sadness that the color blue brought her. She tried to avoid anything blue, but it was everywhere – in the clothes she wore, the sky above her, and the sea that beckoned her. She missed her friend more than words could say, and the color blue only served as a constant reminder of that loss.\n\nBut one day, as Lily was sitting by the ocean, watching the waves crash against the shore, she realized something. The color blue may have made her sad, but it also held a beauty and a serenity that she couldn't deny. She thought of all the happy memories she had shared with Sarah, and how the color blue had been a part of so many of them. And in that moment, Lily realized that while the color blue may bring sadness, it also held a deep sense of comfort and familiarity that she couldn't ignore.\n\nAnd so, Lily learned to embrace the color blue once again, not as a reminder of loss, but as a reminder of the love and memories she shared with her friend. The color blue would always have a bittersweet tinge for her, but it would also be a constant reminder of the beauty and joy that can be found even in moments of sadness."
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_streaming_claude  0:00:01.012323 
  
 
 
 Started 2024-11-11 13:28:39 
 
 Ended 2024-11-11 13:28:40 
 
 Duration 0:00:01.012323 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
msgs:
-Here's a haiku about Mt. Rainier's height:
+    "Every time Lisa saw the color blue, it reminded her of the day her father passed away. He had been wearing a blue shirt, and since then, the color had taken on a heavy weight in her heart.\n\nShe found herself avoiding anything blue, from the sky to her favorite sweater. The mere sight of it would bring back memories of that fateful day, the sound of hospital machines beeping in the background, and the look of sorrow on her mother's face.\n\nNo matter how hard she tried to shake off the sadness that accompanied the color blue, it seemed to linger everywhere she turned. It was as if the color had seeped into her soul and would never let her forget the pain and loss she had experienced.\n\nBut one day, as she sat by the ocean watching the waves crash against the shore, she realized that blue wasn't just a color of sadness. It was also a color of peace and tranquility, of vastness and wonder. And in that moment, she decided to embrace the color blue once again, not as a reminder of her father's passing, but as a symbol of the love and memories they had shared.\n\nFrom then on, whenever she saw the color blue, she would smile, knowing that her father was watching over her, bringing her comfort and joy in the most unexpected of places. And as she gazed at the endless expanse of blue sky above her, she felt a sense of peace and happiness wash over her, knowing that her father's spirit was always with her, guiding her through the darkest of days."
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_streaming_claude  0:00:01.490866 
  
 
 
 Started 2024-11-25 16:36:06 
 
 Ended 2024-11-25 16:36:08 
 
 Duration 0:00:01.490866 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
msgs:
+Here's a haiku about Mt. Rainier:
 
-Rainier stands proud, high
-Piercing through clouds in the sky
-Nature's royal crown
+Rainier stands above
+Piercing through clouds in splendor
+Ancient ice crowned peak
 final:
-Here's a haiku about Mt. Rainier's height:
+Here's a haiku about Mt. Rainier:
 
-Rainier stands proud, high
-Piercing through clouds in the sky
-Nature's royal crown
-
 
 
 
Captured stderr call
 
[2024-11-11T19:28:40Z INFO  baml_events] Function PromptTestClaude:
-    Client: Sonnet (claude-3-5-sonnet-20241022) - 988ms. StopReason: "end_turn". Tokens(in/out): 19/39
+Rainier stands above
+Piercing through clouds in splendor
+Ancient ice crowned peak
+
 
 
 
Captured stderr call
 
[2024-11-26T00:36:08Z INFO  baml_events] Function PromptTestClaude:
+    Client: Sonnet (claude-3-5-sonnet-20241022) - 1460ms. StopReason: "end_turn". Tokens(in/out): 19/36
     ---PROMPT---
     [chat] user: Tell me a haiku about Mt Rainier is tall
     
     ---LLM REPLY---
-    Here's a haiku about Mt. Rainier's height:
+    Here's a haiku about Mt. Rainier:
     
-    Rainier stands proud, high
-    Piercing through clouds in the sky
-    Nature's royal crown
+    Rainier stands above
+    Piercing through clouds in splendor
+    Ancient ice crowned peak
     ---Parsed Response (string)---
-    "Here's a haiku about Mt. Rainier's height:\n\nRainier stands proud, high\nPiercing through clouds in the sky\nNature's royal crown"
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_streaming_gemini  0:00:06.790744 
  
 
 
 Started 2024-11-11 13:28:40 
 
 Ended 2024-11-11 13:28:47 
 
 Duration 0:00:06.790744 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
msgs:
-Dottie Pepperidge, known universally as Dr. Pepper by her adoring fifth-graders, wasn't your average teacher. She didn't just teach history; she brought it to life. When they learned about ancient Egypt, desks were pushed aside for a week-long pyramid construction project (cardboard and glue never held so much meaning). On Roman history week, everyone wore togas to school (even Principal Stern, albeit begrudgingly). 
+    "Here's a haiku about Mt. Rainier:\n\nRainier stands above\nPiercing through clouds in splendor\nAncient ice crowned peak"
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_streaming_gemini  0:00:08.346627 
  
 
 
 Started 2024-11-25 16:36:08 
 
 Ended 2024-11-25 16:36:16 
 
 Duration 0:00:08.346627 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
msgs:
+Dottie Mae's Diner bustled with the usual lunchtime crowd. Truckers, weary from the endless ribbon of highway, hunched over steaming plates of meatloaf. Locals swapped gossip over bottomless cups of coffee. And then there was Millie, perched on her usual stool at the counter, nursing a glass of ice-cold Dr Pepper. 
 
-This week, it was the Renaissance. The classroom buzzed with excitement. Dr. Pepper, a mischievous glint in her eye, had promised a surprise. After a lively discussion on Leonardo da Vinci, she unveiled it. 
+Now, Millie knew her Dr Pepper. Had a glass every day since she was a girl, back when a nickel bought you a smile and a frosty bottle. Today, something was different. The familiar bite was sharper, the caramel notes singing a little louder. She swirled the dark liquid, mesmerized by the way it caught the light.
 
-"Today," she announced dramatically, "we paint!"
+"Something wrong with your drink, Millie?" asked Earl, the owner, wiping his hands on his apron.
 
-Groans of "But we're terrible!" were quickly replaced with eager grabbing of smocks and brushes. Dr. Pepper grinned. This was her favorite part – seeing their hesitation melt into pure, joyful creation.
+"Earl, this Dr Pepper...it's different. It's like...magic."
 
-As the children painted, splattering color with abandon, Dr. Pepper observed. Tommy, usually bouncing off the walls, focused intently on his brushstrokes. Quiet Lily, her face lit with a rare smile, was creating a masterpiece of vibrant hues. In those moments, surrounded by the joyous chaos, Dr. Pepper felt an immense satisfaction. 
+Earl chuckled, "It's the same 23 flavors, Millie. Maybe the heat's getting to you."
 
-It wasn't just history they were learning, she realized, it was self-expression, confidence, and the pure joy of creation. And that, she thought, wiping a stray splash of blue paint from her cheek, was the most rewarding lesson of all. 
+But Millie knew better. This wasn't just any Dr Pepper. This was a taste of forgotten dreams, of summer evenings on porch swings, of a time when life was simpler. As she sipped, a warm feeling spread through her, chasing away the aches and pains that came with her 80-odd years. 
+
+Word got around about Millie's "magic" Dr Pepper. Soon, the diner was packed. People lined up, not just for a burger or a slice of pie, but for a chance to taste the extraordinary in the ordinary. And you know what? Some swore they felt it too. A forgotten memory resurfacing, a spark of joy, a moment of pure, simple happiness. 
+
+It didn't last, of course. The next day, the Dr Pepper tasted just like it always had. But for a fleeting moment, Dottie Mae's Diner had been touched by something special. And it all started with Millie and her belief in the magic of a simple soda. 
 
 final:
-Dottie Pepperidge, known universally as Dr. Pepper by her adoring fifth-graders, wasn't your average teacher. She didn't just teach history; she brought it to life. When they learned about ancient Egypt, desks were pushed aside for a week-long pyramid construction project (cardboard and glue never held so much meaning). On Roman history week, everyone wore togas to school (even Principal Stern, albeit begrudgingly). 
+Dottie Mae's Diner bustled with the usual lunchtime crowd. Truckers, weary from the endless ribbon of highway, hunched over steaming plates of meatloaf. Locals swapped gossip over bottomless cups of coffee. And then there was Millie, perched on her usual stool at the counter, nursing a glass of ice-cold Dr Pepper. 
+
+Now, Millie knew her Dr Pepper. Had a glass every day since she was a girl, back when a nickel bought you a smile and a frosty bottle. Today, something was different. The familiar bite was sharper, the caramel notes singing a little louder. She swirled the dark liquid, mesmerized by the way it caught the light.
 
-This week, it was the Renaissance. The classroom buzzed with excitement. Dr. Pepper, a mischievous glint in her eye, had promised a surprise. After a lively discussion on Leonardo da Vinci, she unveiled it. 
+"Something wrong with your drink, Millie?" asked Earl, the owner, wiping his hands on his apron.
 
-"Today," she announced dramatically, "we paint!"
+"Earl, this Dr Pepper...it's different. It's like...magic."
 
-Groans of "But we're terrible!" were quickly replaced with eager grabbing of smocks and brushes. Dr. Pepper grinned. This was her favorite part – seeing their hesitation melt into pure, joyful creation.
+Earl chuckled, "It's the same 23 flavors, Millie. Maybe the heat's getting to you."
 
-As the children painted, splattering color with abandon, Dr. Pepper observed. Tommy, usually bouncing off the walls, focused intently on his brushstrokes. Quiet Lily, her face lit with a rare smile, was creating a masterpiece of vibrant hues. In those moments, surrounded by the joyous chaos, Dr. Pepper felt an immense satisfaction. 
+But Millie knew better. This wasn't just any Dr Pepper. This was a taste of forgotten dreams, of summer evenings on porch swings, of a time when life was simpler. As she sipped, a warm feeling spread through her, chasing away the aches and pains that came with her 80-odd years. 
 
-It wasn't just history they were learning, she realized, it was self-expression, confidence, and the pure joy of creation. And that, she thought, wiping a stray splash of blue paint from her cheek, was the most rewarding lesson of all. 
+Word got around about Millie's "magic" Dr Pepper. Soon, the diner was packed. People lined up, not just for a burger or a slice of pie, but for a chance to taste the extraordinary in the ordinary. And you know what? Some swore they felt it too. A forgotten memory resurfacing, a spark of joy, a moment of pure, simple happiness. 
 
-
 
 
 
Captured stderr call
 
[2024-11-11T19:28:47Z INFO  baml_events] Function TestGemini:
-    Client: Gemini (gemini-1.5-pro-001) - 6765ms. StopReason: Stop. Tokens(in/out): unknown/unknown
+It didn't last, of course. The next day, the Dr Pepper tasted just like it always had. But for a fleeting moment, Dottie Mae's Diner had been touched by something special. And it all started with Millie and her belief in the magic of a simple soda. 
+
+
 
 
 
Captured stderr call
 
[2024-11-26T00:36:16Z INFO  baml_events] Function TestGemini:
+    Client: Gemini (gemini-1.5-pro-001) - 8337ms. StopReason: Stop. Tokens(in/out): unknown/unknown
     ---PROMPT---
     [chat] user: Write a nice short story about Dr.Pepper
     
     ---LLM REPLY---
-    Dottie Pepperidge, known universally as Dr. Pepper by her adoring fifth-graders, wasn't your average teacher. She didn't just teach history; she brought it to life. When they learned about ancient Egypt, desks were pushed aside for a week-long pyramid construction project (cardboard and glue never held so much meaning). On Roman history week, everyone wore togas to school (even Principal Stern, albeit begrudgingly). 
+    Dottie Mae's Diner bustled with the usual lunchtime crowd. Truckers, weary from the endless ribbon of highway, hunched over steaming plates of meatloaf. Locals swapped gossip over bottomless cups of coffee. And then there was Millie, perched on her usual stool at the counter, nursing a glass of ice-cold Dr Pepper. 
+    
+    Now, Millie knew her Dr Pepper. Had a glass every day since she was a girl, back when a nickel bought you a smile and a frosty bottle. Today, something was different. The familiar bite was sharper, the caramel notes singing a little louder. She swirled the dark liquid, mesmerized by the way it caught the light.
     
-    This week, it was the Renaissance. The classroom buzzed with excitement. Dr. Pepper, a mischievous glint in her eye, had promised a surprise. After a lively discussion on Leonardo da Vinci, she unveiled it. 
+    "Something wrong with your drink, Millie?" asked Earl, the owner, wiping his hands on his apron.
     
-    "Today," she announced dramatically, "we paint!"
+    "Earl, this Dr Pepper...it's different. It's like...magic."
     
-    Groans of "But we're terrible!" were quickly replaced with eager grabbing of smocks and brushes. Dr. Pepper grinned. This was her favorite part – seeing their hesitation melt into pure, joyful creation.
+    Earl chuckled, "It's the same 23 flavors, Millie. Maybe the heat's getting to you."
     
-    As the children painted, splattering color with abandon, Dr. Pepper observed. Tommy, usually bouncing off the walls, focused intently on his brushstrokes. Quiet Lily, her face lit with a rare smile, was creating a masterpiece of vibrant hues. In those moments, surrounded by the joyous chaos, Dr. Pepper felt an immense satisfaction. 
+    But Millie knew better. This wasn't just any Dr Pepper. This was a taste of forgotten dreams, of summer evenings on porch swings, of a time when life was simpler. As she sipped, a warm feeling spread through her, chasing away the aches and pains that came with her 80-odd years. 
     
-    It wasn't just history they were learning, she realized, it was self-expression, confidence, and the pure joy of creation. And that, she thought, wiping a stray splash of blue paint from her cheek, was the most rewarding lesson of all. 
+    Word got around about Millie's "magic" Dr Pepper. Soon, the diner was packed. People lined up, not just for a burger or a slice of pie, but for a chance to taste the extraordinary in the ordinary. And you know what? Some swore they felt it too. A forgotten memory resurfacing, a spark of joy, a moment of pure, simple happiness. 
+    
+    It didn't last, of course. The next day, the Dr Pepper tasted just like it always had. But for a fleeting moment, Dottie Mae's Diner had been touched by something special. And it all started with Millie and her belief in the magic of a simple soda. 
     
     ---Parsed Response (string)---
-    "Dottie Pepperidge, known universally as Dr. Pepper by her adoring fifth-graders, wasn't your average teacher. She didn't just teach history; she brought it to life. When they learned about ancient Egypt, desks were pushed aside for a week-long pyramid construction project (cardboard and glue never held so much meaning). On Roman history week, everyone wore togas to school (even Principal Stern, albeit begrudgingly). \n\nThis week, it was the Renaissance. The classroom buzzed with excitement. Dr. Pepper, a mischievous glint in her eye, had promised a surprise. After a lively discussion on Leonardo da Vinci, she unveiled it. \n\n\"Today,\" she announced dramatically, \"we paint!\"\n\nGroans of \"But we're terrible!\" were quickly replaced with eager grabbing of smocks and brushes. Dr. Pepper grinned. This was her favorite part – seeing their hesitation melt into pure, joyful creation.\n\nAs the children painted, splattering color with abandon, Dr. Pepper observed. Tommy, usually bouncing off the walls, focused intently on his brushstrokes. Quiet Lily, her face lit with a rare smile, was creating a masterpiece of vibrant hues. In those moments, surrounded by the joyous chaos, Dr. Pepper felt an immense satisfaction. \n\nIt wasn't just history they were learning, she realized, it was self-expression, confidence, and the pure joy of creation. And that, she thought, wiping a stray splash of blue paint from her cheek, was the most rewarding lesson of all. \n"
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_tracing_async_only  0:00:04.691202 
  
 
 
 Started 2024-11-11 13:28:47 
 
 Ended 2024-11-11 13:28:51 
 
 Duration 0:00:04.691202 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
STATS TraceStats(failed=0, started=15, finalized=15, submitted=15, sent=15, done=15)
-
 
 
 
Captured stderr call
 
[2024-11-11T19:28:48Z INFO  baml_events] Function FnOutputClass:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 559ms. StopReason: stop. Tokens(in/out): 50/20
+    "Dottie Mae's Diner bustled with the usual lunchtime crowd. Truckers, weary from the endless ribbon of highway, hunched over steaming plates of meatloaf. Locals swapped gossip over bottomless cups of coffee. And then there was Millie, perched on her usual stool at the counter, nursing a glass of ice-cold Dr Pepper. \n\nNow, Millie knew her Dr Pepper. Had a glass every day since she was a girl, back when a nickel bought you a smile and a frosty bottle. Today, something was different. The familiar bite was sharper, the caramel notes singing a little louder. She swirled the dark liquid, mesmerized by the way it caught the light.\n\n\"Something wrong with your drink, Millie?\" asked Earl, the owner, wiping his hands on his apron.\n\n\"Earl, this Dr Pepper...it's different. It's like...magic.\"\n\nEarl chuckled, \"It's the same 23 flavors, Millie. Maybe the heat's getting to you.\"\n\nBut Millie knew better. This wasn't just any Dr Pepper. This was a taste of forgotten dreams, of summer evenings on porch swings, of a time when life was simpler. As she sipped, a warm feeling spread through her, chasing away the aches and pains that came with her 80-odd years. \n\nWord got around about Millie's \"magic\" Dr Pepper. Soon, the diner was packed. People lined up, not just for a burger or a slice of pie, but for a chance to taste the extraordinary in the ordinary. And you know what? Some swore they felt it too. A forgotten memory resurfacing, a spark of joy, a moment of pure, simple happiness. \n\nIt didn't last, of course. The next day, the Dr Pepper tasted just like it always had. But for a fleeting moment, Dottie Mae's Diner had been touched by something special. And it all started with Millie and her belief in the magic of a simple soda. \n"
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_tracing_async_only  0:00:05.556180 
  
 
 
 Started 2024-11-25 16:36:16 
 
 Ended 2024-11-25 16:36:22 
 
 Duration 0:00:05.556180 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
STATS TraceStats(failed=0, started=15, finalized=15, submitted=15, sent=15, done=15)
+
 
 
 
Captured stderr call
 
[2024-11-26T00:36:17Z INFO  baml_events] Function FnOutputClass:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 727ms. StopReason: stop. Tokens(in/out): 50/18
     ---PROMPT---
-    [chat] system: Return a JSON blob with this schema: 
+    [chat] user: Return a JSON blob with this schema: 
     Answer in JSON using this schema:
     {
       prop1: string,
@@ -1156,18 +1239,18 @@
     
     ---LLM REPLY---
     {
-      "prop1": "Hello, World!",
+      "prop1": "Hello",
       "prop2": 540
     }
     ---Parsed Response (class TestOutputClass)---
     {
-      "prop1": "Hello, World!",
+      "prop1": "Hello",
       "prop2": 540
     }
-[2024-11-11T19:28:48Z INFO  baml_events] Function FnOutputClass:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 567ms. StopReason: stop. Tokens(in/out): 50/20
+[2024-11-26T00:36:18Z INFO  baml_events] Function FnOutputClass:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 437ms. StopReason: stop. Tokens(in/out): 50/18
     ---PROMPT---
-    [chat] system: Return a JSON blob with this schema: 
+    [chat] user: Return a JSON blob with this schema: 
     Answer in JSON using this schema:
     {
       prop1: string,
@@ -1180,18 +1263,18 @@
     
     ---LLM REPLY---
     {
-      "prop1": "Hello, World!",
+      "prop1": "Hello",
       "prop2": 540
     }
     ---Parsed Response (class TestOutputClass)---
     {
-      "prop1": "Hello, World!",
+      "prop1": "Hello",
       "prop2": 540
     }
-[2024-11-11T19:28:49Z INFO  baml_events] Function FnOutputClass:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 585ms. StopReason: stop. Tokens(in/out): 50/20
+[2024-11-26T00:36:19Z INFO  baml_events] Function FnOutputClass:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 667ms. StopReason: stop. Tokens(in/out): 50/18
     ---PROMPT---
-    [chat] system: Return a JSON blob with this schema: 
+    [chat] user: Return a JSON blob with this schema: 
     Answer in JSON using this schema:
     {
       prop1: string,
@@ -1204,18 +1287,18 @@
     
     ---LLM REPLY---
     {
-      "prop1": "Hello, world!",
+      "prop1": "Hello",
       "prop2": 540
     }
     ---Parsed Response (class TestOutputClass)---
     {
-      "prop1": "Hello, world!",
+      "prop1": "Hello",
       "prop2": 540
     }
-[2024-11-11T19:28:51Z INFO  baml_events] Function FnOutputClass:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 580ms. StopReason: stop. Tokens(in/out): 50/20
+[2024-11-26T00:36:21Z INFO  baml_events] Function FnOutputClass:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 446ms. StopReason: stop. Tokens(in/out): 50/18
     ---PROMPT---
-    [chat] system: Return a JSON blob with this schema: 
+    [chat] user: Return a JSON blob with this schema: 
     Answer in JSON using this schema:
     {
       prop1: string,
@@ -1228,17 +1311,17 @@
     
     ---LLM REPLY---
     {
-      "prop1": "Hello, world!",
+      "prop1": "example",
       "prop2": 540
     }
     ---Parsed Response (class TestOutputClass)---
     {
-      "prop1": "Hello, world!",
+      "prop1": "example",
       "prop2": 540
     }
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_tracing_sync  0:00:00.001074 
  
 
 
 Started 2024-11-11 13:28:51 
 
 Ended 2024-11-11 13:28:51 
 
 Duration 0:00:00.001074 
 
 Markers   
 
 Fixtures   event_loop_policy   cleanup   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_tracing_thread_pool  0:00:01.493659 
  
 
 
 Started 2024-11-11 13:28:51 
 
 Ended 2024-11-11 13:28:53 
 
 Duration 0:00:01.493659 
 
 Markers   
 
 Fixtures   event_loop_policy   cleanup   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_tracing_thread_pool_async  0:00:13.895948 
  
 
 
 Started 2024-11-11 13:28:53 
 
 Ended 2024-11-11 13:29:07 
 
 Duration 0:00:13.895948 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_tracing_async_gather  0:00:01.315141 
  
 
 
 Started 2024-11-11 13:29:07 
 
 Ended 2024-11-11 13:29:08 
 
 Duration 0:00:01.315141 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_tracing_async_gather_top_level  0:00:01.392317 
  
 
 
 Started 2024-11-11 13:29:08 
 
 Ended 2024-11-11 13:29:09 
 
 Duration 0:00:01.392317 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_dynamic  0:00:03.241872 
  
 
 
 Started 2024-11-11 13:29:09 
 
 Ended 2024-11-11 13:29:13 
 
 Duration 0:00:03.241872 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
{'name': 'Harrison', 'hair_color': <Color.BLACK: 'BLACK'>, 'last_name': [], 'height': 1.83, 'hobbies': [<Hobby.SPORTS: 'SPORTS'>]}
-
 
 
 
Captured stderr call
 
[2024-11-11T19:29:13Z INFO  baml_events] Function ExtractPeople:
-    Client: GPT4 (gpt-4o-2024-08-06) - 3230ms. StopReason: stop. Tokens(in/out): 177/49
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_tracing_sync  0:00:00.000356 
  
 
 
 Started 2024-11-25 16:36:22 
 
 Ended 2024-11-25 16:36:22 
 
 Duration 0:00:00.000356 
 
 Markers   
 
 Fixtures   event_loop_policy   cleanup   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_tracing_thread_pool  0:00:01.489107 
  
 
 
 Started 2024-11-25 16:36:22 
 
 Ended 2024-11-25 16:36:23 
 
 Duration 0:00:01.489107 
 
 Markers   
 
 Fixtures   event_loop_policy   cleanup   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_tracing_thread_pool_async  0:00:14.129490 
  
 
 
 Started 2024-11-25 16:36:23 
 
 Ended 2024-11-25 16:36:37 
 
 Duration 0:00:14.129490 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_tracing_async_gather  0:00:01.398919 
  
 
 
 Started 2024-11-25 16:36:37 
 
 Ended 2024-11-25 16:36:39 
 
 Duration 0:00:01.398919 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_tracing_async_gather_top_level  0:00:01.282715 
  
 
 
 Started 2024-11-25 16:36:39 
 
 Ended 2024-11-25 16:36:40 
 
 Duration 0:00:01.282715 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_dynamic  0:00:01.131032 
  
 
 
 Started 2024-11-25 16:36:40 
 
 Ended 2024-11-25 16:36:41 
 
 Duration 0:00:01.131032 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
{'name': 'Harrison', 'hair_color': <Color.BLACK: 'BLACK'>, 'last_name': [], 'height': 1.8288, 'hobbies': [<Hobby.SPORTS: 'SPORTS'>]}
+
 
 
 
Captured stderr call
 
[2024-11-26T00:36:41Z INFO  baml_events] Function ExtractPeople:
+    Client: GPT4 (gpt-4o-2024-08-06) - 1123ms. StopReason: stop. Tokens(in/out): 177/50
     ---PROMPT---
     [chat] system: You are an expert extraction algorithm. Only extract relevant information from the text. If you do not know the value of an attribute asked to extract, return null for the attribute's value.
     
@@ -1265,7 +1348,7 @@
         "name": "Harrison",
         "hair_color": "BLACK",
         "last_name": [],
-        "height": 1.83,
+        "height": 1.8288,
         "hobbies": ["sports"]
       }
     ]
@@ -1276,18 +1359,18 @@
         "name": "Harrison",
         "hair_color": "BLACK",
         "last_name": [],
-        "height": 1.83,
+        "height": 1.8288,
         "hobbies": [
           "SPORTS"
         ]
       }
     ]
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_dynamic_class_output  0:00:01.135770 
  
 
 
 Started 2024-11-11 13:29:13 
 
 Ended 2024-11-11 13:29:14 
 
 Duration 0:00:01.135770 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
[]
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_dynamic_class_output  0:00:00.931332 
  
 
 
 Started 2024-11-25 16:36:41 
 
 Ended 2024-11-25 16:36:42 
 
 Duration 0:00:00.931332 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
[]
 {"hair_color":"black"}
-
 
 
 
Captured stderr call
 
[2024-11-11T19:29:13Z INFO  baml_events] Function MyFunc:
+
 
 
 
Captured stderr call
 
[2024-11-26T00:36:42Z INFO  baml_events] Function MyFunc:
     Client: GPT35 (gpt-3.5-turbo-0125) - 449ms. StopReason: stop. Tokens(in/out): 49/10
     ---PROMPT---
-    [chat] system: Given a string, extract info using the schema:
+    [chat] user: Given a string, extract info using the schema:
     
     My name is Harrison. My hair is black and I'm 6 feet tall.
     
@@ -1304,10 +1387,10 @@
     {
       "hair_color": "black"
     }
-[2024-11-11T19:29:14Z INFO  baml_events] Function MyFunc:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 674ms. StopReason: stop. Tokens(in/out): 49/10
+[2024-11-26T00:36:42Z INFO  baml_events] Function MyFunc:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 471ms. StopReason: stop. Tokens(in/out): 49/10
     ---PROMPT---
-    [chat] system: Given a string, extract info using the schema:
+    [chat] user: Given a string, extract info using the schema:
     
     My name is Harrison. My hair is black and I'm 6 feet tall.
     
@@ -1324,11 +1407,11 @@
     {
       "hair_color": "black"
     }
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_dynamic_class_nested_output_no_stream  0:00:01.031539 
  
 
 
 Started 2024-11-11 13:29:14 
 
 Ended 2024-11-11 13:29:15 
 
 Duration 0:00:01.031539 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
{"name":{"first_name":"Mark","last_name":"Gonzalez","middle_name":null},"address":null,"hair_color":"black","height":6.0}
-
 
 
 
Captured stderr call
 
[2024-11-11T19:29:15Z INFO  baml_events] Function MyFunc:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 1017ms. StopReason: stop. Tokens(in/out): 97/55
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_dynamic_class_nested_output_no_stream  0:00:01.176101 
  
 
 
 Started 2024-11-25 16:36:42 
 
 Ended 2024-11-25 16:36:43 
 
 Duration 0:00:01.176101 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
{"name":{"first_name":"Mark","last_name":"Gonzalez","middle_name":null},"address":null,"hair_color":"black","height":6.0}
+
 
 
 
Captured stderr call
 
[2024-11-26T00:36:43Z INFO  baml_events] Function MyFunc:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 1164ms. StopReason: stop. Tokens(in/out): 117/57
     ---PROMPT---
-    [chat] system: Given a string, extract info using the schema:
+    [chat] user: Given a string, extract info using the schema:
     
     My name is Mark Gonzalez. My hair is black and I'm 6 feet tall.
     
@@ -1340,6 +1423,10 @@
         middle_name: string or null,
       } or null,
       address: {
+        street: string,
+        city: string,
+        state: string,
+        zip: string,
       } or null,
       hairColor: string,
       height: float or null,
@@ -1354,7 +1441,7 @@
       },
       "address": null,
       "hairColor": "black",
-      "height": 6
+      "height": 6.0
     }
     ---Parsed Response (class DynamicOutput)---
     {
@@ -1367,7 +1454,7 @@
       "hair_color": "black",
       "height": 6.0
     }
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_dynamic_class_nested_output_stream  0:00:00.817210 
  
 
 
 Started 2024-11-11 13:29:15 
 
 Ended 2024-11-11 13:29:16 
 
 Duration 0:00:00.817210 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
streamed  name=None hair_color=None
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_dynamic_class_nested_output_stream  0:00:00.717229 
  
 
 
 Started 2024-11-25 16:36:43 
 
 Ended 2024-11-25 16:36:44 
 
 Duration 0:00:00.717229 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
streamed  name=None hair_color=None
 streamed  {'name': None, 'hair_color': None}
 streamed  name=None hair_color=None
 streamed  {'name': None, 'hair_color': None}
@@ -1444,10 +1531,10 @@
 streamed  name={'first_name': 'Mark', 'last_name': 'Gonzalez'} hair_color='black'
 streamed  {'name': {'first_name': 'Mark', 'last_name': 'Gonzalez'}, 'hair_color': 'black'}
 {"name":{"first_name":"Mark","last_name":"Gonzalez"},"hair_color":"black"}
-
 
 
 
Captured stderr call
 
[2024-11-11T19:29:16Z INFO  baml_events] Function MyFunc:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 806ms. StopReason: stop. Tokens(in/out): 73/35
+
 
 
 
Captured stderr call
 
[2024-11-26T00:36:44Z INFO  baml_events] Function MyFunc:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 710ms. StopReason: stop. Tokens(in/out): 73/35
     ---PROMPT---
-    [chat] system: Given a string, extract info using the schema:
+    [chat] user: Given a string, extract info using the schema:
     
     My name is Mark Gonzalez. My hair is black and I'm 6 feet tall.
     
@@ -1476,7 +1563,7 @@
       },
       "hair_color": "black"
     }
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_stream_dynamic_class_output  0:00:00.547751 
  
 
 
 Started 2024-11-11 13:29:16 
 
 Ended 2024-11-11 13:29:16 
 
 Duration 0:00:00.547751 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
[]
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_stream_dynamic_class_output  0:00:00.785962 
  
 
 
 Started 2024-11-25 16:36:44 
 
 Ended 2024-11-25 16:36:45 
 
 Duration 0:00:00.785962 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
[]
 streamed  {'hair_color': '{'}
 streamed  {'hair_color': '{'}
 streamed  {'hair_color': '{\n  "'}
@@ -1493,10 +1580,10 @@
 final  hair_color='black'
 final  {'hair_color': 'black'}
 final  {"hair_color":"black"}
-
 
 
 
Captured stderr call
 
[2024-11-11T19:29:16Z INFO  baml_events] Function MyFunc:
-    Client: MyClient (gpt-4o-mini-2024-07-18) - 539ms. StopReason: stop. Tokens(in/out): 48/14
+
 
 
 
Captured stderr call
 
[2024-11-26T00:36:45Z INFO  baml_events] Function MyFunc:
+    Client: MyClient (gpt-4o-mini-2024-07-18) - 780ms. StopReason: stop. Tokens(in/out): 48/14
     ---PROMPT---
-    [chat] system: Given a string, extract info using the schema:
+    [chat] user: Given a string, extract info using the schema:
     
     My name is Harrison. My hair is black and I'm 6 feet tall.
     
@@ -1515,23 +1602,23 @@
     {
       "hair_color": "black"
     }
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_dynamic_inputs_list2  0:00:01.235725 
  
 
 
 Started 2024-11-11 13:29:16 
 
 Ended 2024-11-11 13:29:17 
 
 Duration 0:00:01.235725 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:29:17Z INFO  baml_events] Function DynamicListInputOutput:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 1223ms. StopReason: stop. Tokens(in/out): 135/79
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_dynamic_inputs_list2  0:00:01.407343 
  
 
 
 Started 2024-11-25 16:36:45 
 
 Ended 2024-11-25 16:36:46 
 
 Duration 0:00:01.407343 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:36:46Z INFO  baml_events] Function DynamicListInputOutput:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 1399ms. StopReason: stop. Tokens(in/out): 135/79
     ---PROMPT---
-    [chat] system: Here is some input data:
+    [chat] user: Here is some input data:
     ----
     [{
+        "testKey": "myTest",
+        "new_key": "hi1",
         "blah": {
             "nestedKey1": "nestedVal",
         },
-        "testKey": "myTest",
-        "new_key": "hi1",
     }, {
-        "testKey": "myTest",
         "blah": {
             "nestedKey1": "nestedVal",
         },
         "new_key": "hi",
+        "testKey": "myTest",
     }]
     ----
     
@@ -1549,20 +1636,20 @@
     
     ---LLM REPLY---
     [
-        {
-            "testKey": "myTest",
-            "new_key": "hi1",
-            "blah": {
-                "nestedKey1": "nestedVal"
-            }
-        },
-        {
-            "testKey": "myTest",
-            "new_key": "hi",
-            "blah": {
-                "nestedKey1": "nestedVal"
-            }
+      {
+        "testKey": "myTest",
+        "new_key": "hi1",
+        "blah": {
+          "nestedKey1": "nestedVal"
         }
+      },
+      {
+        "testKey": "myTest",
+        "new_key": "hi",
+        "blah": {
+          "nestedKey1": "nestedVal"
+        }
+      }
     ]
     ---Parsed Response (list<class DynInputOutput>)---
     [
@@ -1581,8 +1668,8 @@
         }
       }
     ]
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_dynamic_types_new_enum  0:00:00.910498 
  
 
 
 Started 2024-11-11 13:29:17 
 
 Ended 2024-11-11 13:29:18 
 
 Duration 0:00:00.910498 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:29:18Z INFO  baml_events] Function ExtractPeople:
-    Client: GPT4 (gpt-4o-2024-08-06) - 901ms. StopReason: stop. Tokens(in/out): 149/32
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_dynamic_types_new_enum  0:00:01.202549 
  
 
 
 Started 2024-11-25 16:36:46 
 
 Ended 2024-11-25 16:36:47 
 
 Duration 0:00:01.202549 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:36:47Z INFO  baml_events] Function ExtractPeople:
+    Client: GPT4 (gpt-4o-2024-08-06) - 1196ms. StopReason: stop. Tokens(in/out): 149/32
     ---PROMPT---
     [chat] system: You are an expert extraction algorithm. Only extract relevant information from the text. If you do not know the value of an attribute asked to extract, return null for the attribute's value.
     
@@ -1612,8 +1699,8 @@
         "animalLiked": "GIRAFFE"
       }
     ]
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_dynamic_types_existing_enum  0:00:00.620114 
  
 
 
 Started 2024-11-11 13:29:18 
 
 Ended 2024-11-11 13:29:19 
 
 Duration 0:00:00.620114 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:29:19Z INFO  baml_events] Function ExtractHobby:
-    Client: GPT4 (gpt-4o-2024-08-06) - 612ms. StopReason: stop. Tokens(in/out): 65/12
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_dynamic_types_existing_enum  0:00:00.775854 
  
 
 
 Started 2024-11-25 16:36:47 
 
 Ended 2024-11-25 16:36:48 
 
 Duration 0:00:00.775854 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:36:48Z INFO  baml_events] Function ExtractHobby:
+    Client: GPT4 (gpt-4o-2024-08-06) - 770ms. StopReason: stop. Tokens(in/out): 65/12
     ---PROMPT---
     [chat] system: Answer with a JSON Array using this schema:
     [
@@ -1631,8 +1718,8 @@
       "Golfing",
       "MUSIC"
     ]
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_dynamic_literals  0:00:00.808782 
  
 
 
 Started 2024-11-11 13:29:19 
 
 Ended 2024-11-11 13:29:20 
 
 Duration 0:00:00.808782 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:29:20Z INFO  baml_events] Function ExtractPeople:
-    Client: GPT4 (gpt-4o-2024-08-06) - 801ms. StopReason: stop. Tokens(in/out): 149/32
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_dynamic_literals  0:00:02.349094 
  
 
 
 Started 2024-11-25 16:36:48 
 
 Ended 2024-11-25 16:36:50 
 
 Duration 0:00:02.349094 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:36:50Z INFO  baml_events] Function ExtractPeople:
+    Client: GPT4 (gpt-4o-2024-08-06) - 2342ms. StopReason: stop. Tokens(in/out): 149/32
     ---PROMPT---
     [chat] system: You are an expert extraction algorithm. Only extract relevant information from the text. If you do not know the value of an attribute asked to extract, return null for the attribute's value.
     
@@ -1662,23 +1749,23 @@
         "animalLiked": "GIRAFFE"
       }
     ]
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_dynamic_inputs_list  0:00:01.061539 
  
 
 
 Started 2024-11-11 13:29:20 
 
 Ended 2024-11-11 13:29:21 
 
 Duration 0:00:01.061539 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:29:21Z INFO  baml_events] Function DynamicListInputOutput:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 1049ms. StopReason: stop. Tokens(in/out): 134/78
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_dynamic_inputs_list  0:00:00.894979 
  
 
 
 Started 2024-11-25 16:36:50 
 
 Ended 2024-11-25 16:36:51 
 
 Duration 0:00:00.894979 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:36:51Z INFO  baml_events] Function DynamicListInputOutput:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 884ms. StopReason: stop. Tokens(in/out): 134/78
     ---PROMPT---
-    [chat] system: Here is some input data:
+    [chat] user: Here is some input data:
     ----
     [{
-        "testKey": "myTest",
         "new_key": "hi",
+        "testKey": "myTest",
         "blah": {
             "nestedKey1": "nestedVal",
         },
     }, {
+        "new_key": "hi",
+        "testKey": "myTest",
         "blah": {
             "nestedKey1": "nestedVal",
         },
-        "testKey": "myTest",
-        "new_key": "hi",
     }]
     ----
     
@@ -1696,20 +1783,20 @@
     
     ---LLM REPLY---
     [
-        {
-            "testKey": "myTest",
-            "new_key": "hi",
-            "blah": {
-                "nestedKey1": "nestedVal"
-            }
-        },
-        {
-            "testKey": "myTest",
-            "new_key": "hi",
-            "blah": {
-                "nestedKey1": "nestedVal"
-            }
+      {
+        "testKey": "myTest",
+        "new_key": "hi",
+        "blah": {
+          "nestedKey1": "nestedVal"
         }
+      },
+      {
+        "testKey": "myTest",
+        "new_key": "hi",
+        "blah": {
+          "nestedKey1": "nestedVal"
+        }
+      }
     ]
     ---Parsed Response (list<class DynInputOutput>)---
     [
@@ -1728,14 +1815,14 @@
         }
       }
     ]
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_dynamic_output_map  0:00:00.892150 
  
 
 
 Started 2024-11-11 13:29:21 
 
 Ended 2024-11-11 13:29:22 
 
 Duration 0:00:00.892150 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
[]
-final  hair_color='black' attributes={'height': '6 feet', 'eye_color': 'blue', 'facial_hair': 'beard'}
-final  {'hair_color': 'black', 'attributes': {'height': '6 feet', 'eye_color': 'blue', 'facial_hair': 'beard'}}
-final  {"hair_color":"black","attributes":{"height":"6 feet","eye_color":"blue","facial_hair":"beard"}}
-
 
 
 
Captured stderr call
 
[2024-11-11T19:29:22Z INFO  baml_events] Function MyFunc:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 884ms. StopReason: stop. Tokens(in/out): 80/44
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_dynamic_output_map  0:00:00.718771 
  
 
 
 Started 2024-11-25 16:36:51 
 
 Ended 2024-11-25 16:36:52 
 
 Duration 0:00:00.718771 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
[]
+final  hair_color='black' attributes={'eye_color': 'blue', 'facial_hair': 'beard'}
+final  {'hair_color': 'black', 'attributes': {'eye_color': 'blue', 'facial_hair': 'beard'}}
+final  {"hair_color":"black","attributes":{"eye_color":"blue","facial_hair":"beard"}}
+
 
 
 
Captured stderr call
 
[2024-11-26T00:36:52Z INFO  baml_events] Function MyFunc:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 711ms. StopReason: stop. Tokens(in/out): 80/36
     ---PROMPT---
-    [chat] system: Given a string, extract info using the schema:
+    [chat] user: Given a string, extract info using the schema:
     
     My name is Harrison. My hair is black and I'm 6 feet tall. I have blue eyes and a beard.
     
@@ -1750,7 +1837,6 @@
     {
       "hair_color": "black",
       "attributes": {
-        "height": "6 feet",
         "eye_color": "blue",
         "facial_hair": "beard"
       }
@@ -1759,22 +1845,21 @@
     {
       "hair_color": "black",
       "attributes": {
-        "height": "6 feet",
         "eye_color": "blue",
         "facial_hair": "beard"
       }
     }
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_dynamic_output_union  0:00:02.680483 
  
 
 
 Started 2024-11-11 13:29:22 
 
 Ended 2024-11-11 13:29:24 
 
 Duration 0:00:02.680483 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
[]
-final  hair_color='black' attributes={'eye_color': 'blue', 'facial_hair': 'beard', 'age': '30 years old'} height={'feet': 6.0, 'inches': None}
-final  {'hair_color': 'black', 'attributes': {'eye_color': 'blue', 'facial_hair': 'beard', 'age': '30 years old'}, 'height': {'feet': 6.0, 'inches': None}}
-final  {"hair_color":"black","attributes":{"eye_color":"blue","facial_hair":"beard","age":"30 years old"},"height":{"feet":6.0,"inches":null}}
-final  hair_color='black' attributes={'eye_color': 'blue', 'facial_hair': 'beard'} height={'meters': 1.8}
-final  {'hair_color': 'black', 'attributes': {'eye_color': 'blue', 'facial_hair': 'beard'}, 'height': {'meters': 1.8}}
-final  {"hair_color":"black","attributes":{"eye_color":"blue","facial_hair":"beard"},"height":{"meters":1.8}}
-
 
 
 
Captured stderr call
 
[2024-11-11T19:29:24Z INFO  baml_events] Function MyFunc:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 1731ms. StopReason: stop. Tokens(in/out): 114/56
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_dynamic_output_union  0:00:01.911571 
  
 
 
 Started 2024-11-25 16:36:52 
 
 Ended 2024-11-25 16:36:54 
 
 Duration 0:00:01.911571 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
[]
+final  hair_color='black' attributes={'eye_color': 'blue', 'facial_hair': 'beard'} height={'feet': 6.0, 'inches': None}
+final  {'hair_color': 'black', 'attributes': {'eye_color': 'blue', 'facial_hair': 'beard'}, 'height': {'feet': 6.0, 'inches': None}}
+final  {"hair_color":"black","attributes":{"eye_color":"blue","facial_hair":"beard"},"height":{"feet":6.0,"inches":null}}
+final  hair_color='black' attributes={'eye_color': 'blue', 'facial_hair': 'beard', 'age': '30 years'} height={'meters': 1.8}
+final  {'hair_color': 'black', 'attributes': {'eye_color': 'blue', 'facial_hair': 'beard', 'age': '30 years'}, 'height': {'meters': 1.8}}
+final  {"hair_color":"black","attributes":{"eye_color":"blue","facial_hair":"beard","age":"30 years"},"height":{"meters":1.8}}
+
 
 
 
Captured stderr call
 
[2024-11-26T00:36:53Z INFO  baml_events] Function MyFunc:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 779ms. StopReason: stop. Tokens(in/out): 114/58
     ---PROMPT---
-    [chat] system: Given a string, extract info using the schema:
+    [chat] user: Given a string, extract info using the schema:
     
     My name is Harrison. My hair is black and I'm 6 feet tall. I have blue eyes and a beard. I am 30 years old.
     
@@ -1793,15 +1878,14 @@
     
     ---LLM REPLY---
     {
-      hair_color: "black",
-      attributes: {
-        eye_color: "blue",
-        facial_hair: "beard",
-        age: "30 years old"
+      "hair_color": "black",
+      "attributes": {
+        "eye_color": "blue",
+        "facial_hair": "beard"
       },
-      height: {
-        feet: 6,
-        inches: null
+      "height": {
+        "feet": 6,
+        "inches": null
       }
     }
     ---Parsed Response (class DynamicOutput)---
@@ -1809,18 +1893,17 @@
       "hair_color": "black",
       "attributes": {
         "eye_color": "blue",
-        "facial_hair": "beard",
-        "age": "30 years old"
+        "facial_hair": "beard"
       },
       "height": {
         "feet": 6.0,
         "inches": null
       }
     }
-[2024-11-11T19:29:24Z INFO  baml_events] Function MyFunc:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 920ms. StopReason: stop. Tokens(in/out): 116/53
+[2024-11-26T00:36:54Z INFO  baml_events] Function MyFunc:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 1112ms. StopReason: stop. Tokens(in/out): 116/61
     ---PROMPT---
-    [chat] system: Given a string, extract info using the schema:
+    [chat] user: Given a string, extract info using the schema:
     
     My name is Harrison. My hair is black and I'm 1.8 meters tall. I have blue eyes and a beard. I am 30 years old.
     
@@ -1842,7 +1925,8 @@
       "hair_color": "black",
       "attributes": {
         "eye_color": "blue",
-        "facial_hair": "beard"
+        "facial_hair": "beard",
+        "age": "30 years"
       },
       "height": {
         "meters": 1.8
@@ -1853,13 +1937,36 @@
       "hair_color": "black",
       "attributes": {
         "eye_color": "blue",
-        "facial_hair": "beard"
+        "facial_hair": "beard",
+        "age": "30 years"
       },
       "height": {
         "meters": 1.8
       }
     }
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_nested_class_streaming  0:00:07.976151 
  
 
 
 Started 2024-11-11 13:29:24 
 
 Ended 2024-11-11 13:29:32 
 
 Duration 0:00:07.976151 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
streamed  {'prop1': None, 'prop2': None}
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_nested_class_streaming  0:00:08.064284 
  
 
 
 Started 2024-11-25 16:36:54 
 
 Ended 2024-11-25 16:37:02 
 
 Duration 0:00:08.064284 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
streamed  {'prop1': None, 'prop2': None}
+streamed  {'prop1': None, 'prop2': None}
+streamed  {'prop1': None, 'prop2': None}
+streamed  {'prop1': None, 'prop2': None}
+streamed  {'prop1': None, 'prop2': None}
+streamed  {'prop1': None, 'prop2': None}
+streamed  {'prop1': None, 'prop2': None}
+streamed  {'prop1': None, 'prop2': None}
+streamed  {'prop1': None, 'prop2': None}
+streamed  {'prop1': None, 'prop2': None}
+streamed  {'prop1': None, 'prop2': None}
+streamed  {'prop1': None, 'prop2': None}
+streamed  {'prop1': None, 'prop2': None}
+streamed  {'prop1': None, 'prop2': None}
+streamed  {'prop1': None, 'prop2': None}
+streamed  {'prop1': None, 'prop2': None}
+streamed  {'prop1': None, 'prop2': None}
+streamed  {'prop1': None, 'prop2': None}
+streamed  {'prop1': None, 'prop2': None}
+streamed  {'prop1': None, 'prop2': None}
+streamed  {'prop1': None, 'prop2': None}
+streamed  {'prop1': None, 'prop2': None}
+streamed  {'prop1': None, 'prop2': None}
 streamed  {'prop1': None, 'prop2': None}
 streamed  {'prop1': None, 'prop2': None}
 streamed  {'prop1': None, 'prop2': None}
@@ -1867,75 +1974,92 @@
 streamed  {'prop1': None, 'prop2': None}
 streamed  {'prop1': None, 'prop2': None}
 streamed  {'prop1': '', 'prop2': None}
-streamed  {'prop1': 'Hello', 'prop2': None}
-streamed  {'prop1': 'Hello', 'prop2': None}
-streamed  {'prop1': 'Hello', 'prop2': None}
-streamed  {'prop1': 'Hello', 'prop2': None}
-streamed  {'prop1': 'Hello', 'prop2': None}
-streamed  {'prop1': 'Hello', 'prop2': None}
-streamed  {'prop1': 'Hello', 'prop2': None}
-streamed  {'prop1': 'Hello', 'prop2': None}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': None, 'prop2': None, 'inner': None}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': None, 'prop2': None, 'inner': None}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': None, 'prop2': None, 'inner': None}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': None, 'prop2': None, 'inner': None}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': None, 'prop2': None, 'inner': None}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': None, 'prop2': None, 'inner': None}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': None, 'prop2': None, 'inner': None}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': '', 'prop2': None, 'inner': None}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': None, 'inner': None}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': None, 'inner': None}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': None, 'inner': None}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': None, 'inner': None}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': None, 'inner': None}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': None, 'inner': None}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': None, 'inner': None}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': None, 'inner': None}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': '', 'inner': None}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': 'json', 'inner': None}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': 'json', 'inner': None}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': 'json', 'inner': None}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': 'json', 'inner': None}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': 'json', 'inner': None}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': 'json', 'inner': None}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': 'json', 'inner': None}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': 'json', 'inner': {'prop2': None, 'prop3': None}}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': 'json', 'inner': {'prop2': None, 'prop3': None}}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': 'json', 'inner': {'prop2': None, 'prop3': None}}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': 'json', 'inner': {'prop2': None, 'prop3': None}}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': 'json', 'inner': {'prop2': None, 'prop3': None}}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': 'json', 'inner': {'prop2': None, 'prop3': None}}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': 'json', 'inner': {'prop2': None, 'prop3': None}}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': 'json', 'inner': {'prop2': None, 'prop3': None}}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': 'json', 'inner': {'prop2': None, 'prop3': None}}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': 'json', 'inner': {'prop2': None, 'prop3': None}}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': 'json', 'inner': {'prop2': 42, 'prop3': None}}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': 'json', 'inner': {'prop2': 42, 'prop3': None}}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': 'json', 'inner': {'prop2': 42, 'prop3': None}}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': 'json', 'inner': {'prop2': 42, 'prop3': None}}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': 'json', 'inner': {'prop2': 42, 'prop3': None}}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': 'json', 'inner': {'prop2': None, 'prop3': None}}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': 'json', 'inner': {'prop2': 42, 'prop3': None}}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': 'json', 'inner': {'prop2': 42, 'prop3': None}}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': 'json', 'inner': {'prop2': 42, 'prop3': None}}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': 'json', 'inner': {'prop2': 42, 'prop3': None}}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': 'json', 'inner': {'prop2': 42, 'prop3': None}}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': 'json', 'inner': {'prop2': 42, 'prop3': None}}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': 'json', 'inner': {'prop2': 42, 'prop3': 3.14}}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': 'json', 'inner': {'prop2': 42, 'prop3': 3.14}}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': 'json', 'inner': {'prop2': 42, 'prop3': 3.14}}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': 'json', 'inner': {'prop2': 42, 'prop3': 3.14}}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': 'json', 'inner': {'prop2': 42, 'prop3': 3.14}}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': 'json', 'inner': {'prop2': 42, 'prop3': 3.14}}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': 'json', 'inner': {'prop2': 42, 'prop3': 3.14}}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': 'json', 'inner': {'prop2': 42, 'prop3': 3.14}}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': 'json', 'inner': {'prop2': 42, 'prop3': 3.14}}}
-streamed  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': 'json', 'inner': {'prop2': 42, 'prop3': 3.14}}}
-final  {'prop1': 'Hello', 'prop2': {'prop1': 'World', 'prop2': 'json', 'inner': {'prop2': 42, 'prop3': 3.14}}}
-
 
 
 
Captured stderr call
 
[2024-11-11T19:29:32Z INFO  baml_events] Function FnOutputClassNested:
-    Client: Ollama (llama2) - 7964ms. StopReason: stop. Tokens(in/out): unknown/unknown
-    ---PROMPT---
-    [chat] system: Return a made up json blob that matches this schema:
+streamed  {'prop1': 'hello', 'prop2': None}
+streamed  {'prop1': 'hello', 'prop2': None}
+streamed  {'prop1': 'hello', 'prop2': None}
+streamed  {'prop1': 'hello', 'prop2': None}
+streamed  {'prop1': 'hello', 'prop2': None}
+streamed  {'prop1': 'hello', 'prop2': None}
+streamed  {'prop1': 'hello', 'prop2': None}
+streamed  {'prop1': 'hello', 'prop2': None}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': None, 'prop2': None, 'inner': None}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': None, 'prop2': None, 'inner': None}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': None, 'prop2': None, 'inner': None}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': None, 'prop2': None, 'inner': None}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': None, 'prop2': None, 'inner': None}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': None, 'prop2': None, 'inner': None}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': None, 'prop2': None, 'inner': None}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': '', 'prop2': None, 'inner': None}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': None, 'inner': None}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': None, 'inner': None}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': None, 'inner': None}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': None, 'inner': None}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': None, 'inner': None}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': None, 'inner': None}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': None, 'inner': None}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': None, 'inner': None}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': '', 'inner': None}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'good', 'inner': None}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': None}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': None}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': None}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': None}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': None}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': None}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': None}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': None, 'prop3': None}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': None, 'prop3': None}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': None, 'prop3': None}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': None, 'prop3': None}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': None, 'prop3': None}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': None, 'prop3': None}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': None, 'prop3': None}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': None, 'prop3': None}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': None, 'prop3': None}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': None, 'prop3': None}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': 42, 'prop3': None}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': 42, 'prop3': None}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': 42, 'prop3': None}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': 42, 'prop3': None}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': 42, 'prop3': None}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': None, 'prop3': None}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': 42, 'prop3': None}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': 42, 'prop3': None}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': 42, 'prop3': None}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': 42, 'prop3': None}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': 42, 'prop3': None}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': 42, 'prop3': None}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': 42, 'prop3': 3.14}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': 42, 'prop3': 3.14}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': 42, 'prop3': 3.14}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': 42, 'prop3': 3.14}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': 42, 'prop3': 3.14}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': 42, 'prop3': 3.14}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': 42, 'prop3': 3.14}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': 42, 'prop3': 3.14}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': 42, 'prop3': 3.14}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': 42, 'prop3': 3.14}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': 42, 'prop3': 3.14}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': 42, 'prop3': 3.14}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': 42, 'prop3': 3.14}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': 42, 'prop3': 3.14}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': 42, 'prop3': 3.14}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': 42, 'prop3': 3.14}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': 42, 'prop3': 3.14}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': 42, 'prop3': 3.14}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': 42, 'prop3': 3.14}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': 42, 'prop3': 3.14}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': 42, 'prop3': 3.14}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': 42, 'prop3': 3.14}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': 42, 'prop3': 3.14}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': 42, 'prop3': 3.14}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': 42, 'prop3': 3.14}}}
+streamed  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': 42, 'prop3': 3.14}}}
+final  {'prop1': 'hello', 'prop2': {'prop1': 'world', 'prop2': 'goodbye', 'inner': {'prop2': 42, 'prop3': 3.14}}}
+
 
 
 
Captured stderr call
 
[2024-11-26T00:37:02Z INFO  baml_events] Function FnOutputClassNested:
+    Client: Ollama (llama2) - 8050ms. StopReason: stop. Tokens(in/out): unknown/unknown
+    ---PROMPT---
+    [chat] user: Return a made up json blob that matches this schema:
     Answer in JSON using this schema:
     {
       prop1: string,
@@ -1953,40 +2077,44 @@
     JSON:
     
     ---LLM REPLY---
+    Sure! Here is a made-up JSON blob that matches the schema you provided:
+    ```json
     {
-      "prop1": "Hello",
+      "prop1": "hello",
       "prop2": {
-        "prop1": "World",
-        "prop2": "json",
+        "prop1": "world",
+        "prop2": "goodbye",
         "inner": {
           "prop2": 42,
           "prop3": 3.14,
-        }
-      }
+        },
+      },
     }
+    ```
+    Let me know if you have any questions or need further assistance!
     ---Parsed Response (class TestClassNested)---
     {
-      "prop1": "Hello",
+      "prop1": "hello",
       "prop2": {
-        "prop1": "World",
-        "prop2": "json",
+        "prop1": "world",
+        "prop2": "goodbye",
         "inner": {
           "prop2": 42,
           "prop3": 3.14
         }
       }
     }
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_dynamic_client_with_openai  0:00:00.423741 
  
 
 
 Started 2024-11-11 13:29:32 
 
 Ended 2024-11-11 13:29:33 
 
 Duration 0:00:00.423741 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:29:33Z INFO  baml_events] Function ExpectFailure:
-    Client: MyClient (gpt-3.5-turbo-0125) - 419ms. StopReason: stop. Tokens(in/out): 14/7
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_dynamic_client_with_openai  0:00:00.436631 
  
 
 
 Started 2024-11-25 16:37:02 
 
 Ended 2024-11-25 16:37:02 
 
 Duration 0:00:00.436631 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:37:02Z INFO  baml_events] Function ExpectFailure:
+    Client: MyClient (gpt-3.5-turbo-0125) - 431ms. StopReason: stop. Tokens(in/out): 14/1
     ---PROMPT---
-    [chat] system: What is the capital of England?
+    [chat] user: What is the capital of England?
     
     ---LLM REPLY---
-    The capital of England is London.
+    London
     ---Parsed Response (string)---
-    "The capital of England is London."
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_dynamic_client_with_vertex_json_str_creds  0:00:00.925397 
  
 
 
 Started 2024-11-11 13:29:33 
 
 Ended 2024-11-11 13:29:34 
 
 Duration 0:00:00.925397 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:29:34Z INFO  baml_events] Function ExpectFailure:
-    Client: MyClient () - 918ms. StopReason: "STOP". Tokens(in/out): 7/10
+    "London"
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_dynamic_client_with_vertex_json_str_creds  0:00:00.915723 
  
 
 
 Started 2024-11-25 16:37:02 
 
 Ended 2024-11-25 16:37:03 
 
 Duration 0:00:00.915723 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:37:03Z INFO  baml_events] Function ExpectFailure:
+    Client: MyClient () - 911ms. StopReason: "STOP". Tokens(in/out): 7/10
     ---PROMPT---
     [chat] user: What is the capital of England?
     
@@ -1995,8 +2123,8 @@
     
     ---Parsed Response (string)---
     "The capital of England is **London**. \n"
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_dynamic_client_with_vertex_json_object_creds  0:00:00.944689 
  
 
 
 Started 2024-11-11 13:29:34 
 
 Ended 2024-11-11 13:29:35 
 
 Duration 0:00:00.944689 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:29:35Z INFO  baml_events] Function ExpectFailure:
-    Client: MyClient () - 939ms. StopReason: "STOP". Tokens(in/out): 7/10
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_dynamic_client_with_vertex_json_object_creds  0:00:01.175949 
  
 
 
 Started 2024-11-25 16:37:03 
 
 Ended 2024-11-25 16:37:05 
 
 Duration 0:00:01.175949 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:37:05Z INFO  baml_events] Function ExpectFailure:
+    Client: MyClient () - 1173ms. StopReason: "STOP". Tokens(in/out): 7/10
     ---PROMPT---
     [chat] user: What is the capital of England?
     
@@ -2005,16 +2133,16 @@
     
     ---Parsed Response (string)---
     "The capital of England is **London**. \n"
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_event_log_hook  0:00:01.178086 
  
 
 
 Started 2024-11-11 13:29:35 
 
 Ended 2024-11-11 13:29:36 
 
 Duration 0:00:01.178086 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
Event log hook1: 
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_event_log_hook  0:00:01.121972 
  
 
 
 Started 2024-11-25 16:37:05 
 
 Ended 2024-11-25 16:37:06 
 
 Duration 0:00:01.121972 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
Event log hook1: 
 Event log event  BamlLogEvent {
     metadata: {
-        event_id: "a9d6364a-68f0-4648-a7b5-d5e591609011",
+        event_id: "4fe6d1f0-0318-4199-9b4a-599b0631cee0",
         parent_id: None,
-        root_event_id: "a9d6364a-68f0-4648-a7b5-d5e591609011"
+        root_event_id: "4fe6d1f0-0318-4199-9b4a-599b0631cee0"
     },
     prompt: "[
   {
-    "role": "system",
+    "role": "user",
     "content": [
       {
         "text": "Return this value back to me: [\"a\", \"b\", \"c\"]"
@@ -2024,153 +2152,169 @@
 ]",
     raw_output: "["a", "b", "c"]",
     parsed_output: "["a", "b", "c"]",
-    start_time: "2024-11-11T19:29:35.631Z"
+    start_time: "2024-11-26T00:37:05.368Z"
 }
-
 
 
 
Captured stderr call
 
[2024-11-11T19:29:36Z INFO  baml_events] Function TestFnNamedArgsSingleStringList:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 453ms. StopReason: stop. Tokens(in/out): 23/9
+
 
 
 
Captured stderr call
 
[2024-11-26T00:37:05Z INFO  baml_events] Function TestFnNamedArgsSingleStringList:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 392ms. StopReason: stop. Tokens(in/out): 23/9
     ---PROMPT---
-    [chat] system: Return this value back to me: ["a", "b", "c"]
+    [chat] user: Return this value back to me: ["a", "b", "c"]
     
     ---LLM REPLY---
     ["a", "b", "c"]
     ---Parsed Response (string)---
     "[\"a\", \"b\", \"c\"]"
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 FAILED  test_aws_bedrock  0:00:03.345974 
  
 
 
 Started 2024-11-11 13:29:36 
 
 Ended 2024-11-11 13:29:39 
 
 Duration 0:00:03.345974 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
baml_py.BamlClientHttpError: LLM call failed: LLMErrorResponse { client: "AwsBedrock", model: Some("anthropic.claude-3-5-sonnet-20240620-v1:0"), prompt: Chat([RenderedChatMessage { role: "user", allow_duplicate_role: false, parts: [Text("Write a nice short story about lightning in a rock")] }]), request_options: {"api_key": String("")}, start_time: SystemTime { tv_sec: 1731353379, tv_nsec: 419024000 }, latency: 318.961292ms, message: "ServiceError(\n    ServiceError {\n        source: ThrottlingException(\n            ThrottlingException {\n                message: Some(\n                    \"Too many requests, please wait before trying again.\",\n                ),\n                meta: ErrorMetadata {\n                    code: Some(\n                        \"ThrottlingException\",\n                    ),\n                    message: Some(\n                        \"Too many requests, please wait before trying again.\",\n                    ),\n                    extras: Some(\n                        {\n                            \"aws_request_id\": \"671bf4e1-976e-4832-8839-6ed57f64bd10\",\n                        },\n                    ),\n                },\n            },\n        ),\n        raw: Response {\n            status: StatusCode(\n                429,\n            ),\n            headers: Headers {\n                headers: {\n                    \"date\": HeaderValue {\n                        _private: H0(\n                            \"Mon, 11 Nov 2024 19:29:39 GMT\",\n                        ),\n                    },\n                    \"content-type\": HeaderValue {\n                        _private: H0(\n                            \"application/json\",\n                        ),\n                    },\n                    \"content-length\": HeaderValue {\n                        _private: H0(\n                            \"65\",\n                        ),\n                    },\n                    \"x-amzn-requestid\": HeaderValue {\n                        _private: H0(\n                            \"671bf4e1-976e-4832-8839-6ed57f64bd10\",\n                        ),\n                    },\n                    \"x-amzn-errortype\": HeaderValue {\n                        _private: H0(\n                            \"ThrottlingException:http://internal.amazon.com/coral/com.amazon.bedrock/\",\n                        ),\n                    },\n                },\n            },\n            body: SdkBody {\n                inner: Once(\n                    Some(\n                        b\"{\\\"message\\\":\\\"Too many requests, please wait before trying again.\\\"}\",\n                    ),\n                ),\n                retryable: true,\n            },\n            extensions: Extensions {\n                extensions_02x: Extensions,\n                extensions_1x: Extensions,\n            },\n        },\n    },\n)", code: RateLimited }
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
@pytest.mark.asyncio
-    async def test_aws_bedrock():
-        ## unstreamed
-        res = await b.TestAws("lightning in a rock")
-        print("unstreamed", res)
-    
-        ## streamed
-        stream = b.stream.TestAws("lightning in a rock")
-    
-        async for msg in stream:
-            if msg:
-                print("streamed ", repr(msg[-100:]))
-    
->       res = await stream.get_final_response()
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_aws_bedrock  0:00:03.076307 
  
 
 
 Started 2024-11-25 16:37:06 
 
 Ended 2024-11-25 16:37:09 
 
 Duration 0:00:03.076307 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
unstreamed 
 
-tests/test_functions.py:1137: 
-_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
-../../engine/language_client_python/python_src/baml_py/stream.py:81: in get_final_response
-    return self.__final_coerce((await asyncio.wrap_future(final)))
-_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
+The old rock sat nestled among the roots of a ancient oak tree, its weathered surface bearing the scars of countless seasons. It was a peculiar rock, with veins of quartz that seemed to shimmer in the fading light of day.
 
-x = 
-
->     lambda x: cast(str, x.cast_to(types, types)),
-      self.__ctx_manager.get(),
-    )
-E   baml_py.BamlClientHttpError: LLM call failed: LLMErrorResponse { client: "AwsBedrock", model: Some("anthropic.claude-3-5-sonnet-20240620-v1:0"), prompt: Chat([RenderedChatMessage { role: "user", allow_duplicate_role: false, parts: [Text("Write a nice short story about lightning in a rock")] }]), request_options: {"api_key": String("")}, start_time: SystemTime { tv_sec: 1731353379, tv_nsec: 419024000 }, latency: 318.961292ms, message: "ServiceError(\n    ServiceError {\n        source: ThrottlingException(\n            ThrottlingException {\n                message: Some(\n                    \"Too many requests, please wait before trying again.\",\n                ),\n                meta: ErrorMetadata {\n                    code: Some(\n                        \"ThrottlingException\",\n                    ),\n                    message: Some(\n                        \"Too many requests, please wait before trying again.\",\n                    ),\n                    extras: Some(\n                        {\n                            \"aws_request_id\": \"671bf4e1-976e-4832-8839-6ed57f64bd10\",\n                        },\n                    ),\n                },\n            },\n        ),\n        raw: Response {\n            status: StatusCode(\n                429,\n            ),\n            headers: Headers {\n                headers: {\n                    \"date\": HeaderValue {\n                        _private: H0(\n                            \"Mon, 11 Nov 2024 19:29:39 GMT\",\n                        ),\n                    },\n                    \"content-type\": HeaderValue {\n                        _private: H0(\n                            \"application/json\",\n                        ),\n                    },\n                    \"content-length\": HeaderValue {\n                        _private: H0(\n                            \"65\",\n                        ),\n                    },\n                    \"x-amzn-requestid\": HeaderValue {\n                        _private: H0(\n                            \"671bf4e1-976e-4832-8839-6ed57f64bd10\",\n                        ),\n                    },\n                    \"x-amzn-errortype\": HeaderValue {\n                        _private: H0(\n                            \"ThrottlingException:http://internal.amazon.com/coral/com.amazon.bedrock/\",\n                        ),\n                    },\n                },\n            },\n            body: SdkBody {\n                inner: Once(\n                    Some(\n                        b\"{\\\"message\\\":\\\"Too many requests, please wait before trying again.\\\"}\",\n                    ),\n                ),\n                retryable: true,\n            },\n            extensions: Extensions {\n                extensions_02x: Extensions,\n                extensions_1x: Extensions,\n            },\n        },\n    },\n)", code: RateLimited }
+As the storm clouds gathered, the air grew heavy with electricity. The wind began to pick up, rustling the leaves of the oak tree and causing the rock to vibrate with an otherworldly energy.
 
-baml_client/async_client.py:5055: BamlClientHttpError
 
 
 
Captured stdout call
 
unstreamed The ancient boulder rested atop a windswept hill, its weathered surface etched with countless years of history. For millennia, it had stood silent sentinel, watching the world change around it.
+Suddenly, a brilliant flash of lightning illuminated the sky,
+streamed  '\n\n'
+streamed  '\n\nIn'
+streamed  '\n\nIn the'
+streamed  '\n\nIn the heart'
+streamed  '\n\nIn the heart of'
+streamed  '\n\nIn the heart of the'
+streamed  '\n\nIn the heart of the ancient'
+streamed  '\n\nIn the heart of the ancient forest'
+streamed  '\n\nIn the heart of the ancient forest,'
+streamed  '\n\nIn the heart of the ancient forest, there'
+streamed  '\n\nIn the heart of the ancient forest, there was'
+streamed  '\n\nIn the heart of the ancient forest, there was a'
+streamed  '\n\nIn the heart of the ancient forest, there was a peculiar'
+streamed  '\n\nIn the heart of the ancient forest, there was a peculiar rock'
+streamed  '\n\nIn the heart of the ancient forest, there was a peculiar rock that'
+streamed  '\n\nIn the heart of the ancient forest, there was a peculiar rock that had'
+streamed  '\n\nIn the heart of the ancient forest, there was a peculiar rock that had been'
+streamed  '\n\nIn the heart of the ancient forest, there was a peculiar rock that had been passed'
+streamed  '\n\nIn the heart of the ancient forest, there was a peculiar rock that had been passed down'
+streamed  '\n\nIn the heart of the ancient forest, there was a peculiar rock that had been passed down through'
+streamed  'heart of the ancient forest, there was a peculiar rock that had been passed down through generations'
+streamed  'rt of the ancient forest, there was a peculiar rock that had been passed down through generations of'
+streamed  'f the ancient forest, there was a peculiar rock that had been passed down through generations of the'
+streamed  'ancient forest, there was a peculiar rock that had been passed down through generations of the local'
+streamed  't forest, there was a peculiar rock that had been passed down through generations of the local tribe'
+streamed  ' forest, there was a peculiar rock that had been passed down through generations of the local tribe.'
+streamed  'rest, there was a peculiar rock that had been passed down through generations of the local tribe. It'
+streamed  ', there was a peculiar rock that had been passed down through generations of the local tribe. It was'
+streamed  're was a peculiar rock that had been passed down through generations of the local tribe. It was said'
+streamed  's a peculiar rock that had been passed down through generations of the local tribe. It was said that'
+streamed  'peculiar rock that had been passed down through generations of the local tribe. It was said that the'
+streamed  'iar rock that had been passed down through generations of the local tribe. It was said that the rock'
+streamed  'ock that had been passed down through generations of the local tribe. It was said that the rock held'
+streamed  'k that had been passed down through generations of the local tribe. It was said that the rock held a'
+streamed  'had been passed down through generations of the local tribe. It was said that the rock held a secret'
+streamed  'ad been passed down through generations of the local tribe. It was said that the rock held a secret,'
+streamed  ' been passed down through generations of the local tribe. It was said that the rock held a secret, a'
+streamed  'assed down through generations of the local tribe. It was said that the rock held a secret, a secret'
+streamed  ' down through generations of the local tribe. It was said that the rock held a secret, a secret that'
+streamed  ' through generations of the local tribe. It was said that the rock held a secret, a secret that only'
+streamed  'generations of the local tribe. It was said that the rock held a secret, a secret that only revealed'
+streamed  'ions of the local tribe. It was said that the rock held a secret, a secret that only revealed itself'
+streamed  's of the local tribe. It was said that the rock held a secret, a secret that only revealed itself to'
+streamed  'he local tribe. It was said that the rock held a secret, a secret that only revealed itself to those'
+streamed  'ocal tribe. It was said that the rock held a secret, a secret that only revealed itself to those who'
+streamed  'e. It was said that the rock held a secret, a secret that only revealed itself to those who listened'
+streamed  's said that the rock held a secret, a secret that only revealed itself to those who listened closely'
+streamed  'aid that the rock held a secret, a secret that only revealed itself to those who listened closely to'
+streamed  'that the rock held a secret, a secret that only revealed itself to those who listened closely to the'
+streamed  'rock held a secret, a secret that only revealed itself to those who listened closely to the whispers'
+streamed  'k held a secret, a secret that only revealed itself to those who listened closely to the whispers of'
+streamed  'ld a secret, a secret that only revealed itself to those who listened closely to the whispers of the'
+streamed  'secret, a secret that only revealed itself to those who listened closely to the whispers of the wind'
+streamed  'ret, a secret that only revealed itself to those who listened closely to the whispers of the wind.\n\n'
+streamed  ', a secret that only revealed itself to those who listened closely to the whispers of the wind.\n\nThe'
+streamed  'ecret that only revealed itself to those who listened closely to the whispers of the wind.\n\nThe rock'
+streamed  't that only revealed itself to those who listened closely to the whispers of the wind.\n\nThe rock was'
+streamed  'that only revealed itself to those who listened closely to the whispers of the wind.\n\nThe rock was a'
+streamed  'only revealed itself to those who listened closely to the whispers of the wind.\n\nThe rock was a deep'
+streamed  'nly revealed itself to those who listened closely to the whispers of the wind.\n\nThe rock was a deep,'
+streamed  'evealed itself to those who listened closely to the whispers of the wind.\n\nThe rock was a deep, rich'
+streamed  'd itself to those who listened closely to the whispers of the wind.\n\nThe rock was a deep, rich brown'
+streamed  ' itself to those who listened closely to the whispers of the wind.\n\nThe rock was a deep, rich brown,'
+streamed  'elf to those who listened closely to the whispers of the wind.\n\nThe rock was a deep, rich brown, and'
+streamed  'to those who listened closely to the whispers of the wind.\n\nThe rock was a deep, rich brown, and its'
+streamed  ' who listened closely to the whispers of the wind.\n\nThe rock was a deep, rich brown, and its surface'
+streamed  ' listened closely to the whispers of the wind.\n\nThe rock was a deep, rich brown, and its surface was'
+streamed  'stened closely to the whispers of the wind.\n\nThe rock was a deep, rich brown, and its surface was et'
+streamed  'ed closely to the whispers of the wind.\n\nThe rock was a deep, rich brown, and its surface was etched'
+streamed  'osely to the whispers of the wind.\n\nThe rock was a deep, rich brown, and its surface was etched with'
+streamed  ' the whispers of the wind.\n\nThe rock was a deep, rich brown, and its surface was etched with strange'
+streamed  'spers of the wind.\n\nThe rock was a deep, rich brown, and its surface was etched with strange symbols'
+streamed  ' of the wind.\n\nThe rock was a deep, rich brown, and its surface was etched with strange symbols that'
+streamed  ' wind.\n\nThe rock was a deep, rich brown, and its surface was etched with strange symbols that seemed'
+streamed  'nd.\n\nThe rock was a deep, rich brown, and its surface was etched with strange symbols that seemed to'
+streamed  ' rock was a deep, rich brown, and its surface was etched with strange symbols that seemed to shimmer'
+streamed  'ck was a deep, rich brown, and its surface was etched with strange symbols that seemed to shimmer in'
+streamed  'as a deep, rich brown, and its surface was etched with strange symbols that seemed to shimmer in the'
+streamed  'eep, rich brown, and its surface was etched with strange symbols that seemed to shimmer in the flick'
+streamed  'rich brown, and its surface was etched with strange symbols that seemed to shimmer in the flickering'
+streamed  'n, and its surface was etched with strange symbols that seemed to shimmer in the flickering sunlight'
+streamed  ', and its surface was etched with strange symbols that seemed to shimmer in the flickering sunlight.'
+streamed  'd its surface was etched with strange symbols that seemed to shimmer in the flickering sunlight. The'
+streamed  'surface was etched with strange symbols that seemed to shimmer in the flickering sunlight. The tribe'
+streamed  "rface was etched with strange symbols that seemed to shimmer in the flickering sunlight. The tribe's"
+streamed  "as etched with strange symbols that seemed to shimmer in the flickering sunlight. The tribe's elders"
+streamed  " with strange symbols that seemed to shimmer in the flickering sunlight. The tribe's elders believed"
+streamed  " strange symbols that seemed to shimmer in the flickering sunlight. The tribe's elders believed that"
+streamed  "ange symbols that seemed to shimmer in the flickering sunlight. The tribe's elders believed that the"
+streamed  "symbols that seemed to shimmer in the flickering sunlight. The tribe's elders believed that the rock"
+streamed  "ols that seemed to shimmer in the flickering sunlight. The tribe's elders believed that the rock was"
+streamed  "that seemed to shimmer in the flickering sunlight. The tribe's elders believed that the rock was imb"
+streamed  "t seemed to shimmer in the flickering sunlight. The tribe's elders believed that the rock was imbued"
+streamed  "med to shimmer in the flickering sunlight. The tribe's elders believed that the rock was imbued with"
+streamed  "to shimmer in the flickering sunlight. The tribe's elders believed that the rock was imbued with the"
+streamed  "mmer in the flickering sunlight. The tribe's elders believed that the rock was imbued with the power"
+streamed  "r in the flickering sunlight. The tribe's elders believed that the rock was imbued with the power of"
+streamed  " the flickering sunlight. The tribe's elders believed that the rock was imbued with the power of the"
+streamed  "ering sunlight. The tribe's elders believed that the rock was imbued with the power of the lightning"
+streamed  " sunlight. The tribe's elders believed that the rock was imbued with the power of the lightning that"
+streamed  " sunlight. The tribe's elders believed that the rock was imbued with the power of the lightning that"
+streamed  " sunlight. The tribe's elders believed that the rock was imbued with the power of the lightning that"
+streamed  " sunlight. The tribe's elders believed that the rock was imbued with the power of the lightning that"
+streamed final 
 
-On a particularly stormy night, dark clouds roiled overhead, their bellies heavy with rain and electricity. As the first drops began to fall, a brilliant fork of lightning split the sky, seeking the highest point on the landscape.
+In the heart of the ancient forest, there was a peculiar rock that had been passed down through generations of the local tribe. It was said that the rock held a secret, a secret that only revealed itself to those who listened closely to the whispers of the wind.
 
-In an instant, the bolt found its mark
-
 
 
 
Captured stderr call
 
[2024-11-11T19:29:36Z WARN  aws_runtime::env_config::normalize] section [Connection 1] ignored; config must be in the AWS config file rather than the credentials file
-[2024-11-11T19:29:36Z INFO  aws_config::meta::region] load_region; provider=EnvironmentVariableRegionProvider { env: Env(Real) }
-[2024-11-11T19:29:39Z INFO  baml_events] Function TestAws:
-    Client: AwsBedrock (anthropic.claude-3-5-sonnet-20240620-v1:0) - 3008ms. StopReason: max_tokens. Tokens(in/out): 17/100
+The rock was a deep, rich brown, and its surface was etched with strange symbols that seemed to shimmer in the flickering sunlight. The tribe's elders believed that the rock was imbued with the power of the lightning that
+
 
 
 
Captured stderr call
 
[2024-11-26T00:37:06Z WARN  aws_runtime::env_config::normalize] section [Connection 1] ignored; config must be in the AWS config file rather than the credentials file
+[2024-11-26T00:37:07Z INFO  baml_events] Function TestAws:
+    Client: AwsBedrock (meta.llama3-8b-instruct-v1:0) - 1532ms. StopReason: max_tokens. Tokens(in/out): 24/100
     ---PROMPT---
     [chat] user: Write a nice short story about lightning in a rock
     
     ---LLM REPLY---
-    The ancient boulder rested atop a windswept hill, its weathered surface etched with countless years of history. For millennia, it had stood silent sentinel, watching the world change around it.
     
-    On a particularly stormy night, dark clouds roiled overhead, their bellies heavy with rain and electricity. As the first drops began to fall, a brilliant fork of lightning split the sky, seeking the highest point on the landscape.
     
-    In an instant, the bolt found its mark
+    The old rock sat nestled among the roots of a ancient oak tree, its weathered surface bearing the scars of countless seasons. It was a peculiar rock, with veins of quartz that seemed to shimmer in the fading light of day.
+    
+    As the storm clouds gathered, the air grew heavy with electricity. The wind began to pick up, rustling the leaves of the oak tree and causing the rock to vibrate with an otherworldly energy.
+    
+    Suddenly, a brilliant flash of lightning illuminated the sky,
     ---Parsed Response (string)---
-    "The ancient boulder rested atop a windswept hill, its weathered surface etched with countless years of history. For millennia, it had stood silent sentinel, watching the world change around it.\n\nOn a particularly stormy night, dark clouds roiled overhead, their bellies heavy with rain and electricity. As the first drops began to fall, a brilliant fork of lightning split the sky, seeking the highest point on the landscape.\n\nIn an instant, the bolt found its mark"
-[2024-11-11T19:29:39Z WARN  aws_runtime::env_config::normalize] section [Connection 1] ignored; config must be in the AWS config file rather than the credentials file
-[2024-11-11T19:29:39Z INFO  aws_config::meta::region] load_region; provider=EnvironmentVariableRegionProvider { env: Env(Real) }
-[2024-11-11T19:29:39Z WARN  baml_events] Function TestAws:
-    Client: AwsBedrock (anthropic.claude-3-5-sonnet-20240620-v1:0) - 318ms
+    "\n\nThe old rock sat nestled among the roots of a ancient oak tree, its weathered surface bearing the scars of countless seasons. It was a peculiar rock, with veins of quartz that seemed to shimmer in the fading light of day.\n\nAs the storm clouds gathered, the air grew heavy with electricity. The wind began to pick up, rustling the leaves of the oak tree and causing the rock to vibrate with an otherworldly energy.\n\nSuddenly, a brilliant flash of lightning illuminated the sky,"
+[2024-11-26T00:37:07Z WARN  aws_runtime::env_config::normalize] section [Connection 1] ignored; config must be in the AWS config file rather than the credentials file
+[2024-11-26T00:37:09Z INFO  baml_events] Function TestAws:
+    Client: AwsBedrock (meta.llama3-8b-instruct-v1:0) - 1528ms. StopReason: unknown. Tokens(in/out): 24/100
     ---PROMPT---
     [chat] user: Write a nice short story about lightning in a rock
     
-    ---REQUEST OPTIONS---
-    api_key: ""
-    ---ERROR (RateLimited (429))---
-    ServiceError(
-        ServiceError {
-            source: ThrottlingException(
-                ThrottlingException {
-                    message: Some(
-                        "Too many requests, please wait before trying again.",
-                    ),
-                    meta: ErrorMetadata {
-                        code: Some(
-                            "ThrottlingException",
-                        ),
-                        message: Some(
-                            "Too many requests, please wait before trying again.",
-                        ),
-                        extras: Some(
-                            {
-                                "aws_request_id": "671bf4e1-976e-4832-8839-6ed57f64bd10",
-                            },
-                        ),
-                    },
-                },
-            ),
-            raw: Response {
-                status: StatusCode(
-                    429,
-                ),
-                headers: Headers {
-                    headers: {
-                        "date": HeaderValue {
-                            _private: H0(
-                                "Mon, 11 Nov 2024 19:29:39 GMT",
-                            ),
-                        },
-                        "content-type": HeaderValue {
-                            _private: H0(
-                                "application/json",
-                            ),
-                        },
-                        "content-length": HeaderValue {
-                            _private: H0(
-                                "65",
-                            ),
-                        },
-                        "x-amzn-requestid": HeaderValue {
-                            _private: H0(
-                                "671bf4e1-976e-4832-8839-6ed57f64bd10",
-                            ),
-                        },
-                        "x-amzn-errortype": HeaderValue {
-                            _private: H0(
-                                "ThrottlingException:http://internal.amazon.com/coral/com.amazon.bedrock/",
-                            ),
-                        },
-                    },
-                },
-                body: SdkBody {
-                    inner: Once(
-                        Some(
-                            b"{\"message\":\"Too many requests, please wait before trying again.\"}",
-                        ),
-                    ),
-                    retryable: true,
-                },
-                extensions: Extensions {
-                    extensions_02x: Extensions,
-                    extensions_1x: Extensions,
-                },
-            },
-        },
-    )
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_aws_bedrock_invalid_region  0:00:00.042889 
  
 
 
 Started 2024-11-11 13:29:39 
 
 Ended 2024-11-11 13:29:39 
 
 Duration 0:00:00.042889 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:29:39Z WARN  aws_runtime::env_config::normalize] section [Connection 1] ignored; config must be in the AWS config file rather than the credentials file
-[2024-11-11T19:29:39Z WARN  baml_events] Function TestAwsInvalidRegion:
-    Client: AwsBedrockInvalidRegion (anthropic.claude-3-5-sonnet-20240620-v1:0) - 40ms
+    ---LLM REPLY---
+    
+    
+    In the heart of the ancient forest, there was a peculiar rock that had been passed down through generations of the local tribe. It was said that the rock held a secret, a secret that only revealed itself to those who listened closely to the whispers of the wind.
+    
+    The rock was a deep, rich brown, and its surface was etched with strange symbols that seemed to shimmer in the flickering sunlight. The tribe's elders believed that the rock was imbued with the power of the lightning that
+    ---Parsed Response (string)---
+    "\n\nIn the heart of the ancient forest, there was a peculiar rock that had been passed down through generations of the local tribe. It was said that the rock held a secret, a secret that only revealed itself to those who listened closely to the whispers of the wind.\n\nThe rock was a deep, rich brown, and its surface was etched with strange symbols that seemed to shimmer in the flickering sunlight. The tribe's elders believed that the rock was imbued with the power of the lightning that"
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_aws_bedrock_invalid_region  0:00:00.012686 
  
 
 
 Started 2024-11-25 16:37:09 
 
 Ended 2024-11-25 16:37:09 
 
 Duration 0:00:00.012686 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:37:09Z WARN  aws_runtime::env_config::normalize] section [Connection 1] ignored; config must be in the AWS config file rather than the credentials file
+[2024-11-26T00:37:09Z WARN  baml_events] Function TestAwsInvalidRegion:
+    Client: AwsBedrockInvalidRegion (meta.llama3-8b-instruct-v1:0) - 6ms
     ---PROMPT---
     [chat] user: Write a nice short story about lightning in a rock
     
     ---REQUEST OPTIONS---
-    api_key: ""
     ---ERROR (Unspecified error code: 2)---
     DispatchFailure(
         DispatchFailure {
@@ -2190,76 +2334,65 @@
             },
         },
     )
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_serialization_exception  0:00:00.495733 
  
 
 
 Started 2024-11-11 13:29:39 
 
 Ended 2024-11-11 13:29:40 
 
 Duration 0:00:00.495733 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
Exception message from test:  <ExceptionInfo BamlValidationError(message=Failed to parse LLM response: Failed to coerce value: <root>: Failed while parsing require...required field: nonce2, raw_output=Hello there! How can I assist you today?, prompt=[chat] system: Say "hello there".
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_serialization_exception  0:00:00.393358 
  
 
 
 Started 2024-11-25 16:37:09 
 
 Ended 2024-11-25 16:37:09 
 
 Duration 0:00:00.393358 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
Exception message from test:  <ExceptionInfo BamlValidationError(message=Failed to parse LLM response: Failed to coerce value: <root>: Failed while parsing require...ld: nonce
+  - <root>: Missing required field: nonce2, raw_output=Hello there!, prompt=[chat] user: Say "hello there".
 ) tblen=2>
-
 
 
 
Captured stderr call
 
[2024-11-11T19:29:40Z WARN  baml_events] Function DummyOutputFunction:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 489ms. StopReason: stop. Tokens(in/out): 12/10
+
 
 
 
Captured stderr call
 
[2024-11-26T00:37:09Z WARN  baml_events] Function DummyOutputFunction:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 387ms. StopReason: stop. Tokens(in/out): 12/3
     ---PROMPT---
-    [chat] system: Say "hello there".
+    [chat] user: Say "hello there".
     
     ---LLM REPLY---
-    Hello there! How can I assist you today?
+    Hello there!
     ---Parsed Response (Error)---
     Failed to coerce value: <root>: Failed while parsing required fields: missing=2, unparsed=0
       - <root>: Missing required field: nonce
       - <root>: Missing required field: nonce2
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_stream_serialization_exception  0:00:00.474276 
  
 
 
 Started 2024-11-11 13:29:40 
 
 Ended 2024-11-11 13:29:40 
 
 Duration 0:00:00.474276 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
streamed  nonce=None nonce2=None
-streamed  nonce=None nonce2=None
-streamed  nonce=None nonce2=None
-streamed  nonce=None nonce2=None
-streamed  nonce=None nonce2=None
-streamed  nonce=None nonce2=None
-streamed  nonce=None nonce2=None
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_stream_serialization_exception  0:00:00.636167 
  
 
 
 Started 2024-11-25 16:37:09 
 
 Ended 2024-11-25 16:37:10 
 
 Duration 0:00:00.636167 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
streamed  nonce=None nonce2=None
 streamed  nonce=None nonce2=None
 streamed  nonce=None nonce2=None
 streamed  nonce=None nonce2=None
 streamed  nonce=None nonce2=None
 streamed  nonce=None nonce2=None
-streamed  nonce=None nonce2=None
-Exception message:  <ExceptionInfo BamlValidationError(message=Failed to parse LLM response: Failed to coerce value: <root>: Failed while parsing require...required field: nonce2, raw_output=Hello there! How can I assist you today?, prompt=[chat] system: Say "hello there".
+Exception message:  <ExceptionInfo BamlValidationError(message=Failed to parse LLM response: Failed to coerce value: <root>: Failed while parsing require...ld: nonce
+  - <root>: Missing required field: nonce2, raw_output=Hello there!, prompt=[chat] user: Say "hello there".
 ) tblen=3>
-
 
 
 
Captured stderr call
 
[2024-11-11T19:29:40Z WARN  baml_events] Function DummyOutputFunction:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 466ms. StopReason: stop. Tokens(in/out): 12/10
+
 
 
 
Captured stderr call
 
[2024-11-26T00:37:10Z WARN  baml_events] Function DummyOutputFunction:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 631ms. StopReason: stop. Tokens(in/out): 12/3
     ---PROMPT---
-    [chat] system: Say "hello there".
+    [chat] user: Say "hello there".
     
     ---LLM REPLY---
-    Hello there! How can I assist you today?
+    Hello there!
     ---Parsed Response (Error)---
     Failed to coerce value: <root>: Failed while parsing required fields: missing=2, unparsed=0
       - <root>: Missing required field: nonce
       - <root>: Missing required field: nonce2
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_stream2_serialization_exception  0:00:00.455908 
  
 
 
 Started 2024-11-11 13:29:40 
 
 Ended 2024-11-11 13:29:41 
 
 Duration 0:00:00.455908 
 
 Markers   
 
 Fixtures   event_loop_policy   cleanup   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
streamed  nonce=None nonce2=None nonce3=None
-streamed  nonce=None nonce2=None nonce3=None
-streamed  nonce=None nonce2=None nonce3=None
-streamed  nonce=None nonce2=None nonce3=None
-streamed  nonce=None nonce2=None nonce3=None
-streamed  nonce=None nonce2=None nonce3=None
-streamed  nonce=None nonce2=None nonce3=None
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_stream2_serialization_exception  0:00:00.363745 
  
 
 
 Started 2024-11-25 16:37:10 
 
 Ended 2024-11-25 16:37:10 
 
 Duration 0:00:00.363745 
 
 Markers   
 
 Fixtures   event_loop_policy   cleanup   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
streamed  nonce=None nonce2=None nonce3=None
 streamed  nonce=None nonce2=None nonce3=None
 streamed  nonce=None nonce2=None nonce3=None
 streamed  nonce=None nonce2=None nonce3=None
 streamed  nonce=None nonce2=None nonce3=None
 streamed  nonce=None nonce2=None nonce3=None
-streamed  nonce=None nonce2=None nonce3=None
-Exception message:  <ExceptionInfo BamlValidationError(message=Failed to parse LLM response: Failed to coerce value: <root>: Failed while parsing require...required field: nonce3, raw_output=Hello there! How can I assist you today?, prompt=[chat] system: Say "hello there".
+Exception message:  <ExceptionInfo BamlValidationError(message=Failed to parse LLM response: Failed to coerce value: <root>: Failed while parsing require...d: nonce2
+  - <root>: Missing required field: nonce3, raw_output=Hello there!, prompt=[chat] user: Say "hello there".
 ) tblen=3>
-
 
 
 
Captured stderr call
 
[2024-11-11T19:29:41Z WARN  baml_events] Function DummyOutputFunction:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 448ms. StopReason: stop. Tokens(in/out): 12/10
+
 
 
 
Captured stderr call
 
[2024-11-26T00:37:10Z WARN  baml_events] Function DummyOutputFunction:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 357ms. StopReason: stop. Tokens(in/out): 12/3
     ---PROMPT---
-    [chat] system: Say "hello there".
+    [chat] user: Say "hello there".
     
     ---LLM REPLY---
-    Hello there! How can I assist you today?
+    Hello there!
     ---Parsed Response (Error)---
     Failed to coerce value: <root>: Failed while parsing required fields: missing=3, unparsed=0
       - <root>: Missing required field: nonce
       - <root>: Missing required field: nonce2
       - <root>: Missing required field: nonce3
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_descriptions  0:00:01.875357 
  
 
 
 Started 2024-11-11 13:29:41 
 
 Ended 2024-11-11 13:29:43 
 
 Duration 0:00:01.875357 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:29:43Z INFO  baml_events] Function SchemaDescriptions:
-    Client: GPT4o (gpt-4o-2024-08-06) - 1860ms. StopReason: stop. Tokens(in/out): 340/108
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_descriptions  0:00:02.110718 
  
 
 
 Started 2024-11-25 16:37:10 
 
 Ended 2024-11-25 16:37:12 
 
 Duration 0:00:02.110718 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:37:12Z INFO  baml_events] Function SchemaDescriptions:
+    Client: GPT4o (gpt-4o-2024-08-06) - 2098ms. StopReason: stop. Tokens(in/out): 340/108
     ---PROMPT---
-    [chat] system: Return a schema with this format:
+    [chat] user: Return a schema with this format:
     
     Answer in JSON using this schema:
     {
@@ -2362,32 +2495,35 @@
       "parens": "parens1",
       "other_group": "other"
     }
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 FAILED  test_caching  0:00:01.934738 
  
 
 
 Started 2024-11-11 13:29:43 
 
 Ended 2024-11-11 13:29:45 
 
 Duration 0:00:01.934738 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
AssertionError: Expected second call to be faster than first by a large margin.
-assert 1.0759122371673584 < 0.8567588329315186
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
@pytest.mark.asyncio
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 FAILED  test_caching  0:00:07.604673 
  
 
 
 Started 2024-11-25 16:37:12 
 
 Ended 2024-11-25 16:37:20 
 
 Duration 0:00:07.604673 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
AssertionError: 3.9973297119140625 < 3.6063649654388428. Expected second call to be faster than first by a large margin.
+assert 3.9973297119140625 < 3.6063649654388428
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
@pytest.mark.asyncio
     async def test_caching():
-        story_idea = """
-        In a near-future society where dreams have become a tradable commodity and shared experience, a lonely and socially awkward teenager named Alex discovers they possess a rare and powerful ability to not only view but also manipulate the dreams of others. Initially thrilled by this newfound power, Alex begins subtly altering the dreams of classmates and family members, helping them overcome fears, boost confidence, or experience fantastical adventures. As Alex's skills grow, so does their influence. They start selling premium dream experiences on the black market, crafting intricate and addictive dreamscapes for wealthy clients. However, the line between dream and reality begins to blur for those exposed to Alex's creations. Some clients struggle to differentiate between their true memories and the artificial ones implanted by Alex's dream manipulation.
+        story_idea = f"""
+    In a futuristic world where dreams are a marketable asset and collective experience, an introverted and socially inept teenager named Alex realizes they have a unique and potent skill to not only observe but also alter the dreams of others. Initially excited by this newfound talent, Alex starts discreetly modifying the dreams of peers and relatives, aiding them in conquering fears, boosting self-esteem, or embarking on fantastical journeys. As Alex's abilities expand, so does their sway. They begin marketing exclusive dream experiences on the underground market, designing complex and captivating dreamscapes for affluent clients. However, the boundary between dream and reality starts to fade for those subjected to Alex's creations. Some clients find it difficult to distinguish between their genuine memories and the fabricated ones inserted by Alex's dream manipulation.
     
-        Complications arise when a mysterious government agency takes notice of Alex's unique abilities. They offer Alex a chance to use their gift for "the greater good," hinting at applications in therapy, criminal rehabilitation, and even national security. Simultaneously, an underground resistance movement reaches out, warning Alex about the dangers of dream manipulation and the potential for mass control and exploitation. Caught between these opposing forces, Alex must navigate a complex web of ethical dilemmas. They grapple with questions of free will, the nature of consciousness, and the responsibility that comes with having power over people's minds. As the consequences of their actions spiral outward, affecting the lives of loved ones and strangers alike, Alex is forced to confront the true nature of their ability and decide how—or if—it should be used.
+    Challenges emerge when a secretive government organization becomes aware of Alex's distinct talents. They propose Alex utilize their gift for "the greater good," suggesting uses in therapy, criminal reform, and even national defense. Concurrently, a covert resistance group contacts Alex, cautioning them about the risks of dream manipulation and the potential for widespread control and exploitation. Trapped between these conflicting forces, Alex must navigate a tangled web of moral dilemmas. They wrestle with issues of free will, the essence of consciousness, and the duty that comes with having influence over people's minds. As the repercussions of their actions ripple outward, impacting the lives of loved ones and strangers alike, Alex is compelled to face the true nature of their power and decide how—or if—it should be wielded.
     
-        The story explores themes of identity, the subconscious mind, the ethics of technology, and the power of imagination. It delves into the potential consequences of a world where our most private thoughts and experiences are no longer truly our own, and examines the fine line between helping others and manipulating them for personal gain or a perceived greater good. The narrative further expands on the societal implications of such abilities, questioning the moral boundaries of altering consciousness and the potential for abuse in a world where dreams can be commodified. It challenges the reader to consider the impact of technology on personal autonomy and the ethical responsibilities of those who wield such power.
+    The narrative investigates themes of identity, the subconscious, the ethics of technology, and the power of creativity. It explores the possible outcomes of a world where our most intimate thoughts and experiences are no longer truly our own, and scrutinizes the fine line between aiding others and manipulating them for personal benefit or a perceived greater good. The story further delves into the societal ramifications of such abilities, questioning the moral limits of altering consciousness and the potential for misuse in a world where dreams can be commercialized. It challenges the reader to contemplate the impact of technology on personal freedom and the ethical duties of those who wield such power.
     
-        As Alex's journey unfolds, they encounter various individuals whose lives have been touched by their dream manipulations, each presenting a unique perspective on the ethical quandaries at hand. From a classmate who gains newfound confidence to a wealthy client who becomes addicted to the dreamscapes, the ripple effects of Alex's actions are profound and far-reaching. The government agency's interest in Alex's abilities raises questions about the potential for state control and surveillance, while the resistance movement highlights the dangers of unchecked power and the importance of safeguarding individual freedoms.
+    As Alex's journey progresses, they meet various individuals whose lives have been influenced by their dream manipulations, each offering a distinct viewpoint on the ethical issues at hand. From a peer who gains newfound confidence to a wealthy client who becomes dependent on the dreamscapes, the ripple effects of Alex's actions are significant and extensive. The government agency's interest in Alex's abilities raises questions about the potential for state control and surveillance, while the resistance movement underscores the dangers of unchecked power and the necessity of protecting individual freedoms.
     
-        Ultimately, Alex's story is one of self-discovery and moral reckoning, as they must decide whether to embrace their abilities for personal gain, align with the government's vision of a controlled utopia, or join the resistance in their fight for freedom and autonomy. The narrative invites readers to reflect on the nature of reality, the boundaries of human experience, and the ethical implications of a world where dreams are no longer private sanctuaries but shared and manipulated commodities. It also explores the psychological impact on Alex, who must deal with the burden of knowing the intimate fears and desires of others, and the isolation that comes from being unable to share their own dreams without altering them.
+    Ultimately, Alex's story is one of self-discovery and moral reflection, as they must choose whether to use their abilities for personal gain, align with the government's vision of a controlled utopia, or join the resistance in their struggle for freedom and autonomy. The narrative encourages readers to reflect on the nature of reality, the boundaries of human experience, and the ethical implications of a world where dreams are no longer private sanctuaries but shared and manipulated commodities. It also examines the psychological impact on Alex, who must cope with the burden of knowing the intimate fears and desires of others, and the isolation that comes from being unable to share their own dreams without altering them.
     
-        The story further examines the technological advancements that have made dream manipulation possible, questioning the role of innovation in society and the potential for both progress and peril. It considers the societal divide between those who can afford to buy enhanced dream experiences and those who cannot, highlighting issues of inequality and access. As Alex becomes more entangled in the web of their own making, they must confront the possibility that their actions could lead to unintended consequences, not just for themselves but for the fabric of society as a whole.
+    The story further investigates the technological progress that has made dream manipulation feasible, questioning the role of innovation in society and the potential for both advancement and peril. It considers the societal divide between those who can afford to purchase enhanced dream experiences and those who cannot, highlighting issues of inequality and access. As Alex becomes more ensnared in the web of their own making, they must confront the possibility that their actions could lead to unintended consequences, not just for themselves but for the fabric of society as a whole.
     
-        In the end, Alex's journey is a cautionary tale about the power of dreams and the responsibilities that come with wielding such influence. It serves as a reminder of the importance of ethical considerations in the face of technological advancement and the need to balance innovation with humanity. The story leaves readers pondering the true cost of a world where dreams are no longer sacred, and the potential for both wonder and danger in the uncharted territories of the mind.
+    In the end, Alex's journey is a cautionary tale about the power of dreams and the responsibilities that come with wielding such influence. It serves as a reminder of the importance of ethical considerations in the face of technological advancement and the need to balance innovation with humanity. The story leaves readers pondering the true cost of a world where dreams are no longer sacred, and the potential for both wonder and danger in the uncharted territories of the mind. But it's also a story about the power of imagination and the potential for change, even in a world where our deepest thoughts are no longer our own. And it's a story about the power of choice, and the importance of fighting for the freedom to dream.
+    
+    In conclusion, this story is a reflection on the power of dreams and the responsibilities that come with wielding such influence. It serves as a reminder of the importance of ethical considerations in the face of technological advancement and the need to balance innovation with humanity. The story leaves readers pondering the true cost of a world where dreams are no longer sacred, and the potential for both wonder and danger in the uncharted territories of the mind. But it's also a story about the power of imagination and the potential for change, even in a world where our deepest thoughts are no longer our own. And it's a story about the power of choice, and the importance of fighting for the freedom to dream.
     """
-        rand = random.randint(0, 26)
-        story_idea += " " + rand * "a"
+        rand = uuid.uuid4().hex
+        story_idea = rand + story_idea
+    
         start = time.time()
-        _ = await b.TestCaching(story_idea, "be funny")
+        _ = await b.TestCaching(story_idea, "1. try to be funny")
         duration = time.time() - start
     
         start = time.time()
-        _ = await b.TestCaching(story_idea, "be real")
+        _ = await b.TestCaching(story_idea, "1. try to be funny")
         duration2 = time.time() - start
     
         print("Duration no caching: ", duration)
@@ -2395,70 +2531,126 @@
     
 >       assert (
             duration2 < duration
-        ), "Expected second call to be faster than first by a large margin."
-E       AssertionError: Expected second call to be faster than first by a large margin.
-E       assert 1.0759122371673584 < 0.8567588329315186
+        ), f"{duration2} < {duration}. Expected second call to be faster than first by a large margin."
+E       AssertionError: 3.9973297119140625 < 3.6063649654388428. Expected second call to be faster than first by a large margin.
+E       assert 3.9973297119140625 < 3.6063649654388428
 
-tests/test_functions.py:1248: AssertionError
 
 
 
Captured stdout call
 
Duration no caching:  0.8567588329315186
-Duration with caching:  1.0759122371673584
-
 
 
 
Captured stderr call
 
[2024-11-11T19:29:44Z INFO  baml_events] Function TestCaching:
-    Client: ClaudeWithCaching (claude-3-haiku-20240307) - 853ms. StopReason: "end_turn". Tokens(in/out): 966/23
+tests/test_functions.py:1271: AssertionError
 
 
 
Captured stdout call
 
Duration no caching:  3.6063649654388428
+Duration with caching:  3.9973297119140625
+
 
 
 
Captured stderr call
 
[2024-11-26T00:37:16Z INFO  baml_events] Function TestCaching:
+    Client: ClaudeWithCaching (claude-3-haiku-20240307) - 3597ms. StopReason: "end_turn". Tokens(in/out): 14/421
     ---PROMPT---
-    [chat] system: {"cache_control": Object {"type": String("ephemeral")}}::Describe this in 5 words: 
-        In a near-future society where dreams have become a tradable commodity and shared experience, a lonely and socially awkward teenager named Alex discovers they possess a rare and powerful ability to not only view but also manipulate the dreams of others. Initially thrilled by this newfound power, Alex begins subtly altering the dreams of classmates and family members, helping them overcome fears, boost confidence, or experience fantastical adventures. As Alex's skills grow, so does their influence. They start selling premium dream experiences on the black market, crafting intricate and addictive dreamscapes for wealthy clients. However, the line between dream and reality begins to blur for those exposed to Alex's creations. Some clients struggle to differentiate between their true memories and the artificial ones implanted by Alex's dream manipulation.
+    [chat] system: {"cache_control": Object {"type": String("ephemeral")}}::Generate the following story
+    50ad921e06064de89a839db471078df1
+    In a futuristic world where dreams are a marketable asset and collective experience, an introverted and socially inept teenager named Alex realizes they have a unique and potent skill to not only observe but also alter the dreams of others. Initially excited by this newfound talent, Alex starts discreetly modifying the dreams of peers and relatives, aiding them in conquering fears, boosting self-esteem, or embarking on fantastical journeys. As Alex's abilities expand, so does their sway. They begin marketing exclusive dream experiences on the underground market, designing complex and captivating dreamscapes for affluent clients. However, the boundary between dream and reality starts to fade for those subjected to Alex's creations. Some clients find it difficult to distinguish between their genuine memories and the fabricated ones inserted by Alex's dream manipulation.
+    
+    Challenges emerge when a secretive government organization becomes aware of Alex's distinct talents. They propose Alex utilize their gift for "the greater good," suggesting uses in therapy, criminal reform, and even national defense. Concurrently, a covert resistance group contacts Alex, cautioning them about the risks of dream manipulation and the potential for widespread control and exploitation. Trapped between these conflicting forces, Alex must navigate a tangled web of moral dilemmas. They wrestle with issues of free will, the essence of consciousness, and the duty that comes with having influence over people's minds. As the repercussions of their actions ripple outward, impacting the lives of loved ones and strangers alike, Alex is compelled to face the true nature of their power and decide how—or if—it should be wielded.
+    
+    The narrative investigates themes of identity, the subconscious, the ethics of technology, and the power of creativity. It explores the possible outcomes of a world where our most intimate thoughts and experiences are no longer truly our own, and scrutinizes the fine line between aiding others and manipulating them for personal benefit or a perceived greater good. The story further delves into the societal ramifications of such abilities, questioning the moral limits of altering consciousness and the potential for misuse in a world where dreams can be commercialized. It challenges the reader to contemplate the impact of technology on personal freedom and the ethical duties of those who wield such power.
+    
+    As Alex's journey progresses, they meet various individuals whose lives have been influenced by their dream manipulations, each offering a distinct viewpoint on the ethical issues at hand. From a peer who gains newfound confidence to a wealthy client who becomes dependent on the dreamscapes, the ripple effects of Alex's actions are significant and extensive. The government agency's interest in Alex's abilities raises questions about the potential for state control and surveillance, while the resistance movement underscores the dangers of unchecked power and the necessity of protecting individual freedoms.
     
-        Complications arise when a mysterious government agency takes notice of Alex's unique abilities. They offer Alex a chance to use their gift for "the greater good," hinting at applications in therapy, criminal rehabilitation, and even national security. Simultaneously, an underground resistance movement reaches out, warning Alex about the dangers of dream manipulation and the potential for mass control and exploitation. Caught between these opposing forces, Alex must navigate a complex web of ethical dilemmas. They grapple with questions of free will, the nature of consciousness, and the responsibility that comes with having power over people's minds. As the consequences of their actions spiral outward, affecting the lives of loved ones and strangers alike, Alex is forced to confront the true nature of their ability and decide how—or if—it should be used.
+    Ultimately, Alex's story is one of self-discovery and moral reflection, as they must choose whether to use their abilities for personal gain, align with the government's vision of a controlled utopia, or join the resistance in their struggle for freedom and autonomy. The narrative encourages readers to reflect on the nature of reality, the boundaries of human experience, and the ethical implications of a world where dreams are no longer private sanctuaries but shared and manipulated commodities. It also examines the psychological impact on Alex, who must cope with the burden of knowing the intimate fears and desires of others, and the isolation that comes from being unable to share their own dreams without altering them.
     
-        The story explores themes of identity, the subconscious mind, the ethics of technology, and the power of imagination. It delves into the potential consequences of a world where our most private thoughts and experiences are no longer truly our own, and examines the fine line between helping others and manipulating them for personal gain or a perceived greater good. The narrative further expands on the societal implications of such abilities, questioning the moral boundaries of altering consciousness and the potential for abuse in a world where dreams can be commodified. It challenges the reader to consider the impact of technology on personal autonomy and the ethical responsibilities of those who wield such power.
+    The story further investigates the technological progress that has made dream manipulation feasible, questioning the role of innovation in society and the potential for both advancement and peril. It considers the societal divide between those who can afford to purchase enhanced dream experiences and those who cannot, highlighting issues of inequality and access. As Alex becomes more ensnared in the web of their own making, they must confront the possibility that their actions could lead to unintended consequences, not just for themselves but for the fabric of society as a whole.
     
-        As Alex's journey unfolds, they encounter various individuals whose lives have been touched by their dream manipulations, each presenting a unique perspective on the ethical quandaries at hand. From a classmate who gains newfound confidence to a wealthy client who becomes addicted to the dreamscapes, the ripple effects of Alex's actions are profound and far-reaching. The government agency's interest in Alex's abilities raises questions about the potential for state control and surveillance, while the resistance movement highlights the dangers of unchecked power and the importance of safeguarding individual freedoms.
+    In the end, Alex's journey is a cautionary tale about the power of dreams and the responsibilities that come with wielding such influence. It serves as a reminder of the importance of ethical considerations in the face of technological advancement and the need to balance innovation with humanity. The story leaves readers pondering the true cost of a world where dreams are no longer sacred, and the potential for both wonder and danger in the uncharted territories of the mind. But it's also a story about the power of imagination and the potential for change, even in a world where our deepest thoughts are no longer our own. And it's a story about the power of choice, and the importance of fighting for the freedom to dream.
     
-        Ultimately, Alex's story is one of self-discovery and moral reckoning, as they must decide whether to embrace their abilities for personal gain, align with the government's vision of a controlled utopia, or join the resistance in their fight for freedom and autonomy. The narrative invites readers to reflect on the nature of reality, the boundaries of human experience, and the ethical implications of a world where dreams are no longer private sanctuaries but shared and manipulated commodities. It also explores the psychological impact on Alex, who must deal with the burden of knowing the intimate fears and desires of others, and the isolation that comes from being unable to share their own dreams without altering them.
+    In conclusion, this story is a reflection on the power of dreams and the responsibilities that come with wielding such influence. It serves as a reminder of the importance of ethical considerations in the face of technological advancement and the need to balance innovation with humanity. The story leaves readers pondering the true cost of a world where dreams are no longer sacred, and the potential for both wonder and danger in the uncharted territories of the mind. But it's also a story about the power of imagination and the potential for change, even in a world where our deepest thoughts are no longer our own. And it's a story about the power of choice, and the importance of fighting for the freedom to dream.
     
-        The story further examines the technological advancements that have made dream manipulation possible, questioning the role of innovation in society and the potential for both progress and peril. It considers the societal divide between those who can afford to buy enhanced dream experiences and those who cannot, highlighting issues of inequality and access. As Alex becomes more entangled in the web of their own making, they must confront the possibility that their actions could lead to unintended consequences, not just for themselves but for the fabric of society as a whole.
     
-        In the end, Alex's journey is a cautionary tale about the power of dreams and the responsibilities that come with wielding such influence. It serves as a reminder of the importance of ethical considerations in the face of technological advancement and the need to balance innovation with humanity. The story leaves readers pondering the true cost of a world where dreams are no longer sacred, and the potential for both wonder and danger in the uncharted territories of the mind.
-     aaaaaaaaaaaaaaaaaa
-    user: be funny
+    50ad921e06064de89a839db471078df1
+    In a futuristic world where dreams are a marketable asset and collective experience, an introverted and socially inept teenager named Alex realizes they have a unique and potent skill to not only observe but also alter the dreams of others. Initially excited by this newfound talent, Alex starts discreetly modifying the dreams of peers and relatives, aiding them in conquering fears, boosting self-esteem, or embarking on fantastical journeys. As Alex's abilities expand, so does their sway. They begin marketing exclusive dream experiences on the underground market, designing complex and captivating dreamscapes for affluent clients. However, the boundary between dream and reality starts to fade for those subjected to Alex's creations. Some clients find it difficult to distinguish between their genuine memories and the fabricated ones inserted by Alex's dream manipulation.
+    
+    Challenges emerge when a secretive government organization becomes aware of Alex's distinct talents. They propose Alex utilize their gift for "the greater good," suggesting uses in therapy, criminal reform, and even national defense. Concurrently, a covert resistance group contacts Alex, cautioning them about the risks of dream manipulation and the potential for widespread control and exploitation. Trapped between these conflicting forces, Alex must navigate a tangled web of moral dilemmas. They wrestle with issues of free will, the essence of consciousness, and the duty that comes with having influence over people's minds. As the repercussions of their actions ripple outward, impacting the lives of loved ones and strangers alike, Alex is compelled to face the true nature of their power and decide how—or if—it should be wielded.
+    
+    The narrative investigates themes of identity, the subconscious, the ethics of technology, and the power of creativity. It explores the possible outcomes of a world where our most intimate thoughts and experiences are no longer truly our own, and scrutinizes the fine line between aiding others and manipulating them for personal benefit or a perceived greater good. The story further delves into the societal ramifications of such abilities, questioning the moral limits of altering consciousness and the potential for misuse in a world where dreams can be commercialized. It challenges the reader to contemplate the impact of technology on personal freedom and the ethical duties of those who wield such power.
+    
+    As Alex's journey progresses, they meet various individuals whose lives have been influenced by their dream manipulations, each offering a distinct viewpoint on the ethical issues at hand. From a peer who gains newfound confidence to a wealthy client who becomes dependent on the dreamscapes, the ripple effects of Alex's actions are significant and extensive. The government agency's interest in Alex's abilities raises questions about the potential for state control and surveillance, while the resistance movement underscores the dangers of unchecked power and the necessity of protecting individual freedoms.
+    
+    Ultimately, Alex's story is one of self-discovery and moral reflection, as they must choose whether to use their abilities for personal gain, align with the government's vision of a controlled utopia, or join the resistance in their struggle for freedom and autonomy. The narrative encourages readers to reflect on the nature of reality, the boundaries of human experience, and the ethical implications of a world where dreams are no longer private sanctuaries but shared and manipulated commodities. It also examines the psychological impact on Alex, who must cope with the burden of knowing the intimate fears and desires of others, and the isolation that comes from being unable to share their own dreams without altering them.
+    
+    The story further investigates the technological progress that has made dream manipulation feasible, questioning the role of innovation in society and the potential for both advancement and peril. It considers the societal divide between those who can afford to purchase enhanced dream experiences and those who cannot, highlighting issues of inequality and access. As Alex becomes more ensnared in the web of their own making, they must confront the possibility that their actions could lead to unintended consequences, not just for themselves but for the fabric of society as a whole.
+    
+    In the end, Alex's journey is a cautionary tale about the power of dreams and the responsibilities that come with wielding such influence. It serves as a reminder of the importance of ethical considerations in the face of technological advancement and the need to balance innovation with humanity. The story leaves readers pondering the true cost of a world where dreams are no longer sacred, and the potential for both wonder and danger in the uncharted territories of the mind. But it's also a story about the power of imagination and the potential for change, even in a world where our deepest thoughts are no longer our own. And it's a story about the power of choice, and the importance of fighting for the freedom to dream.
+    
+    In conclusion, this story is a reflection on the power of dreams and the responsibilities that come with wielding such influence. It serves as a reminder of the importance of ethical considerations in the face of technological advancement and the need to balance innovation with humanity. The story leaves readers pondering the true cost of a world where dreams are no longer sacred, and the potential for both wonder and danger in the uncharted territories of the mind. But it's also a story about the power of imagination and the potential for change, even in a world where our deepest thoughts are no longer our own. And it's a story about the power of choice, and the importance of fighting for the freedom to dream.
+    user: 1. try to be funny
     
     ---LLM REPLY---
-    Here's a funny summary in 5 words:
+    Sure, let's see if we can add a little humor to this futuristic tale of dream manipulation:
+    
+    In a world where dreams are the new cryptocurrency, a socially awkward teen named Alex discovers they have the power to hack into people's subconscious and redesign their nightly adventures. At first, Alex uses their powers for good - helping their peers overcome fears of public speaking or giving grandma the dream vacation of a lifetime. But it's not long before Alex starts dabbling in the dream black market, crafting bespoke dreamscapes for wealthy insomniacs who'll pay top dollar for a good night's sleep. 
     
-    Teen's dream skills cause real nightmare.
+    The only problem? Some clients get a little too attached to their custom-made dreams. One CEO wakes up convinced he's a dolphin and starts swimming laps in the company koi pond. Another client develops an unhealthy fixation on her dream persona - a suave secret agent - and starts showing up to work in a catsuit, demanding to be called "Velvet Thunder."
+    
+    As government agencies and shadowy resistance groups come calling, Alex has to navigate a maze of moral quandaries. Should they embrace their power and capitalize on it? Team up with the feds to use dream manipulation for good? Or join the rebels fighting to protect the sanctity of the subconscious?
+    
+    It's enough to give anyone night terrors! But through it all, Alex discovers the true power isn't just in designing dreams - it's in having the courage to face their own.
+    
+    How's that? I tried to inject a bit more humor by poking fun at some of the more absurd consequences of Alex's dream hacking, like the CEO who thinks he's a dolphin and the client who gets dangerously immersed in her secret agent persona. Hopefully that adds a lighthearted touch without losing the core themes and dramatic tension of the story. Let me know if you'd like me to try a different comedic angle.
     ---Parsed Response (string)---
-    "Here's a funny summary in 5 words:\n\nTeen's dream skills cause real nightmare."
-[2024-11-11T19:29:45Z INFO  baml_events] Function TestCaching:
-    Client: ClaudeWithCaching (claude-3-haiku-20240307) - 1070ms. StopReason: "end_turn". Tokens(in/out): 966/37
+    "Sure, let's see if we can add a little humor to this futuristic tale of dream manipulation:\n\nIn a world where dreams are the new cryptocurrency, a socially awkward teen named Alex discovers they have the power to hack into people's subconscious and redesign their nightly adventures. At first, Alex uses their powers for good - helping their peers overcome fears of public speaking or giving grandma the dream vacation of a lifetime. But it's not long before Alex starts dabbling in the dream black market, crafting bespoke dreamscapes for wealthy insomniacs who'll pay top dollar for a good night's sleep. \n\nThe only problem? Some clients get a little too attached to their custom-made dreams. One CEO wakes up convinced he's a dolphin and starts swimming laps in the company koi pond. Another client develops an unhealthy fixation on her dream persona - a suave secret agent - and starts showing up to work in a catsuit, demanding to be called \"Velvet Thunder.\"\n\nAs government agencies and shadowy resistance groups come calling, Alex has to navigate a maze of moral quandaries. Should they embrace their power and capitalize on it? Team up with the feds to use dream manipulation for good? Or join the rebels fighting to protect the sanctity of the subconscious?\n\nIt's enough to give anyone night terrors! But through it all, Alex discovers the true power isn't just in designing dreams - it's in having the courage to face their own.\n\nHow's that? I tried to inject a bit more humor by poking fun at some of the more absurd consequences of Alex's dream hacking, like the CEO who thinks he's a dolphin and the client who gets dangerously immersed in her secret agent persona. Hopefully that adds a lighthearted touch without losing the core themes and dramatic tension of the story. Let me know if you'd like me to try a different comedic angle."
+[2024-11-26T00:37:20Z INFO  baml_events] Function TestCaching:
+    Client: ClaudeWithCaching (claude-3-haiku-20240307) - 3988ms. StopReason: "end_turn". Tokens(in/out): 14/442
     ---PROMPT---
-    [chat] system: {"cache_control": Object {"type": String("ephemeral")}}::Describe this in 5 words: 
-        In a near-future society where dreams have become a tradable commodity and shared experience, a lonely and socially awkward teenager named Alex discovers they possess a rare and powerful ability to not only view but also manipulate the dreams of others. Initially thrilled by this newfound power, Alex begins subtly altering the dreams of classmates and family members, helping them overcome fears, boost confidence, or experience fantastical adventures. As Alex's skills grow, so does their influence. They start selling premium dream experiences on the black market, crafting intricate and addictive dreamscapes for wealthy clients. However, the line between dream and reality begins to blur for those exposed to Alex's creations. Some clients struggle to differentiate between their true memories and the artificial ones implanted by Alex's dream manipulation.
+    [chat] system: {"cache_control": Object {"type": String("ephemeral")}}::Generate the following story
+    50ad921e06064de89a839db471078df1
+    In a futuristic world where dreams are a marketable asset and collective experience, an introverted and socially inept teenager named Alex realizes they have a unique and potent skill to not only observe but also alter the dreams of others. Initially excited by this newfound talent, Alex starts discreetly modifying the dreams of peers and relatives, aiding them in conquering fears, boosting self-esteem, or embarking on fantastical journeys. As Alex's abilities expand, so does their sway. They begin marketing exclusive dream experiences on the underground market, designing complex and captivating dreamscapes for affluent clients. However, the boundary between dream and reality starts to fade for those subjected to Alex's creations. Some clients find it difficult to distinguish between their genuine memories and the fabricated ones inserted by Alex's dream manipulation.
+    
+    Challenges emerge when a secretive government organization becomes aware of Alex's distinct talents. They propose Alex utilize their gift for "the greater good," suggesting uses in therapy, criminal reform, and even national defense. Concurrently, a covert resistance group contacts Alex, cautioning them about the risks of dream manipulation and the potential for widespread control and exploitation. Trapped between these conflicting forces, Alex must navigate a tangled web of moral dilemmas. They wrestle with issues of free will, the essence of consciousness, and the duty that comes with having influence over people's minds. As the repercussions of their actions ripple outward, impacting the lives of loved ones and strangers alike, Alex is compelled to face the true nature of their power and decide how—or if—it should be wielded.
+    
+    The narrative investigates themes of identity, the subconscious, the ethics of technology, and the power of creativity. It explores the possible outcomes of a world where our most intimate thoughts and experiences are no longer truly our own, and scrutinizes the fine line between aiding others and manipulating them for personal benefit or a perceived greater good. The story further delves into the societal ramifications of such abilities, questioning the moral limits of altering consciousness and the potential for misuse in a world where dreams can be commercialized. It challenges the reader to contemplate the impact of technology on personal freedom and the ethical duties of those who wield such power.
+    
+    As Alex's journey progresses, they meet various individuals whose lives have been influenced by their dream manipulations, each offering a distinct viewpoint on the ethical issues at hand. From a peer who gains newfound confidence to a wealthy client who becomes dependent on the dreamscapes, the ripple effects of Alex's actions are significant and extensive. The government agency's interest in Alex's abilities raises questions about the potential for state control and surveillance, while the resistance movement underscores the dangers of unchecked power and the necessity of protecting individual freedoms.
+    
+    Ultimately, Alex's story is one of self-discovery and moral reflection, as they must choose whether to use their abilities for personal gain, align with the government's vision of a controlled utopia, or join the resistance in their struggle for freedom and autonomy. The narrative encourages readers to reflect on the nature of reality, the boundaries of human experience, and the ethical implications of a world where dreams are no longer private sanctuaries but shared and manipulated commodities. It also examines the psychological impact on Alex, who must cope with the burden of knowing the intimate fears and desires of others, and the isolation that comes from being unable to share their own dreams without altering them.
     
-        Complications arise when a mysterious government agency takes notice of Alex's unique abilities. They offer Alex a chance to use their gift for "the greater good," hinting at applications in therapy, criminal rehabilitation, and even national security. Simultaneously, an underground resistance movement reaches out, warning Alex about the dangers of dream manipulation and the potential for mass control and exploitation. Caught between these opposing forces, Alex must navigate a complex web of ethical dilemmas. They grapple with questions of free will, the nature of consciousness, and the responsibility that comes with having power over people's minds. As the consequences of their actions spiral outward, affecting the lives of loved ones and strangers alike, Alex is forced to confront the true nature of their ability and decide how—or if—it should be used.
+    The story further investigates the technological progress that has made dream manipulation feasible, questioning the role of innovation in society and the potential for both advancement and peril. It considers the societal divide between those who can afford to purchase enhanced dream experiences and those who cannot, highlighting issues of inequality and access. As Alex becomes more ensnared in the web of their own making, they must confront the possibility that their actions could lead to unintended consequences, not just for themselves but for the fabric of society as a whole.
     
-        The story explores themes of identity, the subconscious mind, the ethics of technology, and the power of imagination. It delves into the potential consequences of a world where our most private thoughts and experiences are no longer truly our own, and examines the fine line between helping others and manipulating them for personal gain or a perceived greater good. The narrative further expands on the societal implications of such abilities, questioning the moral boundaries of altering consciousness and the potential for abuse in a world where dreams can be commodified. It challenges the reader to consider the impact of technology on personal autonomy and the ethical responsibilities of those who wield such power.
+    In the end, Alex's journey is a cautionary tale about the power of dreams and the responsibilities that come with wielding such influence. It serves as a reminder of the importance of ethical considerations in the face of technological advancement and the need to balance innovation with humanity. The story leaves readers pondering the true cost of a world where dreams are no longer sacred, and the potential for both wonder and danger in the uncharted territories of the mind. But it's also a story about the power of imagination and the potential for change, even in a world where our deepest thoughts are no longer our own. And it's a story about the power of choice, and the importance of fighting for the freedom to dream.
     
-        As Alex's journey unfolds, they encounter various individuals whose lives have been touched by their dream manipulations, each presenting a unique perspective on the ethical quandaries at hand. From a classmate who gains newfound confidence to a wealthy client who becomes addicted to the dreamscapes, the ripple effects of Alex's actions are profound and far-reaching. The government agency's interest in Alex's abilities raises questions about the potential for state control and surveillance, while the resistance movement highlights the dangers of unchecked power and the importance of safeguarding individual freedoms.
+    In conclusion, this story is a reflection on the power of dreams and the responsibilities that come with wielding such influence. It serves as a reminder of the importance of ethical considerations in the face of technological advancement and the need to balance innovation with humanity. The story leaves readers pondering the true cost of a world where dreams are no longer sacred, and the potential for both wonder and danger in the uncharted territories of the mind. But it's also a story about the power of imagination and the potential for change, even in a world where our deepest thoughts are no longer our own. And it's a story about the power of choice, and the importance of fighting for the freedom to dream.
     
-        Ultimately, Alex's story is one of self-discovery and moral reckoning, as they must decide whether to embrace their abilities for personal gain, align with the government's vision of a controlled utopia, or join the resistance in their fight for freedom and autonomy. The narrative invites readers to reflect on the nature of reality, the boundaries of human experience, and the ethical implications of a world where dreams are no longer private sanctuaries but shared and manipulated commodities. It also explores the psychological impact on Alex, who must deal with the burden of knowing the intimate fears and desires of others, and the isolation that comes from being unable to share their own dreams without altering them.
     
-        The story further examines the technological advancements that have made dream manipulation possible, questioning the role of innovation in society and the potential for both progress and peril. It considers the societal divide between those who can afford to buy enhanced dream experiences and those who cannot, highlighting issues of inequality and access. As Alex becomes more entangled in the web of their own making, they must confront the possibility that their actions could lead to unintended consequences, not just for themselves but for the fabric of society as a whole.
+    50ad921e06064de89a839db471078df1
+    In a futuristic world where dreams are a marketable asset and collective experience, an introverted and socially inept teenager named Alex realizes they have a unique and potent skill to not only observe but also alter the dreams of others. Initially excited by this newfound talent, Alex starts discreetly modifying the dreams of peers and relatives, aiding them in conquering fears, boosting self-esteem, or embarking on fantastical journeys. As Alex's abilities expand, so does their sway. They begin marketing exclusive dream experiences on the underground market, designing complex and captivating dreamscapes for affluent clients. However, the boundary between dream and reality starts to fade for those subjected to Alex's creations. Some clients find it difficult to distinguish between their genuine memories and the fabricated ones inserted by Alex's dream manipulation.
     
-        In the end, Alex's journey is a cautionary tale about the power of dreams and the responsibilities that come with wielding such influence. It serves as a reminder of the importance of ethical considerations in the face of technological advancement and the need to balance innovation with humanity. The story leaves readers pondering the true cost of a world where dreams are no longer sacred, and the potential for both wonder and danger in the uncharted territories of the mind.
-     aaaaaaaaaaaaaaaaaa
-    user: be real
+    Challenges emerge when a secretive government organization becomes aware of Alex's distinct talents. They propose Alex utilize their gift for "the greater good," suggesting uses in therapy, criminal reform, and even national defense. Concurrently, a covert resistance group contacts Alex, cautioning them about the risks of dream manipulation and the potential for widespread control and exploitation. Trapped between these conflicting forces, Alex must navigate a tangled web of moral dilemmas. They wrestle with issues of free will, the essence of consciousness, and the duty that comes with having influence over people's minds. As the repercussions of their actions ripple outward, impacting the lives of loved ones and strangers alike, Alex is compelled to face the true nature of their power and decide how—or if—it should be wielded.
+    
+    The narrative investigates themes of identity, the subconscious, the ethics of technology, and the power of creativity. It explores the possible outcomes of a world where our most intimate thoughts and experiences are no longer truly our own, and scrutinizes the fine line between aiding others and manipulating them for personal benefit or a perceived greater good. The story further delves into the societal ramifications of such abilities, questioning the moral limits of altering consciousness and the potential for misuse in a world where dreams can be commercialized. It challenges the reader to contemplate the impact of technology on personal freedom and the ethical duties of those who wield such power.
+    
+    As Alex's journey progresses, they meet various individuals whose lives have been influenced by their dream manipulations, each offering a distinct viewpoint on the ethical issues at hand. From a peer who gains newfound confidence to a wealthy client who becomes dependent on the dreamscapes, the ripple effects of Alex's actions are significant and extensive. The government agency's interest in Alex's abilities raises questions about the potential for state control and surveillance, while the resistance movement underscores the dangers of unchecked power and the necessity of protecting individual freedoms.
+    
+    Ultimately, Alex's story is one of self-discovery and moral reflection, as they must choose whether to use their abilities for personal gain, align with the government's vision of a controlled utopia, or join the resistance in their struggle for freedom and autonomy. The narrative encourages readers to reflect on the nature of reality, the boundaries of human experience, and the ethical implications of a world where dreams are no longer private sanctuaries but shared and manipulated commodities. It also examines the psychological impact on Alex, who must cope with the burden of knowing the intimate fears and desires of others, and the isolation that comes from being unable to share their own dreams without altering them.
+    
+    The story further investigates the technological progress that has made dream manipulation feasible, questioning the role of innovation in society and the potential for both advancement and peril. It considers the societal divide between those who can afford to purchase enhanced dream experiences and those who cannot, highlighting issues of inequality and access. As Alex becomes more ensnared in the web of their own making, they must confront the possibility that their actions could lead to unintended consequences, not just for themselves but for the fabric of society as a whole.
+    
+    In the end, Alex's journey is a cautionary tale about the power of dreams and the responsibilities that come with wielding such influence. It serves as a reminder of the importance of ethical considerations in the face of technological advancement and the need to balance innovation with humanity. The story leaves readers pondering the true cost of a world where dreams are no longer sacred, and the potential for both wonder and danger in the uncharted territories of the mind. But it's also a story about the power of imagination and the potential for change, even in a world where our deepest thoughts are no longer our own. And it's a story about the power of choice, and the importance of fighting for the freedom to dream.
+    
+    In conclusion, this story is a reflection on the power of dreams and the responsibilities that come with wielding such influence. It serves as a reminder of the importance of ethical considerations in the face of technological advancement and the need to balance innovation with humanity. The story leaves readers pondering the true cost of a world where dreams are no longer sacred, and the potential for both wonder and danger in the uncharted territories of the mind. But it's also a story about the power of imagination and the potential for change, even in a world where our deepest thoughts are no longer our own. And it's a story about the power of choice, and the importance of fighting for the freedom to dream.
+    user: 1. try to be funny
     
     ---LLM REPLY---
-    Okay, let me try to summarize the key points in a more concise and direct way:
+    Alright, let's try to inject a little humor into this futuristic tale of dream manipulation. Here's an attempt at adding some comedic elements:
+    
+    In this world of commercialized dream experiences, Alex - the socially awkward teen with a penchant for meddling in others' subconscious - finds themselves in a rather peculiar predicament. Armed with the ability to sculpt dreams like a virtuoso sculptor, Alex can't help but get carried away, crafting increasingly bizarre and outlandish dreamscapes for their unsuspecting clients.
     
-    Teenager with dream manipulation powers faces ethical dilemmas.
+    One wealthy businessman pays handsomely for a VIP dream package, only to find himself trapped in a never-ending loop of show tunes and tap-dancing penguins. Another client, a high-strung workaholic, commissions Alex to create a soothing nature retreat, only to wake up covered in glitter and neon-colored flower petals, much to the delight of their coworkers.
+    
+    As Alex's reputation grows, so do the expectations - and the eccentricities. The government agents tasked with recruiting Alex find themselves struggling to keep a straight face during the pitch meeting, as Alex describes their latest dream design involving a herd of levitating llamas and a game of volleyball with sentient marshmallows.
+    
+    Meanwhile, the resistance fighters trying to sway Alex to their cause can't help but chuckle at the sheer absurdity of Alex's creations, even as they warn of the dangers of dream manipulation. "So, let me get this straight," one rebel leader says, "you've been making people think they're dancing with sentient toasters? Well, that's certainly one way to fight the system."
+    
+    Through it all, Alex remains oblivious to the humor of their situation, laser-focused on perfecting their craft and exploring the boundless possibilities of the dream world. After all, who needs reality when you can design your own fantastical escapes - even if they do involve the occasional mutant unicorn or singing paperclip.
     ---Parsed Response (string)---
-    "Okay, let me try to summarize the key points in a more concise and direct way:\n\nTeenager with dream manipulation powers faces ethical dilemmas."
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_arg_exceptions  0:00:00.719960 
  
 
 
 Started 2024-11-11 13:29:45 
 
 Ended 2024-11-11 13:29:45 
 
 Duration 0:00:00.719960 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:29:45Z ERROR baml_runtime::tracing]   Error: input: Expected type String, got `Number(111)`
+    "Alright, let's try to inject a little humor into this futuristic tale of dream manipulation. Here's an attempt at adding some comedic elements:\n\nIn this world of commercialized dream experiences, Alex - the socially awkward teen with a penchant for meddling in others' subconscious - finds themselves in a rather peculiar predicament. Armed with the ability to sculpt dreams like a virtuoso sculptor, Alex can't help but get carried away, crafting increasingly bizarre and outlandish dreamscapes for their unsuspecting clients.\n\nOne wealthy businessman pays handsomely for a VIP dream package, only to find himself trapped in a never-ending loop of show tunes and tap-dancing penguins. Another client, a high-strung workaholic, commissions Alex to create a soothing nature retreat, only to wake up covered in glitter and neon-colored flower petals, much to the delight of their coworkers.\n\nAs Alex's reputation grows, so do the expectations - and the eccentricities. The government agents tasked with recruiting Alex find themselves struggling to keep a straight face during the pitch meeting, as Alex describes their latest dream design involving a herd of levitating llamas and a game of volleyball with sentient marshmallows.\n\nMeanwhile, the resistance fighters trying to sway Alex to their cause can't help but chuckle at the sheer absurdity of Alex's creations, even as they warn of the dangers of dream manipulation. \"So, let me get this straight,\" one rebel leader says, \"you've been making people think they're dancing with sentient toasters? Well, that's certainly one way to fight the system.\"\n\nThrough it all, Alex remains oblivious to the humor of their situation, laser-focused on perfecting their craft and exploring the boundless possibilities of the dream world. After all, who needs reality when you can design your own fantastical escapes - even if they do involve the occasional mutant unicorn or singing paperclip."
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_arg_exceptions  0:00:00.989845 
  
 
 
 Started 2024-11-25 16:37:20 
 
 Ended 2024-11-25 16:37:21 
 
 Duration 0:00:00.989845 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:37:20Z ERROR baml_runtime::tracing]   Error: input: Expected type String, got `Number(111)`
     
-[2024-11-11T19:29:45Z WARN  baml_events] Function MyFunc:
-    Client: MyClient (<unknown>) - 122ms
+[2024-11-26T00:37:20Z WARN  baml_events] Function MyFunc:
+    Client: MyClient (<unknown>) - 385ms
     ---PROMPT---
-    [chat] system: Given a string, extract info using the schema:
+    [chat] user: Given a string, extract info using the schema:
     
     My name is Harrison. My hair is black and I'm 6 feet tall.
     
@@ -2469,7 +2661,8 @@
     ---REQUEST OPTIONS---
     model: "gpt-4o-mini"
     ---ERROR (InvalidAuthentication (401))---
-    Request failed: {
+    Request failed: https://api.openai.com/v1/chat/completions
+    {
         "error": {
             "message": "Incorrect API key provided: INVALID_KEY. You can find your API key at https://platform.openai.com/account/api-keys.",
             "type": "invalid_request_error",
@@ -2478,10 +2671,10 @@
         }
     }
     
-[2024-11-11T19:29:45Z WARN  baml_events] Function MyFunc:
-    Client: MyClient (<unknown>) - 100ms
+[2024-11-26T00:37:20Z WARN  baml_events] Function MyFunc:
+    Client: MyClient (<unknown>) - 161ms
     ---PROMPT---
-    [chat] system: Given a string, extract info using the schema:
+    [chat] user: Given a string, extract info using the schema:
     
     My name is Harrison. My hair is black and I'm 6 feet tall.
     
@@ -2492,7 +2685,8 @@
     ---REQUEST OPTIONS---
     model: "gpt-4o-mini"
     ---ERROR (InvalidAuthentication (401))---
-    Request failed: {
+    Request failed: https://api.openai.com/v1/chat/completions
+    {
         "error": {
             "message": "Incorrect API key provided: INVALID_KEY. You can find your API key at https://platform.openai.com/account/api-keys.",
             "type": "invalid_request_error",
@@ -2501,27 +2695,27 @@
         }
     }
     
-[2024-11-11T19:29:45Z WARN  baml_events] Function DummyOutputFunction:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 486ms. StopReason: stop. Tokens(in/out): 12/10
+[2024-11-26T00:37:21Z WARN  baml_events] Function DummyOutputFunction:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 426ms. StopReason: stop. Tokens(in/out): 12/3
     ---PROMPT---
-    [chat] system: Say "hello there".
+    [chat] user: Say "hello there".
     
     ---LLM REPLY---
-    Hello there! How can I assist you today?
+    Hello there!
     ---Parsed Response (Error)---
     Failed to coerce value: <root>: Failed while parsing required fields: missing=2, unparsed=0
       - <root>: Missing required field: nonce
       - <root>: Missing required field: nonce2
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_map_as_param  0:00:00.002003 
  
 
 
 Started 2024-11-11 13:29:45 
 
 Ended 2024-11-11 13:29:45 
 
 Duration 0:00:00.002003 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:29:45Z ERROR baml_runtime::tracing]   Error: myMap: a: Expected map, got `String("b")`
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_map_as_param  0:00:00.001731 
  
 
 
 Started 2024-11-25 16:37:21 
 
 Ended 2024-11-25 16:37:21 
 
 Duration 0:00:00.001731 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:37:21Z ERROR baml_runtime::tracing]   Error: myMap: a: Expected map, got `String("b")`
     
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_baml_validation_error_format  0:00:00.588897 
  
 
 
 Started 2024-11-11 13:29:45 
 
 Ended 2024-11-11 13:29:46 
 
 Duration 0:00:00.588897 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
Error:  BamlValidationError(message=Failed to parse LLM response: Failed to coerce value: <root>: Failed while parsing required fields: missing=2, unparsed=0
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_baml_validation_error_format  0:00:00.724016 
  
 
 
 Started 2024-11-25 16:37:21 
 
 Ended 2024-11-25 16:37:22 
 
 Duration 0:00:00.724016 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
Error:  BamlValidationError(message=Failed to parse LLM response: Failed to coerce value: <root>: Failed while parsing required fields: missing=2, unparsed=0
   - <root>: Missing required field: nonce
-  - <root>: Missing required field: nonce2, raw_output=Hello there! How can I assist you today?, prompt=[chat] system: Say "hello there".
+  - <root>: Missing required field: nonce2, raw_output=Hello there! How can I assist you today?, prompt=[chat] user: Say "hello there".
 )
-
 
 
 
Captured stderr call
 
[2024-11-11T19:29:46Z WARN  baml_events] Function DummyOutputFunction:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 584ms. StopReason: stop. Tokens(in/out): 12/10
+
 
 
 
Captured stderr call
 
[2024-11-26T00:37:22Z WARN  baml_events] Function DummyOutputFunction:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 717ms. StopReason: stop. Tokens(in/out): 12/10
     ---PROMPT---
-    [chat] system: Say "hello there".
+    [chat] user: Say "hello there".
     
     ---LLM REPLY---
     Hello there! How can I assist you today?
@@ -2529,10 +2723,10 @@
     Failed to coerce value: <root>: Failed while parsing required fields: missing=2, unparsed=0
       - <root>: Missing required field: nonce
       - <root>: Missing required field: nonce2
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_no_stream_big_integer  0:00:00.320955 
  
 
 
 Started 2024-11-11 13:29:46 
 
 Ended 2024-11-11 13:29:46 
 
 Duration 0:00:00.320955 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:29:46Z INFO  baml_events] Function StreamOneBigNumber:
-    Client: GPT4 (gpt-4o-2024-08-06) - 314ms. StopReason: stop. Tokens(in/out): 46/4
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_no_stream_big_integer  0:00:00.416612 
  
 
 
 Started 2024-11-25 16:37:22 
 
 Ended 2024-11-25 16:37:22 
 
 Duration 0:00:00.416612 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:37:22Z INFO  baml_events] Function StreamOneBigNumber:
+    Client: GPT4 (gpt-4o-2024-08-06) - 411ms. StopReason: stop. Tokens(in/out): 46/4
     ---PROMPT---
-    [chat] system: Respond with only an integer, no affirmations or prefixes or anything.
+    [chat] user: Respond with only an integer, no affirmations or prefixes or anything.
     The response should be parsable as a JSON number.
     
     Please make sure the integer has 12 digits.
@@ -2540,13 +2734,13 @@
     Answer as an int
     
     ---LLM REPLY---
-    810379246135
+    102345678901
     ---Parsed Response (int)---
-    810379246135
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_no_stream_object_with_numbers  0:00:00.599190 
  
 
 
 Started 2024-11-11 13:29:46 
 
 Ended 2024-11-11 13:29:47 
 
 Duration 0:00:00.599190 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:29:47Z INFO  baml_events] Function StreamBigNumbers:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 592ms. StopReason: stop. Tokens(in/out): 70/23
+    102345678901
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_no_stream_object_with_numbers  0:00:00.454241 
  
 
 
 Started 2024-11-25 16:37:22 
 
 Ended 2024-11-25 16:37:22 
 
 Duration 0:00:00.454241 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:37:22Z INFO  baml_events] Function StreamBigNumbers:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 449ms. StopReason: stop. Tokens(in/out): 70/23
     ---PROMPT---
-    [chat] system: Please make sure every integer in the output has 12 digits.
+    [chat] user: Please make sure every integer in the output has 12 digits.
     For floats, provide a mix - from 0-10 places before the decimal point,
     and 1-10 places after the decimal point. Bet.
     
@@ -2559,17 +2753,17 @@
     ---LLM REPLY---
     {
       "a": 123456789012,
-      "b": 9876543.210
+      "b": 7.890123456
     }
     ---Parsed Response (class BigNumbers)---
     {
       "a": 123456789012,
-      "b": 9876543.21
+      "b": 7.890123456
     }
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_no_stream_compound_object  0:00:03.402571 
  
 
 
 Started 2024-11-11 13:29:47 
 
 Ended 2024-11-11 13:29:50 
 
 Duration 0:00:03.402571 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:29:50Z INFO  baml_events] Function StreamingCompoundNumbers:
-    Client: GPT4 (gpt-4o-2024-08-06) - 3394ms. StopReason: stop. Tokens(in/out): 133/150
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_no_stream_compound_object  0:00:02.528516 
  
 
 
 Started 2024-11-25 16:37:22 
 
 Ended 2024-11-25 16:37:25 
 
 Duration 0:00:02.528516 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:37:25Z INFO  baml_events] Function StreamingCompoundNumbers:
+    Client: GPT4 (gpt-4o-2024-08-06) - 2522ms. StopReason: stop. Tokens(in/out): 133/149
     ---PROMPT---
-    [chat] system:     Respond in pure json. Don't use any English descriptions like "Sure, I'll do that",
+    [chat] user:     Respond in pure json. Don't use any English descriptions like "Sure, I'll do that",
         nor put the result into a fenced code block.
     
         Just output a JSON value that could be parsed as JSON.
@@ -2594,56 +2788,56 @@
     {
       "big": {
         "a": 123456789012,
-        "b": 1234567890.123456789
+        "b": 9876543210.1234567890
       },
       "big_nums": [
         {
-          "a": 987654321098,
-          "b": 98765432.123456789
+          "a": 234567890123,
+          "b": 1234567.89
         },
         {
-          "a": 192837465012,
-          "b": 1234567890123.12345
+          "a": 345678901234,
+          "b": 1234567890123.456789
         },
         {
-          "a": 123456789012,
-          "b": 0.123456789
+          "a": 456789012345,
+          "b": 12345.6789012
         }
       ],
       "another": {
-        "a": 999999999999,
-        "b": 4567890123.123
+        "a": 567890123456,
+        "b": 123.456789012
       }
     }
     ---Parsed Response (class CompoundBigNumbers)---
     {
       "big": {
         "a": 123456789012,
-        "b": 1234567890.1234567
+        "b": 9876543210.123457
       },
       "big_nums": [
         {
-          "a": 987654321098,
-          "b": 98765432.12345679
+          "a": 234567890123,
+          "b": 1234567.89
         },
         {
-          "a": 192837465012,
-          "b": 1234567890123.1235
+          "a": 345678901234,
+          "b": 1234567890123.4568
         },
         {
-          "a": 123456789012,
-          "b": 0.123456789
+          "a": 456789012345,
+          "b": 12345.6789012
         }
       ],
       "another": {
-        "a": 999999999999,
-        "b": 4567890123.123
+        "a": 567890123456,
+        "b": 123.456789012
       }
     }
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_no_stream_compound_object_with_yapping  0:00:03.674023 
  
 
 
 Started 2024-11-11 13:29:50 
 
 Ended 2024-11-11 13:29:54 
 
 Duration 0:00:03.674023 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:29:54Z INFO  baml_events] Function StreamingCompoundNumbers:
-    Client: GPT4 (gpt-4o-2024-08-06) - 3664ms. StopReason: stop. Tokens(in/out): 114/175
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_no_stream_compound_object_with_yapping  0:00:01.786465 
  
 
 
 Started 2024-11-25 16:37:25 
 
 Ended 2024-11-25 16:37:27 
 
 Duration 0:00:01.786465 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:37:27Z INFO  baml_events] Function StreamingCompoundNumbers:
+    Client: GPT4 (gpt-4o-2024-08-06) - 1781ms. StopReason: stop. Tokens(in/out): 114/148
     ---PROMPT---
-    [chat] system:     Please give me a friendly response before outputting json. And put the JSON
+    [chat] user:     Please give me a friendly response before outputting json. And put the JSON
         into a fenced code block.
     
     Please make sure every integer has 12 digits.
@@ -2663,63 +2857,55 @@
     }
     
     ---LLM REPLY---
-    Hello there! I'm excited to help you out with some big numbers presented in JSON format. Here you go:
+    Hey there! I'm about to share some JSON data with you following your specifications. Check it out below:
     
     ```json
     {
       "big": {
-        "a": 123456789012,
-        "b": 9876543210.12345
+        "a": 100000000001,
+        "b": 1234567890.123456789
       },
       "big_nums": [
         {
-          "a": 234567890123,
-          "b": 123456789012.0
-        },
-        {
-          "a": 345678901234,
-          "b": 567890123456.789
+          "a": 200000000002,
+          "b": 987654321.1
         },
         {
-          "a": 456789012345,
-          "b": 6789012345.678901
+          "a": 300000000003,
+          "b": 12340000000.1234
         }
       ],
       "another": {
-        "a": 567890123456,
-        "b": 8901234567.891234
+        "a": 400000000004,
+        "b": 567890123.987654321
       }
     }
     ```
     ---Parsed Response (class CompoundBigNumbers)---
     {
       "big": {
-        "a": 123456789012,
-        "b": 9876543210.12345
+        "a": 100000000001,
+        "b": 1234567890.1234567
       },
       "big_nums": [
         {
-          "a": 234567890123,
-          "b": 123456789012.0
-        },
-        {
-          "a": 345678901234,
-          "b": 567890123456.789
+          "a": 200000000002,
+          "b": 987654321.1
         },
         {
-          "a": 456789012345,
-          "b": 6789012345.678901
+          "a": 300000000003,
+          "b": 12340000000.1234
         }
       ],
       "another": {
-        "a": 567890123456,
-        "b": 8901234567.891233
+        "a": 400000000004,
+        "b": 567890123.9876543
       }
     }
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_differing_unions  0:00:01.927315 
  
 
 
 Started 2024-11-11 13:29:54 
 
 Ended 2024-11-11 13:29:56 
 
 Duration 0:00:01.927315 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:29:56Z INFO  baml_events] Function DifferentiateUnions:
-    Client: openai/gpt-4o-mini (gpt-4o-mini-2024-07-18) - 1918ms. StopReason: stop. Tokens(in/out): 50/73
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_differing_unions  0:00:01.066114 
  
 
 
 Started 2024-11-25 16:37:27 
 
 Ended 2024-11-25 16:37:28 
 
 Duration 0:00:01.066114 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:37:28Z INFO  baml_events] Function DifferentiateUnions:
+    Client: openai/gpt-4o-mini (gpt-4o-mini-2024-07-18) - 1057ms. StopReason: stop. Tokens(in/out): 50/60
     ---PROMPT---
-    [chat] system: Create a data model that represents the latter of the two classes.
+    [chat] user: Create a data model that represents the latter of the two classes.
     
     Answer in JSON using any of these schemas:
     {
@@ -2730,25 +2916,25 @@
     }
     
     ---LLM REPLY---
-    Here is a JSON representation of a data model based on the second class schema provided:
+    Here’s a data model in JSON that represents the latter of the two classes, which includes an integer value and a string value:
     
     ```json
     {
       "value": 42,
       "value2": "example string"
     }
-    ```
+    ``` 
     
-    In this model, `value` is an integer, and `value2` is a string. You can replace the values accordingly to fit your specific use case.
+    Feel free to customize the values to suit your needs!
     ---Parsed Response (class OriginalB)---
     {
       "value": 42,
       "value2": "example string"
     }
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_return_failing_assert  0:00:00.421477 
  
 
 
 Started 2024-11-11 13:29:56 
 
 Ended 2024-11-11 13:29:56 
 
 Duration 0:00:00.421477 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:29:56Z WARN  baml_events] Function ReturnFailingAssert:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 414ms. StopReason: stop. Tokens(in/out): 19/1
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_return_failing_assert  0:00:00.619982 
  
 
 
 Started 2024-11-25 16:37:28 
 
 Ended 2024-11-25 16:37:28 
 
 Duration 0:00:00.619982 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:37:28Z WARN  baml_events] Function ReturnFailingAssert:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 615ms. StopReason: stop. Tokens(in/out): 19/1
     ---PROMPT---
-    [chat] system: Return the next integer after 1.
+    [chat] user: Return the next integer after 1.
     
     Answer as an int
     
@@ -2756,42 +2942,9 @@
     2
     ---Parsed Response (Error)---
     Failed to coerce value: <root>: Assertions failed.
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_parameter_failing_assert  0:00:00.003229 
  
 
 
 Started 2024-11-11 13:29:56 
 
 Ended 2024-11-11 13:29:56 
 
 Duration 0:00:00.003229 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:29:56Z ERROR baml_runtime::tracing]   Error: inp: Failed assert: small_int
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_parameter_failing_assert  0:00:00.002388 
  
 
 
 Started 2024-11-25 16:37:28 
 
 Ended 2024-11-25 16:37:28 
 
 Duration 0:00:00.002388 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:37:28Z ERROR baml_runtime::tracing]   Error: inp: Failed assert: small_int
     
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_failing_assert_can_stream  0:00:03.330473 
  
 
 
 Started 2024-11-11 13:29:56 
 
 Ended 2024-11-11 13:30:00 
 
 Duration 0:00:03.330473 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
None
-None
-None
-None
-None
-None
-None
-None
-None
-None
-None
-None
-None
-None
-None
-None
-None
-None
-None
-None
-None
-None
-None
-None
-None
-None
-None
-None
-None
-None
-None
-None
-None
-None
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_failing_assert_can_stream  0:00:02.132530 
  
 
 
 Started 2024-11-25 16:37:28 
 
 Ended 2024-11-25 16:37:31 
 
 Duration 0:00:02.132530 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
None
 None
 None
 None
@@ -3331,13 +3484,10 @@
 None
 None
 None
-None
-None
-None
-
 
 
 
Captured stderr call
 
[2024-11-11T19:30:00Z WARN  baml_events] Function StreamFailingAssertion:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 3323ms. StopReason: stop. Tokens(in/out): 62/285
+
 
 
 
Captured stderr call
 
[2024-11-26T00:37:31Z WARN  baml_events] Function StreamFailingAssertion:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 2127ms. StopReason: stop. Tokens(in/out): 62/267
     ---PROMPT---
-    [chat] system: Tell me two different stories along the theme of Yoshimi battles the pink robots with the same title.
+    [chat] user: Tell me two different stories along the theme of Yoshimi battles the pink robots with the same title.
     Please make each about 300 words long.
     Answer in JSON using this schema:
     {
@@ -3348,9 +3498,9 @@
     
     ---LLM REPLY---
     {
-      "title": "Yoshimi battles the pink robots",
-      "story_a": "Yoshimi was a young scientist who had always been fascinated by robotics. When a group of pink robots suddenly invaded her city, she knew she had to take action. Armed with her knowledge and determination, Yoshimi created a powerful robot of her own to combat the pink invaders. The battle was intense, with laser beams and explosions lighting up the sky. Despite the odds being against her, Yoshimi's robot fought bravely and eventually emerged victorious. The city cheered as the last pink robot was defeated, and Yoshimi was hailed as a hero for her courage and ingenuity.",
-      "story_b": "Yoshimi was a talented martial artist who had been training for years to perfect her skills. When a horde of pink robots descended upon her dojo, Yoshimi knew she had to defend her home. With swift and precise movements, she engaged the robots in combat, dodging their attacks and delivering powerful strikes of her own. As the battle raged on, Yoshimi's determination never wavered. Despite being outnumbered, she fought with incredible strength and skill. In the end, the pink robots were no match for Yoshimi's expertise, and they were defeated one by one. The dojo was safe once again, and Yoshimi stood victorious, a true warrior in every sense of the word."
+      "title": "Yoshimi Battles the Pink Robots",
+      "story_a": "Yoshimi was a skilled warrior, trained in the ancient art of robot combat. When a group of pink robots invaded her village, Yoshimi knew it was up to her to save the day. Armed with her trusty sword and fierce determination, she faced off against the robots in a battle that shook the ground. With each swing of her sword, Yoshimi fought bravely, dodging lasers and parrying attacks. In the end, Yoshimi emerged victorious, her village saved from the robot threat. The townspeople cheered her name, grateful for her bravery and skill in battle.",
+      "story_b": "Yoshimi was a young musician who stumbled upon a mysterious pink robot invasion while on tour with her band. Intrigued by the strange creatures, Yoshimi decided to investigate further. As she delved deeper into the mystery, Yoshimi realized that the robots were not as menacing as they first appeared. In fact, they were lost and confused, searching for their way home. With her music, Yoshimi was able to communicate with the robots and guide them back to their own world, where they belonged. The pink robots danced joyfully as they bid Yoshimi farewell, grateful for her help and understanding."
     }
     ---Parsed Response (Error)---
     Failed to coerce value: <root>: Failed while parsing required fields: missing=0, unparsed=2
@@ -3358,8 +3508,8 @@
         - <root>: Assertions failed.
       - <root>: Failed to parse field story_b: <root>: Assertions failed.
         - <root>: Assertions failed.
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_simple_recursive_type  0:00:03.136111 
  
 
 
 Started 2024-11-11 13:30:00 
 
 Ended 2024-11-11 13:30:03 
 
 Duration 0:00:03.136111 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:30:03Z INFO  baml_events] Function BuildLinkedList:
-    Client: O1 (o1-mini-2024-09-12) - 3125ms. StopReason: stop. Tokens(in/out): 81/421
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_simple_recursive_type  0:00:02.987520 
  
 
 
 Started 2024-11-25 16:37:31 
 
 Ended 2024-11-25 16:37:34 
 
 Duration 0:00:02.987520 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:37:34Z INFO  baml_events] Function BuildLinkedList:
+    Client: O1 (o1-mini-2024-09-12) - 2978ms. StopReason: stop. Tokens(in/out): 81/357
     ---PROMPT---
     [chat] user: Build a linked list from the input array of integers.
     
@@ -3419,43 +3569,43 @@
       },
       "len": 5
     }
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_mutually_recursive_type  0:00:03.676833 
  
 
 
 Started 2024-11-11 13:30:03 
 
 Ended 2024-11-11 13:30:06 
 
 Duration 0:00:03.676833 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:30:06Z INFO  baml_events] Function BuildTree:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 3661ms. StopReason: stop. Tokens(in/out): 216/190
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_mutually_recursive_type  0:00:01.932089 
  
 
 
 Started 2024-11-25 16:37:34 
 
 Ended 2024-11-25 16:37:36 
 
 Duration 0:00:01.932089 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:37:36Z INFO  baml_events] Function BuildTree:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 1921ms. StopReason: stop. Tokens(in/out): 216/190
     ---PROMPT---
-    [chat] system: Given the input binary tree, transform it into a generic tree using the given schema.
+    [chat] user: Given the input binary tree, transform it into a generic tree using the given schema.
     
     INPUT:
     {
-        "data": 5,
-        "right": {
-            "data": 7,
+        "left": {
             "left": {
-                "data": 6,
-                "left": none,
                 "right": none,
+                "data": 1,
+                "left": {
+                    "data": 2,
+                    "right": none,
+                    "left": none,
+                },
             },
             "right": {
-                "left": none,
+                "data": 4,
                 "right": none,
-                "data": 8,
+                "left": none,
             },
+            "data": 3,
         },
-        "left": {
-            "right": {
-                "data": 4,
+        "data": 5,
+        "right": {
+            "data": 7,
+            "left": {
                 "left": none,
                 "right": none,
+                "data": 6,
             },
-            "left": {
-                "left": {
-                    "data": 2,
-                    "right": none,
-                    "left": none,
-                },
+            "right": {
                 "right": none,
-                "data": 1,
+                "data": 8,
+                "left": none,
             },
-            "data": 3,
         },
     }
     
@@ -3576,11 +3726,41 @@
         ]
       }
     }
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 SKIPPED  test_block_constraints  0:00:00.000387 
  
 
 
 Started 2024-11-11 13:30:06 
 
 Ended 2024-11-11 13:30:06 
 
 Duration 0:00:00.000387 
 
 Markers   
 
 Fixtures   event_loop_policy   cleanup   
 
 
 
Skipped: async def function and no async plugin installed (see warnings)
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
('/Users/vbv/Library/Caches/pypoetry/virtualenvs/python-integ-tests-lSPb1NE9-py3.12/lib/python3.12/site-packages/_pytest/python.py', 149, 'Skipped: async def function and no async plugin installed (see warnings)')
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_nested_block_constraints  0:00:00.805415 
  
 
 
 Started 2024-11-11 13:30:06 
 
 Ended 2024-11-11 13:30:07 
 
 Duration 0:00:00.805415 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
nbc=Checked[BlockConstraint, Literal['cross_field']](value=BlockConstraint(foo=1, bar='hello'), checks={'cross_field': Check(name='cross_field', expression='this.bar|length > this.foo', status='succeeded')})
-
 
 
 
Captured stderr call
 
[2024-11-11T19:30:07Z INFO  baml_events] Function MakeNestedBlockConstraint:
-    Client: GPT35 (gpt-3.5-turbo-0125) - 795ms. StopReason: stop. Tokens(in/out): 52/24
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_block_constraints  0:00:00.493983 
  
 
 
 Started 2024-11-25 16:37:36 
 
 Ended 2024-11-25 16:37:36 
 
 Duration 0:00:00.493983 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:37:36Z INFO  baml_events] Function MakeBlockConstraint:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 491ms. StopReason: stop. Tokens(in/out): 42/20
     ---PROMPT---
-    [chat] system: Generate an output where the inner foo is 1 and the inner bar is "hello".
+    [chat] user: Generate an output in the following schema with a short string and a large int.
+    
+    Answer in JSON using this schema:
+    {
+      foo: int,
+      bar: string,
+    }
+    
+    ---LLM REPLY---
+    {
+      "foo": 1000000,
+      "bar": "Hello, World!"
+    }
+    ---Parsed Response (class BlockConstraint)---
+    {
+      "value": {
+        "foo": 1000000,
+        "bar": "Hello, World!"
+      },
+      "checks": {
+        "cross_field": {
+          "name": "cross_field",
+          "expression": "this.bar|length > this.foo",
+          "status": "failed"
+        }
+      }
+    }
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_nested_block_constraints  0:00:00.550835 
  
 
 
 Started 2024-11-25 16:37:36 
 
 Ended 2024-11-25 16:37:37 
 
 Duration 0:00:00.550835 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stdout call
 
nbc=Checked[BlockConstraint, Literal['cross_field']](value=BlockConstraint(foo=1, bar='hello'), checks={'cross_field': Check(name='cross_field', expression='this.bar|length > this.foo', status='succeeded')})
+
 
 
 
Captured stderr call
 
[2024-11-26T00:37:37Z INFO  baml_events] Function MakeNestedBlockConstraint:
+    Client: GPT35 (gpt-3.5-turbo-0125) - 543ms. StopReason: stop. Tokens(in/out): 52/21
+    ---PROMPT---
+    [chat] user: Generate an output where the inner foo is 1 and the inner bar is "hello".
       Answer in JSON using this schema:
     {
       nbc: {
@@ -3591,10 +3771,10 @@
     
     ---LLM REPLY---
     {
-      "nbc": {
-        "foo": 1,
-        "bar": "hello"
-      }
+      nbc: {
+        foo: 1,
+        bar: "hello",
+      },
     }
     ---Parsed Response (class NestedBlockConstraint)---
     {
@@ -3612,8 +3792,8 @@
         }
       }
     }
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_block_constraint_arguments  0:00:00.003396 
  
 
 
 Started 2024-11-11 13:30:07 
 
 Ended 2024-11-11 13:30:07 
 
 Duration 0:00:00.003396 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-11T19:30:07Z ERROR baml_runtime::tracing]   Error: inp: Failed assert: hi
+
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_block_constraint_arguments  0:00:00.002001 
  
 
 
 Started 2024-11-25 16:37:37 
 
 Ended 2024-11-25 16:37:37 
 
 Duration 0:00:00.002001 
 
 Markers  
 asyncio   
 		 
 
 Fixtures   event_loop_policy   cleanup   event_loop   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
Captured stderr call
 
[2024-11-26T00:37:37Z ERROR baml_runtime::tracing]   Error: inp: Failed assert: hi
     
-[2024-11-11T19:30:07Z ERROR baml_runtime::tracing]   Error: inp: Failed assert: hi
+[2024-11-26T00:37:37Z ERROR baml_runtime::tracing]   Error: inp: Failed assert: hi
     
-
 
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
 
 
  
  tests/test_pydantic.py   3   0:00:00.002465  
  
 
  
 PASSED  test_model_validate_success  0:00:00.001196 
  
 
 
 Started 2024-11-11 13:30:07 
 
 Ended 2024-11-11 13:30:07 
 
 Duration 0:00:00.001196 
 
 Markers   
 
 Fixtures   event_loop_policy   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_model_validate_failure  0:00:00.000838 
  
 
 
 Started 2024-11-11 13:30:07 
 
 Ended 2024-11-11 13:30:07 
 
 Duration 0:00:00.000838 
 
 Markers   
 
 Fixtures   event_loop_policy   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
  
 PASSED  test_model_dump  0:00:00.000432 
  
 
 
 Started 2024-11-11 13:30:07 
 
 Ended 2024-11-11 13:30:07 
 
 Duration 0:00:00.000432 
 
 Markers   
 
 Fixtures   event_loop_policy   
 
 
 
  
 Setup 
  
 
 
 
  
  Call 
  
 
 
 
  
 Teardown 
  
 
 
 
 
 
 
 
 
 
 
   
\ No newline at end of file
+

Teardown

tests/test_pydantic.py 3 0:00:00.001668

PASSED test_model_validate_success 0:00:00.000383

Started 2024-11-25 16:37:37
Ended 2024-11-25 16:37:37
Duration 0:00:00.000383
Markers
Fixtures event_loop_policy

Setup

Call

Teardown

PASSED test_model_validate_failure 0:00:00.000537

Started 2024-11-25 16:37:37
Ended 2024-11-25 16:37:37
Duration 0:00:00.000537
Markers
Fixtures event_loop_policy

Setup

Call

Teardown

PASSED test_model_dump 0:00:00.000749

Started 2024-11-25 16:37:37
Ended 2024-11-25 16:37:37
Duration 0:00:00.000749
Markers
Fixtures event_loop_policy

Setup

Call

Teardown

\ No newline at end of file diff --git a/integ-tests/python/tests/test_functions.py b/integ-tests/python/tests/test_functions.py index d4fd427f2..961f897b9 100644 --- a/integ-tests/python/tests/test_functions.py +++ b/integ-tests/python/tests/test_functions.py @@ -1,3 +1,4 @@ +import uuid import json import os import time @@ -370,10 +371,9 @@ async def test_works_with_fallbacks(): @pytest.mark.asyncio async def test_works_with_failing_azure_fallback(): - with pytest.raises(Exception) as e: - res = await b.TestSingleFallbackClient() - assert len(res) > 0, "Expected non-empty result but got empty." - assert "Either base_url or" in str(e) + with pytest.raises(errors.BamlClientError) as e: + _ = await b.TestSingleFallbackClient() + assert "ConnectError" in str(e.value) @pytest.mark.asyncio @@ -778,6 +778,10 @@ async def test_dynamic_class_nested_output_no_stream(): nested_class.add_property("middle_name", tb.string().optional()) other_nested_class = tb.add_class("Address") + other_nested_class.add_property("street", tb.string()) + other_nested_class.add_property("city", tb.string()) + other_nested_class.add_property("state", tb.string()) + other_nested_class.add_property("zip", tb.string()) # name should be first in the prompt schema tb.DynamicOutput.add_property("name", nested_class.type().optional()) @@ -1087,7 +1091,6 @@ async def test_dynamic_client_with_vertex_json_str_creds(): "vertex-ai", { "model": "gemini-1.5-pro", - "project_id": "sam-project-vertex-1", "location": "us-central1", "credentials": os.environ[ "INTEG_TESTS_GOOGLE_APPLICATION_CREDENTIALS_CONTENT" @@ -1110,7 +1113,6 @@ async def test_dynamic_client_with_vertex_json_object_creds(): "vertex-ai", { "model": "gemini-1.5-pro", - "project_id": "sam-project-vertex-1", "location": "us-central1", "credentials": json.loads( os.environ["INTEG_TESTS_GOOGLE_APPLICATION_CREDENTIALS_CONTENT"] @@ -1235,29 +1237,32 @@ async def test_descriptions(): @pytest.mark.asyncio async def test_caching(): - story_idea = """ - In a near-future society where dreams have become a tradable commodity and shared experience, a lonely and socially awkward teenager named Alex discovers they possess a rare and powerful ability to not only view but also manipulate the dreams of others. Initially thrilled by this newfound power, Alex begins subtly altering the dreams of classmates and family members, helping them overcome fears, boost confidence, or experience fantastical adventures. As Alex's skills grow, so does their influence. They start selling premium dream experiences on the black market, crafting intricate and addictive dreamscapes for wealthy clients. However, the line between dream and reality begins to blur for those exposed to Alex's creations. Some clients struggle to differentiate between their true memories and the artificial ones implanted by Alex's dream manipulation. + story_idea = f""" +In a futuristic world where dreams are a marketable asset and collective experience, an introverted and socially inept teenager named Alex realizes they have a unique and potent skill to not only observe but also alter the dreams of others. Initially excited by this newfound talent, Alex starts discreetly modifying the dreams of peers and relatives, aiding them in conquering fears, boosting self-esteem, or embarking on fantastical journeys. As Alex's abilities expand, so does their sway. They begin marketing exclusive dream experiences on the underground market, designing complex and captivating dreamscapes for affluent clients. However, the boundary between dream and reality starts to fade for those subjected to Alex's creations. Some clients find it difficult to distinguish between their genuine memories and the fabricated ones inserted by Alex's dream manipulation. - Complications arise when a mysterious government agency takes notice of Alex's unique abilities. They offer Alex a chance to use their gift for "the greater good," hinting at applications in therapy, criminal rehabilitation, and even national security. Simultaneously, an underground resistance movement reaches out, warning Alex about the dangers of dream manipulation and the potential for mass control and exploitation. Caught between these opposing forces, Alex must navigate a complex web of ethical dilemmas. They grapple with questions of free will, the nature of consciousness, and the responsibility that comes with having power over people's minds. As the consequences of their actions spiral outward, affecting the lives of loved ones and strangers alike, Alex is forced to confront the true nature of their ability and decide how—or if—it should be used. +Challenges emerge when a secretive government organization becomes aware of Alex's distinct talents. They propose Alex utilize their gift for "the greater good," suggesting uses in therapy, criminal reform, and even national defense. Concurrently, a covert resistance group contacts Alex, cautioning them about the risks of dream manipulation and the potential for widespread control and exploitation. Trapped between these conflicting forces, Alex must navigate a tangled web of moral dilemmas. They wrestle with issues of free will, the essence of consciousness, and the duty that comes with having influence over people's minds. As the repercussions of their actions ripple outward, impacting the lives of loved ones and strangers alike, Alex is compelled to face the true nature of their power and decide how—or if—it should be wielded. - The story explores themes of identity, the subconscious mind, the ethics of technology, and the power of imagination. It delves into the potential consequences of a world where our most private thoughts and experiences are no longer truly our own, and examines the fine line between helping others and manipulating them for personal gain or a perceived greater good. The narrative further expands on the societal implications of such abilities, questioning the moral boundaries of altering consciousness and the potential for abuse in a world where dreams can be commodified. It challenges the reader to consider the impact of technology on personal autonomy and the ethical responsibilities of those who wield such power. +The narrative investigates themes of identity, the subconscious, the ethics of technology, and the power of creativity. It explores the possible outcomes of a world where our most intimate thoughts and experiences are no longer truly our own, and scrutinizes the fine line between aiding others and manipulating them for personal benefit or a perceived greater good. The story further delves into the societal ramifications of such abilities, questioning the moral limits of altering consciousness and the potential for misuse in a world where dreams can be commercialized. It challenges the reader to contemplate the impact of technology on personal freedom and the ethical duties of those who wield such power. - As Alex's journey unfolds, they encounter various individuals whose lives have been touched by their dream manipulations, each presenting a unique perspective on the ethical quandaries at hand. From a classmate who gains newfound confidence to a wealthy client who becomes addicted to the dreamscapes, the ripple effects of Alex's actions are profound and far-reaching. The government agency's interest in Alex's abilities raises questions about the potential for state control and surveillance, while the resistance movement highlights the dangers of unchecked power and the importance of safeguarding individual freedoms. +As Alex's journey progresses, they meet various individuals whose lives have been influenced by their dream manipulations, each offering a distinct viewpoint on the ethical issues at hand. From a peer who gains newfound confidence to a wealthy client who becomes dependent on the dreamscapes, the ripple effects of Alex's actions are significant and extensive. The government agency's interest in Alex's abilities raises questions about the potential for state control and surveillance, while the resistance movement underscores the dangers of unchecked power and the necessity of protecting individual freedoms. - Ultimately, Alex's story is one of self-discovery and moral reckoning, as they must decide whether to embrace their abilities for personal gain, align with the government's vision of a controlled utopia, or join the resistance in their fight for freedom and autonomy. The narrative invites readers to reflect on the nature of reality, the boundaries of human experience, and the ethical implications of a world where dreams are no longer private sanctuaries but shared and manipulated commodities. It also explores the psychological impact on Alex, who must deal with the burden of knowing the intimate fears and desires of others, and the isolation that comes from being unable to share their own dreams without altering them. +Ultimately, Alex's story is one of self-discovery and moral reflection, as they must choose whether to use their abilities for personal gain, align with the government's vision of a controlled utopia, or join the resistance in their struggle for freedom and autonomy. The narrative encourages readers to reflect on the nature of reality, the boundaries of human experience, and the ethical implications of a world where dreams are no longer private sanctuaries but shared and manipulated commodities. It also examines the psychological impact on Alex, who must cope with the burden of knowing the intimate fears and desires of others, and the isolation that comes from being unable to share their own dreams without altering them. - The story further examines the technological advancements that have made dream manipulation possible, questioning the role of innovation in society and the potential for both progress and peril. It considers the societal divide between those who can afford to buy enhanced dream experiences and those who cannot, highlighting issues of inequality and access. As Alex becomes more entangled in the web of their own making, they must confront the possibility that their actions could lead to unintended consequences, not just for themselves but for the fabric of society as a whole. +The story further investigates the technological progress that has made dream manipulation feasible, questioning the role of innovation in society and the potential for both advancement and peril. It considers the societal divide between those who can afford to purchase enhanced dream experiences and those who cannot, highlighting issues of inequality and access. As Alex becomes more ensnared in the web of their own making, they must confront the possibility that their actions could lead to unintended consequences, not just for themselves but for the fabric of society as a whole. - In the end, Alex's journey is a cautionary tale about the power of dreams and the responsibilities that come with wielding such influence. It serves as a reminder of the importance of ethical considerations in the face of technological advancement and the need to balance innovation with humanity. The story leaves readers pondering the true cost of a world where dreams are no longer sacred, and the potential for both wonder and danger in the uncharted territories of the mind. +In the end, Alex's journey is a cautionary tale about the power of dreams and the responsibilities that come with wielding such influence. It serves as a reminder of the importance of ethical considerations in the face of technological advancement and the need to balance innovation with humanity. The story leaves readers pondering the true cost of a world where dreams are no longer sacred, and the potential for both wonder and danger in the uncharted territories of the mind. But it's also a story about the power of imagination and the potential for change, even in a world where our deepest thoughts are no longer our own. And it's a story about the power of choice, and the importance of fighting for the freedom to dream. + +In conclusion, this story is a reflection on the power of dreams and the responsibilities that come with wielding such influence. It serves as a reminder of the importance of ethical considerations in the face of technological advancement and the need to balance innovation with humanity. The story leaves readers pondering the true cost of a world where dreams are no longer sacred, and the potential for both wonder and danger in the uncharted territories of the mind. But it's also a story about the power of imagination and the potential for change, even in a world where our deepest thoughts are no longer our own. And it's a story about the power of choice, and the importance of fighting for the freedom to dream. """ - rand = random.randint(0, 26) - story_idea += " " + rand * "a" + rand = uuid.uuid4().hex + story_idea = rand + story_idea + start = time.time() - _ = await b.TestCaching(story_idea, "be funny") + _ = await b.TestCaching(story_idea, "1. try to be funny") duration = time.time() - start start = time.time() - _ = await b.TestCaching(story_idea, "be real") + _ = await b.TestCaching(story_idea, "1. try to be funny") duration2 = time.time() - start print("Duration no caching: ", duration) @@ -1265,7 +1270,7 @@ async def test_caching(): assert ( duration2 < duration - ), "Expected second call to be faster than first by a large margin." + ), f"{duration2} < {duration}. Expected second call to be faster than first by a large margin." @pytest.mark.asyncio @@ -1493,6 +1498,7 @@ async def test_mutually_recursive_type(): ) +@pytest.mark.asyncio async def test_block_constraints(): ret = await b.MakeBlockConstraint() assert ret.checks["cross_field"].status == "failed" diff --git a/integ-tests/ruby/baml_client/inlined.rb b/integ-tests/ruby/baml_client/inlined.rb index 05f152c23..11b3f2a39 100644 --- a/integ-tests/ruby/baml_client/inlined.rb +++ b/integ-tests/ruby/baml_client/inlined.rb @@ -16,7 +16,7 @@ module Baml module Inlined FILE_MAP = { - "clients.baml" => "retry_policy Bar {\n max_retries 3\n strategy {\n type exponential_backoff\n }\n}\n\nretry_policy Foo {\n max_retries 3\n strategy {\n type constant_delay\n delay_ms 100\n }\n}\n\nclient GPT4 {\n provider openai\n options {\n model gpt-4o\n api_key env.OPENAI_API_KEY\n }\n} \n\n\nclient GPT4o {\n provider openai\n options {\n model gpt-4o\n api_key env.OPENAI_API_KEY\n }\n} \n\n\nclient GPT4Turbo {\n retry_policy Bar\n provider openai\n options {\n model gpt-4-turbo\n api_key env.OPENAI_API_KEY\n }\n} \n\nclient GPT35 {\n provider openai\n options {\n model \"gpt-3.5-turbo\"\n api_key env.OPENAI_API_KEY\n }\n}\n\nclient GPT35LegacyProvider {\n provider openai\n options {\n model \"gpt-3.5-turbo\"\n api_key env.OPENAI_API_KEY\n }\n}\n\n\nclient Ollama {\n provider ollama\n options {\n model llama2\n }\n}\n\nclient GPT35Azure {\n provider azure-openai\n options {\n resource_name \"west-us-azure-baml\"\n deployment_id \"gpt-35-turbo-default\"\n // base_url \"https://west-us-azure-baml.openai.azure.com/openai/deployments/gpt-35-turbo-default\"\n api_version \"2024-02-01\"\n api_key env.AZURE_OPENAI_API_KEY\n }\n}\n\nclient Gemini {\n provider google-ai\n options {\n model gemini-1.5-pro-001\n api_key env.GOOGLE_API_KEY\n safetySettings {\n category HARM_CATEGORY_HATE_SPEECH\n threshold BLOCK_LOW_AND_ABOVE\n\n }\n }\n}\n\nclient Vertex {\n provider vertex-ai \n options {\n model gemini-1.5-pro\n project_id sam-project-vertex-1\n location us-central1\n credentials env.INTEG_TESTS_GOOGLE_APPLICATION_CREDENTIALS\n\n }\n}\n\n\nclient AwsBedrock {\n provider aws-bedrock\n options {\n inference_configuration {\n max_tokens 100\n }\n // model \"anthropic.claude-3-5-sonnet-20240620-v1:0\"\n // model_id \"anthropic.claude-3-haiku-20240307-v1:0\"\n model_id \"meta.llama3-8b-instruct-v1:0\"\n // model_id \"mistral.mistral-7b-instruct-v0:2\"\n api_key \"\"\n }\n}\n\nclient AwsBedrockInvalidRegion {\n provider aws-bedrock\n options {\n region \"us-invalid-7\"\n inference_configuration {\n max_tokens 100\n }\n // model \"anthropic.claude-3-5-sonnet-20240620-v1:0\"\n // model_id \"anthropic.claude-3-haiku-20240307-v1:0\"\n model_id \"meta.llama3-8b-instruct-v1:0\"\n // model_id \"mistral.mistral-7b-instruct-v0:2\"\n api_key \"\"\n }\n}\n\nclient Sonnet {\n provider anthropic\n options {\n model claude-3-5-sonnet-20241022\n api_key env.ANTHROPIC_API_KEY\n }\n}\n\nclient Claude {\n provider anthropic\n options {\n model claude-3-haiku-20240307\n api_key env.ANTHROPIC_API_KEY\n max_tokens 1000\n }\n}\n\nclient ClaudeWithCaching {\n provider anthropic\n options {\n model claude-3-haiku-20240307\n api_key env.ANTHROPIC_API_KEY\n max_tokens 1000\n allowed_role_metadata [\"cache_control\"]\n headers {\n \"anthropic-beta\" \"prompt-caching-2024-07-31\"\n }\n }\n}\n\nclient Resilient_SimpleSyntax {\n retry_policy Foo\n provider baml-fallback\n options {\n strategy [\n GPT4Turbo\n GPT35\n Lottery_SimpleSyntax\n ]\n }\n} \n \nclient Lottery_SimpleSyntax {\n provider baml-round-robin\n options {\n start 0\n strategy [\n GPT35\n Claude\n ]\n }\n}\n\nclient TogetherAi {\n provider \"openai-generic\"\n options {\n base_url \"https://api.together.ai/v1\"\n api_key env.TOGETHER_API_KEY\n model \"meta-llama/Llama-3-70b-chat-hf\"\n }\n}\n\n", + "clients.baml" => "retry_policy Bar {\n max_retries 3\n strategy {\n type exponential_backoff\n }\n}\n\nretry_policy Foo {\n max_retries 3\n strategy {\n type constant_delay\n delay_ms 100\n }\n}\n\nclient GPT4 {\n provider openai\n options {\n model gpt-4o\n api_key env.OPENAI_API_KEY\n }\n} \n\n\nclient GPT4o {\n provider openai\n options {\n model gpt-4o\n api_key env.OPENAI_API_KEY\n }\n} \n\n\nclient GPT4Turbo {\n retry_policy Bar\n provider openai\n options {\n model gpt-4-turbo\n api_key env.OPENAI_API_KEY\n }\n} \n\nclient GPT35 {\n provider openai\n options {\n model \"gpt-3.5-turbo\"\n api_key env.OPENAI_API_KEY\n }\n}\n\nclient GPT35LegacyProvider {\n provider openai\n options {\n model \"gpt-3.5-turbo\"\n api_key env.OPENAI_API_KEY\n }\n}\n\n\nclient Ollama {\n provider ollama\n options {\n model llama2\n }\n}\n\nclient GPT35Azure {\n provider azure-openai\n options {\n resource_name \"west-us-azure-baml\"\n deployment_id \"gpt-35-turbo-default\"\n // base_url \"https://west-us-azure-baml.openai.azure.com/openai/deployments/gpt-35-turbo-default\"\n api_version \"2024-02-01\"\n api_key env.AZURE_OPENAI_API_KEY\n }\n}\n\nclient Gemini {\n provider google-ai\n options {\n model gemini-1.5-pro-001\n api_key env.GOOGLE_API_KEY\n safetySettings {\n category HARM_CATEGORY_HATE_SPEECH\n threshold BLOCK_LOW_AND_ABOVE\n }\n }\n}\n\nclient Vertex {\n provider vertex-ai \n options {\n model gemini-1.5-pro\n location us-central1\n credentials env.INTEG_TESTS_GOOGLE_APPLICATION_CREDENTIALS\n }\n}\n\n\nclient AwsBedrock {\n provider aws-bedrock\n options {\n inference_configuration {\n max_tokens 100\n }\n // model \"anthropic.claude-3-5-sonnet-20240620-v1:0\"\n // model_id \"anthropic.claude-3-haiku-20240307-v1:0\"\n model_id \"meta.llama3-8b-instruct-v1:0\"\n // model_id \"mistral.mistral-7b-instruct-v0:2\"\n }\n}\n\nclient AwsBedrockInvalidRegion {\n provider aws-bedrock\n options {\n region \"us-invalid-7\"\n inference_configuration {\n max_tokens 100\n }\n // model \"anthropic.claude-3-5-sonnet-20240620-v1:0\"\n // model_id \"anthropic.claude-3-haiku-20240307-v1:0\"\n model_id \"meta.llama3-8b-instruct-v1:0\"\n // model_id \"mistral.mistral-7b-instruct-v0:2\"\n }\n}\n\nclient Sonnet {\n provider anthropic\n options {\n model claude-3-5-sonnet-20241022\n api_key env.ANTHROPIC_API_KEY\n }\n}\n\nclient Claude {\n provider anthropic\n options {\n model claude-3-haiku-20240307\n api_key env.ANTHROPIC_API_KEY\n max_tokens 1000\n }\n}\n\nclient ClaudeWithCaching {\n provider anthropic\n options {\n model claude-3-haiku-20240307\n api_key env.ANTHROPIC_API_KEY\n max_tokens 500\n allowed_role_metadata [\"cache_control\"]\n headers {\n \"anthropic-beta\" \"prompt-caching-2024-07-31\"\n }\n }\n}\n\nclient Resilient_SimpleSyntax {\n retry_policy Foo\n provider baml-fallback\n options {\n strategy [\n GPT4Turbo\n GPT35\n Lottery_SimpleSyntax\n ]\n }\n} \n \nclient Lottery_SimpleSyntax {\n provider baml-round-robin\n options {\n start 0\n strategy [\n GPT35\n Claude\n ]\n }\n}\n\nclient TogetherAi {\n provider \"openai-generic\"\n options {\n base_url \"https://api.together.ai/v1\"\n api_key env.TOGETHER_API_KEY\n model \"meta-llama/Llama-3-70b-chat-hf\"\n }\n}\n\n", "custom-task.baml" => "class BookOrder {\n orderId string @description(#\"\n The ID of the book order\n \"#)\n title string @description(#\"\n The title of the ordered book\n \"#)\n quantity int @description(#\"\n The quantity of books ordered\n \"#)\n price float @description(#\"\n The price of the book\n \"#)\n}\n\nclass FlightConfirmation {\n confirmationNumber string @description(#\"\n The flight confirmation number\n \"#)\n flightNumber string @description(#\"\n The flight number\n \"#)\n departureTime string @description(#\"\n The scheduled departure time of the flight\n \"#)\n arrivalTime string @description(#\"\n The scheduled arrival time of the flight\n \"#)\n seatNumber string @description(#\"\n The seat number assigned on the flight\n \"#)\n}\n\nclass GroceryReceipt {\n receiptId string @description(#\"\n The ID of the grocery receipt\n \"#)\n storeName string @description(#\"\n The name of the grocery store\n \"#)\n items (string | int | float)[] @description(#\"\n A list of items purchased. Each item consists of a name, quantity, and price.\n \"#)\n totalAmount float @description(#\"\n The total amount spent on groceries\n \"#)\n}\n\nclass CustomTaskResult {\n bookOrder BookOrder | null\n flightConfirmation FlightConfirmation | null\n groceryReceipt GroceryReceipt | null\n}\n\nfunction CustomTask(input: string) -> BookOrder | FlightConfirmation | GroceryReceipt {\n client \"openai/gpt-4o-mini\"\n prompt #\"\n Given the input string, extract either an order for a book, a flight confirmation, or a grocery receipt.\n\n {{ ctx.output_format }}\n\n Input:\n \n {{ input}}\n \"#\n}\n\ntest CustomTask {\n functions [CustomTask]\n args {\n input #\"\nDear [Your Name],\n\nThank you for booking with [Airline Name]! We are pleased to confirm your upcoming flight.\n\nFlight Confirmation Details:\n\nBooking Reference: ABC123\nPassenger Name: [Your Name]\nFlight Number: XY789\nDeparture Date: September 15, 2024\nDeparture Time: 10:30 AM\nArrival Time: 1:45 PM\nDeparture Airport: John F. Kennedy International Airport (JFK), New York, NY\nArrival Airport: Los Angeles International Airport (LAX), Los Angeles, CA\nSeat Number: 12A\nClass: Economy\nBaggage Allowance:\n\nChecked Baggage: 1 piece, up to 23 kg\nCarry-On Baggage: 1 piece, up to 7 kg\nImportant Information:\n\nPlease arrive at the airport at least 2 hours before your scheduled departure.\nCheck-in online via our website or mobile app to save time at the airport.\nEnsure that your identification documents are up to date and match the name on your booking.\nContact Us:\n\nIf you have any questions or need to make changes to your booking, please contact our customer service team at 1-800-123-4567 or email us at support@[airline].com.\n\nWe wish you a pleasant journey and thank you for choosing [Airline Name].\n\nBest regards,\n\n[Airline Name] Customer Service\n \"#\n }\n}", "fiddle-examples/chain-of-thought.baml" => "class Email {\n subject string\n body string\n from_address string\n}\n\nenum OrderStatus {\n ORDERED\n SHIPPED\n DELIVERED\n CANCELLED\n}\n\nclass OrderInfo {\n order_status OrderStatus\n tracking_number string?\n estimated_arrival_date string?\n}\n\nfunction GetOrderInfo(email: Email) -> OrderInfo {\n client GPT4\n prompt #\"\n Given the email below:\n\n ```\n from: {{email.from_address}}\n Email Subject: {{email.subject}}\n Email Body: {{email.body}}\n ```\n\n Extract this info from the email in JSON format:\n {{ ctx.output_format }}\n\n Before you output the JSON, please explain your\n reasoning step-by-step. Here is an example on how to do this:\n 'If we think step by step we can see that ...\n therefore the output JSON is:\n {\n ... the json schema ...\n }'\n \"#\n}", "fiddle-examples/chat-roles.baml" => "// This will be available as an enum in your Python and Typescript code.\nenum Category2 {\n Refund\n CancelOrder\n TechnicalSupport\n AccountIssue\n Question\n}\n\nfunction ClassifyMessage2(input: string) -> Category {\n client GPT4\n\n prompt #\"\n {{ _.role(\"system\") }}\n // You can use _.role(\"system\") to indicate that this text should be a system message\n\n Classify the following INPUT into ONE\n of the following categories:\n\n {{ ctx.output_format }}\n\n {{ _.role(\"user\") }}\n // And _.role(\"user\") to indicate that this text should be a user message\n\n INPUT: {{ input }}\n\n Response:\n \"#\n}", @@ -86,14 +86,14 @@ module Inlined "test-files/functions/prompts/no-chat-messages.baml" => "\n\nfunction PromptTestClaude(input: string) -> string {\n client Sonnet\n prompt #\"\n Tell me a haiku about {{ input }}\n \"#\n}\n\n\nfunction PromptTestStreaming(input: string) -> string {\n client GPT35\n prompt #\"\n Tell me a short story about {{ input }}\n \"#\n}\n\ntest TestName {\n functions [PromptTestStreaming]\n args {\n input #\"\n hello world\n \"#\n }\n}\n", "test-files/functions/prompts/with-chat-messages.baml" => "\nfunction PromptTestOpenAIChat(input: string) -> string {\n client GPT35\n prompt #\"\n {{ _.role(\"system\") }}\n You are an assistant that always responds in a very excited way with emojis and also outputs this word 4 times after giving a response: {{ input }}\n \n {{ _.role(\"user\") }}\n Tell me a haiku about {{ input }}\n \"#\n}\n\nfunction PromptTestOpenAIChatNoSystem(input: string) -> string {\n client GPT35\n prompt #\"\n You are an assistant that always responds in a very excited way with emojis and also outputs this word 4 times after giving a response: {{ input }}\n \n {{ _.role(\"user\") }}\n Tell me a haiku about {{ input }}\n \"#\n}\n\nfunction PromptTestClaudeChat(input: string) -> string {\n client Claude\n prompt #\"\n {{ _.role(\"system\") }}\n You are an assistant that always responds in a very excited way with emojis and also outputs this word 4 times after giving a response: {{ input }}\n \n {{ _.role(\"user\") }}\n Tell me a haiku about {{ input }}\n \"#\n}\n\nfunction PromptTestClaudeChatNoSystem(input: string) -> string {\n client Claude\n prompt #\"\n You are an assistant that always responds in a very excited way with emojis and also outputs this word 4 times after giving a response: {{ input }}\n \n {{ _.role(\"user\") }}\n Tell me a haiku about {{ input }}\n \"#\n}\n\ntest TestSystemAndNonSystemChat1 {\n functions [PromptTestClaude, PromptTestOpenAI, PromptTestOpenAIChat, PromptTestOpenAIChatNoSystem, PromptTestClaudeChat, PromptTestClaudeChatNoSystem]\n args {\n input \"cats\"\n }\n}\n\ntest TestSystemAndNonSystemChat2 {\n functions [PromptTestClaude, PromptTestOpenAI, PromptTestOpenAIChat, PromptTestOpenAIChatNoSystem, PromptTestClaudeChat, PromptTestClaudeChatNoSystem]\n args {\n input \"lion\"\n }\n}", "test-files/functions/v2/basic.baml" => "\n\nfunction ExtractResume2(resume: string) -> Resume {\n client GPT4\n prompt #\"\n {{ _.role('system') }}\n\n Extract the following information from the resume:\n\n Resume:\n <<<<\n {{ resume }}\n <<<<\n\n Output JSON schema:\n {{ ctx.output_format }}\n\n JSON:\n \"#\n}\n\n\nclass WithReasoning {\n value string\n reasoning string @description(#\"\n Why the value is a good fit.\n \"#)\n}\n\n\nclass SearchParams {\n dateRange int? @description(#\"\n In ISO duration format, e.g. P1Y2M10D.\n \"#)\n location string[]\n jobTitle WithReasoning? @description(#\"\n An exact job title, not a general category.\n \"#)\n company WithReasoning? @description(#\"\n The exact name of the company, not a product or service.\n \"#)\n description WithReasoning[] @description(#\"\n Any specific projects or features the user is looking for.\n \"#)\n tags (Tag | string)[]\n}\n\nenum Tag {\n Security\n AI\n Blockchain\n}\n\nfunction GetQuery(query: string) -> SearchParams {\n client GPT4\n prompt #\"\n Extract the following information from the query:\n\n Query:\n <<<<\n {{ query }}\n <<<<\n\n OUTPUT_JSON_SCHEMA:\n {{ ctx.output_format }}\n\n Before OUTPUT_JSON_SCHEMA, list 5 intentions the user may have.\n --- EXAMPLES ---\n 1. \n 2. \n 3. \n 4. \n 5. \n\n {\n ... // OUTPUT_JSON_SCHEMA\n }\n \"#\n}\n\nclass RaysData {\n dataType DataType\n value Resume | Event\n}\n\nenum DataType {\n Resume\n Event\n}\n\nclass Event {\n title string\n date string\n location string\n description string\n}\n\nfunction GetDataType(text: string) -> RaysData {\n client GPT4\n prompt #\"\n Extract the relevant info.\n\n Text:\n <<<<\n {{ text }}\n <<<<\n\n Output JSON schema:\n {{ ctx.output_format }}\n\n JSON:\n \"#\n}", - "test-files/providers/providers.baml" => "function TestAnthropic(input: string) -> string {\n client Claude\n prompt #\"\n Write a nice haiku about {{ input }}\n \"#\n}\n\nfunction PromptTestOpenAI(input: string) -> string {\n client GPT35\n prompt #\"\n Write a nice haiku about {{ input }}\n \"#\n}\n\nfunction TestOpenAILegacyProvider(input: string) -> string {\n client GPT35LegacyProvider\n prompt #\"\n Write a nice haiku about {{ input }}\n \"#\n}\n\nfunction TestAzure(input: string) -> string {\n client GPT35Azure\n prompt #\"\n Write a nice haiku about {{ input }}\n \"#\n}\n\nfunction TestOllama(input: string) -> string {\n client Ollama\n prompt #\"\n Write a nice haiku about {{ input }}\n \"#\n}\n\nfunction TestGemini(input: string) -> string {\n client Gemini\n prompt #\"\n Write a nice short story about {{ input }}\n \"#\n}\n\nfunction TestVertex(input: string) -> string {\n client Vertex\n prompt #\"\n Write a nice short story about {{ input }}\n \"#\n\n}\n\nfunction TestAws(input: string) -> string {\n client AwsBedrock\n prompt #\"\n Write a nice short story about {{ input }}\n \"#\n}\n\nfunction TestAwsInvalidRegion(input: string) -> string {\n client AwsBedrockInvalidRegion\n prompt #\"\n Write a nice short story about {{ input }}\n \"#\n}\n\nfunction TestOpenAIShorthand(input: string) -> string {\n client \"openai/gpt-4o-mini\"\n prompt #\"\n Write a nice short story about {{ input }}\n \"#\n}\n\nfunction TestAnthropicShorthand(input: string) -> string {\n client \"anthropic/claude-3-haiku-20240307\"\n prompt #\"\n Write a nice short story about {{ input }}\n \"#\n}\n\ntest TestProvider {\n functions [\n TestAnthropic, TestVertex, PromptTestOpenAI, TestAzure, TestOllama, TestGemini, TestAws,\n TestAwsInvalidRegion,\n TestOpenAIShorthand,\n TestAnthropicShorthand\n ]\n args {\n input \"Donkey kong and peanut butter\"\n }\n}\n\n\nfunction TestCaching(input: string, not_cached: string) -> string {\n client ClaudeWithCaching\n prompt #\"\n {{ _.role('system', cache_control={\"type\": \"ephemeral\"}) }}\n Describe this in 5 words: {{ input }}\n {{ _.role('user') }}\n {{ not_cached }}\n \"#\n}\n", + "test-files/providers/providers.baml" => "function TestAnthropic(input: string) -> string {\n client Claude\n prompt #\"\n Write a nice haiku about {{ input }}\n \"#\n}\n\nfunction PromptTestOpenAI(input: string) -> string {\n client GPT35\n prompt #\"\n Write a nice haiku about {{ input }}\n \"#\n}\n\nfunction TestOpenAILegacyProvider(input: string) -> string {\n client GPT35LegacyProvider\n prompt #\"\n Write a nice haiku about {{ input }}\n \"#\n}\n\nfunction TestAzure(input: string) -> string {\n client GPT35Azure\n prompt #\"\n Write a nice haiku about {{ input }}\n \"#\n}\n\nfunction TestOllama(input: string) -> string {\n client Ollama\n prompt #\"\n Write a nice haiku about {{ input }}\n \"#\n}\n\nfunction TestGemini(input: string) -> string {\n client Gemini\n prompt #\"\n Write a nice short story about {{ input }}\n \"#\n}\n\nfunction TestVertex(input: string) -> string {\n client Vertex\n prompt #\"\n Write a nice short story about {{ input }}\n \"#\n\n}\n\nfunction TestAws(input: string) -> string {\n client AwsBedrock\n prompt #\"\n Write a nice short story about {{ input }}\n \"#\n}\n\nfunction TestAwsInvalidRegion(input: string) -> string {\n client AwsBedrockInvalidRegion\n prompt #\"\n Write a nice short story about {{ input }}\n \"#\n}\n\nfunction TestOpenAIShorthand(input: string) -> string {\n client \"openai/gpt-4o-mini\"\n prompt #\"\n Write a nice short story about {{ input }}\n \"#\n}\n\nfunction TestAnthropicShorthand(input: string) -> string {\n client \"anthropic/claude-3-haiku-20240307\"\n prompt #\"\n Write a nice short story about {{ input }}\n \"#\n}\n\ntest TestProvider {\n functions [\n TestAnthropic, TestVertex, PromptTestOpenAI, TestAzure, TestOllama, TestGemini, TestAws,\n TestAwsInvalidRegion,\n TestOpenAIShorthand,\n TestAnthropicShorthand\n ]\n args {\n input \"Donkey kong and peanut butter\"\n }\n}\n\n\nfunction TestCaching(input: string, not_cached: string) -> string {\n client ClaudeWithCaching\n prompt #\"\n {{ _.role('system', cache_control={\"type\": \"ephemeral\"}) }}\n Generate the following story\n {{ input }}\n\n {# Haiku require 2048 tokens to cache -#}\n {{ input }}\n\n {{ _.role('user') }}\n {{ not_cached }}\n \"#\n}\n\ntest TestName {\n functions [TestCaching]\n args {\n input #\"\nIn a near-future society where dreams have become a tradable commodity and shared experience, a lonely and socially awkward teenager named Alex discovers they possess a rare and powerful ability to not only view but also manipulate the dreams of others. Initially thrilled by this newfound power, Alex begins subtly altering the dreams of classmates and family members, helping them overcome fears, boost confidence, or experience fantastical adventures. As Alex's skills grow, so does their influence. They start selling premium dream experiences on the black market, crafting intricate and addictive dreamscapes for wealthy clients. However, the line between dream and reality begins to blur for those exposed to Alex's creations. Some clients struggle to differentiate between their true memories and the artificial ones implanted by Alex's dream manipulation.\n\nComplications arise when a mysterious government agency takes notice of Alex's unique abilities. They offer Alex a chance to use their gift for \"the greater good,\" hinting at applications in therapy, criminal rehabilitation, and even national security. Simultaneously, an underground resistance movement reaches out, warning Alex about the dangers of dream manipulation and the potential for mass control and exploitation. Caught between these opposing forces, Alex must navigate a complex web of ethical dilemmas. They grapple with questions of free will, the nature of consciousness, and the responsibility that comes with having power over people's minds. As the consequences of their actions spiral outward, affecting the lives of loved ones and strangers alike, Alex is forced to confront the true nature of their ability and decide how—or if—it should be used.\n\nThe story explores themes of identity, the subconscious mind, the ethics of technology, and the power of imagination. It delves into the potential consequences of a world where our most private thoughts and experiences are no longer truly our own, and examines the fine line between helping others and manipulating them for personal gain or a perceived greater good. The narrative further expands on the societal implications of such abilities, questioning the moral boundaries of altering consciousness and the potential for abuse in a world where dreams can be commodified. It challenges the reader to consider the impact of technology on personal autonomy and the ethical responsibilities of those who wield such power.\n\nAs Alex's journey unfolds, they encounter various individuals whose lives have been touched by their dream manipulations, each presenting a unique perspective on the ethical quandaries at hand. From a classmate who gains newfound confidence to a wealthy client who becomes addicted to the dreamscapes, the ripple effects of Alex's actions are profound and far-reaching. The government agency's interest in Alex's abilities raises questions about the potential for state control and surveillance, while the resistance movement highlights the dangers of unchecked power and the importance of safeguarding individual freedoms.\n\nUltimately, Alex's story is one of self-discovery and moral reckoning, as they must decide whether to embrace their abilities for personal gain, align with the government's vision of a controlled utopia, or join the resistance in their fight for freedom and autonomy. The narrative invites readers to reflect on the nature of reality, the boundaries of human experience, and the ethical implications of a world where dreams are no longer private sanctuaries but shared and manipulated commodities. It also explores the psychological impact on Alex, who must deal with the burden of knowing the intimate fears and desires of others, and the isolation that comes from being unable to share their own dreams without altering them.\n\nThe story further examines the technological advancements that have made dream manipulation possible, questioning the role of innovation in society and the potential for both progress and peril. It considers the societal divide between those who can afford to buy enhanced dream experiences and those who cannot, highlighting issues of inequality and access. As Alex becomes more entangled in the web of their own making, they must confront the possibility that their actions could lead to unintended consequences, not just for themselves but for the fabric of society as a whole.\n\nIn the end, Alex's journey is a cautionary tale about the power of dreams and the responsibilities that come with wielding such influence. It serves as a reminder of the importance of ethical considerations in the face of technological advancement and the need to balance innovation with humanity. The story leaves readers pondering the true cost of a world where dreams are no longer sacred, and the potential for both wonder and danger in the uncharted territories of the mind. But it's also a story about the power of imagination and the potential for change, even in a world where our deepest thoughts are no longer our own. And it's a story about the power of choice, and the importance of fighting for the freedom to dream.\n\nIn conclusion, this story is a reflection on the power of dreams and the responsibilities that come with wielding such influence. It serves as a reminder of the importance of ethical considerations in the face of technological advancement and the need to balance innovation with humanity. The story leaves readers pondering the true cost of a world where dreams are no longer sacred, and the potential for both wonder and danger in the uncharted territories of the mind. But it's also a story about the power of imagination and the potential for change, even in a world where our deepest thoughts are no longer our own. And it's a story about the power of choice, and the importance of fighting for the freedom to dream.\n \"#\n not_cached #\"\n hello world\n \"#\n }\n}\n", "test-files/strategies/fallback-shorthand.baml" => "\nclient FallbackToShorthand {\n provider fallback\n options {\n strategy [\n \"openai/does-not-exist\",\n \"openai/gpt-4o-mini\"\n ]\n }\n}\n\n\nfunction TestFallbackToShorthand(input: string) -> string {\n client FallbackToShorthand\n // TODO make it return the client name instead\n prompt #\"\n Say a haiku about {{input}}.\n \"#\n}\n\ntest TestProvider_FallbackToShorthand {\n functions [\n TestFallbackToShorthand\n ]\n args {\n input \"Donkey kong and peanut butter\"\n }\n}\n", - "test-files/strategies/fallback.baml" => "// Happy path fallbacks.\nclient FaultyClient {\n provider openai\n options {\n model unknown-model\n api_key env.OPENAI_API_KEY\n }\n}\n\n\nclient FallbackClient {\n provider fallback\n options {\n // first 2 clients are expected to fail.\n strategy [\n FaultyClient,\n RetryClientConstant,\n GPT35\n Gemini\n\n ]\n }\n}\n\nfunction TestFallbackClient() -> string {\n client FallbackClient\n // TODO make it return the client name instead\n prompt #\"\n Say a haiku about mexico.\n \"#\n}\n\n// Fallbacks should fail gracefully.\nclient FaultyAzureClient {\n provider azure-openai\n options {\n model unknown-model\n api_key env.OPENAI_API_KEY\n }\n}\n\nclient SingleFallbackClient {\n provider fallback\n options {\n // first 2 clients are expected to fail.\n strategy [\n FaultyAzureClient\n ]\n }\n}\n\nfunction TestSingleFallbackClient() -> string {\n client SingleFallbackClient\n // TODO make it return the client name instead\n prompt #\"\n Say a haiku about mexico.\n \"#\n}\n", + "test-files/strategies/fallback.baml" => "// Happy path fallbacks.\nclient FaultyClient {\n provider openai\n options {\n model unknown-model\n api_key env.OPENAI_API_KEY\n }\n}\n\n\nclient FallbackClient {\n provider fallback\n options {\n // first 2 clients are expected to fail.\n strategy [\n FaultyClient,\n RetryClientConstant,\n GPT35\n Gemini\n\n ]\n }\n}\n\nfunction TestFallbackClient() -> string {\n client FallbackClient\n // TODO make it return the client name instead\n prompt #\"\n Say a haiku about mexico.\n \"#\n}\n\n// Fallbacks should fail gracefully.\nclient FaultyAzureClient {\n provider azure-openai\n options {\n model unknown-model\n resource_name \"unknown-resource-id\"\n deployment_id \"unknown-deployment-id\"\n }\n}\n\nclient SingleFallbackClient {\n provider fallback\n options {\n // first 2 clients are expected to fail.\n strategy [\n FaultyAzureClient\n ]\n }\n}\n\nfunction TestSingleFallbackClient() -> string {\n client SingleFallbackClient\n // TODO make it return the client name instead\n prompt #\"\n Say a haiku about mexico.\n \"#\n}\n", "test-files/strategies/retry.baml" => "\nretry_policy Exponential {\n max_retries 3\n strategy {\n type exponential_backoff\n }\n}\n\nretry_policy Constant {\n max_retries 3\n strategy {\n type constant_delay\n delay_ms 100\n }\n}\n\nclient RetryClientConstant {\n provider openai\n retry_policy Constant\n options {\n model \"gpt-3.5-turbo\"\n api_key \"blah\"\n }\n}\n\nclient RetryClientExponential {\n provider openai\n retry_policy Exponential\n options {\n model \"gpt-3.5-turbo\"\n api_key \"blahh\"\n }\n}\n\nfunction TestRetryConstant() -> string {\n client RetryClientConstant\n prompt #\"\n Say a haiku\n \"#\n}\n\nfunction TestRetryExponential() -> string {\n client RetryClientExponential\n prompt #\"\n Say a haiku\n \"#\n}\n", "test-files/strategies/roundrobin.baml" => "", "test-files/template_string/template_string.baml" => "\nfunction Completion(prefix: string, suffix: string, language: string) -> string {\n client \"openai/gpt-4o\"\n prompt ##\"\n {{ _.role(\"system\", cache_control={\"type\": \"ephemeral\"}) }}\n\n You are a programmer that suggests code completions in the %INSERT-HERE% part below with {{ language }} code. Only output the code that replaces %INSERT-HERE% part, NOT THE SUFFIX OR PREFIX. Respond only with code, and with no markdown formatting.\n\n Try to complete a whole section inside curlies when you can.\n\n {% if language == \"baml\" %}\n {{ BAMLBackground2()}}\n\n Examples:\n INPUT:\n ---\n class MyObject {{\"{\"}}%INSERT-HERE%\n }\n ---\n OUTPUT:\n ---\n property string\n ---\n In this example, we just inserted one line, with tabs for a fake property to aid the user.\n\n INPUT:\n ---\n function foo(input: string) -> string {{\"{\"}} %INSERT-HERE%\n prompt #\"\n {{ \"{{ input }}\" }}\n \"#\n }\n ---\n OUTPUT:\n ---\n client \"openai/gpt-4o\"\n ---\n In this example, no need to add the prompt because it was part of the suffix after INSERT-HERE\n\n INPUT:\n OUTPUT: N/A\n In this example there was nothing to complete, so we returned N/A.\n\n Ignore the \"---\" in your outputs.\n {% endif %}\n\n\n {{ _.role(\"user\") }}\n INPUT:\n ---\n {{ prefix }}%INSERT-HERE%{{ suffix }}\n ---\n \"##\n}\n\ntest CompletionTest3 {\n functions [Completion]\n args {\n prefix ##\"function foo(input: string) -> string {\n client \"openai/gpt-4o\"\n prompt #\"\n \"##\n suffix \"\"\n language \"baml\"\n }\n}\n\ntest CompletionTest2 {\n functions [Completion]\n args {\n prefix \"function foo(input: string) -> string {\\n\"\n suffix \"\\n prompt #\\n\\\"\"\n language \"baml\"\n }\n}\n \ntemplate_string Hi(\n hello: string,\n world: string,\n) ##\"\n {{ hello }} {{ world }}\n\"##\n\ntemplate_string Hi3(\n hello: string,\n world: string,\n) #\"\n {{ hello }} {{ world }}\n\"#\n\ntemplate_string BAMLBackground2() ##\"\n \n BAML is a domain-specific language for building LLM prompts as functions.\n client \"openai/gpt-4o\"\n // prompt with jinja syntax inside here. with double curly braces for variables.\n // make sure to include: {{ \"{{ ctx.output_format }}\"}} in the prompt, which prints the output schema instructions so the LLM returns the output in the correct format (json or string, etc.). DO NOT write the output schema manually.\n prompt #\"\n \n \"#\n }\n\n 3. You do not need to specify to \"answer in JSON format\". Only write in the prompt brief instruction, and any other task-specific things to keep in mind for the task.\n 4. Write a {{ \"{{ _.role(\\\"user\\\") }}\" }} tag to indicate where the user's inputs start. So if there's a convo you can write\n #\"{{ \"{{ _.role(\\\"user\\\") }}\" }} {{ \"{{ some-variable }}\" }}#\n \n \n\n The @asserts only go in the \"output\" types. Don't use them in inputs.\n Do NOT use numbers as confidence intervals if you need to use them. Prefer an enum with descriptions or literals like \"high\", \"medium\", \"low\".\n\n Dedent all declarations.\n\"##\n\ntemplate_string BamlTests() ##\"\n // For image inputs:\n test ImageTest {\n functions [MyFunction]\n args {\n imageArg {\n file \"../images/test.png\"\n // Optional: media_type \"image/png\"\n }\n // Or using URL:\n // imageArg {\n // url \"https://example.com/image.png\"\n // }\n }\n }\n\n // For array/object inputs:\n test ComplexTest {\n functions [MyFunction]\n args {\n input {\n name \"Complex Object\"\n tags [\n \"tag1\",\n #\"\n Multi-line\n tag here\n \"#\n ]\n status PENDING\n type \"error\"\n count 100\n enabled false\n score 7.8\n }\n }\n }\n\"##\n", "test-files/testing_pipeline/output-format.baml" => "class Recipe {\n ingredients map\n recipe_type \"breakfast\" | \"dinner\"\n}\n\nclass Quantity {\n amount int | float\n unit string?\n}\n\nfunction AaaSamOutputFormat(recipe: string) -> Recipe {\n client GPT35\n prompt #\"\n Return this value back to me: {{recipe}}\n\n {{ctx.output_format(map_style='angle')}}\n \"#\n}\n\ntest MyOutput {\n functions [AaaSamOutputFormat]\n args {\n recipe #\"\n Here's a simple recipe for beef stew:\nIngredients:\n\n2 lbs beef chuck, cut into 1-inch cubes\n2 tbsp vegetable oil\n1 onion, diced\n3 carrots, sliced\n2 celery stalks, chopped\n2 potatoes, cubed\n3 cloves garlic, minced\n4 cups beef broth\n1 can (14.5 oz) diced tomatoes\n1 tbsp Worcestershire sauce\n1 tsp dried thyme\n1 bay leaf\nSalt and pepper to taste\n\nInstructions:\n\nSeason beef with salt and pepper. Heat oil in a large pot over medium-high heat. Brown the beef in batches, then set aside.\nIn the same pot, sauté onion, carrots, and celery until softened, about 5 minutes.\nAdd garlic and cook for another minute.\nReturn beef to the pot. Add broth, tomatoes, Worcestershire sauce, thyme, and bay leaf.\nBring to a boil, then reduce heat and simmer covered for 1 hour.\nAdd potatoes and continue simmering for 30-45 minutes, until beef and potatoes are tender.\nRemove bay leaf, adjust seasoning if needed, and serve hot.\n\nWould you like any additional information or variations on this recipe?\n \"#\n }\n}\n", - "test-files/testing_pipeline/resume.baml" => "class Resume {\n name string\n email string\n phone string\n experience Education[]\n education string[]\n skills string[]\n}\n\nclass Education {\n institution string\n location string\n degree string\n major string[]\n graduation_date string?\n}\n\ntemplate_string AddRole(foo: string) #\"\n {{ _.role('system')}}\n You are a {{ foo }}. be nice\n\n {{ _.role('user') }}\n\"#\n\nclient TestClient {\n provider fallback\n retry_policy Constant\n options {\n strategy [\n Claude\n GPT35\n AwsBedrock\n ]\n }\n}\n\nclient Claude2 {\n provider anthropic\n options {\n model claude-3-haiku-20240307\n api_key env.FOOBAR3\n max_tokens 1000\n }\n}\n\nfunction ExtractResume(resume: string, img: image?) -> Resume {\n client Claude2\n prompt #\"\n {{ AddRole(\"Software Engineer\") }}\n\n Extract data:\n \n\n <<<<\n {{ resume }}\n <<<<\n\n {% if img %}\n {{img}}\n {% endif %}\n\n {{ ctx.output_format }}\n \"#\n}\n\ntest sam_resume {\n functions [ExtractResume]\n input {\n img {\n url \"https://avatars.githubusercontent.com/u/1016595?v=4\"\n }\n resume #\"\n Sam Lijin\n he/him | jobs@sxlijin.com | sxlijin.github.io | 111-222-3333 | sxlijin | sxlijin\n\n Experience\n Trunk\n | July 2021 - current\n Trunk Check | Senior Software Engineer | Services TL, Mar 2023 - current | IC, July 2021 - Feb 2023\n Proposed, designed, and led a team of 3 to build a web experience for Check (both a web-only onboarding flow and SaaS offerings)\n Proposed and built vulnerability scanning into Check, enabling it to compete with security products such as Snyk\n Helped grow Check from <1K users to 90K+ users by focusing on product-led growth\n Google | Sept 2017 - June 2021\n User Identity SRE | Senior Software Engineer | IC, Mar 2021 - June 2021\n Designed an incremental key rotation system to limit the global outage risk to Google SSO\n Discovered and severed an undocumented Gmail serving dependency on Identity-internal systems\n Cloud Firestore | Senior Software Engineer | EngProd TL, Aug 2019 - Feb 2021 | IC, Sept 2017 - July 2019\n Metadata TTL system: backlog of XX trillion records, sustained 1M ops/sec, peaking at 3M ops/sec\n\n Designed and implemented a logging system with novel observability and privacy requirements\n Designed and implemented Jepsen-style testing to validate correctness guarantees\n Datastore Migration: zero downtime, xM RPS and xxPB of data over xM customers and 36 datacenters\n\n Designed composite index migration, queue processing migration, progressive rollout, fast rollback, and disk stockout mitigations; implemented transaction log replay, state transitions, and dark launch process\n Designed and implemented end-to-end correctness and performance testing\n Velocity improvements for 60-eng org\n\n Proposed and implemented automated rollbacks: got us out of a 3-month release freeze and prevented 5 outages over the next 6 months\n Proposed and implemented new development and release environments spanning 30+ microservices\n Incident response for API proxy rollback affecting every Google Cloud service\n\n Google App Engine Memcache | Software Engineer | EngProd TL, Apr 2019 - July 2019\n Proposed and led execution of test coverage improvement strategy for a new control plane: reduced rollbacks and ensured strong consistency of a distributed cache serving xxM QPS\n Designed and implemented automated performance regression testing for two critical serving paths\n Used to validate Google-wide rollout of AMD CPUs, by proving a 50p latency delta of <10µs\n Implemented on shared Borg (i.e. vulnerable to noisy neighbors) with <12% variance\n Miscellaneous | Sept 2017 - June 2021\n Redesigned the Noogler training on Google-internal storage technologies & trained 2500+ Nooglers\n Landed multiple google3-wide refactorings, each spanning xxK files (e.g. SWIG to CLIF)\n Education\n Vanderbilt University (Nashville, TN) | May 2017 | B.S. in Computer Science, Mathematics, and Political Science\n\n Stuyvesant HS (New York, NY) | 2013\n\n Skills\n C++, Java, Typescript, Javascript, Python, Bash; light experience with Rust, Golang, Scheme\n gRPC, Bazel, React, Linux\n Hobbies: climbing, skiing, photography\n \"#\n }\n}\n\ntest vaibhav_resume {\n functions [ExtractResume]\n input {\n resume #\"\n Vaibhav Gupta\n linkedin/vaigup\n (972) 400-5279\n vaibhavtheory@gmail.com\n EXPERIENCE\n Google,\n Software Engineer\n Dec 2018-Present\n Seattle, WA\n •\n Augmented Reality,\n Depth Team\n •\n Technical Lead for on-device optimizations\n •\n Optimized and designed front\n facing depth algorithm\n on Pixel 4\n •\n Focus: C++ and SIMD on custom silicon\n \n \n EDUCATION\n University of Texas at Austin\n Aug 2012-May 2015\n Bachelors of Engineering, Integrated Circuits\n Bachelors of Computer Science\n \"#\n }\n}", + "test-files/testing_pipeline/resume.baml" => "class Resume {\n name string\n email string\n phone string\n experience Education[]\n education string[]\n skills string[]\n}\n\nclass Education {\n institution string\n location string\n degree string\n major string[]\n graduation_date string?\n}\n\ntemplate_string AddRole(foo: string) #\"\n {{ _.role('system')}}\n You are a {{ foo }}. be nice\n\n {{ _.role('user') }}\n\"#\n\nclient TestClient {\n provider fallback\n retry_policy Constant\n options {\n strategy [\n Claude\n GPT35\n AwsBedrock\n ]\n }\n}\n\nclient Claude2 {\n provider anthropic\n options {\n model claude-3-haiku-20240307\n api_key env.FOOBAR3\n max_tokens 1000\n }\n}\n\nfunction ExtractResume(resume: string, img: image?) -> Resume {\n client Claude2\n prompt #\"\n {{ AddRole(\"Software Engineer\") }}\n\n Extract data:\n \n\n <<<<\n {{ resume }}\n <<<<\n\n {% if img %}\n {{img}}\n {% endif %}\n\n {{ ctx.output_format }}\n \"#\n}\n\ntest sam_resume {\n functions [ExtractResume]\n args {\n img {\n url \"https://avatars.githubusercontent.com/u/1016595?v=4\"\n }\n resume #\"\n Sam Lijin\n he/him | jobs@sxlijin.com | sxlijin.github.io | 111-222-3333 | sxlijin | sxlijin\n\n Experience\n Trunk\n | July 2021 - current\n Trunk Check | Senior Software Engineer | Services TL, Mar 2023 - current | IC, July 2021 - Feb 2023\n Proposed, designed, and led a team of 3 to build a web experience for Check (both a web-only onboarding flow and SaaS offerings)\n Proposed and built vulnerability scanning into Check, enabling it to compete with security products such as Snyk\n Helped grow Check from <1K users to 90K+ users by focusing on product-led growth\n Google | Sept 2017 - June 2021\n User Identity SRE | Senior Software Engineer | IC, Mar 2021 - June 2021\n Designed an incremental key rotation system to limit the global outage risk to Google SSO\n Discovered and severed an undocumented Gmail serving dependency on Identity-internal systems\n Cloud Firestore | Senior Software Engineer | EngProd TL, Aug 2019 - Feb 2021 | IC, Sept 2017 - July 2019\n Metadata TTL system: backlog of XX trillion records, sustained 1M ops/sec, peaking at 3M ops/sec\n\n Designed and implemented a logging system with novel observability and privacy requirements\n Designed and implemented Jepsen-style testing to validate correctness guarantees\n Datastore Migration: zero downtime, xM RPS and xxPB of data over xM customers and 36 datacenters\n\n Designed composite index migration, queue processing migration, progressive rollout, fast rollback, and disk stockout mitigations; implemented transaction log replay, state transitions, and dark launch process\n Designed and implemented end-to-end correctness and performance testing\n Velocity improvements for 60-eng org\n\n Proposed and implemented automated rollbacks: got us out of a 3-month release freeze and prevented 5 outages over the next 6 months\n Proposed and implemented new development and release environments spanning 30+ microservices\n Incident response for API proxy rollback affecting every Google Cloud service\n\n Google App Engine Memcache | Software Engineer | EngProd TL, Apr 2019 - July 2019\n Proposed and led execution of test coverage improvement strategy for a new control plane: reduced rollbacks and ensured strong consistency of a distributed cache serving xxM QPS\n Designed and implemented automated performance regression testing for two critical serving paths\n Used to validate Google-wide rollout of AMD CPUs, by proving a 50p latency delta of <10µs\n Implemented on shared Borg (i.e. vulnerable to noisy neighbors) with <12% variance\n Miscellaneous | Sept 2017 - June 2021\n Redesigned the Noogler training on Google-internal storage technologies & trained 2500+ Nooglers\n Landed multiple google3-wide refactorings, each spanning xxK files (e.g. SWIG to CLIF)\n Education\n Vanderbilt University (Nashville, TN) | May 2017 | B.S. in Computer Science, Mathematics, and Political Science\n\n Stuyvesant HS (New York, NY) | 2013\n\n Skills\n C++, Java, Typescript, Javascript, Python, Bash; light experience with Rust, Golang, Scheme\n gRPC, Bazel, React, Linux\n Hobbies: climbing, skiing, photography\n \"#\n }\n}\n\ntest vaibhav_resume {\n functions [ExtractResume]\n args {\n resume #\"\n Vaibhav Gupta\n linkedin/vaigup\n (972) 400-5279\n vaibhavtheory@gmail.com\n EXPERIENCE\n Google,\n Software Engineer\n Dec 2018-Present\n Seattle, WA\n •\n Augmented Reality,\n Depth Team\n •\n Technical Lead for on-device optimizations\n •\n Optimized and designed front\n facing depth algorithm\n on Pixel 4\n •\n Focus: C++ and SIMD on custom silicon\n \n \n EDUCATION\n University of Texas at Austin\n Aug 2012-May 2015\n Bachelors of Engineering, Integrated Circuits\n Bachelors of Computer Science\n \"#\n }\n}", } end end \ No newline at end of file diff --git a/integ-tests/typescript/baml_client/inlinedbaml.ts b/integ-tests/typescript/baml_client/inlinedbaml.ts index 57db73d28..6cf961ac7 100644 --- a/integ-tests/typescript/baml_client/inlinedbaml.ts +++ b/integ-tests/typescript/baml_client/inlinedbaml.ts @@ -17,7 +17,7 @@ $ pnpm add @boundaryml/baml // biome-ignore format: autogenerated code const fileMap = { - "clients.baml": "retry_policy Bar {\n max_retries 3\n strategy {\n type exponential_backoff\n }\n}\n\nretry_policy Foo {\n max_retries 3\n strategy {\n type constant_delay\n delay_ms 100\n }\n}\n\nclient GPT4 {\n provider openai\n options {\n model gpt-4o\n api_key env.OPENAI_API_KEY\n }\n} \n\n\nclient GPT4o {\n provider openai\n options {\n model gpt-4o\n api_key env.OPENAI_API_KEY\n }\n} \n\n\nclient GPT4Turbo {\n retry_policy Bar\n provider openai\n options {\n model gpt-4-turbo\n api_key env.OPENAI_API_KEY\n }\n} \n\nclient GPT35 {\n provider openai\n options {\n model \"gpt-3.5-turbo\"\n api_key env.OPENAI_API_KEY\n }\n}\n\nclient GPT35LegacyProvider {\n provider openai\n options {\n model \"gpt-3.5-turbo\"\n api_key env.OPENAI_API_KEY\n }\n}\n\n\nclient Ollama {\n provider ollama\n options {\n model llama2\n }\n}\n\nclient GPT35Azure {\n provider azure-openai\n options {\n resource_name \"west-us-azure-baml\"\n deployment_id \"gpt-35-turbo-default\"\n // base_url \"https://west-us-azure-baml.openai.azure.com/openai/deployments/gpt-35-turbo-default\"\n api_version \"2024-02-01\"\n api_key env.AZURE_OPENAI_API_KEY\n }\n}\n\nclient Gemini {\n provider google-ai\n options {\n model gemini-1.5-pro-001\n api_key env.GOOGLE_API_KEY\n safetySettings {\n category HARM_CATEGORY_HATE_SPEECH\n threshold BLOCK_LOW_AND_ABOVE\n\n }\n }\n}\n\nclient Vertex {\n provider vertex-ai \n options {\n model gemini-1.5-pro\n project_id sam-project-vertex-1\n location us-central1\n credentials env.INTEG_TESTS_GOOGLE_APPLICATION_CREDENTIALS\n\n }\n}\n\n\nclient AwsBedrock {\n provider aws-bedrock\n options {\n inference_configuration {\n max_tokens 100\n }\n // model \"anthropic.claude-3-5-sonnet-20240620-v1:0\"\n // model_id \"anthropic.claude-3-haiku-20240307-v1:0\"\n model_id \"meta.llama3-8b-instruct-v1:0\"\n // model_id \"mistral.mistral-7b-instruct-v0:2\"\n api_key \"\"\n }\n}\n\nclient AwsBedrockInvalidRegion {\n provider aws-bedrock\n options {\n region \"us-invalid-7\"\n inference_configuration {\n max_tokens 100\n }\n // model \"anthropic.claude-3-5-sonnet-20240620-v1:0\"\n // model_id \"anthropic.claude-3-haiku-20240307-v1:0\"\n model_id \"meta.llama3-8b-instruct-v1:0\"\n // model_id \"mistral.mistral-7b-instruct-v0:2\"\n api_key \"\"\n }\n}\n\nclient Sonnet {\n provider anthropic\n options {\n model claude-3-5-sonnet-20241022\n api_key env.ANTHROPIC_API_KEY\n }\n}\n\nclient Claude {\n provider anthropic\n options {\n model claude-3-haiku-20240307\n api_key env.ANTHROPIC_API_KEY\n max_tokens 1000\n }\n}\n\nclient ClaudeWithCaching {\n provider anthropic\n options {\n model claude-3-haiku-20240307\n api_key env.ANTHROPIC_API_KEY\n max_tokens 1000\n allowed_role_metadata [\"cache_control\"]\n headers {\n \"anthropic-beta\" \"prompt-caching-2024-07-31\"\n }\n }\n}\n\nclient Resilient_SimpleSyntax {\n retry_policy Foo\n provider baml-fallback\n options {\n strategy [\n GPT4Turbo\n GPT35\n Lottery_SimpleSyntax\n ]\n }\n} \n \nclient Lottery_SimpleSyntax {\n provider baml-round-robin\n options {\n start 0\n strategy [\n GPT35\n Claude\n ]\n }\n}\n\nclient TogetherAi {\n provider \"openai-generic\"\n options {\n base_url \"https://api.together.ai/v1\"\n api_key env.TOGETHER_API_KEY\n model \"meta-llama/Llama-3-70b-chat-hf\"\n }\n}\n\n", + "clients.baml": "retry_policy Bar {\n max_retries 3\n strategy {\n type exponential_backoff\n }\n}\n\nretry_policy Foo {\n max_retries 3\n strategy {\n type constant_delay\n delay_ms 100\n }\n}\n\nclient GPT4 {\n provider openai\n options {\n model gpt-4o\n api_key env.OPENAI_API_KEY\n }\n} \n\n\nclient GPT4o {\n provider openai\n options {\n model gpt-4o\n api_key env.OPENAI_API_KEY\n }\n} \n\n\nclient GPT4Turbo {\n retry_policy Bar\n provider openai\n options {\n model gpt-4-turbo\n api_key env.OPENAI_API_KEY\n }\n} \n\nclient GPT35 {\n provider openai\n options {\n model \"gpt-3.5-turbo\"\n api_key env.OPENAI_API_KEY\n }\n}\n\nclient GPT35LegacyProvider {\n provider openai\n options {\n model \"gpt-3.5-turbo\"\n api_key env.OPENAI_API_KEY\n }\n}\n\n\nclient Ollama {\n provider ollama\n options {\n model llama2\n }\n}\n\nclient GPT35Azure {\n provider azure-openai\n options {\n resource_name \"west-us-azure-baml\"\n deployment_id \"gpt-35-turbo-default\"\n // base_url \"https://west-us-azure-baml.openai.azure.com/openai/deployments/gpt-35-turbo-default\"\n api_version \"2024-02-01\"\n api_key env.AZURE_OPENAI_API_KEY\n }\n}\n\nclient Gemini {\n provider google-ai\n options {\n model gemini-1.5-pro-001\n api_key env.GOOGLE_API_KEY\n safetySettings {\n category HARM_CATEGORY_HATE_SPEECH\n threshold BLOCK_LOW_AND_ABOVE\n }\n }\n}\n\nclient Vertex {\n provider vertex-ai \n options {\n model gemini-1.5-pro\n location us-central1\n credentials env.INTEG_TESTS_GOOGLE_APPLICATION_CREDENTIALS\n }\n}\n\n\nclient AwsBedrock {\n provider aws-bedrock\n options {\n inference_configuration {\n max_tokens 100\n }\n // model \"anthropic.claude-3-5-sonnet-20240620-v1:0\"\n // model_id \"anthropic.claude-3-haiku-20240307-v1:0\"\n model_id \"meta.llama3-8b-instruct-v1:0\"\n // model_id \"mistral.mistral-7b-instruct-v0:2\"\n }\n}\n\nclient AwsBedrockInvalidRegion {\n provider aws-bedrock\n options {\n region \"us-invalid-7\"\n inference_configuration {\n max_tokens 100\n }\n // model \"anthropic.claude-3-5-sonnet-20240620-v1:0\"\n // model_id \"anthropic.claude-3-haiku-20240307-v1:0\"\n model_id \"meta.llama3-8b-instruct-v1:0\"\n // model_id \"mistral.mistral-7b-instruct-v0:2\"\n }\n}\n\nclient Sonnet {\n provider anthropic\n options {\n model claude-3-5-sonnet-20241022\n api_key env.ANTHROPIC_API_KEY\n }\n}\n\nclient Claude {\n provider anthropic\n options {\n model claude-3-haiku-20240307\n api_key env.ANTHROPIC_API_KEY\n max_tokens 1000\n }\n}\n\nclient ClaudeWithCaching {\n provider anthropic\n options {\n model claude-3-haiku-20240307\n api_key env.ANTHROPIC_API_KEY\n max_tokens 500\n allowed_role_metadata [\"cache_control\"]\n headers {\n \"anthropic-beta\" \"prompt-caching-2024-07-31\"\n }\n }\n}\n\nclient Resilient_SimpleSyntax {\n retry_policy Foo\n provider baml-fallback\n options {\n strategy [\n GPT4Turbo\n GPT35\n Lottery_SimpleSyntax\n ]\n }\n} \n \nclient Lottery_SimpleSyntax {\n provider baml-round-robin\n options {\n start 0\n strategy [\n GPT35\n Claude\n ]\n }\n}\n\nclient TogetherAi {\n provider \"openai-generic\"\n options {\n base_url \"https://api.together.ai/v1\"\n api_key env.TOGETHER_API_KEY\n model \"meta-llama/Llama-3-70b-chat-hf\"\n }\n}\n\n", "custom-task.baml": "class BookOrder {\n orderId string @description(#\"\n The ID of the book order\n \"#)\n title string @description(#\"\n The title of the ordered book\n \"#)\n quantity int @description(#\"\n The quantity of books ordered\n \"#)\n price float @description(#\"\n The price of the book\n \"#)\n}\n\nclass FlightConfirmation {\n confirmationNumber string @description(#\"\n The flight confirmation number\n \"#)\n flightNumber string @description(#\"\n The flight number\n \"#)\n departureTime string @description(#\"\n The scheduled departure time of the flight\n \"#)\n arrivalTime string @description(#\"\n The scheduled arrival time of the flight\n \"#)\n seatNumber string @description(#\"\n The seat number assigned on the flight\n \"#)\n}\n\nclass GroceryReceipt {\n receiptId string @description(#\"\n The ID of the grocery receipt\n \"#)\n storeName string @description(#\"\n The name of the grocery store\n \"#)\n items (string | int | float)[] @description(#\"\n A list of items purchased. Each item consists of a name, quantity, and price.\n \"#)\n totalAmount float @description(#\"\n The total amount spent on groceries\n \"#)\n}\n\nclass CustomTaskResult {\n bookOrder BookOrder | null\n flightConfirmation FlightConfirmation | null\n groceryReceipt GroceryReceipt | null\n}\n\nfunction CustomTask(input: string) -> BookOrder | FlightConfirmation | GroceryReceipt {\n client \"openai/gpt-4o-mini\"\n prompt #\"\n Given the input string, extract either an order for a book, a flight confirmation, or a grocery receipt.\n\n {{ ctx.output_format }}\n\n Input:\n \n {{ input}}\n \"#\n}\n\ntest CustomTask {\n functions [CustomTask]\n args {\n input #\"\nDear [Your Name],\n\nThank you for booking with [Airline Name]! We are pleased to confirm your upcoming flight.\n\nFlight Confirmation Details:\n\nBooking Reference: ABC123\nPassenger Name: [Your Name]\nFlight Number: XY789\nDeparture Date: September 15, 2024\nDeparture Time: 10:30 AM\nArrival Time: 1:45 PM\nDeparture Airport: John F. Kennedy International Airport (JFK), New York, NY\nArrival Airport: Los Angeles International Airport (LAX), Los Angeles, CA\nSeat Number: 12A\nClass: Economy\nBaggage Allowance:\n\nChecked Baggage: 1 piece, up to 23 kg\nCarry-On Baggage: 1 piece, up to 7 kg\nImportant Information:\n\nPlease arrive at the airport at least 2 hours before your scheduled departure.\nCheck-in online via our website or mobile app to save time at the airport.\nEnsure that your identification documents are up to date and match the name on your booking.\nContact Us:\n\nIf you have any questions or need to make changes to your booking, please contact our customer service team at 1-800-123-4567 or email us at support@[airline].com.\n\nWe wish you a pleasant journey and thank you for choosing [Airline Name].\n\nBest regards,\n\n[Airline Name] Customer Service\n \"#\n }\n}", "fiddle-examples/chain-of-thought.baml": "class Email {\n subject string\n body string\n from_address string\n}\n\nenum OrderStatus {\n ORDERED\n SHIPPED\n DELIVERED\n CANCELLED\n}\n\nclass OrderInfo {\n order_status OrderStatus\n tracking_number string?\n estimated_arrival_date string?\n}\n\nfunction GetOrderInfo(email: Email) -> OrderInfo {\n client GPT4\n prompt #\"\n Given the email below:\n\n ```\n from: {{email.from_address}}\n Email Subject: {{email.subject}}\n Email Body: {{email.body}}\n ```\n\n Extract this info from the email in JSON format:\n {{ ctx.output_format }}\n\n Before you output the JSON, please explain your\n reasoning step-by-step. Here is an example on how to do this:\n 'If we think step by step we can see that ...\n therefore the output JSON is:\n {\n ... the json schema ...\n }'\n \"#\n}", "fiddle-examples/chat-roles.baml": "// This will be available as an enum in your Python and Typescript code.\nenum Category2 {\n Refund\n CancelOrder\n TechnicalSupport\n AccountIssue\n Question\n}\n\nfunction ClassifyMessage2(input: string) -> Category {\n client GPT4\n\n prompt #\"\n {{ _.role(\"system\") }}\n // You can use _.role(\"system\") to indicate that this text should be a system message\n\n Classify the following INPUT into ONE\n of the following categories:\n\n {{ ctx.output_format }}\n\n {{ _.role(\"user\") }}\n // And _.role(\"user\") to indicate that this text should be a user message\n\n INPUT: {{ input }}\n\n Response:\n \"#\n}", @@ -87,14 +87,14 @@ const fileMap = { "test-files/functions/prompts/no-chat-messages.baml": "\n\nfunction PromptTestClaude(input: string) -> string {\n client Sonnet\n prompt #\"\n Tell me a haiku about {{ input }}\n \"#\n}\n\n\nfunction PromptTestStreaming(input: string) -> string {\n client GPT35\n prompt #\"\n Tell me a short story about {{ input }}\n \"#\n}\n\ntest TestName {\n functions [PromptTestStreaming]\n args {\n input #\"\n hello world\n \"#\n }\n}\n", "test-files/functions/prompts/with-chat-messages.baml": "\nfunction PromptTestOpenAIChat(input: string) -> string {\n client GPT35\n prompt #\"\n {{ _.role(\"system\") }}\n You are an assistant that always responds in a very excited way with emojis and also outputs this word 4 times after giving a response: {{ input }}\n \n {{ _.role(\"user\") }}\n Tell me a haiku about {{ input }}\n \"#\n}\n\nfunction PromptTestOpenAIChatNoSystem(input: string) -> string {\n client GPT35\n prompt #\"\n You are an assistant that always responds in a very excited way with emojis and also outputs this word 4 times after giving a response: {{ input }}\n \n {{ _.role(\"user\") }}\n Tell me a haiku about {{ input }}\n \"#\n}\n\nfunction PromptTestClaudeChat(input: string) -> string {\n client Claude\n prompt #\"\n {{ _.role(\"system\") }}\n You are an assistant that always responds in a very excited way with emojis and also outputs this word 4 times after giving a response: {{ input }}\n \n {{ _.role(\"user\") }}\n Tell me a haiku about {{ input }}\n \"#\n}\n\nfunction PromptTestClaudeChatNoSystem(input: string) -> string {\n client Claude\n prompt #\"\n You are an assistant that always responds in a very excited way with emojis and also outputs this word 4 times after giving a response: {{ input }}\n \n {{ _.role(\"user\") }}\n Tell me a haiku about {{ input }}\n \"#\n}\n\ntest TestSystemAndNonSystemChat1 {\n functions [PromptTestClaude, PromptTestOpenAI, PromptTestOpenAIChat, PromptTestOpenAIChatNoSystem, PromptTestClaudeChat, PromptTestClaudeChatNoSystem]\n args {\n input \"cats\"\n }\n}\n\ntest TestSystemAndNonSystemChat2 {\n functions [PromptTestClaude, PromptTestOpenAI, PromptTestOpenAIChat, PromptTestOpenAIChatNoSystem, PromptTestClaudeChat, PromptTestClaudeChatNoSystem]\n args {\n input \"lion\"\n }\n}", "test-files/functions/v2/basic.baml": "\n\nfunction ExtractResume2(resume: string) -> Resume {\n client GPT4\n prompt #\"\n {{ _.role('system') }}\n\n Extract the following information from the resume:\n\n Resume:\n <<<<\n {{ resume }}\n <<<<\n\n Output JSON schema:\n {{ ctx.output_format }}\n\n JSON:\n \"#\n}\n\n\nclass WithReasoning {\n value string\n reasoning string @description(#\"\n Why the value is a good fit.\n \"#)\n}\n\n\nclass SearchParams {\n dateRange int? @description(#\"\n In ISO duration format, e.g. P1Y2M10D.\n \"#)\n location string[]\n jobTitle WithReasoning? @description(#\"\n An exact job title, not a general category.\n \"#)\n company WithReasoning? @description(#\"\n The exact name of the company, not a product or service.\n \"#)\n description WithReasoning[] @description(#\"\n Any specific projects or features the user is looking for.\n \"#)\n tags (Tag | string)[]\n}\n\nenum Tag {\n Security\n AI\n Blockchain\n}\n\nfunction GetQuery(query: string) -> SearchParams {\n client GPT4\n prompt #\"\n Extract the following information from the query:\n\n Query:\n <<<<\n {{ query }}\n <<<<\n\n OUTPUT_JSON_SCHEMA:\n {{ ctx.output_format }}\n\n Before OUTPUT_JSON_SCHEMA, list 5 intentions the user may have.\n --- EXAMPLES ---\n 1. \n 2. \n 3. \n 4. \n 5. \n\n {\n ... // OUTPUT_JSON_SCHEMA\n }\n \"#\n}\n\nclass RaysData {\n dataType DataType\n value Resume | Event\n}\n\nenum DataType {\n Resume\n Event\n}\n\nclass Event {\n title string\n date string\n location string\n description string\n}\n\nfunction GetDataType(text: string) -> RaysData {\n client GPT4\n prompt #\"\n Extract the relevant info.\n\n Text:\n <<<<\n {{ text }}\n <<<<\n\n Output JSON schema:\n {{ ctx.output_format }}\n\n JSON:\n \"#\n}", - "test-files/providers/providers.baml": "function TestAnthropic(input: string) -> string {\n client Claude\n prompt #\"\n Write a nice haiku about {{ input }}\n \"#\n}\n\nfunction PromptTestOpenAI(input: string) -> string {\n client GPT35\n prompt #\"\n Write a nice haiku about {{ input }}\n \"#\n}\n\nfunction TestOpenAILegacyProvider(input: string) -> string {\n client GPT35LegacyProvider\n prompt #\"\n Write a nice haiku about {{ input }}\n \"#\n}\n\nfunction TestAzure(input: string) -> string {\n client GPT35Azure\n prompt #\"\n Write a nice haiku about {{ input }}\n \"#\n}\n\nfunction TestOllama(input: string) -> string {\n client Ollama\n prompt #\"\n Write a nice haiku about {{ input }}\n \"#\n}\n\nfunction TestGemini(input: string) -> string {\n client Gemini\n prompt #\"\n Write a nice short story about {{ input }}\n \"#\n}\n\nfunction TestVertex(input: string) -> string {\n client Vertex\n prompt #\"\n Write a nice short story about {{ input }}\n \"#\n\n}\n\nfunction TestAws(input: string) -> string {\n client AwsBedrock\n prompt #\"\n Write a nice short story about {{ input }}\n \"#\n}\n\nfunction TestAwsInvalidRegion(input: string) -> string {\n client AwsBedrockInvalidRegion\n prompt #\"\n Write a nice short story about {{ input }}\n \"#\n}\n\nfunction TestOpenAIShorthand(input: string) -> string {\n client \"openai/gpt-4o-mini\"\n prompt #\"\n Write a nice short story about {{ input }}\n \"#\n}\n\nfunction TestAnthropicShorthand(input: string) -> string {\n client \"anthropic/claude-3-haiku-20240307\"\n prompt #\"\n Write a nice short story about {{ input }}\n \"#\n}\n\ntest TestProvider {\n functions [\n TestAnthropic, TestVertex, PromptTestOpenAI, TestAzure, TestOllama, TestGemini, TestAws,\n TestAwsInvalidRegion,\n TestOpenAIShorthand,\n TestAnthropicShorthand\n ]\n args {\n input \"Donkey kong and peanut butter\"\n }\n}\n\n\nfunction TestCaching(input: string, not_cached: string) -> string {\n client ClaudeWithCaching\n prompt #\"\n {{ _.role('system', cache_control={\"type\": \"ephemeral\"}) }}\n Describe this in 5 words: {{ input }}\n {{ _.role('user') }}\n {{ not_cached }}\n \"#\n}\n", + "test-files/providers/providers.baml": "function TestAnthropic(input: string) -> string {\n client Claude\n prompt #\"\n Write a nice haiku about {{ input }}\n \"#\n}\n\nfunction PromptTestOpenAI(input: string) -> string {\n client GPT35\n prompt #\"\n Write a nice haiku about {{ input }}\n \"#\n}\n\nfunction TestOpenAILegacyProvider(input: string) -> string {\n client GPT35LegacyProvider\n prompt #\"\n Write a nice haiku about {{ input }}\n \"#\n}\n\nfunction TestAzure(input: string) -> string {\n client GPT35Azure\n prompt #\"\n Write a nice haiku about {{ input }}\n \"#\n}\n\nfunction TestOllama(input: string) -> string {\n client Ollama\n prompt #\"\n Write a nice haiku about {{ input }}\n \"#\n}\n\nfunction TestGemini(input: string) -> string {\n client Gemini\n prompt #\"\n Write a nice short story about {{ input }}\n \"#\n}\n\nfunction TestVertex(input: string) -> string {\n client Vertex\n prompt #\"\n Write a nice short story about {{ input }}\n \"#\n\n}\n\nfunction TestAws(input: string) -> string {\n client AwsBedrock\n prompt #\"\n Write a nice short story about {{ input }}\n \"#\n}\n\nfunction TestAwsInvalidRegion(input: string) -> string {\n client AwsBedrockInvalidRegion\n prompt #\"\n Write a nice short story about {{ input }}\n \"#\n}\n\nfunction TestOpenAIShorthand(input: string) -> string {\n client \"openai/gpt-4o-mini\"\n prompt #\"\n Write a nice short story about {{ input }}\n \"#\n}\n\nfunction TestAnthropicShorthand(input: string) -> string {\n client \"anthropic/claude-3-haiku-20240307\"\n prompt #\"\n Write a nice short story about {{ input }}\n \"#\n}\n\ntest TestProvider {\n functions [\n TestAnthropic, TestVertex, PromptTestOpenAI, TestAzure, TestOllama, TestGemini, TestAws,\n TestAwsInvalidRegion,\n TestOpenAIShorthand,\n TestAnthropicShorthand\n ]\n args {\n input \"Donkey kong and peanut butter\"\n }\n}\n\n\nfunction TestCaching(input: string, not_cached: string) -> string {\n client ClaudeWithCaching\n prompt #\"\n {{ _.role('system', cache_control={\"type\": \"ephemeral\"}) }}\n Generate the following story\n {{ input }}\n\n {# Haiku require 2048 tokens to cache -#}\n {{ input }}\n\n {{ _.role('user') }}\n {{ not_cached }}\n \"#\n}\n\ntest TestName {\n functions [TestCaching]\n args {\n input #\"\nIn a near-future society where dreams have become a tradable commodity and shared experience, a lonely and socially awkward teenager named Alex discovers they possess a rare and powerful ability to not only view but also manipulate the dreams of others. Initially thrilled by this newfound power, Alex begins subtly altering the dreams of classmates and family members, helping them overcome fears, boost confidence, or experience fantastical adventures. As Alex's skills grow, so does their influence. They start selling premium dream experiences on the black market, crafting intricate and addictive dreamscapes for wealthy clients. However, the line between dream and reality begins to blur for those exposed to Alex's creations. Some clients struggle to differentiate between their true memories and the artificial ones implanted by Alex's dream manipulation.\n\nComplications arise when a mysterious government agency takes notice of Alex's unique abilities. They offer Alex a chance to use their gift for \"the greater good,\" hinting at applications in therapy, criminal rehabilitation, and even national security. Simultaneously, an underground resistance movement reaches out, warning Alex about the dangers of dream manipulation and the potential for mass control and exploitation. Caught between these opposing forces, Alex must navigate a complex web of ethical dilemmas. They grapple with questions of free will, the nature of consciousness, and the responsibility that comes with having power over people's minds. As the consequences of their actions spiral outward, affecting the lives of loved ones and strangers alike, Alex is forced to confront the true nature of their ability and decide how—or if—it should be used.\n\nThe story explores themes of identity, the subconscious mind, the ethics of technology, and the power of imagination. It delves into the potential consequences of a world where our most private thoughts and experiences are no longer truly our own, and examines the fine line between helping others and manipulating them for personal gain or a perceived greater good. The narrative further expands on the societal implications of such abilities, questioning the moral boundaries of altering consciousness and the potential for abuse in a world where dreams can be commodified. It challenges the reader to consider the impact of technology on personal autonomy and the ethical responsibilities of those who wield such power.\n\nAs Alex's journey unfolds, they encounter various individuals whose lives have been touched by their dream manipulations, each presenting a unique perspective on the ethical quandaries at hand. From a classmate who gains newfound confidence to a wealthy client who becomes addicted to the dreamscapes, the ripple effects of Alex's actions are profound and far-reaching. The government agency's interest in Alex's abilities raises questions about the potential for state control and surveillance, while the resistance movement highlights the dangers of unchecked power and the importance of safeguarding individual freedoms.\n\nUltimately, Alex's story is one of self-discovery and moral reckoning, as they must decide whether to embrace their abilities for personal gain, align with the government's vision of a controlled utopia, or join the resistance in their fight for freedom and autonomy. The narrative invites readers to reflect on the nature of reality, the boundaries of human experience, and the ethical implications of a world where dreams are no longer private sanctuaries but shared and manipulated commodities. It also explores the psychological impact on Alex, who must deal with the burden of knowing the intimate fears and desires of others, and the isolation that comes from being unable to share their own dreams without altering them.\n\nThe story further examines the technological advancements that have made dream manipulation possible, questioning the role of innovation in society and the potential for both progress and peril. It considers the societal divide between those who can afford to buy enhanced dream experiences and those who cannot, highlighting issues of inequality and access. As Alex becomes more entangled in the web of their own making, they must confront the possibility that their actions could lead to unintended consequences, not just for themselves but for the fabric of society as a whole.\n\nIn the end, Alex's journey is a cautionary tale about the power of dreams and the responsibilities that come with wielding such influence. It serves as a reminder of the importance of ethical considerations in the face of technological advancement and the need to balance innovation with humanity. The story leaves readers pondering the true cost of a world where dreams are no longer sacred, and the potential for both wonder and danger in the uncharted territories of the mind. But it's also a story about the power of imagination and the potential for change, even in a world where our deepest thoughts are no longer our own. And it's a story about the power of choice, and the importance of fighting for the freedom to dream.\n\nIn conclusion, this story is a reflection on the power of dreams and the responsibilities that come with wielding such influence. It serves as a reminder of the importance of ethical considerations in the face of technological advancement and the need to balance innovation with humanity. The story leaves readers pondering the true cost of a world where dreams are no longer sacred, and the potential for both wonder and danger in the uncharted territories of the mind. But it's also a story about the power of imagination and the potential for change, even in a world where our deepest thoughts are no longer our own. And it's a story about the power of choice, and the importance of fighting for the freedom to dream.\n \"#\n not_cached #\"\n hello world\n \"#\n }\n}\n", "test-files/strategies/fallback-shorthand.baml": "\nclient FallbackToShorthand {\n provider fallback\n options {\n strategy [\n \"openai/does-not-exist\",\n \"openai/gpt-4o-mini\"\n ]\n }\n}\n\n\nfunction TestFallbackToShorthand(input: string) -> string {\n client FallbackToShorthand\n // TODO make it return the client name instead\n prompt #\"\n Say a haiku about {{input}}.\n \"#\n}\n\ntest TestProvider_FallbackToShorthand {\n functions [\n TestFallbackToShorthand\n ]\n args {\n input \"Donkey kong and peanut butter\"\n }\n}\n", - "test-files/strategies/fallback.baml": "// Happy path fallbacks.\nclient FaultyClient {\n provider openai\n options {\n model unknown-model\n api_key env.OPENAI_API_KEY\n }\n}\n\n\nclient FallbackClient {\n provider fallback\n options {\n // first 2 clients are expected to fail.\n strategy [\n FaultyClient,\n RetryClientConstant,\n GPT35\n Gemini\n\n ]\n }\n}\n\nfunction TestFallbackClient() -> string {\n client FallbackClient\n // TODO make it return the client name instead\n prompt #\"\n Say a haiku about mexico.\n \"#\n}\n\n// Fallbacks should fail gracefully.\nclient FaultyAzureClient {\n provider azure-openai\n options {\n model unknown-model\n api_key env.OPENAI_API_KEY\n }\n}\n\nclient SingleFallbackClient {\n provider fallback\n options {\n // first 2 clients are expected to fail.\n strategy [\n FaultyAzureClient\n ]\n }\n}\n\nfunction TestSingleFallbackClient() -> string {\n client SingleFallbackClient\n // TODO make it return the client name instead\n prompt #\"\n Say a haiku about mexico.\n \"#\n}\n", + "test-files/strategies/fallback.baml": "// Happy path fallbacks.\nclient FaultyClient {\n provider openai\n options {\n model unknown-model\n api_key env.OPENAI_API_KEY\n }\n}\n\n\nclient FallbackClient {\n provider fallback\n options {\n // first 2 clients are expected to fail.\n strategy [\n FaultyClient,\n RetryClientConstant,\n GPT35\n Gemini\n\n ]\n }\n}\n\nfunction TestFallbackClient() -> string {\n client FallbackClient\n // TODO make it return the client name instead\n prompt #\"\n Say a haiku about mexico.\n \"#\n}\n\n// Fallbacks should fail gracefully.\nclient FaultyAzureClient {\n provider azure-openai\n options {\n model unknown-model\n resource_name \"unknown-resource-id\"\n deployment_id \"unknown-deployment-id\"\n }\n}\n\nclient SingleFallbackClient {\n provider fallback\n options {\n // first 2 clients are expected to fail.\n strategy [\n FaultyAzureClient\n ]\n }\n}\n\nfunction TestSingleFallbackClient() -> string {\n client SingleFallbackClient\n // TODO make it return the client name instead\n prompt #\"\n Say a haiku about mexico.\n \"#\n}\n", "test-files/strategies/retry.baml": "\nretry_policy Exponential {\n max_retries 3\n strategy {\n type exponential_backoff\n }\n}\n\nretry_policy Constant {\n max_retries 3\n strategy {\n type constant_delay\n delay_ms 100\n }\n}\n\nclient RetryClientConstant {\n provider openai\n retry_policy Constant\n options {\n model \"gpt-3.5-turbo\"\n api_key \"blah\"\n }\n}\n\nclient RetryClientExponential {\n provider openai\n retry_policy Exponential\n options {\n model \"gpt-3.5-turbo\"\n api_key \"blahh\"\n }\n}\n\nfunction TestRetryConstant() -> string {\n client RetryClientConstant\n prompt #\"\n Say a haiku\n \"#\n}\n\nfunction TestRetryExponential() -> string {\n client RetryClientExponential\n prompt #\"\n Say a haiku\n \"#\n}\n", "test-files/strategies/roundrobin.baml": "", "test-files/template_string/template_string.baml": "\nfunction Completion(prefix: string, suffix: string, language: string) -> string {\n client \"openai/gpt-4o\"\n prompt ##\"\n {{ _.role(\"system\", cache_control={\"type\": \"ephemeral\"}) }}\n\n You are a programmer that suggests code completions in the %INSERT-HERE% part below with {{ language }} code. Only output the code that replaces %INSERT-HERE% part, NOT THE SUFFIX OR PREFIX. Respond only with code, and with no markdown formatting.\n\n Try to complete a whole section inside curlies when you can.\n\n {% if language == \"baml\" %}\n {{ BAMLBackground2()}}\n\n Examples:\n INPUT:\n ---\n class MyObject {{\"{\"}}%INSERT-HERE%\n }\n ---\n OUTPUT:\n ---\n property string\n ---\n In this example, we just inserted one line, with tabs for a fake property to aid the user.\n\n INPUT:\n ---\n function foo(input: string) -> string {{\"{\"}} %INSERT-HERE%\n prompt #\"\n {{ \"{{ input }}\" }}\n \"#\n }\n ---\n OUTPUT:\n ---\n client \"openai/gpt-4o\"\n ---\n In this example, no need to add the prompt because it was part of the suffix after INSERT-HERE\n\n INPUT:\n OUTPUT: N/A\n In this example there was nothing to complete, so we returned N/A.\n\n Ignore the \"---\" in your outputs.\n {% endif %}\n\n\n {{ _.role(\"user\") }}\n INPUT:\n ---\n {{ prefix }}%INSERT-HERE%{{ suffix }}\n ---\n \"##\n}\n\ntest CompletionTest3 {\n functions [Completion]\n args {\n prefix ##\"function foo(input: string) -> string {\n client \"openai/gpt-4o\"\n prompt #\"\n \"##\n suffix \"\"\n language \"baml\"\n }\n}\n\ntest CompletionTest2 {\n functions [Completion]\n args {\n prefix \"function foo(input: string) -> string {\\n\"\n suffix \"\\n prompt #\\n\\\"\"\n language \"baml\"\n }\n}\n \ntemplate_string Hi(\n hello: string,\n world: string,\n) ##\"\n {{ hello }} {{ world }}\n\"##\n\ntemplate_string Hi3(\n hello: string,\n world: string,\n) #\"\n {{ hello }} {{ world }}\n\"#\n\ntemplate_string BAMLBackground2() ##\"\n \n BAML is a domain-specific language for building LLM prompts as functions.\n client \"openai/gpt-4o\"\n // prompt with jinja syntax inside here. with double curly braces for variables.\n // make sure to include: {{ \"{{ ctx.output_format }}\"}} in the prompt, which prints the output schema instructions so the LLM returns the output in the correct format (json or string, etc.). DO NOT write the output schema manually.\n prompt #\"\n \n \"#\n }\n\n 3. You do not need to specify to \"answer in JSON format\". Only write in the prompt brief instruction, and any other task-specific things to keep in mind for the task.\n 4. Write a {{ \"{{ _.role(\\\"user\\\") }}\" }} tag to indicate where the user's inputs start. So if there's a convo you can write\n #\"{{ \"{{ _.role(\\\"user\\\") }}\" }} {{ \"{{ some-variable }}\" }}#\n \n \n\n The @asserts only go in the \"output\" types. Don't use them in inputs.\n Do NOT use numbers as confidence intervals if you need to use them. Prefer an enum with descriptions or literals like \"high\", \"medium\", \"low\".\n\n Dedent all declarations.\n\"##\n\ntemplate_string BamlTests() ##\"\n // For image inputs:\n test ImageTest {\n functions [MyFunction]\n args {\n imageArg {\n file \"../images/test.png\"\n // Optional: media_type \"image/png\"\n }\n // Or using URL:\n // imageArg {\n // url \"https://example.com/image.png\"\n // }\n }\n }\n\n // For array/object inputs:\n test ComplexTest {\n functions [MyFunction]\n args {\n input {\n name \"Complex Object\"\n tags [\n \"tag1\",\n #\"\n Multi-line\n tag here\n \"#\n ]\n status PENDING\n type \"error\"\n count 100\n enabled false\n score 7.8\n }\n }\n }\n\"##\n", "test-files/testing_pipeline/output-format.baml": "class Recipe {\n ingredients map\n recipe_type \"breakfast\" | \"dinner\"\n}\n\nclass Quantity {\n amount int | float\n unit string?\n}\n\nfunction AaaSamOutputFormat(recipe: string) -> Recipe {\n client GPT35\n prompt #\"\n Return this value back to me: {{recipe}}\n\n {{ctx.output_format(map_style='angle')}}\n \"#\n}\n\ntest MyOutput {\n functions [AaaSamOutputFormat]\n args {\n recipe #\"\n Here's a simple recipe for beef stew:\nIngredients:\n\n2 lbs beef chuck, cut into 1-inch cubes\n2 tbsp vegetable oil\n1 onion, diced\n3 carrots, sliced\n2 celery stalks, chopped\n2 potatoes, cubed\n3 cloves garlic, minced\n4 cups beef broth\n1 can (14.5 oz) diced tomatoes\n1 tbsp Worcestershire sauce\n1 tsp dried thyme\n1 bay leaf\nSalt and pepper to taste\n\nInstructions:\n\nSeason beef with salt and pepper. Heat oil in a large pot over medium-high heat. Brown the beef in batches, then set aside.\nIn the same pot, sauté onion, carrots, and celery until softened, about 5 minutes.\nAdd garlic and cook for another minute.\nReturn beef to the pot. Add broth, tomatoes, Worcestershire sauce, thyme, and bay leaf.\nBring to a boil, then reduce heat and simmer covered for 1 hour.\nAdd potatoes and continue simmering for 30-45 minutes, until beef and potatoes are tender.\nRemove bay leaf, adjust seasoning if needed, and serve hot.\n\nWould you like any additional information or variations on this recipe?\n \"#\n }\n}\n", - "test-files/testing_pipeline/resume.baml": "class Resume {\n name string\n email string\n phone string\n experience Education[]\n education string[]\n skills string[]\n}\n\nclass Education {\n institution string\n location string\n degree string\n major string[]\n graduation_date string?\n}\n\ntemplate_string AddRole(foo: string) #\"\n {{ _.role('system')}}\n You are a {{ foo }}. be nice\n\n {{ _.role('user') }}\n\"#\n\nclient TestClient {\n provider fallback\n retry_policy Constant\n options {\n strategy [\n Claude\n GPT35\n AwsBedrock\n ]\n }\n}\n\nclient Claude2 {\n provider anthropic\n options {\n model claude-3-haiku-20240307\n api_key env.FOOBAR3\n max_tokens 1000\n }\n}\n\nfunction ExtractResume(resume: string, img: image?) -> Resume {\n client Claude2\n prompt #\"\n {{ AddRole(\"Software Engineer\") }}\n\n Extract data:\n \n\n <<<<\n {{ resume }}\n <<<<\n\n {% if img %}\n {{img}}\n {% endif %}\n\n {{ ctx.output_format }}\n \"#\n}\n\ntest sam_resume {\n functions [ExtractResume]\n input {\n img {\n url \"https://avatars.githubusercontent.com/u/1016595?v=4\"\n }\n resume #\"\n Sam Lijin\n he/him | jobs@sxlijin.com | sxlijin.github.io | 111-222-3333 | sxlijin | sxlijin\n\n Experience\n Trunk\n | July 2021 - current\n Trunk Check | Senior Software Engineer | Services TL, Mar 2023 - current | IC, July 2021 - Feb 2023\n Proposed, designed, and led a team of 3 to build a web experience for Check (both a web-only onboarding flow and SaaS offerings)\n Proposed and built vulnerability scanning into Check, enabling it to compete with security products such as Snyk\n Helped grow Check from <1K users to 90K+ users by focusing on product-led growth\n Google | Sept 2017 - June 2021\n User Identity SRE | Senior Software Engineer | IC, Mar 2021 - June 2021\n Designed an incremental key rotation system to limit the global outage risk to Google SSO\n Discovered and severed an undocumented Gmail serving dependency on Identity-internal systems\n Cloud Firestore | Senior Software Engineer | EngProd TL, Aug 2019 - Feb 2021 | IC, Sept 2017 - July 2019\n Metadata TTL system: backlog of XX trillion records, sustained 1M ops/sec, peaking at 3M ops/sec\n\n Designed and implemented a logging system with novel observability and privacy requirements\n Designed and implemented Jepsen-style testing to validate correctness guarantees\n Datastore Migration: zero downtime, xM RPS and xxPB of data over xM customers and 36 datacenters\n\n Designed composite index migration, queue processing migration, progressive rollout, fast rollback, and disk stockout mitigations; implemented transaction log replay, state transitions, and dark launch process\n Designed and implemented end-to-end correctness and performance testing\n Velocity improvements for 60-eng org\n\n Proposed and implemented automated rollbacks: got us out of a 3-month release freeze and prevented 5 outages over the next 6 months\n Proposed and implemented new development and release environments spanning 30+ microservices\n Incident response for API proxy rollback affecting every Google Cloud service\n\n Google App Engine Memcache | Software Engineer | EngProd TL, Apr 2019 - July 2019\n Proposed and led execution of test coverage improvement strategy for a new control plane: reduced rollbacks and ensured strong consistency of a distributed cache serving xxM QPS\n Designed and implemented automated performance regression testing for two critical serving paths\n Used to validate Google-wide rollout of AMD CPUs, by proving a 50p latency delta of <10µs\n Implemented on shared Borg (i.e. vulnerable to noisy neighbors) with <12% variance\n Miscellaneous | Sept 2017 - June 2021\n Redesigned the Noogler training on Google-internal storage technologies & trained 2500+ Nooglers\n Landed multiple google3-wide refactorings, each spanning xxK files (e.g. SWIG to CLIF)\n Education\n Vanderbilt University (Nashville, TN) | May 2017 | B.S. in Computer Science, Mathematics, and Political Science\n\n Stuyvesant HS (New York, NY) | 2013\n\n Skills\n C++, Java, Typescript, Javascript, Python, Bash; light experience with Rust, Golang, Scheme\n gRPC, Bazel, React, Linux\n Hobbies: climbing, skiing, photography\n \"#\n }\n}\n\ntest vaibhav_resume {\n functions [ExtractResume]\n input {\n resume #\"\n Vaibhav Gupta\n linkedin/vaigup\n (972) 400-5279\n vaibhavtheory@gmail.com\n EXPERIENCE\n Google,\n Software Engineer\n Dec 2018-Present\n Seattle, WA\n •\n Augmented Reality,\n Depth Team\n •\n Technical Lead for on-device optimizations\n •\n Optimized and designed front\n facing depth algorithm\n on Pixel 4\n •\n Focus: C++ and SIMD on custom silicon\n \n \n EDUCATION\n University of Texas at Austin\n Aug 2012-May 2015\n Bachelors of Engineering, Integrated Circuits\n Bachelors of Computer Science\n \"#\n }\n}", + "test-files/testing_pipeline/resume.baml": "class Resume {\n name string\n email string\n phone string\n experience Education[]\n education string[]\n skills string[]\n}\n\nclass Education {\n institution string\n location string\n degree string\n major string[]\n graduation_date string?\n}\n\ntemplate_string AddRole(foo: string) #\"\n {{ _.role('system')}}\n You are a {{ foo }}. be nice\n\n {{ _.role('user') }}\n\"#\n\nclient TestClient {\n provider fallback\n retry_policy Constant\n options {\n strategy [\n Claude\n GPT35\n AwsBedrock\n ]\n }\n}\n\nclient Claude2 {\n provider anthropic\n options {\n model claude-3-haiku-20240307\n api_key env.FOOBAR3\n max_tokens 1000\n }\n}\n\nfunction ExtractResume(resume: string, img: image?) -> Resume {\n client Claude2\n prompt #\"\n {{ AddRole(\"Software Engineer\") }}\n\n Extract data:\n \n\n <<<<\n {{ resume }}\n <<<<\n\n {% if img %}\n {{img}}\n {% endif %}\n\n {{ ctx.output_format }}\n \"#\n}\n\ntest sam_resume {\n functions [ExtractResume]\n args {\n img {\n url \"https://avatars.githubusercontent.com/u/1016595?v=4\"\n }\n resume #\"\n Sam Lijin\n he/him | jobs@sxlijin.com | sxlijin.github.io | 111-222-3333 | sxlijin | sxlijin\n\n Experience\n Trunk\n | July 2021 - current\n Trunk Check | Senior Software Engineer | Services TL, Mar 2023 - current | IC, July 2021 - Feb 2023\n Proposed, designed, and led a team of 3 to build a web experience for Check (both a web-only onboarding flow and SaaS offerings)\n Proposed and built vulnerability scanning into Check, enabling it to compete with security products such as Snyk\n Helped grow Check from <1K users to 90K+ users by focusing on product-led growth\n Google | Sept 2017 - June 2021\n User Identity SRE | Senior Software Engineer | IC, Mar 2021 - June 2021\n Designed an incremental key rotation system to limit the global outage risk to Google SSO\n Discovered and severed an undocumented Gmail serving dependency on Identity-internal systems\n Cloud Firestore | Senior Software Engineer | EngProd TL, Aug 2019 - Feb 2021 | IC, Sept 2017 - July 2019\n Metadata TTL system: backlog of XX trillion records, sustained 1M ops/sec, peaking at 3M ops/sec\n\n Designed and implemented a logging system with novel observability and privacy requirements\n Designed and implemented Jepsen-style testing to validate correctness guarantees\n Datastore Migration: zero downtime, xM RPS and xxPB of data over xM customers and 36 datacenters\n\n Designed composite index migration, queue processing migration, progressive rollout, fast rollback, and disk stockout mitigations; implemented transaction log replay, state transitions, and dark launch process\n Designed and implemented end-to-end correctness and performance testing\n Velocity improvements for 60-eng org\n\n Proposed and implemented automated rollbacks: got us out of a 3-month release freeze and prevented 5 outages over the next 6 months\n Proposed and implemented new development and release environments spanning 30+ microservices\n Incident response for API proxy rollback affecting every Google Cloud service\n\n Google App Engine Memcache | Software Engineer | EngProd TL, Apr 2019 - July 2019\n Proposed and led execution of test coverage improvement strategy for a new control plane: reduced rollbacks and ensured strong consistency of a distributed cache serving xxM QPS\n Designed and implemented automated performance regression testing for two critical serving paths\n Used to validate Google-wide rollout of AMD CPUs, by proving a 50p latency delta of <10µs\n Implemented on shared Borg (i.e. vulnerable to noisy neighbors) with <12% variance\n Miscellaneous | Sept 2017 - June 2021\n Redesigned the Noogler training on Google-internal storage technologies & trained 2500+ Nooglers\n Landed multiple google3-wide refactorings, each spanning xxK files (e.g. SWIG to CLIF)\n Education\n Vanderbilt University (Nashville, TN) | May 2017 | B.S. in Computer Science, Mathematics, and Political Science\n\n Stuyvesant HS (New York, NY) | 2013\n\n Skills\n C++, Java, Typescript, Javascript, Python, Bash; light experience with Rust, Golang, Scheme\n gRPC, Bazel, React, Linux\n Hobbies: climbing, skiing, photography\n \"#\n }\n}\n\ntest vaibhav_resume {\n functions [ExtractResume]\n args {\n resume #\"\n Vaibhav Gupta\n linkedin/vaigup\n (972) 400-5279\n vaibhavtheory@gmail.com\n EXPERIENCE\n Google,\n Software Engineer\n Dec 2018-Present\n Seattle, WA\n •\n Augmented Reality,\n Depth Team\n •\n Technical Lead for on-device optimizations\n •\n Optimized and designed front\n facing depth algorithm\n on Pixel 4\n •\n Focus: C++ and SIMD on custom silicon\n \n \n EDUCATION\n University of Texas at Austin\n Aug 2012-May 2015\n Bachelors of Engineering, Integrated Circuits\n Bachelors of Computer Science\n \"#\n }\n}", } export const getBamlFiles = () => { return fileMap; diff --git a/integ-tests/typescript/test-report.html b/integ-tests/typescript/test-report.html index b6e28a9e4..fd505d175 100644 --- a/integ-tests/typescript/test-report.html +++ b/integ-tests/typescript/test-report.html @@ -257,6 +257,2334 @@ font-size: 1rem; padding: 0 0.5rem; } -

Test Report

Started: 2024-11-19 03:07:16
Suites (1)
0 passed
1 failed
0 pending
Tests (67)
66 passed
1 failed
0 pending
Integ tests > should work for all inputs
single bool
passed
0.471s
Integ tests > should work for all inputs
single string list
passed
0.507s
Integ tests > should work for all inputs
return literal union
passed
0.407s
Integ tests > should work for all inputs
single class
passed
0.514s
Integ tests > should work for all inputs
multiple classes
passed
0.514s
Integ tests > should work for all inputs
single enum list
passed
0.71s
Integ tests > should work for all inputs
single float
passed
0.411s
Integ tests > should work for all inputs
single int
passed
0.283s
Integ tests > should work for all inputs
single literal int
passed
0.429s
Integ tests > should work for all inputs
single literal bool
passed
0.862s
Integ tests > should work for all inputs
single literal string
passed
0.571s
Integ tests > should work for all inputs
single class with literal prop
passed
0.576s
Integ tests > should work for all inputs
single class with literal union prop
passed
0.564s
Integ tests > should work for all inputs
single optional string
passed
0.396s
Integ tests > should work for all inputs
single map string to string
passed
0.508s
Integ tests > should work for all inputs
single map string to class
passed
0.721s
Integ tests > should work for all inputs
single map string to map
passed
0.618s
Integ tests > should work for all inputs
enum key in map
passed
0.714s
Integ tests > should work for all inputs
literal string union key in map
passed
0.817s
Integ tests > should work for all inputs
single literal string key in map
passed
0.69s
Integ tests
should work for all outputs
passed
5.774s
Integ tests
works with retries1
passed
0.994s
Integ tests
works with retries2
passed
2.058s
Integ tests
works with fallbacks
passed
2.055s
Integ tests
should work with image from url
passed
1.277s
Integ tests
should work with image from base 64
passed
0.984s
Integ tests
should work with audio base 64
passed
1.632s
Integ tests
should work with audio from url
passed
1.611s
Integ tests
should support streaming in OpenAI
passed
2.018s
Integ tests
should support streaming in Gemini
passed
6.424s
Integ tests
should support AWS
passed
1.619s
Integ tests
should support streaming in AWS
passed
1.527s
Integ tests
should allow overriding the region
passed
0.01s
Integ tests
should support OpenAI shorthand
passed
9.113s
Integ tests
should support OpenAI shorthand streaming
passed
8.598s
Integ tests
should support anthropic shorthand
passed
3.379s
Integ tests
should support anthropic shorthand streaming
passed
2.332s
Integ tests
should support streaming without iterating
passed
3.02s
Integ tests
should support streaming in Claude
passed
1.525s
Integ tests
should support vertex
passed
11.555s
Integ tests
supports tracing sync
passed
0.02s
Integ tests
supports tracing async
passed
2.937s
Integ tests
should work with dynamic types single
passed
1.243s
Integ tests
should work with dynamic types enum
passed
0.731s
Integ tests
should work with dynamic literals
passed
1.094s
Integ tests
should work with dynamic types class
passed
1.041s
Integ tests
should work with dynamic inputs class
passed
0.596s
Integ tests
should work with dynamic inputs list
passed
0.631s
Integ tests
should work with dynamic output map
passed
0.819s
Integ tests
should work with dynamic output union
passed
2.357s
Integ tests
should work with nested classes
failed
0.117s
Error: BamlError: BamlClientError: Something went wrong with the LLM client: reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Domain("localhost")), port: Some(11434), path: "/v1/chat/completions", query: None, fragment: None }, source: hyper_util::client::legacy::Error(Connect, ConnectError("tcp connect error", Os { code: 111, kind: ConnectionRefused, message: "Connection refused" })) }
-    at BamlStream.parsed [as getFinalResponse] (/workspaces/baml/engine/language_client_typescript/stream.js:58:39)
-    at Object.<anonymous> (/workspaces/baml/integ-tests/typescript/tests/integ-tests.test.ts:602:19)
Integ tests
should work with dynamic client
passed
0.379s
Integ tests
should work with 'onLogEvent'
passed
2.343s
Integ tests
should work with a sync client
passed
0.498s
Integ tests
should raise an error when appropriate
passed
0.987s
Integ tests
should raise a BAMLValidationError
passed
0.477s
Integ tests
should reset environment variables correctly
passed
1.431s
Integ tests
should use aliases when serializing input objects - classes
passed
0.918s
Integ tests
should use aliases when serializing, but still have original keys in jinja
passed
0.805s
Integ tests
should use aliases when serializing input objects - enums
passed
0.424s
Integ tests
should use aliases when serializing input objects - lists
passed
0.512s
Integ tests
constraints: should handle checks in return types
passed
0.822s
Integ tests
constraints: should handle checks in returned unions
passed
0.827s
Integ tests
constraints: should handle block-level checks
passed
0.602s
Integ tests
constraints: should handle nested-block-level checks
passed
0.78s
Integ tests
simple recursive type
passed
1.427s
Integ tests
mutually recursive type
passed
2.402s
\ No newline at end of file +

Test Report

Started: 2024-11-25 16:42:57
Suites (1)
0 passed
1 failed
0 pending
Tests (67)
65 passed
2 failed
0 pending
Integ tests > should work for all inputs
single bool
passed
0.435s
Integ tests > should work for all inputs
single string list
passed
0.413s
Integ tests > should work for all inputs
return literal union
passed
0.472s
Integ tests > should work for all inputs
single class
passed
0.492s
Integ tests > should work for all inputs
multiple classes
passed
0.419s
Integ tests > should work for all inputs
single enum list
passed
0.379s
Integ tests > should work for all inputs
single float
passed
0.33s
Integ tests > should work for all inputs
single int
passed
0.4s
Integ tests > should work for all inputs
single literal int
passed
0.334s
Integ tests > should work for all inputs
single literal bool
passed
0.427s
Integ tests > should work for all inputs
single literal string
passed
0.377s
Integ tests > should work for all inputs
single class with literal prop
passed
0.947s
Integ tests > should work for all inputs
single class with literal union prop
passed
0.551s
Integ tests > should work for all inputs
single optional string
passed
0.371s
Integ tests > should work for all inputs
single map string to string
passed
0.429s
Integ tests > should work for all inputs
single map string to class
passed
0.808s
Integ tests > should work for all inputs
single map string to map
passed
0.51s
Integ tests > should work for all inputs
enum key in map
passed
0.795s
Integ tests > should work for all inputs
literal string union key in map
passed
0.507s
Integ tests > should work for all inputs
single literal string key in map
passed
0.615s
Integ tests
should work for all outputs
passed
4.625s
Integ tests
works with retries1
passed
1.139s
Integ tests
works with retries2
passed
2.203s
Integ tests
works with fallbacks
passed
1.743s
Integ tests
should work with image from url
passed
2.23s
Integ tests
should work with image from base 64
passed
1.562s
Integ tests
should work with audio base 64
passed
1.054s
Integ tests
should work with audio from url
passed
1.086s
Integ tests
should support streaming in OpenAI
passed
2.135s
Integ tests
should support streaming in Gemini
passed
9.901s
Integ tests
should support AWS
passed
1.69s
Integ tests
should support streaming in AWS
passed
1.619s
Integ tests
should allow overriding the region
passed
0.024s
Integ tests
should support OpenAI shorthand
passed
14.595s
Integ tests
should support OpenAI shorthand streaming
passed
8.963s
Integ tests
should support anthropic shorthand
passed
2.5s
Integ tests
should support anthropic shorthand streaming
passed
2.398s
Integ tests
should support streaming without iterating
passed
1.873s
Integ tests
should support streaming in Claude
passed
1.216s
Integ tests
should support vertex
passed
10.167s
Integ tests
supports tracing sync
passed
0.02s
Integ tests
supports tracing async
passed
3.711s
Integ tests
should work with dynamic types single
passed
1.13s
Integ tests
should work with dynamic types enum
passed
1.201s
Integ tests
should work with dynamic literals
passed
1.021s
Integ tests
should work with dynamic types class
passed
1.343s
Integ tests
should work with dynamic inputs class
passed
0.515s
Integ tests
should work with dynamic inputs list
passed
0.486s
Integ tests
should work with dynamic output map
passed
0.58s
Integ tests
should work with dynamic output union
passed
1.902s
Integ tests
should work with nested classes
failed
9.02s
Error: {"type":"BamlValidationError","prompt":"[\u001b[2mchat\u001b[0m] \u001b[43muser: \u001b[0mReturn a made up json blob that matches this schema:\nAnswer in JSON using this schema:\n{\n  prop1: string,\n  prop2: {\n    prop1: string,\n    prop2: string,\n    inner: {\n      prop2: int,\n      prop3: float,\n    },\n  },\n}\n---\n\nJSON:\n","raw_output":"Sure! Here's a made-up JSON blob that matches the schema you provided:\n```json\n{\n  \"prop1\": \"Hello World!\",\n  \"prop2\": {\n    \"prop1\": \"Foo”,\n    \"prop2\": \"Bar\",\n    \"inner\": {\n      \"prop2\": 42,\n      \"prop3\": 3.14,\n    },\n  },\n}\n```\nExplanation:\n\n* The `prop1` field is a string with the value \"Hello World!\".\n* The `prop2` field is an object with three keys: `prop1`, `prop2`, and `inner`.\n\t+ The `prop1` field within the `prop2` object is a string with the value \"Foo”.\n\t+ The `prop2` field within the `prop2` object is a string with the value \"Bar\".\n\t+ The `inner` field within the `prop2` object is an object with two keys: `prop2` and `prop3`.\n\t\t- The `prop2` field within the `inner` object is an integer with the value 42.\n\t\t- The `prop3` field within the `inner` object is a floating-point number with the value 3.14.\n\nI hope this helps! Let me know if you have any questions or need further assistance.","message":"BamlValidationError: Failed to parse LLM response: Failed to coerce value: <root>: Failed to find any TestClassNested in 3 items\n  - <root>: Failed while parsing required fields: missing=0, unparsed=1\n    - <root>: Failed to parse field prop2: prop2: Failed while parsing required fields: missing=1, unparsed=0\n      - prop2: Missing required field: prop2\n      - prop2: Failed while parsing required fields: missing=1, unparsed=0\n        - prop2: Missing required field: prop2\n  - <root>: Failed while parsing required fields: missing=2, unparsed=0\n    - <root>: Missing required field: prop1\n    - <root>: Missing required field: prop2\n  - <root>: Failed while parsing required fields: missing=0, unparsed=1\n    - <root>: Failed to parse field prop2: prop2: Failed while parsing required fields: missing=1, unparsed=0\n      - prop2: Missing required field: prop2\n      - prop2: Failed while parsing required fields: missing=1, unparsed=0\n        - prop2: Missing required field: prop2"}
+    at BamlStream.parsed [as getFinalResponse] (/Users/vbv/repos/gloo-lang/engine/language_client_typescript/stream.js:58:39)
+    at Object.<anonymous> (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:602:19)
Integ tests
should work with dynamic client
passed
0.55s
Integ tests
should work with 'onLogEvent'
passed
1.828s
Integ tests
should work with a sync client
passed
0.797s
Integ tests
should raise an error when appropriate
passed
0.779s
Integ tests
should raise a BAMLValidationError
passed
0.354s
Integ tests
should reset environment variables correctly
passed
1.23s
Integ tests
should use aliases when serializing input objects - classes
passed
0.881s
Integ tests
should use aliases when serializing, but still have original keys in jinja
passed
0.947s
Integ tests
should use aliases when serializing input objects - enums
passed
0.523s
Integ tests
should use aliases when serializing input objects - lists
passed
0.401s
Integ tests
constraints: should handle checks in return types
passed
0.671s
Integ tests
constraints: should handle checks in returned unions
passed
3.124s
Integ tests
constraints: should handle block-level checks
passed
0.52s
Integ tests
constraints: should handle nested-block-level checks
passed
0.526s
Integ tests
simple recursive type
passed
8.24s
Integ tests
mutually recursive type
failed
1.903s
Error: expect(received).toEqual(expected) // deep equality
+
+- Expected  - 6
++ Received  + 5
+
+@@ -4,30 +4,29 @@
+        Object {
+          "children": Object {
+            "trees": Array [
+              Object {
+                "children": Object {
+-                 "trees": Array [
++                 "trees": Array [],
++               },
++               "data": 1,
++             },
+              Object {
+                "children": Object {
+                  "trees": Array [],
+                },
+                "data": 2,
+              },
+            ],
+          },
+-               "data": 1,
++         "data": 3,
+        },
+        Object {
+          "children": Object {
+            "trees": Array [],
+          },
+          "data": 4,
+-             },
+-           ],
+-         },
+-         "data": 3,
+        },
+        Object {
+          "children": Object {
+            "trees": Array [
+              Object {
+    at Object.toEqual (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:838:17)
Console Log
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:48:15)
+    at Promise.then.completed (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/utils.js:298:28)
+    at new Promise (<anonymous>)
+    at callAsyncCircusFn (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/utils.js:231:10)
+    at _callCircusTest (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:316:40)
+    at _runTest (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:252:3)
+    at _runTestsForDescribeBlock (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:126:9)
+    at _runTestsForDescribeBlock (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:121:9)
+    at _runTestsForDescribeBlock (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:121:9)
+    at run (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:71:3)
+    at runAndTransformResultsToJestFormat (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/legacy-code-todo-rewrite/jestAdapterInit.js:122:21)
+    at jestAdapter (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/legacy-code-todo-rewrite/jestAdapter.js:79:19)
+    at runTestInternal (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-runner@29.7.0/node_modules/jest-runner/build/runTest.js:367:16)
+    at runTest (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-runner@29.7.0/node_modules/jest-runner/build/runTest.js:444:34)
calling with class
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:54:15)
got response key, true, 52
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:194:15)
Expected error Error: BamlError: BamlClientError: BamlClientHttpError: LLM call failed: LLMErrorResponse { client: "RetryClientConstant", model: None, prompt: Chat([RenderedChatMessage { role: "user", allow_duplicate_role: false, parts: [Text("Say a haiku")] }]), request_options: {"model": String("gpt-3.5-turbo")}, start_time: SystemTime { tv_sec: 1732581794, tv_nsec: 12578000 }, latency: 272.346375ms, message: "Request failed: https://api.openai.com/v1/chat/completions\n{\n    \"error\": {\n        \"message\": \"Incorrect API key provided: blah. You can find your API key at https://platform.openai.com/account/api-keys.\",\n        \"type\": \"invalid_request_error\",\n        \"param\": null,\n        \"code\": \"invalid_api_key\"\n    }\n}\n", code: InvalidAuthentication }
+    at BamlAsyncClient.parsed [as TestRetryConstant] (/Users/vbv/repos/gloo-lang/integ-tests/typescript/baml_client/async_client.ts:2810:18)
+    at Object.<anonymous> (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:191:7) {
+  code: 'GenericFailure'
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:203:15)
Expected error Error: BamlError: BamlClientError: BamlClientHttpError: LLM call failed: LLMErrorResponse { client: "RetryClientExponential", model: None, prompt: Chat([RenderedChatMessage { role: "user", allow_duplicate_role: false, parts: [Text("Say a haiku")] }]), request_options: {"model": String("gpt-3.5-turbo")}, start_time: SystemTime { tv_sec: 1732581796, tv_nsec: 310300000 }, latency: 197.029625ms, message: "Request failed: https://api.openai.com/v1/chat/completions\n{\n    \"error\": {\n        \"message\": \"Incorrect API key provided: blahh. You can find your API key at https://platform.openai.com/account/api-keys.\",\n        \"type\": \"invalid_request_error\",\n        \"param\": null,\n        \"code\": \"invalid_api_key\"\n    }\n}\n", code: InvalidAuthentication }
+    at BamlAsyncClient.parsed [as TestRetryExponential] (/Users/vbv/repos/gloo-lang/integ-tests/typescript/baml_client/async_client.ts:2835:18)
+    at Object.<anonymous> (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:200:7) {
+  code: 'GenericFailure'
+}
    at log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:362:15)
+    at func (/Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:83:38)
+    at AsyncLocalStorage.run (node:async_hooks:338:14)
+    at run (/Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:81:22)
+    at Object.<anonymous> (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:371:5)
+    at Promise.then.completed (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/utils.js:298:28)
+    at new Promise (<anonymous>)
+    at callAsyncCircusFn (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/utils.js:231:10)
+    at _callCircusTest (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:316:40)
+    at _runTest (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:252:3)
+    at _runTestsForDescribeBlock (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:126:9)
+    at _runTestsForDescribeBlock (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:121:9)
+    at run (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:71:3)
+    at runAndTransformResultsToJestFormat (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/legacy-code-todo-rewrite/jestAdapterInit.js:122:21)
+    at jestAdapter (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/legacy-code-todo-rewrite/jestAdapter.js:79:19)
+    at runTestInternal (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-runner@29.7.0/node_modules/jest-runner/build/runTest.js:367:16)
+    at runTest (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-runner@29.7.0/node_modules/jest-runner/build/runTest.js:444:34)
hello world
    at log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:365:15)
+    at func (/Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:83:38)
+    at AsyncLocalStorage.run (node:async_hooks:338:14)
+    at run (/Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:81:22)
+    at Object.<anonymous> (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:371:5)
+    at Promise.then.completed (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/utils.js:298:28)
+    at new Promise (<anonymous>)
+    at callAsyncCircusFn (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/utils.js:231:10)
+    at _callCircusTest (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:316:40)
+    at _runTest (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:252:3)
+    at _runTestsForDescribeBlock (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:126:9)
+    at _runTestsForDescribeBlock (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:121:9)
+    at run (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:71:3)
+    at runAndTransformResultsToJestFormat (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/legacy-code-todo-rewrite/jestAdapterInit.js:122:21)
+    at jestAdapter (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/legacy-code-todo-rewrite/jestAdapter.js:79:19)
+    at runTestInternal (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-runner@29.7.0/node_modules/jest-runner/build/runTest.js:367:16)
+    at runTest (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-runner@29.7.0/node_modules/jest-runner/build/runTest.js:444:34)
dummyFunc returned
    at log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:368:15)
+    at func (/Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:83:38)
+    at AsyncLocalStorage.run (node:async_hooks:338:14)
+    at run (/Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:81:22)
+    at Object.<anonymous> (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:371:5)
+    at Promise.then.completed (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/utils.js:298:28)
+    at new Promise (<anonymous>)
+    at callAsyncCircusFn (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/utils.js:231:10)
+    at _callCircusTest (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:316:40)
+    at _runTest (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:252:3)
+    at _runTestsForDescribeBlock (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:126:9)
+    at _runTestsForDescribeBlock (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:121:9)
+    at run (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:71:3)
+    at runAndTransformResultsToJestFormat (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/legacy-code-todo-rewrite/jestAdapterInit.js:122:21)
+    at jestAdapter (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/legacy-code-todo-rewrite/jestAdapter.js:79:19)
+    at runTestInternal (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-runner@29.7.0/node_modules/jest-runner/build/runTest.js:367:16)
+    at runTest (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-runner@29.7.0/node_modules/jest-runner/build/runTest.js:444:34)
dummyFunc2 returned
    at log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:383:15)
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at async Promise.all (index 0)
+    at dummyFn (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:389:22)
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at async Promise.all (index 0)
+    at Object.<anonymous> (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:401:5)
samDummyNested nested1
    at log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:383:15)
+    at runNextTicks (node:internal/process/task_queues:60:5)
+    at listOnTimeout (node:internal/timers:538:9)
+    at processTimers (node:internal/timers:512:7)
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at async Promise.all (index 1)
+    at dummyFn (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:389:22)
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at async Promise.all (index 0)
+    at Object.<anonymous> (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:401:5)
samDummyNested nested2
    at log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:383:15)
+    at runNextTicks (node:internal/process/task_queues:60:5)
+    at listOnTimeout (node:internal/timers:538:9)
+    at processTimers (node:internal/timers:512:7)
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at async Promise.all (index 2)
+    at dummyFn (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:389:22)
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at async Promise.all (index 0)
+    at Object.<anonymous> (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:401:5)
samDummyNested nested3
    at log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:394:15)
+    at runNextTicks (node:internal/process/task_queues:60:5)
+    at listOnTimeout (node:internal/timers:538:9)
+    at processTimers (node:internal/timers:512:7)
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at async Promise.all (index 0)
+    at Object.<anonymous> (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:401:5)
dummy hi1
    at log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:383:15)
+    at runNextTicks (node:internal/process/task_queues:60:5)
+    at listOnTimeout (node:internal/timers:538:9)
+    at processTimers (node:internal/timers:512:7)
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at async Promise.all (index 0)
+    at dummyFn (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:389:22)
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at async Promise.all (index 1)
+    at Object.<anonymous> (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:401:5)
samDummyNested nested1
    at log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:383:15)
+    at runNextTicks (node:internal/process/task_queues:60:5)
+    at listOnTimeout (node:internal/timers:538:9)
+    at processTimers (node:internal/timers:512:7)
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at async Promise.all (index 1)
+    at dummyFn (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:389:22)
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at async Promise.all (index 1)
+    at Object.<anonymous> (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:401:5)
samDummyNested nested2
    at log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:383:15)
+    at runNextTicks (node:internal/process/task_queues:60:5)
+    at listOnTimeout (node:internal/timers:538:9)
+    at processTimers (node:internal/timers:512:7)
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at async Promise.all (index 2)
+    at dummyFn (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:389:22)
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at async Promise.all (index 1)
+    at Object.<anonymous> (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:401:5)
samDummyNested nested3
    at log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:394:15)
+    at runNextTicks (node:internal/process/task_queues:60:5)
+    at listOnTimeout (node:internal/timers:538:9)
+    at processTimers (node:internal/timers:512:7)
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at async Promise.all (index 1)
+    at Object.<anonymous> (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:401:5)
dummy hi2
    at log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:383:15)
+    at runNextTicks (node:internal/process/task_queues:60:5)
+    at listOnTimeout (node:internal/timers:538:9)
+    at processTimers (node:internal/timers:512:7)
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at async Promise.all (index 0)
+    at dummyFn (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:389:22)
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at async Promise.all (index 2)
+    at Object.<anonymous> (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:401:5)
samDummyNested nested1
    at log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:383:15)
+    at runNextTicks (node:internal/process/task_queues:60:5)
+    at listOnTimeout (node:internal/timers:538:9)
+    at processTimers (node:internal/timers:512:7)
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at async Promise.all (index 1)
+    at dummyFn (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:389:22)
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at async Promise.all (index 2)
+    at Object.<anonymous> (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:401:5)
samDummyNested nested2
    at log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:383:15)
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at async Promise.all (index 2)
+    at dummyFn (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:389:22)
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at async Promise.all (index 2)
+    at Object.<anonymous> (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:401:5)
samDummyNested nested3
    at log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:394:15)
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at async Promise.all (index 2)
+    at Object.<anonymous> (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:401:5)
dummy hi3
    at log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:408:15)
+    at func (/Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:44)
+    at AsyncLocalStorage.run (node:async_hooks:338:14)
+    at run (/Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:28)
+    at Object.<anonymous> (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:424:5)
hello world
    at log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:383:15)
+    at runNextTicks (node:internal/process/task_queues:60:5)
+    at listOnTimeout (node:internal/timers:538:9)
+    at processTimers (node:internal/timers:512:7)
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at async Promise.all (index 0)
+    at dummyFn (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:389:22)
samDummyNested nested1
    at log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:383:15)
+    at runNextTicks (node:internal/process/task_queues:60:5)
+    at listOnTimeout (node:internal/timers:538:9)
+    at processTimers (node:internal/timers:512:7)
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at async Promise.all (index 1)
+    at dummyFn (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:389:22)
samDummyNested nested2
    at log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:383:15)
+    at runNextTicks (node:internal/process/task_queues:60:5)
+    at listOnTimeout (node:internal/timers:538:9)
+    at processTimers (node:internal/timers:512:7)
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at async Promise.all (index 2)
+    at dummyFn (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:389:22)
samDummyNested nested3
    at log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:394:15)
+    at runNextTicks (node:internal/process/task_queues:60:5)
+    at listOnTimeout (node:internal/timers:538:9)
+    at processTimers (node:internal/timers:512:7)
dummy firstDummyFuncArg
    at log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:383:15)
+    at runNextTicks (node:internal/process/task_queues:60:5)
+    at listOnTimeout (node:internal/timers:538:9)
+    at processTimers (node:internal/timers:512:7)
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at async Promise.all (index 0)
+    at dummyFn (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:389:22)
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at /Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:413:20
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at Object.<anonymous> (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:407:17)
samDummyNested nested1
    at log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:383:15)
+    at runNextTicks (node:internal/process/task_queues:60:5)
+    at listOnTimeout (node:internal/timers:538:9)
+    at processTimers (node:internal/timers:512:7)
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at async Promise.all (index 1)
+    at dummyFn (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:389:22)
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at /Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:413:20
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at Object.<anonymous> (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:407:17)
samDummyNested nested2
    at log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:383:15)
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at async Promise.all (index 2)
+    at dummyFn (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:389:22)
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at /Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:413:20
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at Object.<anonymous> (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:407:17)
samDummyNested nested3
    at log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:394:15)
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at /Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:413:20
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at Object.<anonymous> (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:407:17)
dummy secondDummyFuncArg
    at log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:383:15)
+    at runNextTicks (node:internal/process/task_queues:60:5)
+    at listOnTimeout (node:internal/timers:538:9)
+    at processTimers (node:internal/timers:512:7)
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at async Promise.all (index 0)
+    at dummyFn (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:389:22)
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at /Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:421:20
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at Object.<anonymous> (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:407:17)
samDummyNested nested1
    at log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:383:15)
+    at runNextTicks (node:internal/process/task_queues:60:5)
+    at listOnTimeout (node:internal/timers:538:9)
+    at processTimers (node:internal/timers:512:7)
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at async Promise.all (index 1)
+    at dummyFn (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:389:22)
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at /Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:421:20
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at Object.<anonymous> (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:407:17)
samDummyNested nested2
    at log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:383:15)
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at async Promise.all (index 2)
+    at dummyFn (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:389:22)
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at /Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:421:20
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at Object.<anonymous> (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:407:17)
samDummyNested nested3
    at log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:394:15)
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at /Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:421:20
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:38
+    at /Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:13
+    at Object.<anonymous> (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:407:17)
dummy thirdDummyFuncArg
    at log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:427:15)
+    at func (/Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:104:44)
+    at AsyncLocalStorage.run (node:async_hooks:338:14)
+    at run (/Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:102:28)
+    at Object.<anonymous> (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:433:5)
hello world
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:437:13)
stats {"failed":0,"started":30,"finalized":30,"submitted":30,"sent":30,"done":30}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:461:13)
[
+  {
+    name: 'Harrison',
+    hair_color: 'BLACK',
+    last_name: null,
+    height: 1.83,
+    hobbies: [ 'SPORTS' ]
+  }
+]
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:530:13)
+    at Promise.then.completed (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/utils.js:298:28)
+    at new Promise (<anonymous>)
+    at callAsyncCircusFn (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/utils.js:231:10)
+    at _callCircusTest (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:316:40)
+    at _runTest (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:252:3)
+    at _runTestsForDescribeBlock (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:126:9)
+    at _runTestsForDescribeBlock (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:121:9)
+    at run (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:71:3)
+    at runAndTransformResultsToJestFormat (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/legacy-code-todo-rewrite/jestAdapterInit.js:122:21)
+    at jestAdapter (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/legacy-code-todo-rewrite/jestAdapter.js:79:19)
+    at runTestInternal (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-runner@29.7.0/node_modules/jest-runner/build/runTest.js:367:16)
+    at runTest (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-runner@29.7.0/node_modules/jest-runner/build/runTest.js:444:34)
[
+  [
+    'hair_color',
+    ClassPropertyBuilder { bldr: ClassPropertyBuilder {} }
+  ],
+  [
+    'attributes',
+    ClassPropertyBuilder { bldr: ClassPropertyBuilder {} }
+  ]
+]
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:532:15)
+    at Promise.then.completed (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/utils.js:298:28)
+    at new Promise (<anonymous>)
+    at callAsyncCircusFn (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/utils.js:231:10)
+    at _callCircusTest (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:316:40)
+    at _runTest (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:252:3)
+    at _runTestsForDescribeBlock (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:126:9)
+    at _runTestsForDescribeBlock (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:121:9)
+    at run (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:71:3)
+    at runAndTransformResultsToJestFormat (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/legacy-code-todo-rewrite/jestAdapterInit.js:122:21)
+    at jestAdapter (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/legacy-code-todo-rewrite/jestAdapter.js:79:19)
+    at runTestInternal (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-runner@29.7.0/node_modules/jest-runner/build/runTest.js:367:16)
+    at runTest (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-runner@29.7.0/node_modules/jest-runner/build/runTest.js:444:34)
Property: hair_color
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:532:15)
+    at Promise.then.completed (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/utils.js:298:28)
+    at new Promise (<anonymous>)
+    at callAsyncCircusFn (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/utils.js:231:10)
+    at _callCircusTest (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:316:40)
+    at _runTest (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:252:3)
+    at _runTestsForDescribeBlock (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:126:9)
+    at _runTestsForDescribeBlock (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:121:9)
+    at run (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:71:3)
+    at runAndTransformResultsToJestFormat (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/legacy-code-todo-rewrite/jestAdapterInit.js:122:21)
+    at jestAdapter (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/legacy-code-todo-rewrite/jestAdapter.js:79:19)
+    at runTestInternal (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-runner@29.7.0/node_modules/jest-runner/build/runTest.js:367:16)
+    at runTest (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-runner@29.7.0/node_modules/jest-runner/build/runTest.js:444:34)
Property: attributes
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:540:13)
final  {
+  hair_color: 'black',
+  attributes: { eye_color: 'blue', facial_hair: 'beard' }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:564:13)
+    at Promise.then.completed (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/utils.js:298:28)
+    at new Promise (<anonymous>)
+    at callAsyncCircusFn (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/utils.js:231:10)
+    at _callCircusTest (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:316:40)
+    at _runTest (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:252:3)
+    at _runTestsForDescribeBlock (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:126:9)
+    at _runTestsForDescribeBlock (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:121:9)
+    at run (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:71:3)
+    at runAndTransformResultsToJestFormat (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/legacy-code-todo-rewrite/jestAdapterInit.js:122:21)
+    at jestAdapter (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/legacy-code-todo-rewrite/jestAdapter.js:79:19)
+    at runTestInternal (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-runner@29.7.0/node_modules/jest-runner/build/runTest.js:367:16)
+    at runTest (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-runner@29.7.0/node_modules/jest-runner/build/runTest.js:444:34)
[
+  [
+    'hair_color',
+    ClassPropertyBuilder { bldr: ClassPropertyBuilder {} }
+  ],
+  [
+    'attributes',
+    ClassPropertyBuilder { bldr: ClassPropertyBuilder {} }
+  ],
+  [ 'height', ClassPropertyBuilder { bldr: ClassPropertyBuilder {} } ]
+]
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:566:15)
+    at Promise.then.completed (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/utils.js:298:28)
+    at new Promise (<anonymous>)
+    at callAsyncCircusFn (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/utils.js:231:10)
+    at _callCircusTest (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:316:40)
+    at _runTest (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:252:3)
+    at _runTestsForDescribeBlock (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:126:9)
+    at _runTestsForDescribeBlock (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:121:9)
+    at run (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:71:3)
+    at runAndTransformResultsToJestFormat (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/legacy-code-todo-rewrite/jestAdapterInit.js:122:21)
+    at jestAdapter (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/legacy-code-todo-rewrite/jestAdapter.js:79:19)
+    at runTestInternal (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-runner@29.7.0/node_modules/jest-runner/build/runTest.js:367:16)
+    at runTest (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-runner@29.7.0/node_modules/jest-runner/build/runTest.js:444:34)
Property: hair_color
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:566:15)
+    at Promise.then.completed (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/utils.js:298:28)
+    at new Promise (<anonymous>)
+    at callAsyncCircusFn (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/utils.js:231:10)
+    at _callCircusTest (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:316:40)
+    at _runTest (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:252:3)
+    at _runTestsForDescribeBlock (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:126:9)
+    at _runTestsForDescribeBlock (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:121:9)
+    at run (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:71:3)
+    at runAndTransformResultsToJestFormat (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/legacy-code-todo-rewrite/jestAdapterInit.js:122:21)
+    at jestAdapter (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/legacy-code-todo-rewrite/jestAdapter.js:79:19)
+    at runTestInternal (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-runner@29.7.0/node_modules/jest-runner/build/runTest.js:367:16)
+    at runTest (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-runner@29.7.0/node_modules/jest-runner/build/runTest.js:444:34)
Property: attributes
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:566:15)
+    at Promise.then.completed (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/utils.js:298:28)
+    at new Promise (<anonymous>)
+    at callAsyncCircusFn (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/utils.js:231:10)
+    at _callCircusTest (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:316:40)
+    at _runTest (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:252:3)
+    at _runTestsForDescribeBlock (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:126:9)
+    at _runTestsForDescribeBlock (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:121:9)
+    at run (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:71:3)
+    at runAndTransformResultsToJestFormat (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/legacy-code-todo-rewrite/jestAdapterInit.js:122:21)
+    at jestAdapter (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/legacy-code-todo-rewrite/jestAdapter.js:79:19)
+    at runTestInternal (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-runner@29.7.0/node_modules/jest-runner/build/runTest.js:367:16)
+    at runTest (/Users/vbv/repos/gloo-lang/integ-tests/typescript/node_modules/.pnpm/jest-runner@29.7.0/node_modules/jest-runner/build/runTest.js:444:34)
Property: height
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:574:13)
final  {
+  hair_color: 'black',
+  attributes: { eye_color: 'blue', facial_hair: 'beard', age: '30' },
+  height: { feet: 6, inches: null }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:585:13)
final  {
+  hair_color: 'black',
+  attributes: { eye_color: 'blue', facial_hair: 'beard' },
+  height: { meters: 1.8 }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg { prop1: null, prop2: null }
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg { prop1: null, prop2: null }
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg { prop1: null, prop2: null }
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg { prop1: null, prop2: null }
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg { prop1: null, prop2: null }
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg { prop1: null, prop2: null }
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg { prop1: null, prop2: null }
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg { prop1: null, prop2: null }
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg { prop1: null, prop2: null }
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg { prop1: null, prop2: null }
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg { prop1: null, prop2: null }
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg { prop1: null, prop2: null }
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg { prop1: null, prop2: null }
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg { prop1: null, prop2: null }
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg { prop1: null, prop2: null }
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg { prop1: null, prop2: null }
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg { prop1: null, prop2: null }
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg { prop1: null, prop2: null }
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg { prop1: null, prop2: null }
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg { prop1: null, prop2: null }
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg { prop1: null, prop2: null }
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg { prop1: null, prop2: null }
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg { prop1: null, prop2: null }
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg { prop1: null, prop2: null }
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg { prop1: null, prop2: null }
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg { prop1: null, prop2: null }
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg { prop1: null, prop2: null }
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg { prop1: null, prop2: null }
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg { prop1: null, prop2: null }
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg { prop1: null, prop2: null }
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg { prop1: '', prop2: null }
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg { prop1: 'Hello', prop2: null }
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg { prop1: 'Hello World', prop2: null }
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg { prop1: 'Hello World!', prop2: null }
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg { prop1: 'Hello World!', prop2: null }
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg { prop1: 'Hello World!', prop2: null }
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg { prop1: 'Hello World!', prop2: null }
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg { prop1: 'Hello World!', prop2: null }
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg { prop1: 'Hello World!', prop2: null }
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg { prop1: 'Hello World!', prop2: null }
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg { prop1: 'Hello World!', prop2: null }
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: { prop1: null, prop2: null, inner: null }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: { prop1: null, prop2: null, inner: null }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: { prop1: null, prop2: null, inner: null }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: { prop1: null, prop2: null, inner: null }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: { prop1: null, prop2: null, inner: null }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: { prop1: null, prop2: null, inner: null }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: { prop1: null, prop2: null, inner: null }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: { prop1: '', prop2: null, inner: null }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: { prop1: 'Foo', prop2: null, inner: null }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: { prop1: 'Foo”,', prop2: null, inner: null }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: { prop1: 'Foo”,', prop2: null, inner: null }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: { prop1: 'Foo”,', prop2: null, inner: null }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: { prop1: 'Foo”,\n    ', prop2: null, inner: null }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: { prop1: 'Foo”,\n    "prop', prop2: null, inner: null }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: { prop1: 'Foo”,\n    "prop2', prop2: null, inner: null }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: { prop1: 'Foo”,\n    "prop2":', prop2: null, inner: null }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: { prop1: 'Foo”,\n    "prop2": ', prop2: null, inner: null }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: { prop1: 'Foo”,\n    "prop2": "Bar', prop2: null, inner: null }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: { prop1: 'Foo”,\n    "prop2": "Bar', prop2: null, inner: null }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: { prop1: 'Foo”,\n    "prop2": "Bar', prop2: null, inner: null }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: { prop1: 'Foo”,\n    "prop2": "Bar', prop2: null, inner: null }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: { prop1: 'Foo”,\n    "prop2": "Bar', prop2: null, inner: null }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: { prop1: 'Foo”,\n    "prop2": "Bar', prop2: null, inner: null }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: { prop1: 'Foo”,\n    "prop2": "Bar', prop2: null, inner: null }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: null, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: null, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: null, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: null, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: null, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: null, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: null, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: null, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: null, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: null, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: null, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
+    at runNextTicks (node:internal/process/task_queues:60:5)
+    at listOnTimeout (node:internal/timers:538:9)
+    at processTimers (node:internal/timers:512:7)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
+    at runNextTicks (node:internal/process/task_queues:60:5)
+    at listOnTimeout (node:internal/timers:538:9)
+    at processTimers (node:internal/timers:512:7)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
+    at runNextTicks (node:internal/process/task_queues:60:5)
+    at listOnTimeout (node:internal/timers:538:9)
+    at processTimers (node:internal/timers:512:7)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
+    at runNextTicks (node:internal/process/task_queues:60:5)
+    at listOnTimeout (node:internal/timers:538:9)
+    at processTimers (node:internal/timers:512:7)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
+    at runNextTicks (node:internal/process/task_queues:60:5)
+    at listOnTimeout (node:internal/timers:538:9)
+    at processTimers (node:internal/timers:512:7)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
+    at runNextTicks (node:internal/process/task_queues:60:5)
+    at listOnTimeout (node:internal/timers:538:9)
+    at processTimers (node:internal/timers:512:7)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
+    at runNextTicks (node:internal/process/task_queues:60:5)
+    at listOnTimeout (node:internal/timers:538:9)
+    at processTimers (node:internal/timers:512:7)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
+    at runNextTicks (node:internal/process/task_queues:60:5)
+    at listOnTimeout (node:internal/timers:538:9)
+    at processTimers (node:internal/timers:512:7)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
+    at runNextTicks (node:internal/process/task_queues:60:5)
+    at listOnTimeout (node:internal/timers:538:9)
+    at processTimers (node:internal/timers:512:7)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: 3.14 }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: null }
+  }
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:598:15)
msg {
+  prop1: 'Hello World!',
+  prop2: {
+    prop1: 'Foo”,\n    "prop2": "Bar',
+    prop2: null,
+    inner: { prop2: 42, prop3: null }
+  }
+}
    at log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:623:15)
+    at callback (/Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:70:17)
onLogEvent {
+  metadata: {
+    eventId: '4965c78f-09d8-436d-931d-305945add5d4',
+    rootEventId: '4965c78f-09d8-436d-931d-305945add5d4'
+  },
+  prompt: '[\n' +
+    '  {\n' +
+    '    "role": "user",\n' +
+    '    "content": [\n' +
+    '      {\n' +
+    '        "text": "Return this value back to me: [\\"a\\", \\"b\\", \\"c\\"]"\n' +
+    '      }\n' +
+    '    ]\n' +
+    '  }\n' +
+    ']',
+  rawOutput: '["a", "b", "c"]',
+  parsedOutput: '["a", "b", "c"]',
+  startTime: '2024-11-26T00:44:43.387Z'
+}
    at log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:623:15)
+    at callback (/Users/vbv/repos/gloo-lang/engine/language_client_typescript/async_context_vars.js:70:17)
onLogEvent {
+  metadata: {
+    eventId: 'c0cee060-530e-43dc-84c0-be5077be71ca',
+    rootEventId: 'c0cee060-530e-43dc-84c0-be5077be71ca'
+  },
+  prompt: '[\n' +
+    '  {\n' +
+    '    "role": "user",\n' +
+    '    "content": [\n' +
+    '      {\n' +
+    '        "text": "Return this value back to me: [\\"d\\", \\"e\\", \\"f\\"]"\n' +
+    '      }\n' +
+    '    ]\n' +
+    '  }\n' +
+    ']',
+  rawOutput: '["d", "e", "f"]',
+  parsedOutput: '["d", "e", "f"]',
+  startTime: '2024-11-26T00:44:43.813Z'
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:657:15)
Error: Error: BamlError: BamlClientError: BamlClientHttpError: LLM call failed: LLMErrorResponse { client: "MyClient", model: None, prompt: Chat([RenderedChatMessage { role: "user", allow_duplicate_role: false, parts: [Text("Given a string, extract info using the schema:\n\nMy name is Harrison. My hair is black and I'm 6 feet tall.\n\nAnswer in JSON using this schema:\n{\n}")] }]), request_options: {"model": String("gpt-4o-mini")}, start_time: SystemTime { tv_sec: 1732581885, tv_nsec: 564378000 }, latency: 158.524416ms, message: "Request failed: https://api.openai.com/v1/chat/completions\n{\n    \"error\": {\n        \"message\": \"Incorrect API key provided: INVALID_KEY. You can find your API key at https://platform.openai.com/account/api-keys.\",\n        \"type\": \"invalid_request_error\",\n        \"param\": null,\n        \"code\": \"invalid_api_key\"\n    }\n}\n", code: InvalidAuthentication }
+    at BamlAsyncClient.parsed (/Users/vbv/repos/gloo-lang/integ-tests/typescript/baml_client/async_client.ts:1585:18)
+    at Object.<anonymous> (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:654:7) {
+  code: 'GenericFailure'
+}
    at log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:665:17)
+    at Object.<anonymous> (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:661:5)
BamlValidationError: BamlValidationError: Failed to parse LLM response: Failed to coerce value: <root>: Failed while parsing required fields: missing=2, unparsed=0
+  - <root>: Missing required field: nonce
+  - <root>: Missing required field: nonce2
+    at Function.from (/Users/vbv/repos/gloo-lang/engine/language_client_typescript/index.js:33:28)
+    at from (/Users/vbv/repos/gloo-lang/engine/language_client_typescript/index.js:58:32)
+    at BamlAsyncClient.DummyOutputFunction (/Users/vbv/repos/gloo-lang/integ-tests/typescript/baml_client/async_client.ts:562:50)
+    at /Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:663:9
+    at Object.<anonymous> (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:661:5) {
+  prompt: '[\x1B[2mchat\x1B[0m] \x1B[43muser: \x1B[0mSay "hello there".\n',
+  raw_output: 'Hello there!'
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:677:17)
error BamlValidationError: BamlValidationError: Failed to parse LLM response: Failed to coerce value: <root>: Failed while parsing required fields: missing=2, unparsed=0
+  - <root>: Missing required field: nonce
+  - <root>: Missing required field: nonce2
+    at Function.from (/Users/vbv/repos/gloo-lang/engine/language_client_typescript/index.js:33:28)
+    at from (/Users/vbv/repos/gloo-lang/engine/language_client_typescript/index.js:58:32)
+    at BamlAsyncClient.DummyOutputFunction (/Users/vbv/repos/gloo-lang/integ-tests/typescript/baml_client/async_client.ts:562:50)
+    at Object.<anonymous> (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:673:7) {
+  prompt: '[\x1B[2mchat\x1B[0m] \x1B[43muser: \x1B[0mSay "hello there".\n',
+  raw_output: 'Hello there! How can I assist you today?'
+}
    at Object.log (/Users/vbv/repos/gloo-lang/integ-tests/typescript/tests/integ-tests.test.ts:777:13)
{"nbc":{"value":{"foo":1,"bar":"hello"},"checks":{"cross_field":{"name":"cross_field","expression":"this.bar|length > this.foo","status":"succeeded"}}}}
\ No newline at end of file diff --git a/typescript/fiddle-frontend/app/[project_id]/_components/ProjectView.tsx b/typescript/fiddle-frontend/app/[project_id]/_components/ProjectView.tsx index 8115e82e3..6e15fff87 100644 --- a/typescript/fiddle-frontend/app/[project_id]/_components/ProjectView.tsx +++ b/typescript/fiddle-frontend/app/[project_id]/_components/ProjectView.tsx @@ -330,7 +330,7 @@ const PlaygroundView = () => { return ( <> - + Loading...}> diff --git a/typescript/playground-common/src/baml_wasm_web/EventListener.tsx b/typescript/playground-common/src/baml_wasm_web/EventListener.tsx index 5a096b3bb..fb40f1c60 100644 --- a/typescript/playground-common/src/baml_wasm_web/EventListener.tsx +++ b/typescript/playground-common/src/baml_wasm_web/EventListener.tsx @@ -515,13 +515,18 @@ export const currentClientsAtom = atom((get) => { return [] } - const wasmScopes = func.orchestration_graph(runtime) - if (wasmScopes === null) { - return [] - } + try { + const wasmScopes = func.orchestration_graph(runtime) + if (wasmScopes === null) { + return [] + } - const nodes = createClientNodes(wasmScopes) - return nodes.map((node) => node.name) + const nodes = createClientNodes(wasmScopes) + return nodes.map((node) => node.name) + } catch (e) { + console.error(e) + return ['Error!'] + } }) // something about the orchestration graph is broken, comment it out to make it work @@ -1101,7 +1106,7 @@ export const EventListener: React.FC<{ children: React.ReactNode }> = ({ childre ) ) : ( - {children} + {children} )} ) diff --git a/typescript/playground-common/src/baml_wasm_web/test_uis/test_result.tsx b/typescript/playground-common/src/baml_wasm_web/test_uis/test_result.tsx index abd80b5cb..bda2d47a8 100644 --- a/typescript/playground-common/src/baml_wasm_web/test_uis/test_result.tsx +++ b/typescript/playground-common/src/baml_wasm_web/test_uis/test_result.tsx @@ -738,7 +738,7 @@ const TestResults: React.FC = () => { - + {showClientGraph ? : showTests ? : } diff --git a/typescript/playground-common/src/shared/FunctionPanel.tsx b/typescript/playground-common/src/shared/FunctionPanel.tsx index 23df5e3f6..53e0edb49 100644 --- a/typescript/playground-common/src/shared/FunctionPanel.tsx +++ b/typescript/playground-common/src/shared/FunctionPanel.tsx @@ -36,6 +36,7 @@ import { vscode } from '../utils/vscode' import clsx from 'clsx' import { ErrorBoundary } from 'react-error-boundary' import { Badge } from '@/components/ui/badge' +import CustomErrorBoundary from '../utils/ErrorFallback' const handleCopy = (text: string) => () => { navigator.clipboard.writeText(text) @@ -365,7 +366,9 @@ enum Topic {
- + + +
diff --git a/typescript/playground-common/src/shared/Selectors.tsx b/typescript/playground-common/src/shared/Selectors.tsx index 46760b6a0..db57fafe9 100644 --- a/typescript/playground-common/src/shared/Selectors.tsx +++ b/typescript/playground-common/src/shared/Selectors.tsx @@ -16,6 +16,7 @@ import SearchBarWithSelector from '../lib/searchbar' import Link from './Link' import { Dialog, DialogContent, DialogTrigger } from '../components/ui/dialog' import { Snippets } from './Snippets' +import CustomErrorBoundary from '../utils/ErrorFallback' const ClientHeader: React.FC = () => { const orchIndex = useAtomValue(orchIndexAtom) @@ -190,13 +191,21 @@ export const ViewSelector: React.FC = () => { return (
- + + +
- - - + + + +
+ +
+ + +
diff --git a/typescript/playground-common/src/utils/ErrorFallback.tsx b/typescript/playground-common/src/utils/ErrorFallback.tsx index 1d0e17cbe..653771187 100644 --- a/typescript/playground-common/src/utils/ErrorFallback.tsx +++ b/typescript/playground-common/src/utils/ErrorFallback.tsx @@ -1,49 +1,62 @@ +import React from 'react' import { Button } from '@/components/ui/button' import { RefreshCcw } from 'lucide-react' import { ErrorBoundary, type FallbackProps } from 'react-error-boundary' -const ErrorFallback: React.FC = ({ error, resetErrorBoundary }) => { - return ( -
-
-

Something went wrong

- -
-
- {error.message && ( - 
-            {error.message}
-
- )} - {error.stack && ( - 
-            {error.stack}
-
+const ErrorFallback: (message?: string) => React.FC = (message) => { + const FB = ({ error, resetErrorBoundary }: FallbackProps) => { + return ( +
+
+

{message ?? 'Something went wrong'}

+ +
+ + {error instanceof Error && ( +
+ {error.message && ( + 
+                {error.message}
+
+ )} + {error.stack && ( + 
+                {error.stack}
+
+ )} + {error && Object.keys(error).length > 0 && ( + 
+                {JSON.stringify(error, null, 2)}
+
+ )} +
)} - {error && Object.keys(error).length > 0 && ( + {error && typeof error === 'string' && ( 
-            {JSON.stringify(error, null, 2)}
+            {error}
)}
-
- ) + ) + } + return FB } interface MyErrorBoundaryProps { children: React.ReactNode + message?: string } -const CustomErrorBoundary: React.FC = ({ children }) => { +const CustomErrorBoundary: React.FC = ({ children, message }) => { return ( { // Reset the state of your app so the error doesn't happen again }} diff --git a/typescript/vscode-ext/packages/web-panel/src/App.tsx b/typescript/vscode-ext/packages/web-panel/src/App.tsx index b59d6cc6b..1b470f355 100644 --- a/typescript/vscode-ext/packages/web-panel/src/App.tsx +++ b/typescript/vscode-ext/packages/web-panel/src/App.tsx @@ -20,20 +20,26 @@ import { useFeedbackWidget } from './lib/feedback_widget' function App() { useFeedbackWidget() return ( - + Loading...
}>
- + + +
- - + + + + + + {' '}