feat: make ruby work again (#650)

- ruby now supports streaming and tracing
- ruby does _not_ yet support: custom trace decorators, images, or
dynamic types
- ruby compilation is not baked into the default `cargo build` - too
much stuff requires ruby-specific compiler flags
- `baml` gem 0.1.10 has been released and contains the latest stuff
- `release_language_client_ruby` now works
- ruby codegen has been updated, and needs to be shipped in the vscode
extension
- fix: codegen for the default `baml_client` used to automatically
happen in `baml_src`, it now happens in `baml_src/..`
- chore: delete integ-tests2

.envrc

@@ -1 +1,3 @@
 PATH_add tools/
+
+eval "$(mise activate bash)"

.github/workflows/release_language_client_ruby.yaml

@@ -2,14 +2,15 @@ name: Release engine/language_client_ruby
 
 on:
   workflow_dispatch: {}
-  #push:
-  #  branches:
-  #    - sam/rust-llm
 
 permissions:
   contents: read
   id-token: write
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   build:
     strategy:
@@ -23,6 +24,9 @@ jobs:
           #- x64-mingw-ucrt
 
     runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: engine/language_client_ruby
     steps:
       - uses: rubygems/configure-rubygems-credentials@main
         with:
@@ -31,8 +35,6 @@ jobs:
 
       - uses: actions/checkout@v4
 
-      - run: cd engine/language_client_ruby
-
       - uses: oxidize-rb/actions/setup-ruby-and-rust@main
         with:
           rubygems: latest
@@ -103,13 +105,10 @@ jobs:
             gem install rb_sys -v $version
           fi
 
-      - run: |
-          echo "Switching to top-level workspace - we need to be able to resolve the root-level Cargo.toml"
-          cd ..
-          pwd
-
       - name: Build gem
         shell: bash
+        working-directory: engine
+        if: ${{ matrix.platform == 'x86_64-linux' }}
         run: |
           : Compile gem
           echo "Docker Working Directory: $(pwd)"
@@ -123,6 +122,22 @@ jobs:
             --build \
             -- sudo yum install -y perl-IPC-Cmd
 
+      - name: Build gem
+        shell: bash
+        working-directory: engine
+        if: ${{ matrix.platform != 'x86_64-linux' }}
+        run: |
+          : Compile gem
+          echo "Docker Working Directory: $(pwd)"
+          set -x
+
+          rb-sys-dock \
+            --platform ${{ matrix.platform }} \
+            --mount-toolchains \
+            --directory language_client_ruby \
+            --ruby-versions 3.3,3.2,3.1,3.0,2.7 \
+            --build
+
       #################################################################################################################
       #
       #  END: these steps are copied from https://github.com/oxidize-rb/actions/blob/main/cross-gem/action.yml

.gitignore

@@ -122,6 +122,7 @@ $RECYCLE.BIN/
 /dist
 /node_modules
 /out/
+engine/language_client_ruby/**/*.bundle
 engine/target/
 Cargo.lock
 Icon

.mise.toml

@@ -1,2 +1,2 @@
 [tools]
-ruby = "3.3.0"
+ruby = "3.1"

engine/Cargo.toml

@@ -66,5 +66,11 @@ license-file = "LICENSE"
 [workspace.metadata.workspaces]
 allow_branch = "canary"
 
+[profile.dev]
+lto = false
+
+[profile.dev2]
+inherits = "dev"
+
 [profile.release]
 lto = true

engine/baml-lib/jsonish/src/jsonish/parser/markdown_parser.rs

@@ -12,8 +12,10 @@ pub fn parse<'a>(str: &'a str, options: &ParseOptions) -> Result<Vec<(String, Va
     let mut remaining = str;
     // Find regex for markdown blocks (```<tag><EOF|newline>)
 
-    let md_tag_start = regex::Regex::new(r"```([a-zA-Z0-9 ]+)(?:\n|$)").expect("Invalid regex");
-    let md_tag_end = regex::Regex::new(r"```(?:\n|$)").expect("Invalid regex");
+    let md_tag_start = regex::Regex::new(r"```([a-zA-Z0-9 ]+)(?:\n|$)")
+        .map_err(|e| anyhow::Error::from(e).context("Failed to build regex for md-tag-start"))?;
+    let md_tag_end = regex::Regex::new(r"```(?:\n|$)")
+        .map_err(|e| anyhow::Error::from(e).context("Failed to build regex for md-tag-end"))?;
 
     let mut should_loop = true;
 

engine/baml-runtime/src/cli/generate.rs

@@ -30,26 +30,30 @@ impl GenerateArgs {
         if generated.is_empty() {
             let client_type = caller_type.into();
             // Normally `baml_client` is added via the generator, but since we're not running the generator, we need to add it manually.
-            let output_dir = src_dir.join("..").join("baml_client");
+            let output_dir_relative_to_baml_src = PathBuf::from("..");
             let generate_output = runtime.generate_client(
                 &client_type,
                 &internal_baml_codegen::GeneratorArgs::new(
-                    &output_dir,
+                    &output_dir_relative_to_baml_src.join("baml_client"),
                     &self.from,
                     all_files.iter(),
                 )?,
             )?;
-            println!("Generated 1 baml_client at {}", output_dir.display());
+            println!(
+                "Generated 1 baml_client at {}",
+                generate_output.output_dir.display()
+            );
 
             println!(
                 r#"
 You can automatically generate a client by adding the following to any one of your BAML files:
 
 generator my_client {{
     output_type "{}"
-    output_dir "../"
+    output_dir "{}"
 }}"#,
-                generate_output.client_type.to_string()
+                generate_output.client_type.to_string(),
+                output_dir_relative_to_baml_src.join("").display(),
             );
         } else {
             println!("Generated {} baml_client", generated.len());

engine/language-client-codegen/askama.toml

@@ -7,4 +7,3 @@ dirs = [
 # whitespace can be either preserve, suppress, or minimize
 # suppress and minimize are both too aggressive for us
 whitespace = "preserve"
-escape = "none"

engine/language-client-codegen/src/lib.rs

@@ -31,7 +31,7 @@ fn relative_path_to_baml_src(path: &PathBuf, baml_src: &PathBuf) -> Result<PathB
 
 impl GeneratorArgs {
     pub fn new<'i>(
-        output_dir: impl Into<PathBuf>,
+        output_dir_relative_to_baml_src: impl Into<PathBuf>,
         baml_src_dir: impl Into<PathBuf>,
         input_files: impl IntoIterator<Item = (&'i PathBuf, &'i String)>,
     ) -> Result<Self> {
@@ -42,7 +42,7 @@ impl GeneratorArgs {
             .collect::<Result<_>>()?;
 
         Ok(Self {
-            output_dir_relative_to_baml_src: output_dir.into(),
+            output_dir_relative_to_baml_src: output_dir_relative_to_baml_src.into(),
             baml_src_dir: baml_src.clone(),
             // for the key, whhich is the name, just get the filename
             input_file_map,

engine/language-client-codegen/src/python/templates/inlinedbaml.py.j2

@@ -1,7 +1,7 @@
 
 file_map = {
-    {% for k in file_map %}
-    {{ k.0 }}: {{ k.1 }},
+    {% for (path, contents) in file_map %}
+    {{ path }}: {{ contents }},
     {%- endfor %}
 }
 

engine/language-client-codegen/src/ruby/field_type.rs

@@ -20,7 +20,7 @@ impl ToRuby for FieldType {
                 TypeValue::String => "String".to_string(),
                 TypeValue::Null => "NilClass".to_string(),
                 // TODO: Create Baml::Types::Image
-                TypeValue::Image => "Baml::Types::Image".to_string(),
+                TypeValue::Image => "Baml::Image".to_string(),
             },
             FieldType::Union(inner) => format!(
                 // https://sorbet.org/docs/union-types