fix: build ruby by default (#823)

`cargo build` in `engine/` will now surface build errors in the Ruby
code.

This will require users to have `ruby >= 3.1` installed on their system,
which we'll enforce using `mise` and `direnv` - I've added instructions
for how to do this in `engine/baml-runtime/build.rs`, which will
complain loudly, and with helpful error messages, if the checks it
applies on a user's machine don't pass.

Also update our CI and release workflows to build with Ruby (windows and
arm64-linux do not build Ruby, due to weird issues with both)

- releases building:
https://github.com/BoundaryML/baml/actions/runs/10068806151/job/27834787301
- releases pushing:
https://github.com/BoundaryML/baml/actions/runs/10068719096/job/27834639984

.github/examples/release_language_client_ruby.yaml → .github/workflows/build-ruby-release.reusable.yaml

@@ -1,38 +1,42 @@
 name: Release engine/language_client_ruby
 
 on:
-  workflow_dispatch: {}
+  workflow_call: {}
 
 permissions:
   contents: read
   id-token: write
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
+  # suffix is important to prevent a concurrency deadlock with the calling workflow
+  group: ${{ github.workflow }}-${{ github.ref }}-build-ruby
   cancel-in-progress: true
 
 jobs:
   build:
     strategy:
       fail-fast: false
       matrix:
-        platform:
-          - x86_64-linux
-          - aarch64-linux
-          - x86_64-darwin
-          - arm64-darwin
+        _:
+          - platform: x86_64-linux
+            # This is necessary because rb-sys-dock depends on manylinux2014,
+            # which is based on CentOS 7 which is EOL as of July 2024 Once
+            # rake-compiler-dock switches to manylinux_2_28 and rb-sys-dock
+            # picks that up, we can pick their updates up and then drop this.
+            # See https://github.com/oxidize-rb/rb-sys/issues/402 and
+            # https://github.com/rake-compiler/rake-compiler-dock/issues/122
+            # for more details.
+            rb-sys-dock-setup: ./x86-64_linux-setup.sh
+          - platform: aarch64-linux
+          - platform: x86_64-darwin
+          - platform: arm64-darwin
           #- x64-mingw-ucrt
 
     runs-on: ubuntu-latest
     defaults:
       run:
         working-directory: engine/language_client_ruby
     steps:
-      - uses: rubygems/configure-rubygems-credentials@main
-        with:
-          # https://rubygems.org/profile/oidc/api_key_roles/rg_oidc_akr_p6x4xz53qtk948na3bgy
-          role-to-assume: rg_oidc_akr_p6x4xz53qtk948na3bgy
-
       - uses: actions/checkout@v4
 
       - uses: oxidize-rb/actions/setup-ruby-and-rust@main
@@ -65,34 +69,6 @@ jobs:
           echo "RB_SYS_DOCK_CACHE_DIR=$rb_sys_dock_cache_dir" >> $GITHUB_ENV
           echo "rb_sys_version=$rb_sys_version" >> $GITHUB_OUTPUT
 
-      #- name: Setup caching
-      #  uses: actions/cache@v4
-      #  with:
-      #    path: |
-      #      ${{ env.RB_SYS_DOCK_CACHE_DIR }}
-      #      tmp/rb-sys-dock/${{ inputs.platform }}/target
-      #    key: rb-sys-dock-${{ inputs.cache-version }}-${{ inputs.platform }}-${{ hashFiles('**/Gemfile.lock', '**/Cargo.lock') }}
-      #    save-always: ${{ inputs.cache-save-always == 'true' }}
-      #    restore-keys: |
-      #      rb-sys-dock-${{ inputs.cache-version }}-${{ inputs.platform }}-
-
-      #- name: Install cargo-cache
-      #  uses: oxidize-rb/actions/cargo-binstall@v1
-      #  id: install-cargo-cache
-      #  if: inputs.cargo-cache-clean == 'true'
-      #  with:
-      #    crate: cargo-cache
-      #    version: 0.8.3
-      #    strategies: quick-install
-
-      #- name: Clean the cargo cache
-      #  if: inputs.cargo-cache-clean == 'true'
-      #  uses: oxidize-rb/actions/post-run@v1
-      #  with:
-      #    run: cargo-cache --autoclean
-      #    cwd: ${{ inputs.working-directory }}
-      #    always: ${{ inputs.cache-save-always == 'true' }}
-
       - name: Setup rb-sys
         shell: bash
         run: |
@@ -108,35 +84,24 @@ jobs:
       - name: Build gem
         shell: bash
         working-directory: engine
-        if: ${{ matrix.platform == 'x86_64-linux' }}
         run: |
           : Compile gem
           echo "Docker Working Directory: $(pwd)"
           set -x
 
+          # Unfortunately we can't actually parallelize the Ruby versions
+          # because they get bundled into the same gem
           rb-sys-dock \
-            --platform ${{ matrix.platform }} \
+            --platform ${{ matrix._.platform }} \
             --mount-toolchains \
             --directory language_client_ruby \
-            --ruby-versions 3.3,3.2,3.1,3.0,2.7 \
+            --ruby-versions 3.3,3.2,3.1 \
             --build \
-            -- sudo yum install -y perl-IPC-Cmd
-
-      - name: Build gem
+            -- ${{ matrix._.rb-sys-dock-setup }}
+          
+      - name: Show built gem
         shell: bash
-        working-directory: engine
-        if: ${{ matrix.platform != 'x86_64-linux' }}
-        run: |
-          : Compile gem
-          echo "Docker Working Directory: $(pwd)"
-          set -x
-
-          rb-sys-dock \
-            --platform ${{ matrix.platform }} \
-            --mount-toolchains \
-            --directory language_client_ruby \
-            --ruby-versions 3.3,3.2,3.1,3.0,2.7 \
-            --build
+        run: find pkg -name '*.gem'
 
       #################################################################################################################
       #
@@ -147,7 +112,14 @@ jobs:
       #
       #################################################################################################################
 
-      - run: |
-          for i in $(ls pkg/*.gem); do
-            gem push $i
-          done
+      - name: Upload Ruby artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: gem-${{ matrix._.platform }}
+          path: engine/language_client_ruby/pkg/*.gem
+          if-no-files-found: error
+
+      # - run: |
+      #     for i in $(ls pkg/*.gem); do
+      #       gem push $i
+      #     done

.github/workflows/primary.yml

@@ -46,10 +46,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: pnpm/action-setup@v3
-        with:
-          version: 9.0.6
-          run_install: false
+      - uses: jdx/mise-action@v2
       - uses: actions/setup-node@v4
         with:
           node-version: 20
@@ -81,6 +78,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+      - uses: jdx/mise-action@v2
       - uses: dtolnay/rust-toolchain@stable
         with:
           toolchain: stable

.github/workflows/release.yml

@@ -28,6 +28,7 @@ jobs:
     name: Build WASM
     steps:
       - uses: actions/checkout@v4
+      - uses: jdx/mise-action@v2
       - uses: dtolnay/rust-toolchain@stable
         with:
           toolchain: stable
@@ -38,10 +39,6 @@ jobs:
       - name: Bindgen
         run: cargo install -f wasm-bindgen-cli@0.2.92
         working-directory: engine/baml-schema-wasm
-      - uses: pnpm/action-setup@v3
-        with:
-          version: 8
-          run_install: false
       # Set up Node.js
       - name: Setup Node.js
         uses: actions/setup-node@v4
@@ -97,6 +94,9 @@ jobs:
           path: typescript/vscode-ext/packages/web-panel/dist
           if-no-files-found: error
 
+  build-ruby-release:
+    uses: ./.github/workflows/build-ruby-release.reusable.yaml
+
   build-release:
     strategy:
       fail-fast: false
@@ -105,6 +105,7 @@ jobs:
           - target: aarch64-apple-darwin
             host: macos-14
             node_build: pnpm build --target aarch64-apple-darwin
+            mise: true
 
           # Disabled as python is not supported on aarch64 windows
           # - target: aarch64-pc-windows-msvc
@@ -117,20 +118,27 @@ jobs:
             # need a new version of manylinux to build crates on arm64-linux
             container: ghcr.io/rust-cross/manylinux_2_28-cross:aarch64
             node_build: pnpm build --target aarch64-unknown-linux-gnu --use-napi-cross
+            cargo_args: -p baml-typescript-ffi -p baml-python-ffi
+            baml_build_help: 'off'
 
           - target: x86_64-apple-darwin
             host: macos-latest
             node_build: pnpm build --target x86_64-apple-darwin
+            mise: true
 
           - target: x86_64-pc-windows-msvc
             host: windows-latest
             node_build: pnpm build --target x86_64-pc-windows-msvc
+            setup-python-architecture: x64
+            cargo_args: -p baml-typescript-ffi -p baml-python-ffi
+            baml_build_help: 'off'
 
           - target: x86_64-unknown-linux-gnu
             host: ubuntu-latest
             # Using any of the manylinux containers breaks a bunch of stuff: actions/checkout here, pnpm build there - not worth it.
             container: null
             node_build: pnpm build --target x86_64-unknown-linux-gnu --use-napi-cross
+            mise: true
 
     name: Build ${{ matrix._.target }}
     runs-on: ${{ matrix._.host }}
@@ -141,17 +149,23 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: "3.8"
-          architecture: ${{ matrix._.host == 'windows-latest' && 'x64' || null }}
+          architecture: ${{ matrix._.setup-python-architecture }}
+
+      - uses: jdx/mise-action@v2
+        if: ${{ matrix._.mise }}
+
       - uses: pnpm/action-setup@v3
         with:
           version: 9.0.6
           run_install: false
+
       - uses: actions/setup-node@v4
         with:
           node-version: 20
           cache: pnpm
           cache-dependency-path: |
             engine/language_client_typescript/pnpm-lock.yaml
+
       # Install rust
       - uses: dtolnay/rust-toolchain@stable
         with:
@@ -171,8 +185,10 @@ jobs:
           cache-on-failure: true
 
       - name: Build Rust
-        run: cargo build --release --target ${{ matrix._.target }}
+        run: cargo build --release --target ${{ matrix._.target }} ${{ matrix._.cargo_args }}
         working-directory: engine
+        env:
+          BAML_BUILD_HELP: ${{ matrix._.baml_build_help }}
 
       # Build Node
       - name: PNPM Build
@@ -202,11 +218,21 @@ jobs:
           name: bindings-${{ matrix._.target }}
           path: engine/language_client_typescript/*.node
           if-no-files-found: error
+
+  # placeholder fan-in step
+  assert-all-builds-passed:
+    runs-on: ubuntu-latest
+    needs:
+      - build-release
+      - build-ruby-release
+      - build-wasm
+    steps:
+      - run: true
 
   publish-to-pypi:
     environment: release
     if: github.ref_type == 'tag'
-    needs: [build-release, build-wasm]
+    needs: [assert-all-builds-passed] 
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -227,7 +253,7 @@ jobs:
   publish-to-npm:
     environment: release
     if: github.ref_type == 'tag'
-    needs: [build-release, build-wasm]
+    needs: [assert-all-builds-passed] 
     runs-on: ubuntu-latest
     env:
       NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
@@ -274,6 +300,35 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+
+  publish-to-rubygems:
+    environment: release
+    if: github.ref_type == 'tag'
+    needs: [assert-all-builds-passed] 
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: rubygems/configure-rubygems-credentials@main
+        with:
+          # https://rubygems.org/profile/oidc/api_key_roles/rg_oidc_akr_p6x4xz53qtk948na3bgy
+          role-to-assume: rg_oidc_akr_p6x4xz53qtk948na3bgy
+
+      - uses: jdx/mise-action@v2
+
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: gem-*
+          path: engine/language_client_ruby/pkg/
+          merge-multiple: true
+
+      - working-directory: engine/language_client_ruby
+        run: |
+          set -euxo pipefail
+          find pkg
+          for i in $(ls pkg/*.gem); do
+            gem push $i
+          done
 
   # publish-vscode:
   #   environment: release
@@ -331,7 +386,11 @@ jobs:
   release-github:
     runs-on: ubuntu-latest
     if: github.ref_type == 'tag'
-    needs: [publish-to-pypi, publish-to-npm]
+    needs: 
+      - publish-to-pypi
+      - publish-to-npm
+      - publish-to-rubygems
+      # - publish-vscode
     steps:
       - uses: actions/checkout@v4
 

.mise.toml

@@ -1,2 +1,5 @@
 [tools]
+node = "latest"
 ruby = "3.1"
+pnpm = "latest"
+poetry = "latest"

engine/Cargo.toml

@@ -6,7 +6,7 @@ members = [
   "baml-fmt",
   "baml-runtime",
   "baml-schema-wasm",
-  "language-client-codegen",
+  "language_client_codegen",
   "language_client_python",
   "language_client_ruby/ext/ruby_ffi",
   "language_client_typescript",
@@ -21,10 +21,9 @@ default-members = [
   # and it's OK to not build this by default because we have to build ts separately (this may
   # also have something to do with resolver=2 and how duplicate dep builds are handled)
   # "baml-schema-wasm",
-  "language-client-codegen",
+  "language_client_codegen",
   "language_client_python",
-  # requires ruby to be installed, does not build for <3.1
-  # "language_client_ruby/ext/ruby_ffi",
+  "language_client_ruby/ext/ruby_ffi",
   "language_client_typescript",
 ]
 
@@ -55,7 +54,7 @@ walkdir = "2.5.0"
 web-time = "1.1.0"
 
 baml-types = { path = "baml-lib/baml-types" }
-internal-baml-codegen = { path = "language-client-codegen" }
+internal-baml-codegen = { path = "language_client_codegen" }
 internal-baml-core = { path = "baml-lib/baml-core" }
 internal-baml-jinja = { path = "baml-lib/jinja" }