From 528057a029ada62f0fc870e75467f4c3989ee107 Mon Sep 17 00:00:00 2001 From: Greg Hale Date: Wed, 16 Oct 2024 12:22:49 -0700 Subject: [PATCH] Get typescript integ test working with constraint --- engine/baml-lib/baml-types/src/baml_value.rs | 37 +++++++++++++++++++ engine/baml-lib/baml-types/src/constraint.rs | 33 +++++++++++++---- .../src/internal/llm_client/mod.rs | 2 +- .../src/typescript/templates/index.ts.j2 | 3 +- .../src/types/function_results.rs | 2 +- .../python/baml_client/partial_types.py | 2 +- integ-tests/python/baml_client/types.py | 16 ++++---- integ-tests/typescript/baml_client/index.ts | 3 +- integ-tests/typescript/baml_client/types.ts | 22 +++++------ integ-tests/typescript/test-report.html | 2 +- .../typescript/tests/integ-tests.test.ts | 8 ++++ 11 files changed, 95 insertions(+), 35 deletions(-) diff --git a/engine/baml-lib/baml-types/src/baml_value.rs b/engine/baml-lib/baml-types/src/baml_value.rs index 8835bb9f6..892082f5d 100644 --- a/engine/baml-lib/baml-types/src/baml_value.rs +++ b/engine/baml-lib/baml-types/src/baml_value.rs @@ -533,3 +533,40 @@ fn add_checks<'a, S: SerializeMap>( } Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + use serde_json; + use crate::JinjaExpression; + + #[test] + fn test_baml_value_with_meta_serialization() { + let baml_value: BamlValueWithMeta> = + BamlValueWithMeta::String("hi".to_string(), vec![]); + let baml_value_2: BamlValueWithMeta> = + BamlValueWithMeta::Class( + "ContactInfo".to_string(), + vec![ + ("primary".to_string(), BamlValueWithMeta::Class( + "PhoneNumber".to_string(), + vec![ + ("value".to_string(), BamlValueWithMeta::String( + "123-456-7890".to_string(), + vec![ + ResponseCheck { + name: "foo".to_string(), + expression: "foo".to_string(), + status: "succeeded".to_string(), + } + ] + )) + ].into_iter().collect(), + vec![] + )) + ].into_iter().collect(), + vec![]); + assert!(serde_json::to_value(baml_value).is_ok()); + assert!(serde_json::to_value(baml_value_2).is_ok()); + } +} diff --git a/engine/baml-lib/baml-types/src/constraint.rs b/engine/baml-lib/baml-types/src/constraint.rs index 15ab3ae01..063c13a0a 100644 --- a/engine/baml-lib/baml-types/src/constraint.rs +++ b/engine/baml-lib/baml-types/src/constraint.rs @@ -16,22 +16,39 @@ pub enum ConstraintLevel { /// The user-visible schema for a failed check. #[derive(Clone, Debug, serde::Serialize)] pub struct ResponseCheck { - pub name: Option, + pub name: String, pub expression: String, pub status: String, } impl ResponseCheck { - pub fn from_constraint_result((Constraint{ level, expression, label }, succeeded): (Constraint, bool)) -> Option { - match level { - ConstraintLevel::Check => { - let status = if succeeded { "succeeded".to_string() } else { "failed".to_string() }; - Some( ResponseCheck { + /// Convert a Constraint and its status to a ResponseCheck. + /// Returns `None` if the Constraint is not a check (i.e., + /// if it doesn't meet the invariants that level==Check and + /// label==Some). + pub fn from_check_result( + ( + Constraint { + level, + expression, + label, + }, + succeeded, + ): (Constraint, bool), + ) -> Option { + match (level, label) { + (ConstraintLevel::Check, Some(label)) => { + let status = if succeeded { + "succeeded".to_string() + } else { + "failed".to_string() + }; + Some(ResponseCheck { name: label, expression: expression.0, - status + status, }) - }, + } _ => None, } } diff --git a/engine/baml-runtime/src/internal/llm_client/mod.rs b/engine/baml-runtime/src/internal/llm_client/mod.rs index 2bafebbcb..4b57aa05d 100644 --- a/engine/baml-runtime/src/internal/llm_client/mod.rs +++ b/engine/baml-runtime/src/internal/llm_client/mod.rs @@ -43,7 +43,7 @@ pub fn parsed_value_to_response(baml_value: BamlValueWithFlags) -> Result Err(anyhow::anyhow!("Failed assertion: {:?}", err)), None => Ok(baml_value_with_meta.map_meta(|cs| { cs.into_iter() - .filter_map(|res| ResponseCheck::from_constraint_result(res)) + .filter_map(|res| ResponseCheck::from_check_result(res)) .collect() })), } diff --git a/engine/language_client_codegen/src/typescript/templates/index.ts.j2 b/engine/language_client_codegen/src/typescript/templates/index.ts.j2 index 23c77ec6f..677c84088 100644 --- a/engine/language_client_codegen/src/typescript/templates/index.ts.j2 +++ b/engine/language_client_codegen/src/typescript/templates/index.ts.j2 @@ -5,6 +5,5 @@ export { b } from "./sync_client" {% endif %} export * from "./types" export * from "./tracing" -export * from "./checked" export { resetBamlEnvVars } from "./globals" -export { BamlValidationError } from "@boundaryml/baml" +export { BamlValidationError, Checked } from "@boundaryml/baml" diff --git a/engine/language_client_typescript/src/types/function_results.rs b/engine/language_client_typescript/src/types/function_results.rs index 866819af8..934b67c17 100644 --- a/engine/language_client_typescript/src/types/function_results.rs +++ b/engine/language_client_typescript/src/types/function_results.rs @@ -23,6 +23,6 @@ impl FunctionResult { .parsed_content() .map_err(|e| from_anyhow_error(e))?; - Ok(serde_json::json!(BamlValue::from(parsed))) + Ok(serde_json::to_value(parsed)?) } } diff --git a/integ-tests/python/baml_client/partial_types.py b/integ-tests/python/baml_client/partial_types.py index 7a1f5a17c..fdfe99431 100644 --- a/integ-tests/python/baml_client/partial_types.py +++ b/integ-tests/python/baml_client/partial_types.py @@ -21,7 +21,7 @@ from . import types -from .types import Checks__too_big, Checks__valid_email, Checks__valid_phone_number, Checks__earth_aged__no_infants, Checks__unreasonably_certain, Checks__regex_bad__regex_good__trivial, Checks__young_enough +from .types import Checks__valid_phone_number, Checks__earth_aged__no_infants, Checks__unreasonably_certain, Checks__too_big, Checks__valid_email, Checks__young_enough, Checks__regex_bad__regex_good__trivial ############################################################################### diff --git a/integ-tests/python/baml_client/types.py b/integ-tests/python/baml_client/types.py index 40e0c19ba..a43438179 100644 --- a/integ-tests/python/baml_client/types.py +++ b/integ-tests/python/baml_client/types.py @@ -119,9 +119,6 @@ class TestEnum(str, Enum): F = "F" G = "G" -class Checks__too_big(BaseModel): - too_big: baml_py.Check - class Checks__earth_aged__no_infants(BaseModel): earth_aged: baml_py.Check no_infants: baml_py.Check @@ -129,19 +126,22 @@ class Checks__earth_aged__no_infants(BaseModel): class Checks__young_enough(BaseModel): young_enough: baml_py.Check -class Checks__valid_email(BaseModel): - valid_email: baml_py.Check +class Checks__valid_phone_number(BaseModel): + valid_phone_number: baml_py.Check class Checks__unreasonably_certain(BaseModel): unreasonably_certain: baml_py.Check class Checks__regex_bad__regex_good__trivial(BaseModel): - regex_bad: baml_py.Check regex_good: baml_py.Check + regex_bad: baml_py.Check trivial: baml_py.Check -class Checks__valid_phone_number(BaseModel): - valid_phone_number: baml_py.Check +class Checks__too_big(BaseModel): + too_big: baml_py.Check + +class Checks__valid_email(BaseModel): + valid_email: baml_py.Check class BigNumbers(BaseModel): diff --git a/integ-tests/typescript/baml_client/index.ts b/integ-tests/typescript/baml_client/index.ts index 6e7c5fa80..bdc7d19a1 100644 --- a/integ-tests/typescript/baml_client/index.ts +++ b/integ-tests/typescript/baml_client/index.ts @@ -20,6 +20,5 @@ export { b } from "./async_client" export * from "./types" export * from "./tracing" -export * from "./checked" export { resetBamlEnvVars } from "./globals" -export { BamlValidationError } from "@boundaryml/baml" \ No newline at end of file +export { BamlValidationError, Checked } from "@boundaryml/baml" \ No newline at end of file diff --git a/integ-tests/typescript/baml_client/types.ts b/integ-tests/typescript/baml_client/types.ts index c4416acba..693d7ab8c 100644 --- a/integ-tests/typescript/baml_client/types.ts +++ b/integ-tests/typescript/baml_client/types.ts @@ -120,19 +120,23 @@ export interface Checks__valid_email { valid_email: Check } -export interface Checks__earth_aged__no_infants { - no_infants: Check - earth_aged: Check +export interface Checks__regex_bad__regex_good__trivial { + regex_bad: Check + trivial: Check + regex_good: Check +} + +export interface Checks__young_enough { + young_enough: Check } export interface Checks__unreasonably_certain { unreasonably_certain: Check } -export interface Checks__regex_bad__regex_good__trivial { - trivial: Check - regex_bad: Check - regex_good: Check +export interface Checks__earth_aged__no_infants { + earth_aged: Check + no_infants: Check } export interface Checks__valid_phone_number { @@ -143,10 +147,6 @@ export interface Checks__too_big { too_big: Check } -export interface Checks__young_enough { - young_enough: Check -} - export interface BigNumbers { a: number b: number diff --git a/integ-tests/typescript/test-report.html b/integ-tests/typescript/test-report.html index 53c1c981b..961d81cae 100644 --- a/integ-tests/typescript/test-report.html +++ b/integ-tests/typescript/test-report.html @@ -257,4 +257,4 @@ font-size: 1rem; padding: 0 0.5rem; } -

Test Report

Started: 2024-10-15 16:10:11
Suites (1)
0 passed
1 failed
0 pending
Tests (0)
0 passed
0 failed
0 pending
\ No newline at end of file +

Test Report

Started: 2024-10-16 12:20:12
Suites (1)
1 passed
0 failed
0 pending
Tests (47)
1 passed
0 failed
46 pending
Integ tests > should work for all inputs
single bool
pending
0s
Integ tests > should work for all inputs
single string list
pending
0s
Integ tests > should work for all inputs
single class
pending
0s
Integ tests > should work for all inputs
multiple classes
pending
0s
Integ tests > should work for all inputs
single enum list
pending
0s
Integ tests > should work for all inputs
single float
pending
0s
Integ tests > should work for all inputs
single int
pending
0s
Integ tests > should work for all inputs
single optional string
pending
0s
Integ tests > should work for all inputs
single map string to string
pending
0s
Integ tests > should work for all inputs
single map string to class
pending
0s
Integ tests > should work for all inputs
single map string to map
pending
0s
Integ tests
should work for all outputs
pending
0s
Integ tests
works with retries1
pending
0s
Integ tests
works with retries2
pending
0s
Integ tests
works with fallbacks
pending
0s
Integ tests
should work with image from url
pending
0s
Integ tests
should work with image from base 64
pending
0s
Integ tests
should work with audio base 64
pending
0s
Integ tests
should work with audio from url
pending
0s
Integ tests
should support streaming in OpenAI
pending
0s
Integ tests
should support streaming in Gemini
pending
0s
Integ tests
should support AWS
pending
0s
Integ tests
should support streaming in AWS
pending
0s
Integ tests
should support OpenAI shorthand
pending
0s
Integ tests
should support OpenAI shorthand streaming
pending
0s
Integ tests
should support anthropic shorthand
pending
0s
Integ tests
should support anthropic shorthand streaming
pending
0s
Integ tests
should support streaming without iterating
pending
0s
Integ tests
should support streaming in Claude
pending
0s
Integ tests
should support vertex
pending
0s
Integ tests
supports tracing sync
pending
0s
Integ tests
supports tracing async
pending
0s
Integ tests
should work with dynamic types single
pending
0s
Integ tests
should work with dynamic types enum
pending
0s
Integ tests
should work with dynamic types class
pending
0s
Integ tests
should work with dynamic inputs class
pending
0s
Integ tests
should work with dynamic inputs list
pending
0s
Integ tests
should work with dynamic output map
pending
0s
Integ tests
should work with dynamic output union
pending
0s
Integ tests
should work with nested classes
pending
0s
Integ tests
should work with dynamic client
pending
0s
Integ tests
should work with 'onLogEvent'
pending
0s
Integ tests
should work with a sync client
pending
0s
Integ tests
should raise an error when appropriate
pending
0s
Integ tests
should raise a BAMLValidationError
pending
0s
Integ tests
should reset environment variables correctly
pending
0s
Integ tests
should include checks
passed
1.298s
\ No newline at end of file diff --git a/integ-tests/typescript/tests/integ-tests.test.ts b/integ-tests/typescript/tests/integ-tests.test.ts index 5c9aba440..b2eba7a32 100644 --- a/integ-tests/typescript/tests/integ-tests.test.ts +++ b/integ-tests/typescript/tests/integ-tests.test.ts @@ -641,6 +641,14 @@ describe('Integ tests', () => { ) expect(people.length).toBeGreaterThan(0) }) + + it('should include checks', async() => { + const res = await b.ExtractContactInfo("Reach me at 333-333-4444. If that doesn't work, me@hellovai.com!") + expect(res.primary.value.value).toEqual("333-333-4444"); + expect(res.primary.value.checks.valid_phone_number.status).toEqual("succeeded"); + expect(res.secondary?.value.value).toEqual("me@hellovai.com"); + expect(res.secondary?.value.checks.valid_email.status).toEqual("succeeded"); + }) }) interface MyInterface {