From 41848682f56943ca5688ff8cc294994a374fd30f Mon Sep 17 00:00:00 2001 From: Edward Zhang Date: Sat, 28 Dec 2024 02:57:27 -0800 Subject: [PATCH 1/4] Fix issue where traceAsync in JS does not return original value --- engine/language_client_typescript/async_context_vars.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/engine/language_client_typescript/async_context_vars.js b/engine/language_client_typescript/async_context_vars.js index 218dfcd5d..89466d054 100644 --- a/engine/language_client_typescript/async_context_vars.js +++ b/engine/language_client_typescript/async_context_vars.js @@ -78,7 +78,7 @@ class BamlCtxManager { [`arg${i}`]: arg, // generic way to label args }), {}); const [mng, span] = this.startTrace(name, params); - this.ctx.run(mng, () => { + return this.ctx.run(mng, () => { try { const response = func(...args); this.endTrace(span, response); @@ -99,7 +99,7 @@ class BamlCtxManager { [`arg${i}`]: arg, // generic way to label args }), {}); const [mng, span] = this.startTrace(name, params); - await this.ctx.run(mng, async () => { + return await this.ctx.run(mng, async () => { try { const response = await func(...args); this.endTrace(span, response); From 8faf23e90d6bec5b7825870146a3b7c1bfbca517 Mon Sep 17 00:00:00 2001 From: Edward Zhang Date: Sat, 28 Dec 2024 03:53:01 -0800 Subject: [PATCH 2/4] Handle endTrace with no return value --- engine/language_client_typescript/async_context_vars.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/language_client_typescript/async_context_vars.js b/engine/language_client_typescript/async_context_vars.js index 89466d054..a6f234d0f 100644 --- a/engine/language_client_typescript/async_context_vars.js +++ b/engine/language_client_typescript/async_context_vars.js @@ -51,7 +51,7 @@ class BamlCtxManager { return; } try { - span.finish(response, manager); + span.finish(response === undefined ? {} : response, manager); } catch (e) { console.error('BAML internal error', e); From b1debcdff220aca0295cf376993ab1125c310101 Mon Sep 17 00:00:00 2001 From: Aaron Villalpando Date: Mon, 30 Dec 2024 14:50:07 -0800 Subject: [PATCH 3/4] Fix trace functions not returning objects. Co-authored-by: kyzyx --- .../typescript_src/async_context_vars.ts | 6 +-- integ-tests/typescript/test-report.html | 51 +------------------ integ-tests/typescript/tests/tracing.test.ts | 44 ++++++++++++++++ 3 files changed, 48 insertions(+), 53 deletions(-) create mode 100644 integ-tests/typescript/tests/tracing.test.ts diff --git a/engine/language_client_typescript/typescript_src/async_context_vars.ts b/engine/language_client_typescript/typescript_src/async_context_vars.ts index 78487a088..809f5f3ed 100644 --- a/engine/language_client_typescript/typescript_src/async_context_vars.ts +++ b/engine/language_client_typescript/typescript_src/async_context_vars.ts @@ -57,7 +57,7 @@ export class BamlCtxManager { return } try { - span.finish(response, manager) + span.finish(response === undefined ? null : response, manager) } catch (e) { console.error('BAML internal error', e) } @@ -89,7 +89,7 @@ export class BamlCtxManager { {}, ) const [mng, span] = this.startTrace(name, params) - this.ctx.run(mng, () => { + return this.ctx.run(mng, () => { try { const response = func(...args) this.endTrace(span, response) @@ -113,7 +113,7 @@ export class BamlCtxManager { {}, ) const [mng, span] = this.startTrace(name, params) - await this.ctx.run(mng, async () => { + return await this.ctx.run(mng, async () => { try { const response = await func(...args) this.endTrace(span, response) diff --git a/integ-tests/typescript/test-report.html b/integ-tests/typescript/test-report.html index 19102aad1..9b2a7b338 100644 --- a/integ-tests/typescript/test-report.html +++ b/integ-tests/typescript/test-report.html @@ -257,53 +257,4 @@ font-size: 1rem; padding: 0 0.5rem; } -<<<<<<< HEAD -

Test Report

Started: 2024-12-18 19:35:20
Suites (1)
0 passed
1 failed
0 pending
Tests (83)
79 passed
4 failed
0 pending
Integ tests > should work for all inputs
single bool
passed
0.361s
Integ tests > should work for all inputs
single string list
passed
0.444s
Integ tests > should work for all inputs
return literal union
passed
0.537s
Integ tests > should work for all inputs
single class
passed
0.442s
Integ tests > should work for all inputs
multiple classes
passed
0.685s
Integ tests > should work for all inputs
single enum list
passed
0.35s
Integ tests > should work for all inputs
single float
passed
0.339s
Integ tests > should work for all inputs
single int
passed
0.436s
Integ tests > should work for all inputs
single literal int
passed
0.511s
Integ tests > should work for all inputs
single literal bool
passed
0.387s
Integ tests > should work for all inputs
single literal string
passed
0.384s
Integ tests > should work for all inputs
single class with literal prop
passed
0.559s
Integ tests > should work for all inputs
single class with literal union prop
passed
0.78s
Integ tests > should work for all inputs
single optional string
passed
0.652s
Integ tests > should work for all inputs
single map string to string
passed
0.763s
Integ tests > should work for all inputs
single map string to class
passed
0.635s
Integ tests > should work for all inputs
single map string to map
passed
0.549s
Integ tests > should work for all inputs
enum key in map
passed
0.615s
Integ tests > should work for all inputs
literal string union key in map
passed
0.821s
Integ tests > should work for all inputs
single literal string key in map
passed
0.471s
Integ tests > should work for all inputs
primitive union alias
passed
0.553s
Integ tests > should work for all inputs
map alias
passed
1s
Integ tests > should work for all inputs
alias union
passed
1.357s
Integ tests > should work for all inputs
alias pointing to recursive class
passed
1.143s
Integ tests > should work for all inputs
class pointing to alias that points to recursive class
passed
1.105s
Integ tests > should work for all inputs
recursive class with alias indirection
passed
1.023s
Integ tests > should work for all inputs
merge alias attributes
passed
0.605s
Integ tests > should work for all inputs
return alias with merged attrs
passed
0.521s
Integ tests > should work for all inputs
alias with multiple attrs
passed
0.514s
Integ tests > should work for all inputs
simple recursive map alias
passed
1.231s
Integ tests > should work for all inputs
simple recursive map alias
passed
0.608s
Integ tests > should work for all inputs
recursive alias cycles
passed
0.821s
Integ tests > should work for all inputs
json type alias cycle
passed
5.814s
Integ tests
should work for all outputs
passed
5.424s
Integ tests
works with retries1
passed
1.155s
Integ tests
works with retries2
passed
2.451s
Integ tests
works with fallbacks
passed
1.945s
Integ tests
should work with image from url
passed
1.536s
Integ tests
should work with image from base 64
passed
1.436s
Integ tests
should work with audio base 64
passed
1.739s
Integ tests
should work with audio from url
passed
1.739s
Integ tests
should support streaming in OpenAI
passed
2.323s
Integ tests
should support streaming in Gemini
passed
7.924s
Integ tests
should support AWS
passed
1.583s
Integ tests
should support streaming in AWS
passed
1.614s
Integ tests
should allow overriding the region
passed
0.045s
Integ tests
should support OpenAI shorthand
passed
13.65s
Integ tests
should support OpenAI shorthand streaming
passed
9.065s
Integ tests
should support anthropic shorthand
passed
3.223s
Integ tests
should support anthropic shorthand streaming
passed
3.834s
Integ tests
should support streaming without iterating
passed
5.387s
Integ tests
should support streaming in Claude
passed
0.927s
Integ tests
should support azure
failed
1.087s
Error: expect(received).toContain(expected) // indexOf
-
-Expected substring: "donkey"
-Received string:    "barrel-tossing ape
-king of jungle, climbs with ease
-rescues damsel fair"
-    at Object.toContain (/workspaces/baml/integ-tests/typescript/tests/integ-tests.test.ts:440:31)
Integ tests
should support azure streaming
passed
1.005s
Integ tests
should fail if azure is not configured
failed
0.051s
Error: expect(received).rejects.toThrow(expected)
-
-Expected substring: "BamlClientError"
-Received message:   "BamlError: BamlInvalidArgumentError: BAML function TestAzureFailure does not exist in baml_src/ (did you typo it?): function `TestAzureFailure` not found. Did you mean one of: TestAzure, ExpectFailure, ExtractResume, NestedAlias, TestAnthropic?"
-
-      2508 |         __baml_options__?.clientRegistry,
-      2509 |       )
-    > 2510 |       return raw.parsed() as string
-           |                  ^
-      2511 |     } catch (error: any) {
-      2512 |       const bamlError = createBamlValidationError(error);
-      2513 |       if (bamlError instanceof BamlValidationError) {
-
-      at BamlAsyncClient.parsed [as TestAzureFailure] (baml_client/async_client.ts:2510:18)
-      at tests/integ-tests.test.ts:455:7
-      at Object.<anonymous> (tests/integ-tests.test.ts:454:5)
-    at Object.toThrow (/workspaces/baml/integ-tests/typescript/node_modules/.pnpm/expect@29.7.0/node_modules/expect/build/index.js:218:22)
-    at Object.toThrow (/workspaces/baml/integ-tests/typescript/tests/integ-tests.test.ts:456:16)
-    at Promise.then.completed (/workspaces/baml/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/utils.js:298:28)
-    at new Promise (<anonymous>)
-    at callAsyncCircusFn (/workspaces/baml/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/utils.js:231:10)
-    at _callCircusTest (/workspaces/baml/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:316:40)
-    at _runTest (/workspaces/baml/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:252:3)
-    at _runTestsForDescribeBlock (/workspaces/baml/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:126:9)
-    at _runTestsForDescribeBlock (/workspaces/baml/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:121:9)
-    at run (/workspaces/baml/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/run.js:71:3)
-    at runAndTransformResultsToJestFormat (/workspaces/baml/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/legacy-code-todo-rewrite/jestAdapterInit.js:122:21)
-    at jestAdapter (/workspaces/baml/integ-tests/typescript/node_modules/.pnpm/jest-circus@29.7.0/node_modules/jest-circus/build/legacy-code-todo-rewrite/jestAdapter.js:79:19)
-    at runTestInternal (/workspaces/baml/integ-tests/typescript/node_modules/.pnpm/jest-runner@29.7.0/node_modules/jest-runner/build/runTest.js:367:16)
-    at runTest (/workspaces/baml/integ-tests/typescript/node_modules/.pnpm/jest-runner@29.7.0/node_modules/jest-runner/build/runTest.js:444:34)
Integ tests
should support vertex
failed
0.001s
Error: BamlError: Failed to read service account file: 
-
-Caused by:
-    No such file or directory (os error 2)
Integ tests
supports tracing sync
passed
0.014s
Integ tests
supports tracing async
passed
6.818s
Integ tests
should work with dynamic types single
passed
1.551s
Integ tests
should work with dynamic types enum
passed
1.222s
Integ tests
should work with dynamic literals
passed
1.523s
Integ tests
should work with dynamic types class
passed
1.831s
Integ tests
should work with dynamic inputs class
passed
0.532s
Integ tests
should work with dynamic inputs list
passed
0.492s
Integ tests
should work with dynamic output map
passed
0.841s
Integ tests
should work with dynamic output union
passed
1.729s
Integ tests
should work with nested classes
failed
0.11s
Error: BamlError: BamlClientError: Something went wrong with the LLM client: reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Domain("localhost")), port: Some(11434), path: "/v1/chat/completions", query: None, fragment: None }, source: hyper_util::client::legacy::Error(Connect, ConnectError("tcp connect error", Os { code: 111, kind: ConnectionRefused, message: "Connection refused" })) }
-    at BamlStream.parsed [as getFinalResponse] (/workspaces/baml/engine/language_client_typescript/stream.js:58:39)
-    at Object.<anonymous> (/workspaces/baml/integ-tests/typescript/tests/integ-tests.test.ts:723:19)
Integ tests
should work with dynamic client
passed
0.38s
Integ tests
should work with 'onLogEvent'
passed
1.649s
Integ tests
should work with a sync client
passed
0.417s
Integ tests
should raise an error when appropriate
passed
1.066s
Integ tests
should raise a BAMLValidationError
passed
0.576s
Integ tests
should reset environment variables correctly
passed
1.738s
Integ tests
should use aliases when serializing input objects - classes
passed
0.819s
Integ tests
should use aliases when serializing, but still have original keys in jinja
passed
1.127s
Integ tests
should use aliases when serializing input objects - enums
passed
0.508s
Integ tests
should use aliases when serializing input objects - lists
passed
0.385s
Integ tests
constraints: should handle checks in return types
passed
0.712s
Integ tests
constraints: should handle checks in returned unions
passed
0.771s
Integ tests
constraints: should handle block-level checks
passed
0.591s
Integ tests
constraints: should handle nested-block-level checks
passed
0.615s
Integ tests
simple recursive type
passed
2.564s
Integ tests
mutually recursive type
passed
2.154s
-======= -

Test Report

Started: 2024-12-19 17:53:14
Suites (1)
0 passed
1 failed
0 pending
Tests (71)
69 passed
2 failed
0 pending
Integ tests > should work for all inputs
single bool
passed
0.4s
Integ tests > should work for all inputs
single string list
passed
0.513s
Integ tests > should work for all inputs
return literal union
passed
0.41s
Integ tests > should work for all inputs
optional list and map
passed
2.148s
Integ tests > should work for all inputs
single class
passed
0.614s
Integ tests > should work for all inputs
multiple classes
passed
0.513s
Integ tests > should work for all inputs
single enum list
passed
0.92s
Integ tests > should work for all inputs
single float
passed
0.403s
Integ tests > should work for all inputs
single int
passed
0.516s
Integ tests > should work for all inputs
single literal int
passed
0.41s
Integ tests > should work for all inputs
single literal bool
passed
0.369s
Integ tests > should work for all inputs
single literal string
passed
0.433s
Integ tests > should work for all inputs
single class with literal prop
passed
0.616s
Integ tests > should work for all inputs
single class with literal union prop
passed
0.877s
Integ tests > should work for all inputs
single optional string
passed
0.365s
Integ tests > should work for all inputs
single map string to string
passed
0.594s
Integ tests > should work for all inputs
single map string to class
passed
0.635s
Integ tests > should work for all inputs
single map string to map
passed
0.545s
Integ tests > should work for all inputs
enum key in map
passed
1.022s
Integ tests > should work for all inputs
literal string union key in map
passed
0.683s
Integ tests > should work for all inputs
single literal string key in map
passed
0.921s
Integ tests
should work for all outputs
passed
5.118s
Integ tests
works with retries1
passed
1.151s
Integ tests
works with retries2
passed
2.431s
Integ tests
works with fallbacks
passed
1.946s
Integ tests
should work with image from url
passed
4.301s
Integ tests
should work with image from base 64
passed
1.843s
Integ tests
should work with audio base 64
passed
2.046s
Integ tests
should work with audio from url
passed
1.845s
Integ tests
should support streaming in OpenAI
passed
2.228s
Integ tests
should support streaming in Gemini
passed
10.231s
Integ tests
should support AWS
passed
3.004s
Integ tests
should support streaming in AWS
passed
1.517s
Integ tests
should allow overriding the region
passed
0.077s
Integ tests
should support OpenAI shorthand
passed
16.429s
Integ tests
should support OpenAI shorthand streaming
passed
12.635s
Integ tests
should support anthropic shorthand
passed
6.513s
Integ tests
should support anthropic shorthand streaming
passed
2.57s
Integ tests
should support streaming without iterating
passed
5.048s
Integ tests
should support streaming in Claude
passed
1.126s
Integ tests
should support azure
passed
0.979s
Integ tests
should support azure streaming
passed
0.905s
Integ tests
should fail if azure is not configured
passed
0.047s
Integ tests
should support vertex
failed
0.001s
Error: BamlError: Failed to read service account file: 
-
-Caused by:
-    No such file or directory (os error 2)
Integ tests
supports tracing sync
passed
0.013s
Integ tests
supports tracing async
passed
3.442s
Integ tests
should work with dynamic types single
passed
1.321s
Integ tests
should work with dynamic types enum
passed
1.027s
Integ tests
should work with dynamic literals
passed
1.079s
Integ tests
should work with dynamic types class
passed
1.461s
Integ tests
should work with dynamic inputs class
passed
0.532s
Integ tests
should work with dynamic inputs list
passed
0.605s
Integ tests
should work with dynamic output map
passed
0.849s
Integ tests
should work with dynamic output union
passed
1.919s
Integ tests
should work with nested classes
failed
0.107s
Error: BamlError: BamlClientError: Something went wrong with the LLM client: reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Domain("localhost")), port: Some(11434), path: "/v1/chat/completions", query: None, fragment: None }, source: hyper_util::client::legacy::Error(Connect, ConnectError("tcp connect error", Os { code: 111, kind: ConnectionRefused, message: "Connection refused" })) }
-    at BamlStream.parsed [as getFinalResponse] (/workspaces/baml/engine/language_client_typescript/stream.js:58:39)
-    at Object.<anonymous> (/workspaces/baml/integ-tests/typescript/tests/integ-tests.test.ts:645:19)
Integ tests
should work with dynamic client
passed
0.502s
Integ tests
should work with 'onLogEvent'
passed
1.996s
Integ tests
should work with a sync client
passed
0.666s
Integ tests
should raise an error when appropriate
passed
1.072s
Integ tests
should raise a BAMLValidationError
passed
0.462s
Integ tests
should reset environment variables correctly
passed
1.536s
Integ tests
should use aliases when serializing input objects - classes
passed
0.82s
Integ tests
should use aliases when serializing, but still have original keys in jinja
passed
0.781s
Integ tests
should use aliases when serializing input objects - enums
passed
0.649s
Integ tests
should use aliases when serializing input objects - lists
passed
0.411s
Integ tests
constraints: should handle checks in return types
passed
0.717s
Integ tests
constraints: should handle checks in returned unions
passed
0.824s
Integ tests
constraints: should handle block-level checks
passed
0.71s
Integ tests
constraints: should handle nested-block-level checks
passed
0.707s
Integ tests
simple recursive type
passed
2.164s
Integ tests
mutually recursive type
passed
2.253s
->>>>>>> canary +

Test Report

Started: 2024-12-30 14:48:03
Suites (1)
1 passed
0 failed
0 pending
Tests (4)
4 passed
0 failed
0 pending
Trace Test
traceSync with Return
passed
0.016s
Trace Test
traceSync with no return
passed
0.001s
Trace Test
traceAsync with Return
passed
0.001s
Trace Test
traceAsync with no return
passed
0.003s
\ No newline at end of file diff --git a/integ-tests/typescript/tests/tracing.test.ts b/integ-tests/typescript/tests/tracing.test.ts new file mode 100644 index 000000000..7af942fc2 --- /dev/null +++ b/integ-tests/typescript/tests/tracing.test.ts @@ -0,0 +1,44 @@ +import { traceSync, traceAsync } from '../baml_client/tracing'; + + +function fnWithReturn() : string { + console.log("fnWithReturn"); + return "Hello, world"; +} + +function fnWithNoReturn() : void { + console.log("fnWithNoReturn"); +} + +async function asyncfnWithReturn() : Promise { + console.log("asyncfnWithReturn"); + return Promise.resolve("Hello, world"); +} + +async function asyncfnWithNoReturn() : Promise { + console.log("asyncfnWithNoReturn"); +} + + +describe('Trace Test', () => { + test('traceSync with Return', async () => { + const tracedFnWithReturn = traceSync("tracedFnWithReturn", fnWithReturn); + const result = tracedFnWithReturn(); + expect(result).toBeDefined(); + expect(result).toBe('Hello, world'); + }); + test('traceSync with no return', async () => { + const tracedFnWithNoReturn = traceSync("tracedFnWithNoReturn", fnWithNoReturn); + tracedFnWithNoReturn(); + }); + test('traceAsync with Return', async () => { + const tracedFnWithReturn = traceAsync("tracedAsyncFnWithReturn", asyncfnWithReturn); + const result = await tracedFnWithReturn(); + expect(result).toBeDefined(); + expect(result).toBe('Hello, world'); + }); + test('traceAsync with no return', async () => { + const tracedFnWithNoReturn = traceAsync("tracedAsyncFnWithNoReturn", asyncfnWithNoReturn); + await tracedFnWithNoReturn(); + }); +}); \ No newline at end of file From db9e0b63802af446846d41b62c5e8fdd7b883961 Mon Sep 17 00:00:00 2001 From: Aaron Villalpando Date: Mon, 30 Dec 2024 14:55:24 -0800 Subject: [PATCH 4/4] regenerate --- engine/language_client_typescript/async_context_vars.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/language_client_typescript/async_context_vars.js b/engine/language_client_typescript/async_context_vars.js index a6f234d0f..f0bf2e6e4 100644 --- a/engine/language_client_typescript/async_context_vars.js +++ b/engine/language_client_typescript/async_context_vars.js @@ -51,7 +51,7 @@ class BamlCtxManager { return; } try { - span.finish(response === undefined ? {} : response, manager); + span.finish(response === undefined ? null : response, manager); } catch (e) { console.error('BAML internal error', e);