feat(trace): implement otlp http backend (#7)

Signed-off-by: GALLLASMILAN <gallas.milan@gmail.com>

.dockerignore

@@ -11,7 +11,8 @@ temp
 Dockerfile
 coverage
 .husky
-.env.testing
+.env.testing.local
+.env.testing.docker
 .env
 .travis.yml
 scripts

.env.testing → .env.testing.docker

@@ -1,17 +1,18 @@
-PORT=3000
+PORT=4318
 AUTH_KEY=valid-api-key
 FASTIFY_BODY_LIMIT=10485760
 
 REDIS_URL=redis://redis:6379/0
 MONGODB_URL=mongodb://mongo:mongo@mongo:27017
 DATA_EXPIRATION_IN_DAYS=7
 
-POST_TEST=4002
-
 MLFLOW_API_URL=http://mlflow:8080/
 MLFLOW_AUTHORIZATION=BASE_AUTH
 MLFLOW_USERNAME=admin
 MLFLOW_PASSWORD=password
 MLFLOW_DEFAULT_EXPERIMENT_ID=0
 MLFLOW_TRACE_DELETE_IN_BATCHES_CRON_PATTERN="0 */1 * * * *"
 MLFLOW_TRACE_DELETE_IN_BATCHES_BATCH_SIZE=100
+
+BEE_FRAMEWORK_INSTRUMENTATION_METRICS_ENABLED=false
+BEE_FRAMEWORK_INSTRUMENTATION_ENABLED=true

.env.testing.local

@@ -0,0 +1,4 @@
+BEE_FRAMEWORK_INSTRUMENTATION_METRICS_ENABLED=false
+BEE_FRAMEWORK_INSTRUMENTATION_ENABLED=true
+PORT=4318
+AUTH_KEY=valid-api-key

.github/workflows/main.yml

@@ -0,0 +1,38 @@
+name: Lint, Build, Test
+
+on:
+  push:
+    branches: ['main']
+  pull_request:
+    branches: ['main']
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  main:
+    timeout-minutes: 20
+    name: Lint & Build & Test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Enable Corepack
+        run: corepack enable
+      - name: Use Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version-file: '.nvmrc'
+          cache: 'yarn'
+      - name: Install dependencies
+        run: yarn install
+      - name: Code Lint
+        run: yarn lint
+      - name: Code Format
+        run: yarn format
+      - name: Commits Lint
+        run: yarn commitlint --verbose --from "${{ github.event.pull_request.base.sha || github.event.commits[0].id }}" --to "${{ github.event.pull_request.head.sha || github.event.head_commit.id }}"
+      - name: Build
+        run: yarn build

CONTRIBUTING.md

@@ -44,7 +44,8 @@ yarn install
 5.  **Setup environmental variables:** You should set the following variables in your `.env` file in the repository’s root.
 
 - copy [.env.example](./.env.example) and fill the right values.
-- The [.env.testing](./.env.testing) is used for integration API testing in docker
+- The [.env.testing.local](./.env.testing.local) is used directly in the vitest runtime for all tests running locally.
+- The [.env.testing.docker](./.env.testing.docker) is used for integration API testing in docker.
 
 6.  **Run migrations** You should run database migrations to prepare the necessary data for a valid application run.
 

README.md

@@ -22,11 +22,12 @@
    - [Running the server via Docker](#running-the-server-via-docker)
    - [Running the Server via Node.js](#running-the-server-via-nodejs)
    - [Development Mode](#development-mode)
-4. [🧪 Run tests](#-run-tests)
-5. [📣 Publishing](#-publishing)
-6. [Code of conduct](#code-of-conduct)
-7. [Legal notice](#legal-notice)
-8. [📖 Docs](#-docs)
+4. [🪑 Setup with OpenTelemetry](#-setup-with-opentelemetry)
+5. [🧪 Run tests](#-run-tests)
+6. [📣 Publishing](#-publishing)
+7. [Code of conduct](#code-of-conduct)
+8. [Legal notice](#legal-notice)
+9. [📖 Docs](#-docs)
 
 ## 👩‍💻 Get started with Observe
 
@@ -131,6 +132,13 @@ For development mode with hot-reloading, use:
 yarn dev
 ```
 
+### 🪑 Setup with OpenTelemetry
+
+The Observe service can be used only with the [bee-agent-framework](https://github.com/i-am-bee/bee-agent-framework).
+See [OpenTelemetry Instrumentation in Bee-Agent-Framework](https://github.com/i-am-bee/bee-agent-framework/blob/main/docs/instrumentation.md) for instructions on how to enable the instrumentation there.
+
+For more info on setting up observe service in the [OpenTelemetry collector](https://opentelemetry.io/docs/collector/) read the [Configuration](https://opentelemetry.io/docs/collector/configuration/#exporters) page.
+
 ### 🧪 Run tests
 
 > This project uses integration API tests that require a running observe instance to be executed.

compose.yml

@@ -7,11 +7,11 @@ services:
         GIT_TAG: ${TAG:-testing}
         BUILD_DATETIME: ${BUILD_DATETIME:-}
     ports:
-      - '${OBSERVE_API_EXPOSED_PORT:-4002}:3000'
+      - '${OBSERVE_API_EXPOSED_PORT:-4318}:4318'
     env_file:
-      - .env.testing
+      - .env.testing.docker
     healthcheck:
-      test: wget --no-verbose --tries=1 --spider http://0.0.0.0:3000/health || exit 1
+      test: wget --no-verbose --tries=1 --spider http://0.0.0.0:4318/health || exit 1
       interval: 10s
       timeout: 5s
       retries: 5
@@ -24,5 +24,5 @@ services:
       [
         'sh',
         '-c',
-        "while ! curl --silent --fail http://observe_api:3000/health; do echo 'Waiting for API...'; sleep 5; done"
+        "while ! curl --silent --fail http://observe_api:4318/health; do echo 'Waiting for API...'; sleep 5; done"
       ]

package.json

@@ -6,7 +6,8 @@
   "description": "Observability API server for bee-agent-framework",
   "type": "module",
   "scripts": {
-    "build": "rm -rf dist && tsc",
+    "build": "rm -rf dist && tsc && cp -R src/protos dist/protos",
+    "proto:generate": "proto-loader-gen-types --defaults --oneofs --longs=Number --enums=String --grpcLib=@grpc/grpc-js --outDir=./src/types/generated ./src/protos/*.proto && node scripts/add_js_extensions.js",
     "start:infra": "docker compose -f compose-before.yml up -d mongo redis mlflow",
     "start:dev": "tsx watch ./src/index.ts | pino-pretty --singleLine",
     "stop:infra": "docker compose -f compose-before.yml down",
@@ -30,34 +31,42 @@
     "@commitlint/config-conventional": "^19.5.0",
     "@commitlint/types": "^19.5.0",
     "@fastify/type-provider-json-schema-to-ts": "^3.0.0",
+    "@opentelemetry/exporter-trace-otlp-proto": "^0.55.0",
+    "@opentelemetry/instrumentation": "^0.55.0",
+    "@opentelemetry/sdk-node": "^0.55.0",
+    "@opentelemetry/semantic-conventions": "^1.28.0",
     "@release-it/conventional-changelog": "^8.0.1",
     "@types/dotenv-safe": "^8.1.6",
     "@types/node": "^20.16.5",
     "@typescript-eslint/eslint-plugin": "^7.13.0",
     "@typescript-eslint/parser": "^7.13.0",
     "@vitest/coverage-v8": "^1.6.0",
+    "bee-agent-framework": "^0.0.45",
     "eslint": "^8.56.0",
     "eslint-config-prettier": "^9.1.0",
     "eslint-plugin-import": "^2.29.1",
     "eslint-plugin-prettier": "^5.1.3",
     "globals": "^15.5.0",
     "husky": "^9.0.11",
     "lint-staged": "^15.2.7",
+    "ollama": "^0.5.10",
     "pino-pretty": "^11.1.0",
     "prettier": "^3.3.2",
     "release-it": "^17.6.0",
     "ts-node": "^10.9.2",
     "tsx": "^4.11.0",
     "typescript": "^5.4.5",
     "typescript-eslint": "^7.13.0",
-    "vitest": "^1.6.0"
+    "vitest": "^2.1.3"
   },
   "dependencies": {
     "@fastify/auth": "^4.6.1",
     "@fastify/request-context": "^5.1.0",
     "@fastify/swagger": "^8.14.0",
     "@fastify/swagger-ui": "^3.0.0",
     "@godaddy/terminus": "^4.12.1",
+    "@grpc/grpc-js": "^1.12.2",
+    "@grpc/proto-loader": "^0.7.13",
     "@mikro-orm/cli": "6.2.9",
     "@mikro-orm/core": "6.2.9",
     "@mikro-orm/migrations-mongodb": "6.2.9",
@@ -71,7 +80,9 @@
     "http-status-codes": "^2.3.0",
     "ioredis": "^5.4.1",
     "json-schema-to-ts": "^3.1.0",
-    "pino": "^9.2.0"
+    "pino": "^9.2.0",
+    "protobufjs": "^7.4.0",
+    "typescript-json-schema": "^0.65.1"
   },
   "resolutions": {
     "@types/node": "20.16.5"

scripts/add_js_extensions.js

@@ -0,0 +1,43 @@
+/**
+ * Copyright 2024 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+const directory = path.resolve(__dirname, '../src/types/generated');
+
+async function addJsExtensions(dir) {
+  const files = await fs.promises.readdir(dir);
+  for (const file of files) {
+    const filePath = path.join(dir, file);
+    const stat = await fs.promises.lstat(filePath);
+    if (stat.isDirectory()) {
+      await addJsExtensions(filePath);
+    } else if (file.endsWith('.ts')) {
+      let content = await fs.promises.readFile(filePath, 'utf8');
+      content = content.replace(/(from\s+['"](\.?\.\/[^'"]+))(?<!\.js)(?=['"])/g, '$1.js');
+      await fs.promises.writeFile(filePath, content, 'utf8');
+    }
+  }
+}
+
+addJsExtensions(directory)
+  .then(() => console.log('Added .js extensions to imports in generated TypeScript files.'))
+  .catch(console.error);

scripts/copyright.sh

@@ -24,9 +24,9 @@ if ! command -v nwa &> /dev/null && command -v go &> /dev/null; then
 fi
 
 if command -v nwa &> /dev/null; then
-  nwa add -l apache -c "IBM Corp." scripts executor test src examples
+  nwa add -l apache -c "IBM Corp." scripts/**/*.{js,sh} src/**/*.{ts,js}
 elif command -v docker &> /dev/null; then
-  docker run -it -v "${PWD}:/src" ghcr.io/b1nary-gr0up/nwa:main add -l apache -c "IBM Corp." scripts executor test src examples
+  docker run -it -v "${PWD}:/src" ghcr.io/b1nary-gr0up/nwa:main add -l apache -c "IBM Corp." scripts src
 else
   echo "Error: 'nwa' is not available. Either install it manually or install go/docker."
   exit 1

scripts/publish.sh

@@ -1,4 +1,3 @@
-#!/bin/bash
 # Copyright 2024 IBM Corp.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 set -e
 
 REPOSITORY="docker.io"

scripts/test_local.sh

@@ -32,7 +32,7 @@ docker compose build
 if timeout 120 yarn start:infra 2>/dev/null ; then
     echo '🆗 docker containers are working and ready to test'
 else
-    echo '❌ There is some error with the docker compose up command. Check your environments in .env.testing and see docker logs ...'
+    echo '❌ There is some error with the docker compose up command. Check your environments in .env.testing.docker and see docker logs ...'
     docker compose logs
     exit 1;
 fi
@@ -44,7 +44,7 @@ yarn mikro-orm-esm migration:up
 if timeout 120 docker compose up -d 2>/dev/null ; then
     echo '🆗 docker containers are working and ready to test'
 else
-    echo '❌ There is some error with the docker compose up command. Check your environments in .env.testing and see docker logs ...'
+    echo '❌ There is some error with the docker compose up command. Check your environments in .env.testing.docker and see docker logs ...'
     docker compose logs
     exit 1;
 fi

src/app.ts

@@ -27,6 +27,7 @@ import { swaggerPlugin } from './utils/swagger.js';
 import { constants } from './utils/constants.js';
 import traceModule from './trace/trace.module.js';
 import spanModule from './span/span.module.js';
+import systemModule from './system/system.module.js';
 import { LogLabels, fastifyLogger, getCorrelationId } from './utils/logger.js';
 import { contextPlugin } from './utils/context.js';
 import { errorPlugin } from './utils/error.js';
@@ -79,8 +80,9 @@ export async function bootstrap({ port }: BootstrapOptions) {
 
   // Declare all routes
   app.after(() => {
-    app.register(traceModule);
-    app.register(spanModule);
+    app.register(traceModule, { prefix: '/v1' });
+    app.register(spanModule, { prefix: '/v1' });
+    app.register(systemModule);
   });
 
   const url = await app.listen({ port, host: '0.0.0.0' });

src/mlflow/utils/assembly-trace-span-tree.ts

@@ -30,12 +30,9 @@ export function assemblyTraceSpanTree({
   requestId: string;
   trace: Trace;
 }): TraceMlflowTreeItem[] {
-  const createSpanTree = (
-    treeItem: TreeItem,
-    parentId: string | null = null
-  ): TraceMlflowTreeItem[] => {
+  const createSpanTree = (treeItem: TreeItem, parentId: string): TraceMlflowTreeItem[] => {
     const span = {
-      name: treeItem.id,
+      name: treeItem.name,
       context: {
         trace_id: requestId,
         span_id: treeItem.id
@@ -69,13 +66,15 @@ export function assemblyTraceSpanTree({
     return [span, ...treeItem.children.flatMap((item) => createSpanTree(item, treeItem.id))];
   };
 
+  const mainTreeItem = trace.tree[0];
+
   return [
     // the main trace span - only one top span must be defined
     {
-      name: 'trace',
+      name: mainTreeItem.name,
       context: {
         trace_id: requestId,
-        span_id: trace.id
+        span_id: mainTreeItem.id
       },
       parent_id: null,
       attributes: {
@@ -90,6 +89,6 @@ export function assemblyTraceSpanTree({
       events: [],
       status_message: ''
     },
-    ...trace.tree.flatMap((treeItem) => createSpanTree(treeItem, trace.id))
+    ...mainTreeItem.children.flatMap((treeItem) => createSpanTree(treeItem, mainTreeItem.id))
   ];
 }