Feature: DAPR Integration

.dapr/appconfig-dev.yaml

@@ -0,0 +1,19 @@
+apiVersion: dapr.io/v1alpha1
+kind: Configuration
+metadata:
+  name: appconfig
+  namespace: default
+spec:
+  tracing:
+    samplingRate: "1"
+    stdout: true
+  features:
+    - name: SchedulerReminders
+      enabled: true
+  #    zipkin:
+  #      endpointAddress: http://localhost:9411/api/v2/spans
+  secrets:
+    scopes:
+      - storeName: secretstore-local
+        defaultAccess: allow
+        deniedSecrets: [ ]

.dapr/components/configstore.yaml

@@ -0,0 +1,20 @@
+# Reference: https://docs.dapr.io/reference/components-reference/supported-configuration-stores/redis-configuration-store/
+
+apiVersion: dapr.io/v1alpha1
+kind: Component
+metadata:
+  name: configstore
+  namespace: development
+spec:
+  type: configuration.redis
+  metadata:
+    - name: redisHost
+      secretKeyRef:
+        name: REDIS_URI
+        key: REDIS_URI
+    - name: redisPassword
+      secretKeyRef:
+        name: REDIS_PASSWORD
+        key: REDIS_PASSWORD
+auth:
+  secretStore: secretstore-local

.dapr/components/pubsub-redis.yaml

@@ -0,0 +1,25 @@
+# Reference: https://docs.dapr.io/reference/components-reference/supported-pubsub/setup-redis-pubsub/
+
+apiVersion: dapr.io/v1alpha1
+kind: Component
+metadata:
+  name: pubsub-redis
+  namespace: development
+spec:
+  type: pubsub.redis
+  version: v1
+  metadata:
+    - name: redisHost
+      secretKeyRef:
+        name: REDIS_URI
+        key: REDIS_URI
+    - name: redisPassword
+      secretKeyRef:
+        name: REDIS_PASSWORD
+        key: REDIS_PASSWORD
+    - name: consumerID
+      value: "{appID}"
+    - name: concurrency
+      value: "10"
+auth:
+  secretStore: secretstore-local

.dapr/components/secretstore-env.yaml

@@ -0,0 +1,13 @@
+# Reference: https://docs.dapr.io/reference/components-reference/supported-secret-stores/file-secret-store/
+
+apiVersion: dapr.io/v1alpha1
+kind: Component
+metadata:
+  name: secretstore-local
+  namespace: development
+spec:
+  type: secretstores.local.env
+  version: v1
+  metadata:
+    - name: prefix
+      value: "SECRETS_"

.dapr/components/statestore.yaml

@@ -0,0 +1,23 @@
+# Reference: https://docs.dapr.io/reference/components-reference/supported-state-stores/setup-redis/
+
+apiVersion: dapr.io/v1alpha1
+kind: Component
+metadata:
+  name: statestore
+  namespace: development
+spec:
+  type: state.redis
+  version: v1
+  metadata:
+    - name: redisHost
+      secretKeyRef:
+        name: REDIS_URI
+        key: REDIS_URI
+    - name: redisPassword
+      secretKeyRef:
+        name: REDIS_PASSWORD
+        key: REDIS_PASSWORD
+    - name: actorStateStore
+      value: "true"
+auth:
+  secretStore: secretstore-local

.dapr/components/subscriptions.yaml

@@ -0,0 +1,12 @@
+apiVersion: dapr.io/v2alpha1
+kind: Subscription
+metadata:
+  name: pubsub-subscription
+spec:
+  topic: budLitellmMessages
+  routes:
+    default: /notifications
+  pubsubname: pubsub-redis
+  deadLetterTopic: poisonMessages
+scopes:
+  - budlitellm

.dockerignore

@@ -9,3 +9,4 @@ tests
 .devcontainer
 *.tgz
 log.txt
+budlitellm

Dockerfile.bud

@@ -11,7 +11,7 @@ WORKDIR /app
 
 # Install build dependencies
 RUN apt-get clean && apt-get update && \
-    apt-get install -y gcc python3-dev && \
+    apt-get install -y gcc python3-dev git && \
     rm -rf /var/lib/apt/lists/*
 
 RUN pip install --upgrade pip && \
@@ -21,7 +21,7 @@ RUN pip install --upgrade pip && \
 COPY . .
 
 # Build Admin UI
-RUN chmod +x build_admin_ui.sh && ./build_admin_ui.sh
+RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
 
 # Build the package
 RUN rm -rf dist/* && python -m build
@@ -44,13 +44,13 @@ RUN pip uninstall PyJWT -y
 RUN pip install PyJWT --no-cache-dir
 
 # Build Admin UI
-RUN chmod +x build_admin_ui.sh && ./build_admin_ui.sh
+RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
 
 # Runtime stage
 FROM $LITELLM_RUNTIME_IMAGE AS runtime
 
 # Update dependencies and clean up - handles debian security issue
-RUN apt-get update && apt-get upgrade -y && rm -rf /var/lib/apt/lists/* 
+RUN apt-get update && apt-get install -y git && apt-get upgrade -y && rm -rf /var/lib/apt/lists/* 
 
 WORKDIR /app
 # Copy the current directory contents into the container at /app
@@ -64,10 +64,14 @@ COPY --from=builder /wheels/ /wheels/
 # Install the built wheel using pip; again using a wildcard if it's the only file
 RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
 
+# ensure gptcache is installed
+RUN pip install git+https://github.com/BudEcosystem/BudServeGPTCache.git#egg=gptcache
+RUN pip install git+https://github.com/BudEcosystem/bud-microframe.git#egg=budmicroframe
+
 # Generate prisma client
 RUN prisma generate
-RUN chmod +x entrypoint.sh
+# RUN chmod +x entrypoint.sh
 
-EXPOSE 4000/tcp
+EXPOSE 4010/tcp
 
-ENTRYPOINT ["sh", "-c", "cp /app/config/config.yaml /app && litellm --config /app/config.yaml --port 4000"]
+# ENTRYPOINT ["sh", "-c", "litellm --config /app/litellm_config.yaml --port 4000"]

deploy/docker-compose-dev.yaml

@@ -0,0 +1,70 @@
+include:
+  - ./docker-compose-redis.yaml
+
+services:
+  bud-litellm-app:
+    image: bud-microframe/$APP_NAME:$NAMESPACE
+    profiles:
+      - app
+    container_name: bud-mf-$NAMESPACE-$APP_NAME
+    build:
+      context: ..
+      dockerfile: ./Dockerfile.bud
+    # command: [ "litellm", "--config", "/app/litellm_config.yaml", "--port", $APP_PORT ]
+    # command: sh -c "cd litellm/proxy && litellm --config /app/litellm_config.yaml --port $APP_PORT"
+    command: sh -c "tail -f /dev/null"
+    # command : sh -c "alembic -c ./budcluster/alembic.ini upgrade head && tail -f /dev/null" # && uvicorn $APP_NAME.main:app --host 0.0.0.0 --port $APP_PORT --reload"
+    # ports:
+    #   - "$DAPR_GRPC_PORT:$DAPR_GRPC_PORT" # Dapr instances communicate over gRPC so gRPC port needs to be exposed
+    #   - "$DAPR_HTTP_PORT:$DAPR_HTTP_PORT" # Expose Dapr HTTP port for service invocation
+    #   - "$APP_PORT:$APP_PORT" # Expose app port for debugging purposes
+    volumes:
+      - ../:/app/
+      - ../cache:/app/cache
+    env_file:
+      - path: ../.env
+        required: true
+    depends_on:
+      - bud-litellm-redis
+      - bud-litellm-placement
+    network_mode: host
+  bud-litellm-sidecar:
+    container_name: bud-mf-$NAMESPACE-$APP_NAME-dapr
+    image: "daprio/daprd:edge"
+    command: [
+      "./daprd",
+      "--app-id", "$APP_NAME",
+      "--app-port", "$APP_PORT",
+      "--dapr-http-port", "$DAPR_HTTP_PORT",
+      "--dapr-grpc-port", "$DAPR_GRPC_PORT",
+      "--placement-host-address", "$DAPR_PLACEMENT_HOST:$DAPR_PLACEMENT_PORT", # Dapr's placement service can be reach via the docker DNS entry
+      "--metrics-port", "$DAPR_METRICS_PORT",
+      "--resources-path", "/components",
+      "--config", "/config/appconfig.yaml",
+      "--log-as-json"
+    ]
+    env_file:
+      - path: ../.env
+        required: true
+    volumes:
+      # - "../crypto-keys:/crypto-keys"
+      - "${DAPR_COMPONENTS:-../.dapr/components/}:/components" # Mount the components folder for the runtime to use. The mounted location must match the --resources-path argument.
+      - "${DAPR_APP_CONFIG:-../.dapr/appconfig-dev.yaml}:/config/appconfig.yaml" # Mount the config file for the runtime to use. The mounted location must match the --config argument.
+      - ./:/app/
+    network_mode: "host"
+    # ports:
+    #   - "$DAPR_GRPC_PORT:$DAPR_GRPC_PORT"
+    #   - "$DAPR_HTTP_PORT:$DAPR_HTTP_PORT"
+    # network_mode: "service:app"
+  bud-litellm-placement:
+    container_name: bud-mf-$NAMESPACE-$APP_NAME-placement
+    image: "daprio/placement:edge"
+    command: [ "./placement", "--port", "$DAPR_PLACEMENT_PORT" ]
+    ports:
+      - "$DAPR_PLACEMENT_PORT:$DAPR_PLACEMENT_PORT"
+    networks:
+      - bud-litellm-network
+networks:
+  bud-litellm-network:
+    name: bud-mf-$NAMESPACE-$APP_NAME
+    driver: bridge

deploy/docker-compose-redis.yaml

@@ -0,0 +1,28 @@
+services:
+  bud-litellm-redis:
+    container_name: bud-mf-$NAMESPACE-litellm-redis
+    image: redis:alpine
+    ports:
+      - "$REDIS_PORT:$REDIS_PORT"
+    command:
+      - /bin/sh
+      - -c
+      - redis-server --requirepass "${SECRETS_REDIS_PASSWORD:?REDIS_PASSWORD variable is not set}" --port ${REDIS_PORT}
+    sysctls:
+      net.core.somaxconn: 1024
+    healthcheck:
+      test: [ "CMD", "redis-cli", "--raw", "incr", "ping" ]
+    volumes:
+      - budlitellm_redis_data:/data
+    # networks:
+    #   - bud-microframe-nw
+    # network_mode: "host"
+    networks:
+      - bud-litellm-network
+
+volumes:
+  budlitellm_redis_data:
+networks:
+  bud-litellm-network:
+    name: bud-mf-$NAMESPACE-$APP_NAME
+    driver: bridge

deploy/start_dev.sh

@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+
+#
+#  -----------------------------------------------------------------------------
+#  Copyright (c) 2024 Bud Ecosystem Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#  -----------------------------------------------------------------------------
+#
+
+DAPR_COMPONENTS="../.dapr/components/"
+DAPR_APP_CONFIG="../.dapr/appconfig-dev.yaml"
+
+DOCKER_COMPOSE_FILE="./deploy/docker-compose-dev.yaml"
+BUILD_FLAG=""
+DETACH_FLAG=""
+
+function display_help() {
+    echo "Usage: $0 [options]"
+    echo
+    echo "Options:"
+    echo "  --dapr-components        Set the dapr components folder path, this should be relative to the deploy directory (default: $DAPR_COMPONENTS)"
+    echo "  --dapr-app-config        Set the dapr app config path, this should be relative to the deploy directory (default: $DAPR_APP_CONFIG)"
+    echo "  -f FILE                  Specify the Docker Compose file to use, this should be relative to your current directory (default: $DOCKER_COMPOSE_FILE)"
+    echo "  --build                  Include this flag to force a rebuild of the Docker containers"
+    echo "  --skip-app               Include this flag to skip the app container"
+    echo "  -d                       Include this flag to detach and run the containers in background"
+    echo "  --help                   Display this help message and exit"
+    echo
+    echo "Example:"
+    echo "  $0 -f docker-compose-local.yaml --build"
+    echo "  This will use 'docker-compose-local.yaml' and force a rebuild of the containers."
+    echo
+    exit 0
+}
+
+# Parse arguments
+while [[ "$#" -gt 0 ]]; do
+    case $1 in
+        --dapr-components) DAPR_COMPONENTS="$2"; shift ;;
+        --dapr-app-config) DAPR_APP_CONFIG="$2"; shift ;;
+        -f) DOCKER_COMPOSE_FILE="$2"; shift ;;
+        --build) BUILD_FLAG="--build" ;;
+        --skip-app) SKIP_APP_FLAG="true" ;;
+        -d) DETACH_FLAG="-d" ;;
+        --help) display_help ;;
+        *) echo "Unknown parameter passed: $1"; show_help; exit 1 ;;
+    esac
+    shift
+done
+
+set -a
+source ./.env
+set +a
+
+export REDIS_PORT=$(echo "${SECRETS_REDIS_URI:-redis:6379}" | cut -d':' -f2)
+
+: ${APP_NAME:?Application name is required, use --app-name flag to specify the name.}
+
+# Print the environment variables
+echo "****************************************************"
+echo "*                                                  *"
+echo "*         Starting Microservice Environment        *"
+echo "*                                                  *"
+echo "****************************************************"
+echo ""
+echo "🛠 App Name            : $APP_NAME"
+echo "🌐 App Port             : $APP_PORT"
+echo "🔑 Redis Uri           : $SECRETS_REDIS_URI"
+echo "🌍 Dapr HTTP Port      : $DAPR_HTTP_PORT"
+echo "🌍 Dapr gRPC Port      : $DAPR_GRPC_PORT"
+echo "🛠 Namespace            : $NAMESPACE"
+echo "📊 Log Level           : $LOG_LEVEL"
+echo "🗂 Config Store Name    : $CONFIGSTORE_NAME"
+echo "🔐 Secret Store Name   : $SECRETSTORE_NAME"
+echo "🛠 Dapr Components     : $DAPR_COMPONENTS"
+echo "🛠 Dapr App Config     : $DAPR_APP_CONFIG"
+echo "🛠 Docker Compose File : $DOCKER_COMPOSE_FILE"
+echo "🚀 Build flag          : $BUILD_FLAG"
+echo ""
+echo "****************************************************"
+
+# Bring up Docker Compose
+echo "Bringing up Docker Compose with file: $DOCKER_COMPOSE_FILE"
+if [ -z "$SKIP_APP_FLAG" ]; then
+    docker compose --profile app -f "$DOCKER_COMPOSE_FILE" up $BUILD_FLAG $DETACH_FLAG
+else
+    docker compose -f "$DOCKER_COMPOSE_FILE" up $BUILD_FLAG $DETACH_FLAG
+fi