Merge pull request #471 from dh4340/pandora_image_gen

Pandora: Addition of Further AI and Ollama Integration

src/ghosts.pandora/Dockerfile

@@ -1,17 +1,17 @@
-# Use the slim variant of Python 3.12
-FROM python:3.12-slim
+# Use the slim variant of Python 3.10
+FROM python:3.10-slim
 
 # Set the working directory in the container
 WORKDIR /app
 
 # Install required system packages and Python dependencies
 RUN apt-get update && \
-    apt-get install --no-install-recommends -y gcc libffi-dev libgl1-mesa-glx libglib2.0-0 && \
+    apt-get install --no-install-recommends -y gcc libffi-dev libgl1-mesa-glx libglib2.0-0 ffmpeg espeak-ng && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
 # Copy only the requirements file first
-COPY requirements.txt ./
+COPY requirements.txt .
 
 # Install Python dependencies
 RUN pip install --no-cache-dir -r requirements.txt

src/ghosts.pandora/app/app_logging.py

@@ -2,57 +2,109 @@
 import os
 from logging.handlers import TimedRotatingFileHandler
 
-# Define log directory and file
-LOG_DIR = "logs"
-LOG_FILE = "app.log"
+from config.config import (FILE_LOGGING, LOG_DIR, LOG_FILE,
+                           LOG_JSON_FORMATTING, LOG_LEVEL)
+from pythonjsonlogger import jsonlogger
 
-# Create log directory if it doesn't exist
-if not os.path.exists(LOG_DIR):
-    os.makedirs(LOG_DIR)
+# Create log directory dynamically if needed
+os.makedirs(LOG_DIR, exist_ok=True)
 
 
-# Set up logging
 def setup_logger(
-    logger_name: str, log_file: str = LOG_FILE, level: int = logging.INFO
+    logger_name: str,
+    log_file: str = None,
+    level: int = None,
+    enable_file_logging: bool = FILE_LOGGING,
 ) -> logging.Logger:
     """
-    Set up a logger that logs messages to both console and file.
+    Set up a logger that logs messages to both console and optionally to a file.
 
     Args:
         logger_name (str): Name of the logger.
-        log_file (str, optional): Path to the log file. Defaults to "app.log".
-        level (int, optional): Logging level. Defaults to logging.INFO.
+        log_file (str, optional): Path to the log file. Defaults to LOG_FILE or "app.log".
+        level (int, optional): Logging level. Defaults to LOG_LEVEL or logging.INFO.
+        enable_file_logging (bool, optional): Enable or disable file logging. Defaults to True.
 
     Returns:
         logging.Logger: Configured logger instance.
     """
-    # Create a logger
+    log_file = log_file or LOG_FILE or "app.log"
+    level = level if level is not None else LOG_LEVEL or logging.INFO
+
+    # Get the logger instance
     logger = logging.getLogger(logger_name)
+
+    # Prevent duplicate handlers
+    if logger.hasHandlers():
+        logger.handlers.clear()
+
+    # Set logging level
     logger.setLevel(level)
 
-    # Formatter for logs
-    formatter = logging.Formatter(
-        "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
-    )
+    # JSON formatter for both console and file
+    json_formatter = jsonlogger.JsonFormatter(LOG_JSON_FORMATTING)
 
     # Console handler
     console_handler = logging.StreamHandler()
-    console_handler.setFormatter(formatter)
+    console_handler.setFormatter(json_formatter)
+    logger.addHandler(console_handler)
 
-    # File handler with rotation
-    file_handler = TimedRotatingFileHandler(
-        os.path.join(LOG_DIR, log_file), when="midnight", interval=1
-    )
-    file_handler.setFormatter(formatter)
-    file_handler.suffix = "%Y-%m-%d"  # Rotate logs daily
-
-    # Avoid duplicate logging if the logger is already configured
-    if not logger.hasHandlers():
-        logger.addHandler(console_handler)
-        logger.addHandler(file_handler)
+    # File handler (if enabled)
+    if enable_file_logging:
+        try:
+            file_handler = TimedRotatingFileHandler(
+                os.path.join(LOG_DIR, log_file),
+                when="midnight",
+                interval=1,
+                backupCount=30,
+            )
+            file_handler.suffix = "%Y-%m-%d"
+            file_handler.setFormatter(json_formatter)
+            logger.addHandler(file_handler)
+        except Exception as e:
+            logger.error(f"Failed to add file handler for logger '{logger_name}': {e}")
 
     return logger
 
 
-# Usage example
-# app_logger = setup_logger('app_logger')
+def configure_uvicorn_logging():
+    """
+    Configures Uvicorn logging to use JSON formatting and enable logging to a file.
+    """
+    # Disable colour output in Uvicorn by setting this environment variable
+    os.environ["UVICORN_NO_COLOR"] = "1"
+
+    json_formatter = jsonlogger.JsonFormatter(LOG_JSON_FORMATTING)
+    console_handler = logging.StreamHandler()
+    console_handler.setFormatter(json_formatter)
+
+    # File handler (if enabled)
+    file_handler = None
+    if FILE_LOGGING:
+        try:
+            file_handler = TimedRotatingFileHandler(
+                os.path.join(LOG_DIR, LOG_FILE),
+                when="midnight",
+                interval=1,
+                backupCount=30,
+            )
+            file_handler.suffix = "%Y-%m-%d"
+            file_handler.setFormatter(json_formatter)
+        except Exception as e:
+            logging.error(f"Failed to configure Uvicorn file handler: {e}")
+
+    # Apply the handlers to Uvicorn loggers
+    for uvicorn_logger_name in ["uvicorn", "uvicorn.error", "uvicorn.access"]:
+        uv_logger = logging.getLogger(uvicorn_logger_name)
+        uv_logger.handlers.clear()  # Prevent duplicate handlers
+        uv_logger.addHandler(console_handler)
+        if file_handler:
+            uv_logger.addHandler(file_handler)
+        uv_logger.propagate = False  # Avoid duplicate logs in root logger
+        uv_logger.setLevel(LOG_LEVEL or logging.INFO)
+
+    # Ensure that no colour formatting is applied by disabling colour in Uvicorn logs
+    uv_logger = logging.getLogger("uvicorn")
+    for handler in uv_logger.handlers:
+        if isinstance(handler, logging.StreamHandler):
+            handler.setFormatter(json_formatter)  # Apply JSON formatter

src/ghosts.pandora/app/config/config.py

@@ -1,11 +1,38 @@
-import app_logging
-
-# Initialize logger
-logger = app_logging.setup_logger("app_logger")
+# Define log directory and file
+LOG_DIR = "logs"
+LOG_FILE = "app.log"
+LOG_LEVEL = "DEBUG"
+FILE_LOGGING = True
+LOG_JSON_FORMATTING = "%(asctime)s %(name)s %(levelname)s %(message)s"
 
+# AI Configuration
 OLLAMA_ENABLED = True
 OLLAMA_API_URL = "http://ollama:11434/api/generate"
 OLLAMA_TIMEOUT = 60
+HTML_MODEL = "web_content"
+IMAGE_MODEL = "llama3.2"
+JSON_MODEL = "llama3.2"
+PPT_MODEL = "llama3.2"
+SCRIPT_MODEL = "llama3.2"
+STYLESHEET_MODEL = "llama3.2"
+TEXT_MODEL = "llama3.2"
+VOICE_MODEL = "llama3.2"
+XLSX_MODEL = "llama3.2"
+PDF_MODEL = "llama3.2"
+CSV_MODEL = "llama3.2"
+
+# Voice Configuration
+VOICE_GENERATION_ENABLED = False
+
+# Image Configuration
+IMAGE_GENERATION_MODEL = "stabilityai/sdxl-turbo"
+DIFFUSERS_LOCAL_FILES_ONLY = True
+
+# Video Configuration
+VIDEO_GENERATION_ENABLED = False
+
+# Faker config (not used currently)
+FAKER_LOCALE = ["en_US", "en_GB"]
 
 # List of available endpoints
 endpoints = [

src/ghosts.pandora/app/main.py

@@ -1,56 +1,38 @@
-import uvicorn
-from fastapi import FastAPI, Response, HTTPException
-
-from routes import (
-    zip_routes,
-    binary_routes,
-    json_routes,
-    csv_routes,
-    text_routes,
-    stylesheet_routes,
-    script_routes,
-    image_routes,
-    html_routes,
-    onenote_routes,
-    doc_routes,
-    executable_routes,
-    iso_routes,
-    ppt_routes,
-    xlsx_routes,
-    mp4_routes,
-    video_routes,
-    payload_routes,
-    pdf_routes,
-    unknown_routes,
-)
 import configparser
-import os
-import app_logging
 import inspect
-from config.config import OPENAPI_METADATA
+import os
+
+import uvicorn
+from app_logging import configure_uvicorn_logging, setup_logger
+from config.config import LOG_LEVEL, OPENAPI_METADATA
+from fastapi import FastAPI, HTTPException, Response
+from routes import (archive_routes, binary_routes, csv_routes, doc_routes,
+                    executable_routes, html_routes, image_routes, iso_routes,
+                    json_routes, onenote_routes, payload_routes, pdf_routes,
+                    ppt_routes, script_routes, stylesheet_routes, text_routes,
+                    unknown_routes, video_routes, voice_routes, xlsx_routes)
 
 # Unset proxy environment variables
 os.environ.pop("http_proxy", None)
 os.environ.pop("https_proxy", None)
 os.environ.pop("HTTP_PROXY", None)
 os.environ.pop("HTTPS_PROXY", None)
 
+logger = setup_logger(__name__)
 
-# Initialize logger
-logger = app_logging.setup_logger("app_logger")
+configure_uvicorn_logging()
 
-# Load configuration
+# Load configuration (original)
 config = configparser.ConfigParser()
 config.read(os.path.join("app", "config", "app.config"))
 
 # Initialize FastAPI with OpenAPI metadata from config
-app = FastAPI(
-    **OPENAPI_METADATA,  # Unpacking the OpenAPI metadata
-)
+app = FastAPI(**OPENAPI_METADATA)
 
 
 # Include routers
-app.include_router(zip_routes.router)
+app.include_router(archive_routes.router)
+app.include_router(voice_routes.router)
 app.include_router(binary_routes.router)
 app.include_router(json_routes.router)
 app.include_router(csv_routes.router)
@@ -62,7 +44,6 @@
 app.include_router(doc_routes.router)
 app.include_router(ppt_routes.router)
 app.include_router(xlsx_routes.router)
-app.include_router(mp4_routes.router)
 app.include_router(video_routes.router)
 app.include_router(pdf_routes.router)
 app.include_router(payload_routes.router)
@@ -102,7 +83,7 @@ def file_type_handler(path: str) -> Response:
         "gif": image_routes.return_image,
         "jpg": image_routes.return_image,
         "jpeg": image_routes.return_image,
-        "ico": image_routes.return_image,
+        # "ico": image_routes.return_image,
         # Documents
         "doc": doc_routes.return_doc_file,
         "docx": doc_routes.return_doc_file,
@@ -113,7 +94,9 @@ def file_type_handler(path: str) -> Response:
         "odt": doc_routes.return_doc_file,
         "one": onenote_routes.return_onenote,
         # Video files
-        "mp4": mp4_routes.return_mp4,
+        "mp4": video_routes.return_video,
+        # Sound files
+        "mp3": voice_routes.generate_synthesised_conversation,
         # Spreadsheets
         "xls": xlsx_routes.return_xlsx,
         "xlsx": xlsx_routes.return_xlsx,
@@ -135,9 +118,8 @@ def file_type_handler(path: str) -> Response:
         "ppsm": ppt_routes.return_ppt,
         "odp": ppt_routes.return_ppt,
         # Compressed Files
-        "tar": zip_routes.return_tar,
-        "gz": zip_routes.return_gz,
-        "zip": zip_routes.return_zip,
+        "tar": archive_routes.return_tar,
+        "zip": archive_routes.return_zip,
         # Executables
         "exe": executable_routes.return_exe,
         "msi": executable_routes.return_msi,
@@ -153,17 +135,14 @@ def file_type_handler(path: str) -> Response:
         None: text_routes.return_text,
     }
 
-    # Check if the file type has a corresponding handler
     if file_type in handler_mapping:
         logger.info(f"File type '{file_type}' is supported.")
         handler_function = handler_mapping[file_type]
 
-        # Prepare parameters if necessary (depends on your handler's requirements)
         params = {}
         if "path" in inspect.signature(handler_function).parameters:
             params["path"] = path
 
-        # Call the handler function
         try:
             return handler_function(**params)
         except Exception as e:
@@ -176,6 +155,12 @@ def file_type_handler(path: str) -> Response:
 
 app.include_router(unknown_routes.router)
 
-# Run the app with Uvicorn
 if __name__ == "__main__":
-    uvicorn.run(app, host="0.0.0.0", port=80)
+    uvicorn.run(
+        app,
+        host="0.0.0.0",
+        port=80,
+        log_config=None,
+        log_level=LOG_LEVEL,
+        colorize=False,
+    )