diff --git a/.agent/workflows/deploy_to_production.md b/.agent/workflows/deploy_to_production.md new file mode 100644 index 0000000..c3a72be --- /dev/null +++ b/.agent/workflows/deploy_to_production.md @@ -0,0 +1,31 @@ +--- +description: How to deploy the AI Hub application to the production server (192.168.68.113) +--- + +This workflow automates the deployment of the Cortex Hub to the production server located at `192.168.68.113`. +The production server uses `cortex-hub` inside `/home/coder/project/cortex-hub`. + +Follow these steps carefully: + +1. Sync the local codebase to a temporary directory on the production server via `rsync`. + * The destination is `/tmp/cortex-hub/`. + * We exclude `.git`, `node_modules`, `ui/client-app/node_modules`, and Python cache directories. +// turbo +2. Copy the synced files into the actual project directory replacing the old ones. + * We first clean `nginx.conf` and other related files if needed to ensure pure overwrite. +// turbo +3. Run the local deployment script on the production server. + * `cd /home/coder/project/cortex-hub` + * `sudo bash local_deployment.sh` + +### Automated Script Example + +You can also use the single bash script stored in the repository, `remote_deploy.sh`, which performs all these steps. + +1. Ensure `sshpass` is installed (`sudo apt-get install sshpass`) +2. Run `./remote_deploy.sh` + +```bash +# To run this workflow manually via the script: +bash /app/remote_deploy.sh +``` diff --git a/.agent/workflows/envoy_control_plane.md b/.agent/workflows/envoy_control_plane.md new file mode 100644 index 0000000..34ac523 --- /dev/null +++ b/.agent/workflows/envoy_control_plane.md @@ -0,0 +1,69 @@ +--- +description: How to manage Envoy resources using the EnvoyControlPlane API (192.168.68.90:8090) +--- + +This workflow describes how to interact with the custom Envoy Control Plane API to manage Envoy configurations, specifically retrieving and upserting Listeners and Clusters. + +### Base URL +The API is available internally at `http://192.168.68.90:8090`. + +### 1. View Existing Resources +You can fetch lists or single resources in JSON or YAML format. + +**List all clusters:** +```bash +# Returns a JSON document with "enabled" and "disabled" arrays of resources. +curl -s http://192.168.68.90:8090/list-clusters | jq +``` + +**List all listeners:** +```bash +curl -s http://192.168.68.90:8090/list-listeners | jq +``` + +**Get a specific resource (e.g. in YAML format for easy reading):** +```bash +# To get a cluster named "_ai_unified_server" +curl -s "http://192.168.68.90:8090/get-cluster?name=_ai_unified_server&format=yaml" + +# To get a listener +curl -s "http://192.168.68.90:8090/get-listener?name=listener_0&format=yaml" +``` + +### 2. Upsert a Cluster +To update or create a new Cluster, you must POST a specific JSON structure to `/add-cluster`. + +```bash +# 1. Prepare your raw Envoy Cluster YAML configuration (e.g., cluster.yaml) +# 2. Embed it into a JSON request wrapper +CLUSTER_YAML=$(cat cluster.yaml) +jq -n --arg name "my-cluster-name" --arg yaml "$CLUSTER_YAML" '{ + name: $name, + yaml: $yaml, + upsert: true +}' > request.json + +# 3. Send the POST Request +curl -X POST -H "Content-Type: application/json" -d @request.json http://192.168.68.90:8090/add-cluster +``` + +### 3. Upsert a Listener +Similarly, to update or create a listener, POST to `/add-listener`. + +```bash +# 1. Prepare your Listener YAML configuration (e.g., listener.yaml) +LISTENER_YAML=$(cat listener.yaml) +jq -n --arg name "listener_0" --arg yaml "$LISTENER_YAML" '{ + name: $name, + yaml: $yaml, + upsert: true +}' > request.json + +# 2. Send the POST Request +curl -X POST -H "Content-Type: application/json" -d @request.json http://192.168.68.90:8090/add-listener +``` + +### Additional Available Endpoints +* **Disable/Enable**: `/disable-cluster` / `/disable-listener` and `/enable-cluster` / `/enable-listener`. Requires JSON body `{"name": "resource_name"}`. +* **Remove**: `/remove-cluster` / `/remove-listener`. Permanently deletes a disabled resource. +* **State Management**: `/flush-to-db` saves memory state to the database, `/load-from-db` forces memory state to align with the database. diff --git a/ai-hub/Dockerfile b/ai-hub/Dockerfile index 4d9a211..049400f 100644 --- a/ai-hub/Dockerfile +++ b/ai-hub/Dockerfile @@ -20,4 +20,4 @@ # 6. Define the command to run the application # --host 0.0.0.0 makes the server accessible from outside the container -CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"] \ No newline at end of file +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000", "--proxy-headers", "--forwarded-allow-ips", "*"] \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 90990ce..206af75 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,26 @@ version: '3.8' services: - # The backend service for your AI Hub + # Unified Frontend and Nginx Gateway (Production Build) + ai-frontend: + build: ./ui/client-app + container_name: ai_unified_frontend + restart: unless-stopped + ports: + - "8002:80" + volumes: + - ./nginx.conf:/etc/nginx/nginx.conf:ro + deploy: + resources: + limits: + cpus: '0.50' + memory: 512M + reservations: + memory: 128M + depends_on: + - ai-hub + + # AI Hub Backend Service ai-hub: build: ./ai-hub container_name: ai_hub_service @@ -13,26 +32,14 @@ - OIDC_SERVER_URL=https://auth.jerxie.com - OIDC_REDIRECT_URI=https://ai.jerxie.com/api/v1/users/login/callback volumes: - # Mount the named volume to the /app/data directory in the container - ai_hub_data:/app/data:rw - ports: - # Expose the AI Hub's port - - "8002:8000" - - # The frontend service for your React application, served by Nginx - ai-ui: - # Use a build context to find the React app's Dockerfile - # This assumes your React project is in a subdirectory named 'react-app' - build: ./ui/client-app - container_name: ai_frontend_service - restart: unless-stopped - - environment: - - REACT_APP_API_BASE_URL=https://ai.jerxie.com/api/v1 - - PORT=8000 - ports: - # Map host port 8003 to container port 8000 - - "8003:8000" + deploy: + resources: + limits: + cpus: '1.0' + memory: 1G + reservations: + memory: 256M # Define the named volume for the AI hub's data volumes: diff --git a/docs/deployment_reference.md b/docs/deployment_reference.md new file mode 100644 index 0000000..af59fe4 --- /dev/null +++ b/docs/deployment_reference.md @@ -0,0 +1,101 @@ +# Cortex Hub: Deployment & CI/CD Architecture Reference + +This document serves as the comprehensive reference guide for the deployment architecture of the Cortex Hub AI system. It maps the journey of a request from the external internet to the backend code, details the containerized architecture, and outlines the automated deployment (CI/CD) paths, to help debug future production issues. + +## 1. High-Level Architecture Flow + +When an external user visits `https://ai.jerxie.com`, the request cascades through the following components: + +```mermaid +flowchart TD + A[External User] -->|HTTPS :443| B(Envoy Control Plane Proxy\nHost IP: 192.168.68.90 :10001) + B -->|HTTP :80| C{Nginx Gateway\nContainer: ai_unified_frontend\nHost IP: 192.168.68.113 :8002} + + C -->|Static Files / JS / CSS| D[React Frontend Build] + C -->|/api/v1/* Routing| E(FastAPI Backend\nContainer: ai_hub_service\nExposed: :8000 internally) +``` + +### Components Breakdown + +**1. Envoy Proxy** (`192.168.68.90`): +* **Role**: Primary Edge Proxy. Terminates external SSL (HTTPS), handles external SNI routing (matching `ai.jerxie.com`), and passes decrypted traffic internally. +* **Control API**: Envoy configurations are manipulated dynamically at `http://192.168.68.90:8090/`. +* **Important Note**: When Envoy forwards traffic internally, the scheme changes from `HTTPS` to `HTTP`. + +**2. Nginx Gateway** (`192.168.68.113` - `ai_unified_frontend` container): +* **Role**: Application Gateway. Handles the static React frontend natively, and reverses proxies all `/api/v1/` calls directly to the Python backend. +* **Network Binding**: Bound to host port `8002` (internal container port `80`) to avoid port collisions with other host services. +* **Protocol Map**: Converts connection headers between "upgrade" (WebSockets) and "close" (Standard HTTP), and passes Envoy's original scheme (`X-Forwarded-Proto`) along so FastAPI knows it originated as HTTPS. + +**3. Uvicorn / FastAPI Backend** (`ai_hub_service` container): +* **Role**: Core application logic. Needs to be aware it is sitting behind proxies to compute accurate callback URLs for tasks like **OIDC Login**, which inherently depend on accurate external domains and schemas. +* **Configuration**: Runs with `--proxy-headers` and `--forwarded-allow-ips="*"` to ensure it trusts `X-Forwarded-Proto` variables injected by Nginx. + +--- + +## 2. CI/CD Pipeline & Deployment Scripts + +The code moves from development to production through a formalized sequence, managed by dedicated local shell scripts. + +### The Scripts + +1. **`remote_deploy.sh` (The Triggger)** + * **Where it runs**: *Locally on your dev machine* + * **What it does**: + 1. Uses `rsync` over SSH (`sshpass`) to securely copy local workspace (`/app/`) changes onto the production server `192.168.68.113` under a temporary `/tmp/` directory. + 2. It specifically excludes massive or unnecessary folders (`.git`, `node_modules`, `__pycache__`). + 3. Overwrites the destination project folder (`/home/coder/project/cortex-hub`) taking care to retain system permissions. + 4. SSH triggers the `local_deployment.sh` script centrally on the production server. + +2. **`local_deployment.sh` (The Builder)** + * **Where it runs**: *Server 192.168.68.113* + * **What it does**: + 1. Destroys the old running containers. + 2. Triggers Docker Compose (`docker compose up -d --build --remove-orphans`) to rebuild the application context and discard deprecated container setups (e.g., when the UI shifted into Nginx). + 3. Performs automated database migrations running parallel idempotent logic (`app/db/migrate.py`) via the `Uvicorn` startup lifecycle. + +### How to Release +You just run: +```bash +bash /app/remote_deploy.sh +``` + +--- + +## 3. Reference Troubleshooting & Known Pitfalls + +If production encounters a bug or routing fails, these are the historically primary offenders: + +### 1. OIDC Redirection Errors (Auth Flips to HTTP) +**Symptoms**: OAuth login fails because Dex/Auth redirects the user back to `http://ai.jerxie.com/api/...` instead of `https://`. +**Root Cause**: FastAPI believes it's serving HTTP because Nginx didn't forward the Envoy proxy scheme. +**Verification Check**: +* Check `app/ai-hub/Dockerfile`: Ensure `uvicorn` terminates with `--proxy-headers --forwarded-allow-ips "*"`. +* Check `nginx.conf`: Ensure `proxy_set_header X-Forwarded-Proto` relies on the HTTP dynamically captured from Envoy, NOT a hard-coded `$scheme` string. + +### 2. WebSocket Hanging / HTTP Request Failing +**Symptoms**: Normal API calls return strange transport errors, or WebSocket voice channels refuse to upgrade. +**Root Cause**: Misconfigured HTTP `Upgrade` logic. Setting `Connection "upgrade"` unconditionally on an Nginx location breaks normal HTTP REST calls. +**Verification Check**: +* Check `nginx.conf` mappings. It must have: + ```nginx + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + ``` + ... and passed safely into the `proxy_set_header Connection $connection_upgrade;` field. + +### 3. "Port is already allocated" (Container Fails to Deploy) +**Symptoms**: Backend loads normally, but Frontend acts as `dead` or `Exit 1` silently during `remote_deploy.sh`. +**Root Cause**: Binding collisions on the production host `192.168.68.113`. (e.g., trying to bind Nginx to host port `8000` when another container `python3_11` uses it). +**Verification Check**: +* Run `docker ps -a` on `192.168.68.113`. +* Ensure the frontend port (`8002:80`) mapped inside `docker-compose.yml` does not overlap with any live container binding arrays. + +### 4. Envoy Cluster Sync Outages +**Symptoms**: `curl -v https://ai.jerxie.com` dumps a generic 404 or `503 Service Unavailable` with `server: envoy`. +**Root Cause**: The Envoy FilterChain (Listener SNI Map) doesn't trace back to a correct, valid Docker IP:Port allocation. +**Verification Check**: +* Query the Control Plane API: `curl -s http://192.168.68.90:8090/get-cluster?name=_ai_unified_server`. +* Make sure `portValue` in the JSON Endpoint equates to the one published in `docker-compose.yml` (`8002` vs `8000`). If mismatched, you must format a JSON package and `POST` it to `/add-cluster` utilizing the EnvoryControlPlane workflow. diff --git a/local_deployment.sh b/local_deployment.sh index 278f298..c84def1 100644 --- a/local_deployment.sh +++ b/local_deployment.sh @@ -15,51 +15,47 @@ # 8. Performs cleanup of old, unused Docker images. # --- Configuration --- -# Set the absolute path to your project directory. -PROJECT_DIR="/home/coder/project/cortex-hub" +# Set the project directory to the directory where this script is located. +PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &> /dev/null && pwd)" +# --- Helper Function --- +# Find the correct docker-compose command (modern plugin or standalone v1) +if docker compose version &> /dev/null; then + DOCKER_CMD="docker compose" +else + DOCKER_CMD="docker-compose" +fi + # --- Script Execution --- echo "๐Ÿš€ Starting AI Hub deployment process..." -# Check and install Docker Compose if not found. -echo "๐Ÿ” Checking for Docker Compose..." -if ! command -v docker-compose &> /dev/null; then - echo "โš ๏ธ Docker Compose not found. Installing..." - sudo curl -L "https://github.com/docker/compose/releases/download/v2.20.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose - sudo chmod +x /usr/local/bin/docker-compose - echo "โœ… Docker Compose installed." -else - echo "โœ… Docker Compose is already installed." -fi - - # Navigate to the project directory. Exit if the directory doesn't exist. cd "$PROJECT_DIR" || { echo "Error: Project directory '$PROJECT_DIR' not found. Exiting."; exit 1; } - - # Stop and remove any existing containers to ensure a clean deployment. echo "๐Ÿ›‘ Stopping and removing old Docker containers and networks..." -docker-compose down || true +sudo $DOCKER_CMD down || true # Pull the latest images if they are hosted on a registry. -# This step is optional and can be commented out if you only build from source. # echo "๐Ÿ“ฅ Pulling latest Docker images..." -# docker-compose pull +# sudo $DOCKER_CMD pull # Build new images and start the services. The `--build` flag ensures # the images are re-built from their respective Dockerfiles. +# The `--remove-orphans` flag ensures old service containers are cleaned up. echo "๐Ÿ—๏ธ Building and starting new containers..." -sudo docker-compose up -d --build +sudo $DOCKER_CMD up -d --build --remove-orphans + +echo "โœ… Containers started! Checking status..." +sudo docker ps --filter "name=ai_" echo "โœ… Deployment complete! The AI Hub application is now running." # --- Post-Deployment Cleanup --- echo "๐Ÿงน Cleaning up unused Docker resources..." -# Remove dangling images (images without a tag). The '|| true' prevents the script -# from exiting if there are no dangling images to remove. +# Remove dangling images (images without a tag). sudo docker system prune -f || true echo "โœจ Cleanup finished." \ No newline at end of file diff --git a/nginx.conf b/nginx.conf new file mode 100644 index 0000000..1e0b9dc --- /dev/null +++ b/nginx.conf @@ -0,0 +1,62 @@ +worker_processes 1; + +events { worker_connections 1024; } + +http { + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + map $http_x_forwarded_proto $proxy_x_forwarded_proto { + default $http_x_forwarded_proto; + '' $scheme; + } + + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + + # Backend upstream + upstream backend_service { + server ai-hub:8000; + } + + server { + listen 80; + server_name localhost; + + # Increase the max body size for audio uploads + client_max_body_size 50M; + + # Frontend: Serve the static production build directly + location / { + root /usr/share/nginx/html; + index index.html index.htm; + try_files $uri /index.html; + } + + # Backend: /api/v1 (proxied to the backend container) + location /api/v1 { + proxy_pass http://backend_service; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Host $host; + proxy_cache_bypass $http_upgrade; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $proxy_x_forwarded_proto; + } + + # Health check + location /health { + return 200 'OK'; + } + } +} diff --git a/remote_deploy.sh b/remote_deploy.sh new file mode 100755 index 0000000..2784a66 --- /dev/null +++ b/remote_deploy.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# Description: Automates deployment from the local environment to the production host 192.168.68.113 + +HOST="192.168.68.113" +USER="axieyangb" +PASS="a6163484a" +REMOTE_TMP="/tmp/cortex-hub/" +REMOTE_PROJ="/home/coder/project/cortex-hub" + +echo "Checking if sshpass is installed..." +if ! command -v sshpass &> /dev/null; then + echo "sshpass could not be found, installing..." + sudo apt-get update && sudo apt-get install -y sshpass +fi + +# 1. Sync local codebase to temporary directory on remote server +echo "Syncing local files to production..." +sshpass -p "$PASS" rsync -avz \ + --exclude '.git' \ + --exclude 'node_modules' \ + --exclude 'ui/client-app/node_modules' \ + --exclude 'ui/client-app/build' \ + --exclude 'ai-hub/__pycache__' \ + --exclude '.venv' \ + -e "ssh -o StrictHostKeyChecking=no" /app/ "$USER@$HOST:$REMOTE_TMP" + +if [ $? -ne 0 ]; then + echo "Rsync failed! Exiting." + exit 1 +fi + +# 2. Copy the synced files into the actual project directory replacing the old ones +echo "Overwriting production project files..." +sshpass -p "$PASS" ssh -o StrictHostKeyChecking=no "$USER@$HOST" << EOF + echo '$PASS' | sudo -S rm -rf $REMOTE_PROJ/nginx.conf + echo '$PASS' | sudo -S cp -r ${REMOTE_TMP}* $REMOTE_PROJ/ + echo '$PASS' | sudo -S chown -R $USER:$USER $REMOTE_PROJ +EOF + +# 3. Rebuild and restart services remotely +echo "Deploying on production server..." +sshpass -p "$PASS" ssh -o StrictHostKeyChecking=no "$USER@$HOST" << EOF + cd $REMOTE_PROJ + echo '$PASS' | sudo -S bash local_deployment.sh +EOF + +echo "Done! The new code is deployed to $HOST." diff --git a/ui/client-app/Dockerfile b/ui/client-app/Dockerfile index 38f34ce..6ee3227 100644 --- a/ui/client-app/Dockerfile +++ b/ui/client-app/Dockerfile @@ -1,24 +1,35 @@ -# Use an official Node.js image to run the app -FROM node:18-alpine +# Stage 1: Build the React application +FROM node:18-alpine AS build_stage -# Set the working directory inside the container WORKDIR /app -# Copy package.json and package-lock.json (or yarn.lock) +# Copy dependency files COPY package*.json ./ # Install dependencies RUN npm install -# Copy the rest of the application source code +# Copy application source COPY . . -# Set environment variables for the application -ENV HOST=0.0.0.0 -ENV PORT=8000 +# Set the API base URL for the production build +# Since everything is unified under Nginx on the same host, we use a relative path. +ENV REACT_APP_API_BASE_URL=/api/v1 -# Expose the application port -EXPOSE 8000 +# Build the production-ready static files +RUN npm run build -# Command to start the application -CMD ["npm", "start"] \ No newline at end of file +# Stage 2: Serve the static files using Nginx +FROM nginx:alpine + +# Remove default Nginx static files +RUN rm -rf /usr/share/nginx/html/* + +# Copy the build output from the first stage +COPY --from=build_stage /app/build /usr/share/nginx/html + +# Expose port 80 (Nginx default) +EXPOSE 80 + +# Start Nginx +CMD ["nginx", "-g", "daemon off;"] \ No newline at end of file