Newer
Older
cortex-hub / ai-hub / integration_tests / test_provider_config.py
import os
import httpx
import pytest

BASE_URL = os.getenv("SYNC_TEST_BASE_URL", "http://127.0.0.1:8002/api/v1")

def _headers():
    return {
        "X-User-ID": os.environ.get("SYNC_TEST_USER_ID", "")
    }

def test_verify_llm_success_gemini():
    """
    Test 1: Check LLM verification with a valid API Key and Model format directly.
    """
    payload = {
        "provider_name": "gemini",
        "api_key": os.getenv("GEMINI_API_KEY", "fallback_key_if_missing"),
        "model": "gemini/gemini-3-flash-preview"
    }
    
    with httpx.Client(timeout=15.0) as client:
        r = client.post(f"{BASE_URL}/users/me/config/verify_llm", headers=_headers(), json=payload)
        
        assert r.status_code == 200, f"Expected 200 OK: {r.text}"
        data = r.json()
        
        # Depending on if the key works for litellm.acompletion('Hello') right now
        # If it returns success: True or False doesn't matter for the HTTP connection, 
        # but logically it should be processed without 400s or 500s.
        # If the key provided by the user is real, 'success' will be True.
        assert "success" in data, "Response missing 'success'"

def test_verify_llm_failure_invalid_key():
    """
    Test 2: Check LLM verification with an explicitly invalid API Key fails gracefully.
    """
    payload = {
        "provider_name": "gemini",
        "api_key": "junk_invalid_key_123",
        "model": "gemini/gemini-3-flash-preview"
    }
    
    with httpx.Client(timeout=15.0) as client:
        r = client.post(f"{BASE_URL}/users/me/config/verify_llm", headers=_headers(), json=payload)
        
        assert r.status_code == 200, f"Expected 200 OK: {r.text}"
        data = r.json()
        assert data["success"] is False, "Verification should fail for an invalid key."
        assert "message" in data and len(data["message"]) > 0

def test_update_user_llm_preferences():
    """
    Test 3: Update LLM user preferences fully and ensure they save.
    """
    payload = {
        "llm": {
            "active_provider": "gemini",
            "providers": {
                "gemini": {
                    "api_key": os.getenv("GEMINI_API_KEY", "fallback_key_if_missing"),
                    "model": "gemini/gemini-3-flash-preview"
                }
            }
        },
        "tts": {},
        "stt": {},
        "statuses": {}
    }
    
    with httpx.Client(timeout=10.0) as client:
        r = client.put(f"{BASE_URL}/users/me/config", headers=_headers(), json=payload)
        assert r.status_code == 200, f"Expected 200 OK: {r.text}"

        # Fetch config to verify
        r_get = client.get(f"{BASE_URL}/users/me/config", headers=_headers())
        assert r_get.status_code == 200
        saved_prefs = r_get.json()
        
        # Verify provider configs
        assert "preferences" in saved_prefs
        assert "llm" in saved_prefs["preferences"]
        assert saved_prefs["preferences"]["llm"]["active_provider"] == "gemini"
        
        # The API likely masks the key on GET requests inside `merge_user_config`
        # We ensure it exists
        saved_gemini = saved_prefs["preferences"]["llm"]["providers"].get("gemini", {})
        assert "api_key" in saved_gemini
        assert "model" in saved_gemini
        assert saved_gemini["model"] == "gemini/gemini-3-flash-preview"

def test_verify_llm_success_gemini_masked_key_fallback():
    """
    Test 4: Verify that a masked key (***) correctly pulls the true key from 
    the user's DB configuration automatically.
    """
    payload = {
        "provider_name": "gemini",
        "api_key": "***",
        "model": "gemini/gemini-3-flash-preview"
    }
    
    with httpx.Client(timeout=15.0) as client:
        r = client.post(f"{BASE_URL}/users/me/config/verify_llm", headers=_headers(), json=payload)
        
        # If DB lookup succeeds, it executes test using the real key saved in Test 3
        assert r.status_code == 200, f"Expected 200 OK: {r.text}"
        data = r.json()
        assert "success" in data

def test_verify_llm_unrecognized_provider():
    """
    Test 5: Check behavior for a completely unrecognized provider name.
    """
    payload = {
        "provider_name": "non_existent_provider_xyz",
        "api_key": "123",
        "model": "unknown_model"
    }
    with httpx.Client(timeout=10.0) as client:
        r = client.post(f"{BASE_URL}/users/me/config/verify_llm", headers=_headers(), json=payload)
        
        assert r.status_code == 200
        data = r.json()
        # Should gracefully fail
        assert data["success"] is False

def test_get_provider_models():
    """
    Test 6: Check retrieving the list of models for a provider via GET /users/me/config/models
    """
    with httpx.Client(timeout=10.0) as client:
        r = client.get(f"{BASE_URL}/users/me/config/models", params={"provider_name": "gemini"}, headers=_headers())
        assert r.status_code == 200, f"Expected 200 OK: {r.text}"
        
        models = r.json()
        assert isinstance(models, list), "Response should be a list"
        
        # We expect at least some gemini models to be returned 
        # (e.g. gemini-3-flash-preview, gemini-2.5-flash, etc.)
        assert len(models) > 0, "No models returned for provider 'gemini'"
        
        # Verify the structure of the returned objects
        assert "model_name" in models[0], "Missing 'model_name' key in model object"