diff --git a/ai-hub/app/app.py b/ai-hub/app/app.py index 3ea1803..1cc6705 100644 --- a/ai-hub/app/app.py +++ b/ai-hub/app/app.py @@ -64,6 +64,9 @@ # Launch periodic Ghost Mirror cleanup asyncio.create_task(_periodic_mirror_cleanup(orchestrator)) + + # Launch periodic LLM provider health check + asyncio.create_task(_periodic_provider_health_check()) except Exception as e: logger.error(f"[M6] Failed to start gRPC server: {e}") @@ -111,6 +114,76 @@ await asyncio.sleep(3600) # Run every hour +async def _periodic_provider_health_check(): + """Periodically tests actual connectivity to all system-level LLM providers.""" + await asyncio.sleep(60) # Initial delay to let the system boot fully + from app.core.providers.factory import get_llm_provider + from sqlalchemy.orm.attributes import flag_modified + import copy + + while True: + try: + from app.db.session import get_db_session + from app.db import models + from app.config import settings + + with get_db_session() as db: + super_admin_email = settings.SUPER_ADMINS[0] if settings.SUPER_ADMINS else None + if not super_admin_email: + await asyncio.sleep(3600) + continue + + admin_user = db.query(models.User).filter(models.User.email == super_admin_email).first() + if admin_user and admin_user.preferences: + prefs = copy.deepcopy(admin_user.preferences) + statuses = prefs.get("statuses", {}) + + llm_providers = prefs.get("llm", {}).get("providers", {}) + # Also inject hardcoded defaults if not in DB yet + if "deepseek" not in llm_providers and settings.DEEPSEEK_API_KEY: + llm_providers["deepseek"] = {"api_key": settings.DEEPSEEK_API_KEY, "model": settings.DEEPSEEK_MODEL_NAME} + if "gemini" not in llm_providers and settings.GEMINI_API_KEY: + llm_providers["gemini"] = {"api_key": settings.GEMINI_API_KEY, "model": settings.GEMINI_MODEL_NAME} + + changed = False + for p_name, p_data in list(llm_providers.items()): + api_key = p_data.get("api_key") + if not api_key or "***" in api_key or api_key.lower() in ["none", ""]: + continue + + status_key = f"llm_{p_name}" + new_status = "error" + try: + llm = get_llm_provider( + provider_name=p_name, + model_name=p_data.get("model") or "", + api_key_override=api_key + ) + # Short timeout to avoid blocking the loop heavily + async def test_llm(): + return await llm.acompletion(prompt="Hello") + await asyncio.wait_for(test_llm(), timeout=10.0) + new_status = "success" + except Exception as e: + logger.error(f"[Health Check] LLM {p_name} background test failed: {e}") + new_status = "error" + + if statuses.get(status_key) != new_status: + statuses[status_key] = new_status + changed = True + + if changed: + prefs["statuses"] = statuses + admin_user.preferences = prefs + flag_modified(admin_user, "preferences") + db.commit() + logger.info("[Health Check] System LLM statuses updated.") + + except Exception as e: + logger.error(f"[Health Check] Background task error: {e}") + + await asyncio.sleep(300) # Re-check every 5 minutes + def create_app() -> FastAPI: """ Factory function to create and configure the FastAPI application. diff --git a/ai-hub/app/core/services/preference.py b/ai-hub/app/core/services/preference.py index e4e22c4..30d0e04 100644 --- a/ai-hub/app/core/services/preference.py +++ b/ai-hub/app/core/services/preference.py @@ -143,12 +143,15 @@ p_data["api_key"] = self.mask_key(p_data["api_key"]) return masked_dict + merged_statuses = copy.deepcopy(system_statuses) + merged_statuses.update(user_statuses) + return schemas.ConfigResponse( preferences=schemas.UserPreferences( llm=mask_section_prefs(llm_prefs), tts=mask_section_prefs(tts_prefs), stt=mask_section_prefs(stt_prefs), - statuses=user.preferences.get("statuses", {}) if user.preferences else {} + statuses=merged_statuses ), effective=effective )