diff --git a/ai-hub/app/api/routes/agent_update.py b/ai-hub/app/api/routes/agent_update.py index 05a6a31..f63c1db 100644 --- a/ai-hub/app/api/routes/agent_update.py +++ b/ai-hub/app/api/routes/agent_update.py @@ -13,7 +13,7 @@ import tarfile import logging from fastapi import APIRouter, HTTPException, Request -from fastapi.responses import StreamingResponse, JSONResponse +from fastapi.responses import StreamingResponse, JSONResponse, Response logger = logging.getLogger(__name__) @@ -168,8 +168,8 @@ with open(installer_path, "rb") as f: content = f.read() - return StreamingResponse( - iter([content]), + return Response( + content=content, media_type="text/x-python", headers={"Content-Disposition": "attachment; filename=bootstrap_installer.py"} ) diff --git a/ai-hub/app/core/templates/provisioning/provision.py.j2 b/ai-hub/app/core/templates/provisioning/provision.py.j2 index 4876fb5..7cafb28 100644 --- a/ai-hub/app/core/templates/provisioning/provision.py.j2 +++ b/ai-hub/app/core/templates/provisioning/provision.py.j2 @@ -17,9 +17,19 @@ f.write("""{{ config_yaml }}""") # 3. Download bootstrap_installer.py -print("[*] Downloading installer...") installer_url = "{{ base_url }}/api/v1/agent/installer" -urllib.request.urlretrieve(installer_url, "bootstrap_installer.py") +print(f"[*] Downloading installer from {installer_url} ...") +try: + urllib.request.urlretrieve(installer_url, "bootstrap_installer.py") +except Exception as e: + print(f"❌ Failed to download installer: {e}") + sys.exit(1) + +size = os.path.getsize("bootstrap_installer.py") +if size < 100: + with open("bootstrap_installer.py") as f: content = f.read() + print(f"❌ Downloaded file is too small or corrupt ({size} bytes): {content}") + sys.exit(1) # 4. Run installer with --daemon (or --non-interactive) print("[*] Bootstrapping agent...") @@ -30,6 +40,12 @@ "--token", "{{ invite_token }}", "--node-id", "{{ node_id }}" ] -subprocess.run(cmd) - -print("✅ Provisioning complete! Node should be online in the Mesh Dashboard shortly.") +try: + subprocess.run(cmd, check=True) + print("✅ Provisioning complete! Node should be online in the Mesh Dashboard shortly.") +except subprocess.CalledProcessError as e: + print(f"❌ Provisioning failed! Installer exited with code {e.returncode}") + sys.exit(e.returncode) +except Exception as e: + print(f"❌ Provisioning crashed: {e}") + sys.exit(1) diff --git a/ai-hub/native_hub.log b/ai-hub/native_hub.log index 4c5ff27..0b59be4 100644 --- a/ai-hub/native_hub.log +++ b/ai-hub/native_hub.log @@ -5,7 +5,7 @@ INFO:app.core.tools.registry:Registered dynamic tool plugin: 'mesh_terminal_control' INFO:app.core.tools.registry:Registered dynamic tool plugin: 'mesh_wait_tasks' INFO:app.core.tools.registry:Registered dynamic tool plugin: 'read_skill_artifact' -INFO: Started server process [6031] +INFO: Started server process [78674] INFO: Waiting for application startup. INFO:app.db.migrate:Starting database migrations... INFO:app.db.migrate:Column 'audio_path' already exists in 'messages'. @@ -30,6 +30,12 @@ INFO:app.core.orchestration.scheduler:[Scheduler] Agent background services (Zombie Sweeper & CRON) started. INFO:app.core.skills.bootstrap:Checking for system skills bootstrapping... INFO:app.core.skills.bootstrap:System skills bootstrap completed. +ERROR:app.core.orchestration.scheduler:[Scheduler] Zombie Sweeper iteration failed: (sqlite3.OperationalError) no such column: agent_instances.total_runs +[SQL: SELECT agent_instances.id AS agent_instances_id, agent_instances.template_id AS agent_instances_template_id, agent_instances.session_id AS agent_instances_session_id, agent_instances.mesh_node_id AS agent_instances_mesh_node_id, agent_instances.status AS agent_instances_status, agent_instances.current_workspace_jail AS agent_instances_current_workspace_jail, agent_instances.last_heartbeat AS agent_instances_last_heartbeat, agent_instances.total_runs AS agent_instances_total_runs, agent_instances.successful_runs AS agent_instances_successful_runs, agent_instances.total_tokens_accumulated AS agent_instances_total_tokens_accumulated, agent_instances.total_input_tokens AS agent_instances_total_input_tokens, agent_instances.total_output_tokens AS agent_instances_total_output_tokens, agent_instances.total_running_time_seconds AS agent_instances_total_running_time_seconds, agent_instances.tool_call_counts AS agent_instances_tool_call_counts +FROM agent_instances +WHERE agent_instances.status = ? AND agent_instances.last_heartbeat < ?] +[parameters: ('active', '2026-03-24 04:57:40.128841')] +(Background on this error at: https://sqlalche.me/e/20/e3q8) INFO: Application startup complete. INFO: Uvicorn running on http://0.0.0.0:8000 (Press CTRL+C to quit) ✅ Loading configuration from app/config.yaml @@ -78,749 +84,220 @@ - VERSION: 1.0.0 ------------------------------------ Creating database tables... -INFO: 127.0.0.1:44394 - "HEAD /api/v1/users/login/local HTTP/1.1" 405 Method Not Allowed -INFO: 127.0.0.1:44406 - "POST /api/v1/users/login/local HTTP/1.1" 200 OK +INFO: 127.0.0.1:40396 - "HEAD /api/v1/users/login/local HTTP/1.1" 405 Method Not Allowed +INFO: 127.0.0.1:40412 - "POST /api/v1/users/login/local HTTP/1.1" 200 OK INFO:app.core.services.preference:Saving updated global preferences via admin 915a44b3-7ab9-4670-bb86-cb5ae31304bc 🏠 Configuration synchronized to app/config.yaml -INFO: 127.0.0.1:44406 - "PUT /api/v1/users/me/config HTTP/1.1" 200 OK -INFO: 127.0.0.1:44406 - "POST /api/v1/users/admin/groups HTTP/1.1" 409 Conflict -INFO: 127.0.0.1:44406 - "GET /api/v1/users/admin/groups HTTP/1.1" 200 OK -INFO: 127.0.0.1:44406 - "PUT /api/v1/users/admin/groups/75fb001c-25a2-4f40-97e8-6d9ce38f1c2c HTTP/1.1" 200 OK -INFO: 127.0.0.1:44406 - "PUT /api/v1/users/admin/users/915a44b3-7ab9-4670-bb86-cb5ae31304bc/group HTTP/1.1" 200 OK -INFO: 127.0.0.1:44406 - "POST /api/v1/nodes/admin?admin_id=915a44b3-7ab9-4670-bb86-cb5ae31304bc HTTP/1.1" 409 Conflict +INFO: 127.0.0.1:40412 - "PUT /api/v1/users/me/config HTTP/1.1" 200 OK +INFO: 127.0.0.1:40412 - "POST /api/v1/users/admin/groups HTTP/1.1" 409 Conflict +INFO: 127.0.0.1:40412 - "GET /api/v1/users/admin/groups HTTP/1.1" 200 OK +INFO: 127.0.0.1:40412 - "PUT /api/v1/users/admin/groups/75fb001c-25a2-4f40-97e8-6d9ce38f1c2c HTTP/1.1" 200 OK +INFO: 127.0.0.1:40412 - "PUT /api/v1/users/admin/users/915a44b3-7ab9-4670-bb86-cb5ae31304bc/group HTTP/1.1" 200 OK +INFO: 127.0.0.1:40412 - "POST /api/v1/nodes/admin?admin_id=915a44b3-7ab9-4670-bb86-cb5ae31304bc HTTP/1.1" 409 Conflict INFO:app.core.services.node_registry:[📋] NodeRegistry: Deregistered test-node-1 -INFO: 127.0.0.1:44406 - "DELETE /api/v1/nodes/admin/test-node-1?admin_id=915a44b3-7ab9-4670-bb86-cb5ae31304bc HTTP/1.1" 200 OK +INFO: 127.0.0.1:40412 - "DELETE /api/v1/nodes/admin/test-node-1?admin_id=915a44b3-7ab9-4670-bb86-cb5ae31304bc HTTP/1.1" 200 OK INFO:app.api.routes.nodes:[admin] Created node 'test-node-1' by admin 915a44b3-7ab9-4670-bb86-cb5ae31304bc -INFO: 127.0.0.1:44406 - "POST /api/v1/nodes/admin?admin_id=915a44b3-7ab9-4670-bb86-cb5ae31304bc HTTP/1.1" 200 OK -INFO: 127.0.0.1:44406 - "POST /api/v1/nodes/admin?admin_id=915a44b3-7ab9-4670-bb86-cb5ae31304bc HTTP/1.1" 409 Conflict +INFO: 127.0.0.1:40412 - "POST /api/v1/nodes/admin?admin_id=915a44b3-7ab9-4670-bb86-cb5ae31304bc HTTP/1.1" 200 OK +INFO: 127.0.0.1:40412 - "POST /api/v1/nodes/admin?admin_id=915a44b3-7ab9-4670-bb86-cb5ae31304bc HTTP/1.1" 409 Conflict INFO:app.core.services.node_registry:[📋] NodeRegistry: Deregistered test-node-2 -INFO: 127.0.0.1:44406 - "DELETE /api/v1/nodes/admin/test-node-2?admin_id=915a44b3-7ab9-4670-bb86-cb5ae31304bc HTTP/1.1" 200 OK +INFO: 127.0.0.1:40412 - "DELETE /api/v1/nodes/admin/test-node-2?admin_id=915a44b3-7ab9-4670-bb86-cb5ae31304bc HTTP/1.1" 200 OK INFO:app.api.routes.nodes:[admin] Created node 'test-node-2' by admin 915a44b3-7ab9-4670-bb86-cb5ae31304bc [NodeRegistry] DB mark-offline failed for test-node-2: UPDATE statement on table 'agent_nodes' expected to update 1 row(s); 0 were matched. -INFO: 127.0.0.1:44406 - "POST /api/v1/nodes/admin?admin_id=915a44b3-7ab9-4670-bb86-cb5ae31304bc HTTP/1.1" 200 OK -INFO: 127.0.0.1:44406 - "POST /api/v1/users/admin/groups HTTP/1.1" 409 Conflict +INFO: 127.0.0.1:40412 - "POST /api/v1/nodes/admin?admin_id=915a44b3-7ab9-4670-bb86-cb5ae31304bc HTTP/1.1" 200 OK +INFO: 127.0.0.1:40412 - "POST /api/v1/users/admin/groups HTTP/1.1" 409 Conflict INFO:app.api.routes.agent_update:[AgentUpdate] Version check → 1.0.77 -INFO: 127.0.0.1:44410 - "GET /api/v1/agent/version HTTP/1.1" 200 OK INFO:app.api.routes.agent_update:[AgentUpdate] Version check → 1.0.77 -INFO: 127.0.0.1:44414 - "GET /api/v1/agent/version HTTP/1.1" 200 OK +INFO: 127.0.0.1:40420 - "GET /api/v1/agent/version HTTP/1.1" 200 OK +INFO: 127.0.0.1:40416 - "GET /api/v1/agent/version HTTP/1.1" 200 OK INFO:app.core.grpc.services.grpc_server:[gRPC] Incoming RPC Call: /agent.AgentOrchestrator/SyncConfiguration INFO:app.core.grpc.services.grpc_server:[gRPC] Incoming RPC Call: /agent.AgentOrchestrator/SyncConfiguration -INFO:app.core.grpc.services.grpc_server:[🔑] SyncConfiguration REQUEST from test-node-1 (token prefix: WHNo...) -INFO:app.core.grpc.services.grpc_server:[🔑] SyncConfiguration REQUEST from test-node-2 (token prefix: ZMfp...) -INFO:app.core.grpc.services.grpc_server:[🔑] Token validated for test-node-1 (owner: 915a44b3-7ab9-4670-bb86-cb5ae31304bc) -INFO:app.core.grpc.services.grpc_server:[🔑] Handshake successful for test-node-1 (owner: 915a44b3-7ab9-4670-bb86-cb5ae31304bc) +INFO:app.core.grpc.services.grpc_server:[🔑] SyncConfiguration REQUEST from test-node-1 (token prefix: nw-b...) +INFO:app.core.grpc.services.grpc_server:[🔑] SyncConfiguration REQUEST from test-node-2 (token prefix: 3GoS...) INFO:app.core.grpc.services.grpc_server:[🔑] Token validated for test-node-2 (owner: 915a44b3-7ab9-4670-bb86-cb5ae31304bc) -INFO:app.core.services.node_registry:[📋] NodeRegistry: Registered test-node-1 (owner: 915a44b3-7ab9-4670-bb86-cb5ae31304bc) | Stats enabled INFO:app.core.grpc.services.grpc_server:[🔑] Handshake successful for test-node-2 (owner: 915a44b3-7ab9-4670-bb86-cb5ae31304bc) -INFO:app.core.grpc.services.grpc_server:[gRPC] Incoming RPC Call: /agent.AgentOrchestrator/ReportHealth +INFO:app.core.grpc.services.grpc_server:[🔑] Token validated for test-node-1 (owner: 915a44b3-7ab9-4670-bb86-cb5ae31304bc) INFO:app.core.services.node_registry:[📋] NodeRegistry: Registered test-node-2 (owner: 915a44b3-7ab9-4670-bb86-cb5ae31304bc) | Stats enabled +INFO:app.core.grpc.services.grpc_server:[🔑] Handshake successful for test-node-1 (owner: 915a44b3-7ab9-4670-bb86-cb5ae31304bc) +INFO:app.core.grpc.services.grpc_server:[gRPC] Incoming RPC Call: /agent.AgentOrchestrator/ReportHealth +INFO:app.core.services.node_registry:[📋] NodeRegistry: Registered test-node-1 (owner: 915a44b3-7ab9-4670-bb86-cb5ae31304bc) | Stats enabled INFO:app.core.grpc.services.grpc_server:[gRPC] Incoming RPC Call: /agent.AgentOrchestrator/TaskStream INFO:app.core.grpc.services.grpc_server:[gRPC] Incoming RPC Call: /agent.AgentOrchestrator/ReportHealth INFO:app.core.grpc.services.grpc_server:[gRPC] Incoming RPC Call: /agent.AgentOrchestrator/TaskStream -INFO:app.core.grpc.services.grpc_server:[*] Node test-node-1 Attempting to establish TaskStream... INFO:app.core.grpc.services.grpc_server:[*] Node test-node-2 Attempting to establish TaskStream... -INFO:app.core.grpc.services.grpc_server:[*] Node test-node-1 Online (TaskStream established) INFO:app.core.grpc.services.grpc_server:[*] Node test-node-2 Online (TaskStream established) - [📁🔄] Triggering Resync Check for test-node-1... +INFO:app.core.grpc.services.grpc_server:[*] Node test-node-1 Attempting to establish TaskStream... +INFO:app.core.grpc.services.grpc_server:[*] Node test-node-1 Online (TaskStream established) [📁🔄] Triggering Resync Check for test-node-2... + [📁🔄] Triggering Resync Check for test-node-1... + [📁⚠️] Failed to fetch active sessions for node reconciliation: (sqlite3.OperationalError) no such column: sessions.auto_clear_history +[SQL: SELECT sessions.id AS sessions_id, sessions.user_id AS sessions_user_id, sessions.title AS sessions_title, sessions.provider_name AS sessions_provider_name, sessions.stt_provider_name AS sessions_stt_provider_name, sessions.tts_provider_name AS sessions_tts_provider_name, sessions.feature_name AS sessions_feature_name, sessions.created_at AS sessions_created_at, sessions.is_archived AS sessions_is_archived, sessions.is_cancelled AS sessions_is_cancelled, sessions.sync_workspace_id AS sessions_sync_workspace_id, sessions.attached_node_ids AS sessions_attached_node_ids, sessions.node_sync_status AS sessions_node_sync_status, sessions.sync_config AS sessions_sync_config, sessions.restrict_skills AS sessions_restrict_skills, sessions.allowed_skill_names AS sessions_allowed_skill_names, sessions.system_prompt_override AS sessions_system_prompt_override, sessions.is_locked AS sessions_is_locked, sessions.auto_clear_history AS sessions_auto_clear_history +FROM sessions +WHERE sessions.is_archived = 0 AND sessions.sync_workspace_id IS NOT NULL] +(Background on this error at: https://sqlalche.me/e/20/e3q8) + [📁⚠️] Failed to fetch active sessions for node reconciliation: (sqlite3.OperationalError) no such column: sessions.auto_clear_history +[SQL: SELECT sessions.id AS sessions_id, sessions.user_id AS sessions_user_id, sessions.title AS sessions_title, sessions.provider_name AS sessions_provider_name, sessions.stt_provider_name AS sessions_stt_provider_name, sessions.tts_provider_name AS sessions_tts_provider_name, sessions.feature_name AS sessions_feature_name, sessions.created_at AS sessions_created_at, sessions.is_archived AS sessions_is_archived, sessions.is_cancelled AS sessions_is_cancelled, sessions.sync_workspace_id AS sessions_sync_workspace_id, sessions.attached_node_ids AS sessions_attached_node_ids, sessions.node_sync_status AS sessions_node_sync_status, sessions.sync_config AS sessions_sync_config, sessions.restrict_skills AS sessions_restrict_skills, sessions.allowed_skill_names AS sessions_allowed_skill_names, sessions.system_prompt_override AS sessions_system_prompt_override, sessions.is_locked AS sessions_is_locked, sessions.auto_clear_history AS sessions_auto_clear_history +FROM sessions +WHERE sessions.is_archived = 0 AND sessions.sync_workspace_id IS NOT NULL] +(Background on this error at: https://sqlalche.me/e/20/e3q8) ================================================== 📡 CORTEX MESH DASHBOARD | 2 Nodes Online -------------------------------------------------- - 🟢 test-node-1 | Workers: 0 | Running: 0 tasks - Capabilities: {'has_sudo': 'true', 'has_display': 'false', 'os_release': '6.10.11-linuxkit', 'arch': 'aarch64', 'os': 'linux', 'gpu': 'none', 'local_ip': '172.27.0.2', 'is_root': 'false', 'shell': 'v1'} 🟢 test-node-2 | Workers: 0 | Running: 0 tasks Capabilities: {'has_sudo': 'true', 'has_display': 'false', 'os_release': '6.10.11-linuxkit', 'arch': 'aarch64', 'os': 'linux', 'gpu': 'none', 'local_ip': '172.27.0.2', 'is_root': 'false', 'shell': 'v1'} -================================================== - - [📁🧹] Running Mirror Cleanup. Active Sessions: 18 - [📁🧹] Running Mirror Cleanup. Active Sessions: 4 - [📁🗑️] Purged orphaned ghost mirror: 30 - [📁⚠️] Failed to purge orphaned mirror 30: [Errno 2] No such file or directory: './data/mirrors/30' - -================================================== -📡 CORTEX MESH DASHBOARD | 2 Nodes Online --------------------------------------------------- 🟢 test-node-1 | Workers: 0 | Running: 0 tasks Capabilities: {'has_sudo': 'true', 'has_display': 'false', 'os_release': '6.10.11-linuxkit', 'arch': 'aarch64', 'os': 'linux', 'gpu': 'none', 'local_ip': '172.27.0.2', 'is_root': 'false', 'shell': 'v1'} - 🟢 test-node-2 | Workers: 0 | Running: 0 tasks - Capabilities: {'has_sudo': 'true', 'has_display': 'false', 'os_release': '6.10.11-linuxkit', 'arch': 'aarch64', 'os': 'linux', 'gpu': 'none', 'local_ip': '172.27.0.2', 'is_root': 'false', 'shell': 'v1'} ================================================== -INFO:app.api.routes.nodes:[admin] Created node 'test-agent-node-ec22f68d' by admin 915a44b3-7ab9-4670-bb86-cb5ae31304bc -INFO: 127.0.0.1:41294 - "POST /api/v1/nodes/admin?admin_id=915a44b3-7ab9-4670-bb86-cb5ae31304bc HTTP/1.1" 200 OK -INFO:app.core.grpc.services.assistant:[📁📤] Workspace session-31-11d0eb16 prepared on server for offline node test-agent-node-ec22f68d -INFO: 127.0.0.1:41294 - "POST /api/v1/agents/deploy HTTP/1.1" 200 OK -INFO: 127.0.0.1:41294 - "GET /api/v1/sessions/31 HTTP/1.1" 200 OK -INFO: 127.0.0.1:41294 - "GET /api/v1/agents/a8bd22bf-9812-4702-95a9-13afaa5ac038/triggers HTTP/1.1" 200 OK +ERROR:app.app:[📁🧹] Ghost Mirror periodic cleanup fail: (sqlite3.OperationalError) no such column: sessions.auto_clear_history +[SQL: SELECT sessions.id AS sessions_id, sessions.user_id AS sessions_user_id, sessions.title AS sessions_title, sessions.provider_name AS sessions_provider_name, sessions.stt_provider_name AS sessions_stt_provider_name, sessions.tts_provider_name AS sessions_tts_provider_name, sessions.feature_name AS sessions_feature_name, sessions.created_at AS sessions_created_at, sessions.is_archived AS sessions_is_archived, sessions.is_cancelled AS sessions_is_cancelled, sessions.sync_workspace_id AS sessions_sync_workspace_id, sessions.attached_node_ids AS sessions_attached_node_ids, sessions.node_sync_status AS sessions_node_sync_status, sessions.sync_config AS sessions_sync_config, sessions.restrict_skills AS sessions_restrict_skills, sessions.allowed_skill_names AS sessions_allowed_skill_names, sessions.system_prompt_override AS sessions_system_prompt_override, sessions.is_locked AS sessions_is_locked, sessions.auto_clear_history AS sessions_auto_clear_history +FROM sessions +WHERE sessions.sync_workspace_id IS NOT NULL] +(Background on this error at: https://sqlalche.me/e/20/e3q8) +INFO:app.api.routes.nodes:[admin] Created node 'test-agent-node-3f60acad' by admin 915a44b3-7ab9-4670-bb86-cb5ae31304bc +[📁⚠️] Mirror Cleanup Thread Error: (sqlite3.OperationalError) no such column: sessions.auto_clear_history +[SQL: SELECT sessions.id AS sessions_id, sessions.user_id AS sessions_user_id, sessions.title AS sessions_title, sessions.provider_name AS sessions_provider_name, sessions.stt_provider_name AS sessions_stt_provider_name, sessions.tts_provider_name AS sessions_tts_provider_name, sessions.feature_name AS sessions_feature_name, sessions.created_at AS sessions_created_at, sessions.is_archived AS sessions_is_archived, sessions.is_cancelled AS sessions_is_cancelled, sessions.sync_workspace_id AS sessions_sync_workspace_id, sessions.attached_node_ids AS sessions_attached_node_ids, sessions.node_sync_status AS sessions_node_sync_status, sessions.sync_config AS sessions_sync_config, sessions.restrict_skills AS sessions_restrict_skills, sessions.allowed_skill_names AS sessions_allowed_skill_names, sessions.system_prompt_override AS sessions_system_prompt_override, sessions.is_locked AS sessions_is_locked, sessions.auto_clear_history AS sessions_auto_clear_history +FROM sessions +WHERE sessions.is_archived = 0 AND sessions.sync_workspace_id IS NOT NULL] +(Background on this error at: https://sqlalche.me/e/20/e3q8) +INFO: 127.0.0.1:44190 - "POST /api/v1/nodes/admin?admin_id=915a44b3-7ab9-4670-bb86-cb5ae31304bc HTTP/1.1" 200 OK +INFO: 127.0.0.1:44190 - "POST /api/v1/agents/deploy HTTP/1.1" 500 Internal Server Error +ERROR: Exception in ASGI application +Traceback (most recent call last): + File "/home/vscode/.local/lib/python3.11/site-packages/sqlalchemy/engine/base.py", line 1967, in _exec_single_context + self.dialect.do_execute( + File "/home/vscode/.local/lib/python3.11/site-packages/sqlalchemy/engine/default.py", line 952, in do_execute + cursor.execute(statement, parameters) +sqlite3.OperationalError: table sessions has no column named auto_clear_history -================================================== -📡 CORTEX MESH DASHBOARD | 2 Nodes Online --------------------------------------------------- - 🟢 test-node-1 | Workers: 0 | Running: 0 tasks - Capabilities: {'has_sudo': 'true', 'has_display': 'false', 'os_release': '6.10.11-linuxkit', 'arch': 'aarch64', 'os': 'linux', 'gpu': 'none', 'local_ip': '172.27.0.2', 'is_root': 'false', 'shell': 'v1'} - 🟢 test-node-2 | Workers: 0 | Running: 0 tasks - Capabilities: {'has_sudo': 'true', 'has_display': 'false', 'os_release': '6.10.11-linuxkit', 'arch': 'aarch64', 'os': 'linux', 'gpu': 'none', 'local_ip': '172.27.0.2', 'is_root': 'false', 'shell': 'v1'} -================================================== +The above exception was the direct cause of the following exception: -INFO:app.core.orchestration.scheduler:[Scheduler] INTERVAL WAKEUP: Triggering Agent a8bd22bf-9812-4702-95a9-13afaa5ac038 (Wait: 5s, Elapsed: 63909884476s) -INFO:app.core.services.rag:[RAG] Mesh Context gathered. Length: 237 chars. -INFO:app.core.services.rag:[RAG] Mesh Context excerpt: Attached Agent Nodes (Infrastructure): -- Node ID: test-agent-node-ec22f68d - Name: Agent Test Node - Description: No description provided. - Status: offline - Terminal Sandbox Mode: PERMISSIVE - AI Re... -WARNING:app.core.services.prompt:Prompt with slug 'rag-pipeline' not found. -INFO:root:[Architect] Starting autonomous loop (Turn 1). Prompt Size: 121 chars across 2 messages. -INFO:root:[Architect] Msg 0 (system): You are a cron agent. Run shell tasks periodically.... -INFO:root:[Architect] Msg 1 (user): Hello test agent! Just reply the word 'Acknowledged' and nothing else.... -17:41:15 - LiteLLM:INFO: utils.py:3895 - -LiteLLM completion() model= gemini-3-flash-preview; provider = gemini -INFO:LiteLLM: -LiteLLM completion() model= gemini-3-flash-preview; provider = gemini -INFO:root:[Architect] First chunk received after 1.12s -[AgentExecutor] Task 4.2: Idempotency check for a8bd22bf-9812-4702-95a9-13afaa5ac038 in /tmp/cortex/agent_77046c0c/ -[AgentExecutor] Starting run for a8bd22bf-9812-4702-95a9-13afaa5ac038 with provider 'gemini'. Prompt length: 70 -INFO: 127.0.0.1:55284 - "GET /api/v1/sessions/31/messages HTTP/1.1" 200 OK -INFO: 127.0.0.1:55284 - "GET /api/v1/agents HTTP/1.1" 200 OK -INFO: 127.0.0.1:55284 - "GET /api/v1/agents/a8bd22bf-9812-4702-95a9-13afaa5ac038/triggers HTTP/1.1" 200 OK -INFO: 127.0.0.1:55284 - "PATCH /api/v1/agents/a8bd22bf-9812-4702-95a9-13afaa5ac038/status HTTP/1.1" 200 OK -INFO: 127.0.0.1:55284 - "PATCH /api/v1/agents/a8bd22bf-9812-4702-95a9-13afaa5ac038/config HTTP/1.1" 200 OK -INFO: 127.0.0.1:55284 - "DELETE /api/v1/agents/a8bd22bf-9812-4702-95a9-13afaa5ac038 HTTP/1.1" 200 OK -INFO:app.core.services.node_registry:[📋] NodeRegistry: Deregistered test-agent-node-ec22f68d -INFO: 127.0.0.1:55284 - "DELETE /api/v1/nodes/admin/test-agent-node-ec22f68d?admin_id=915a44b3-7ab9-4670-bb86-cb5ae31304bc HTTP/1.1" 200 OK -INFO: 127.0.0.1:55286 - "GET /api/v1/speech/voices HTTP/1.1" 200 OK +Traceback (most recent call last): + File "/home/vscode/.local/lib/python3.11/site-packages/uvicorn/protocols/http/httptools_impl.py", line 416, in run_asgi + result = await app( # type: ignore[func-returns-value] + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/vscode/.local/lib/python3.11/site-packages/uvicorn/middleware/proxy_headers.py", line 60, in __call__ + return await self.app(scope, receive, send) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/vscode/.local/lib/python3.11/site-packages/fastapi/applications.py", line 1160, in __call__ + await super().__call__(scope, receive, send) + File "/home/vscode/.local/lib/python3.11/site-packages/starlette/applications.py", line 107, in __call__ + await self.middleware_stack(scope, receive, send) + File "/home/vscode/.local/lib/python3.11/site-packages/starlette/middleware/errors.py", line 186, in __call__ + raise exc + File "/home/vscode/.local/lib/python3.11/site-packages/starlette/middleware/errors.py", line 164, in __call__ + await self.app(scope, receive, _send) + File "/home/vscode/.local/lib/python3.11/site-packages/starlette/middleware/cors.py", line 87, in __call__ + await self.app(scope, receive, send) + File "/home/vscode/.local/lib/python3.11/site-packages/starlette/middleware/exceptions.py", line 63, in __call__ + await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send) + File "/home/vscode/.local/lib/python3.11/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app + raise exc + File "/home/vscode/.local/lib/python3.11/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app + await app(scope, receive, sender) + File "/home/vscode/.local/lib/python3.11/site-packages/fastapi/middleware/asyncexitstack.py", line 18, in __call__ + await self.app(scope, receive, send) + File "/home/vscode/.local/lib/python3.11/site-packages/starlette/routing.py", line 716, in __call__ + await self.middleware_stack(scope, receive, send) + File "/home/vscode/.local/lib/python3.11/site-packages/starlette/routing.py", line 736, in app + await route.handle(scope, receive, send) + File "/home/vscode/.local/lib/python3.11/site-packages/starlette/routing.py", line 290, in handle + await self.app(scope, receive, send) + File "/home/vscode/.local/lib/python3.11/site-packages/fastapi/routing.py", line 119, in app + await wrap_app_handling_exceptions(app, request)(scope, receive, send) + File "/home/vscode/.local/lib/python3.11/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app + raise exc + File "/home/vscode/.local/lib/python3.11/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app + await app(scope, receive, sender) + File "/home/vscode/.local/lib/python3.11/site-packages/fastapi/routing.py", line 105, in app + response = await f(request) + ^^^^^^^^^^^^^^^^ + File "/home/vscode/.local/lib/python3.11/site-packages/fastapi/routing.py", line 431, in app + raw_response = await run_endpoint_function( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/vscode/.local/lib/python3.11/site-packages/fastapi/routing.py", line 315, in run_endpoint_function + return await run_in_threadpool(dependant.call, **values) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/vscode/.local/lib/python3.11/site-packages/starlette/concurrency.py", line 32, in run_in_threadpool + return await anyio.to_thread.run_sync(func) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/vscode/.local/lib/python3.11/site-packages/anyio/to_thread.py", line 63, in run_sync + return await get_async_backend().run_sync_in_worker_thread( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/vscode/.local/lib/python3.11/site-packages/anyio/_backends/_asyncio.py", line 2502, in run_sync_in_worker_thread + return await future + ^^^^^^^^^^^^ + File "/home/vscode/.local/lib/python3.11/site-packages/anyio/_backends/_asyncio.py", line 986, in run + result = context.run(func, *args) + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/app/ai-hub/app/api/routes/agents.py", line 298, in deploy_agent + db.flush() + File "/home/vscode/.local/lib/python3.11/site-packages/sqlalchemy/orm/session.py", line 4331, in flush + self._flush(objects) + File "/home/vscode/.local/lib/python3.11/site-packages/sqlalchemy/orm/session.py", line 4466, in _flush + with util.safe_reraise(): + File "/home/vscode/.local/lib/python3.11/site-packages/sqlalchemy/util/langhelpers.py", line 121, in __exit__ + raise exc_value.with_traceback(exc_tb) + File "/home/vscode/.local/lib/python3.11/site-packages/sqlalchemy/orm/session.py", line 4427, in _flush + flush_context.execute() + File "/home/vscode/.local/lib/python3.11/site-packages/sqlalchemy/orm/unitofwork.py", line 466, in execute + rec.execute(self) + File "/home/vscode/.local/lib/python3.11/site-packages/sqlalchemy/orm/unitofwork.py", line 642, in execute + util.preloaded.orm_persistence.save_obj( + File "/home/vscode/.local/lib/python3.11/site-packages/sqlalchemy/orm/persistence.py", line 93, in save_obj + _emit_insert_statements( + File "/home/vscode/.local/lib/python3.11/site-packages/sqlalchemy/orm/persistence.py", line 1233, in _emit_insert_statements + result = connection.execute( + ^^^^^^^^^^^^^^^^^^^ + File "/home/vscode/.local/lib/python3.11/site-packages/sqlalchemy/engine/base.py", line 1419, in execute + return meth( + ^^^^^ + File "/home/vscode/.local/lib/python3.11/site-packages/sqlalchemy/sql/elements.py", line 527, in _execute_on_connection + return connection._execute_clauseelement( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/vscode/.local/lib/python3.11/site-packages/sqlalchemy/engine/base.py", line 1641, in _execute_clauseelement + ret = self._execute_context( + ^^^^^^^^^^^^^^^^^^^^^^ + File "/home/vscode/.local/lib/python3.11/site-packages/sqlalchemy/engine/base.py", line 1846, in _execute_context + return self._exec_single_context( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/vscode/.local/lib/python3.11/site-packages/sqlalchemy/engine/base.py", line 1986, in _exec_single_context + self._handle_dbapi_exception( + File "/home/vscode/.local/lib/python3.11/site-packages/sqlalchemy/engine/base.py", line 2363, in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e + File "/home/vscode/.local/lib/python3.11/site-packages/sqlalchemy/engine/base.py", line 1967, in _exec_single_context + self.dialect.do_execute( + File "/home/vscode/.local/lib/python3.11/site-packages/sqlalchemy/engine/default.py", line 952, in do_execute + cursor.execute(statement, parameters) +sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) table sessions has no column named auto_clear_history +[SQL: INSERT INTO sessions (user_id, title, provider_name, stt_provider_name, tts_provider_name, feature_name, created_at, is_archived, is_cancelled, sync_workspace_id, attached_node_ids, node_sync_status, sync_config, restrict_skills, allowed_skill_names, system_prompt_override, is_locked, auto_clear_history) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)] +[parameters: ('agent-system', None, 'gemini', None, None, 'agent_harness', '2026-03-24 05:00:51.756379', 0, 0, None, '["test-agent-node-3f60acad"]', '{}', '{}', 0, '[]', 'You are a cron agent. Run shell tasks periodically.', 1, 0)] +(Background on this error at: https://sqlalche.me/e/20/e3q8) +INFO: 127.0.0.1:44206 - "GET /api/v1/speech/voices HTTP/1.1" 200 OK INFO:app.api.routes.tts:Using TTS provider: GeminiTTSProvider for user=915a44b3-7ab9-4670-bb86-cb5ae31304bc INFO:app.core.providers.tts.gemini:TTS request [model=gemini-2.5-flash-preview-tts, vertex=False]: 'Hello from integration test audio pipeline.' INFO:app.core.services.tts:Successfully gathered audio data for all 1 chunks. INFO:app.core.services.tts:Concatenated 1 chunks into a single PCM stream. -INFO: 127.0.0.1:55288 - "POST /api/v1/speech?stream=false HTTP/1.1" 200 OK +INFO: 127.0.0.1:44218 - "POST /api/v1/speech?stream=false HTTP/1.1" 200 OK INFO:app.api.routes.stt:Received transcription request for file: test_audio_pipeline.wav INFO:app.api.routes.stt:Resolving STT. user_id=915a44b3-7ab9-4670-bb86-cb5ae31304bc, provider=google_gemini INFO:app.api.routes.stt:Using STT provider: GoogleSTTProvider -INFO:app.core.services.stt:Starting transcription for audio data (175290 bytes). - -================================================== -📡 CORTEX MESH DASHBOARD | 2 Nodes Online --------------------------------------------------- - 🟢 test-node-1 | Workers: 0 | Running: 0 tasks - Capabilities: {'has_sudo': 'true', 'has_display': 'false', 'os_release': '6.10.11-linuxkit', 'arch': 'aarch64', 'os': 'linux', 'gpu': 'none', 'local_ip': '172.27.0.2', 'is_root': 'false', 'shell': 'v1'} - 🟢 test-node-2 | Workers: 0 | Running: 0 tasks - Capabilities: {'has_sudo': 'true', 'has_display': 'false', 'os_release': '6.10.11-linuxkit', 'arch': 'aarch64', 'os': 'linux', 'gpu': 'none', 'local_ip': '172.27.0.2', 'is_root': 'false', 'shell': 'v1'} -================================================== - +INFO:app.core.services.stt:Starting transcription for audio data (165690 bytes). INFO:app.core.services.stt:Transcribed audio. Length: 43 characters. -INFO: 127.0.0.1:55288 - "POST /api/v1/stt/transcribe HTTP/1.1" 200 OK -INFO: 127.0.0.1:45688 - "GET /api/v1/nodes/test-node-1/status HTTP/1.1" 200 OK -INFO: 127.0.0.1:45688 - "POST /api/v1/sessions/ HTTP/1.1" 200 OK -[📁📤] Initiating Workspace Push for Session session-32-1511b094 to test-node-1 - [📁] Sync Status from test-node-1: Synchronized -[📁📤] Initiating Workspace Push for Session session-32-1511b094 to test-node-2 - [📁] Sync Status from test-node-2: Synchronized -INFO: 127.0.0.1:45688 - "POST /api/v1/sessions/32/nodes HTTP/1.1" 200 OK -[📁✏️] AI Write: case1_c501c142.txt (Session: session-32-1511b094) -> Dispatching to 2 nodes -INFO: 127.0.0.1:45688 - "POST /api/v1/nodes/test-node-1/fs/touch HTTP/1.1" 200 OK - [📁] Sync Status from test-node-1: File written - [📁] Sync Status from test-node-2: File written - [📁📥] Received Manifest from test-node-2 for session-32-1511b094 - [📁📥] Received Manifest from test-node-1 for session-32-1511b094 -INFO: 127.0.0.1:45688 - "GET /api/v1/nodes/test-node-2/fs/cat?path=case1_c501c142.txt&session_id=session-32-1511b094 HTTP/1.1" 200 OK -INFO: 127.0.0.1:45688 - "GET /api/v1/nodes/test-node-1/fs/cat?path=case1_c501c142.txt&session_id=session-32-1511b094 HTTP/1.1" 200 OK -[📁✏️] AI Write: case2_4e925763.txt (Session: session-32-1511b094) -> Dispatching to 2 nodes -INFO: 127.0.0.1:45688 - "POST /api/v1/nodes/test-node-1/fs/touch HTTP/1.1" 200 OK - [📁] Sync Status from test-node-1: File written - [📁] Sync Status from test-node-2: File written - [📁📥] Received Manifest from test-node-2 for session-32-1511b094 - [📁📥] Received Manifest from test-node-1 for session-32-1511b094 -INFO: 127.0.0.1:45688 - "GET /api/v1/nodes/test-node-2/fs/cat?path=case2_4e925763.txt&session_id=session-32-1511b094 HTTP/1.1" 200 OK -INFO: 127.0.0.1:45688 - "GET /api/v1/nodes/test-node-1/fs/cat?path=case2_4e925763.txt&session_id=session-32-1511b094 HTTP/1.1" 200 OK -[📁✏️] AI Write: case3_f2c109da.txt (Session: session-32-1511b094) -> Dispatching to 2 nodes -INFO: 127.0.0.1:45688 - "POST /api/v1/nodes/test-node-1/fs/touch HTTP/1.1" 200 OK - [📁] Sync Status from test-node-2: File written - [📁] Sync Status from test-node-1: File written - [📁📥] Received Manifest from test-node-1 for session-32-1511b094 - [📁📥] Received Manifest from test-node-2 for session-32-1511b094 -INFO: 127.0.0.1:45688 - "GET /api/v1/nodes/test-node-2/fs/cat?path=case3_f2c109da.txt&session_id=session-32-1511b094 HTTP/1.1" 200 OK -[📁🗑️] AI Remove: case3_f2c109da.txt (Session: session-32-1511b094) -> Dispatching to 2 nodes -INFO: 127.0.0.1:45688 - "POST /api/v1/nodes/test-node-1/fs/rm HTTP/1.1" 200 OK - [📁] Sync Status from test-node-1: Deleted case3_f2c109da.txt - [📁] Sync Status from test-node-2: Deleted case3_f2c109da.txt - [📁📥] Received Manifest from test-node-2 for session-32-1511b094 - [📁📥] Received Manifest from test-node-1 for session-32-1511b094 -INFO: 127.0.0.1:45688 - "GET /api/v1/nodes/test-node-2/fs/cat?path=case3_f2c109da.txt&session_id=session-32-1511b094 HTTP/1.1" 404 Not Found -INFO: 127.0.0.1:45688 - "GET /api/v1/nodes/test-node-1/fs/cat?path=case3_f2c109da.txt&session_id=session-32-1511b094 HTTP/1.1" 404 Not Found -[📁✏️] AI Write: case4_61657f07.txt (Session: session-32-1511b094) -> Dispatching to 2 nodes -INFO: 127.0.0.1:45688 - "POST /api/v1/nodes/test-node-1/fs/touch HTTP/1.1" 200 OK - [📁] Sync Status from test-node-1: File written - [📁📥] Received Manifest from test-node-1 for session-32-1511b094 - [📁] Sync Status from test-node-2: File written - [📁📥] Received Manifest from test-node-2 for session-32-1511b094 -INFO: 127.0.0.1:45688 - "GET /api/v1/nodes/test-node-2/fs/cat?path=case4_61657f07.txt&session_id=session-32-1511b094 HTTP/1.1" 200 OK -[📁🗑️] AI Remove: case4_61657f07.txt (Session: session-32-1511b094) -> Dispatching to 2 nodes -INFO: 127.0.0.1:45688 - "POST /api/v1/nodes/test-node-2/fs/rm HTTP/1.1" 200 OK - [📁] Sync Status from test-node-2: Deleted case4_61657f07.txt - [📁] Sync Status from test-node-1: Deleted case4_61657f07.txt - [📁📥] Received Manifest from test-node-1 for session-32-1511b094 - [📁📥] Received Manifest from test-node-2 for session-32-1511b094 -INFO: 127.0.0.1:45688 - "GET /api/v1/nodes/test-node-1/fs/cat?path=case4_61657f07.txt&session_id=session-32-1511b094 HTTP/1.1" 404 Not Found -INFO: 127.0.0.1:45688 - "GET /api/v1/nodes/test-node-1/fs/cat?path=case4_61657f07.txt&session_id=session-32-1511b094 HTTP/1.1" 404 Not Found -[📁✏️] AI Write: case9_latency_7e626973.txt (Session: session-32-1511b094) -> Dispatching to 2 nodes -INFO: 127.0.0.1:45688 - "POST /api/v1/nodes/test-node-1/fs/touch HTTP/1.1" 200 OK - [📁] Sync Status from test-node-1: File written - [📁] Sync Status from test-node-2: File written - [📁📥] Received Manifest from test-node-1 for session-32-1511b094 - [📁📥] Received Manifest from test-node-2 for session-32-1511b094 -INFO: 127.0.0.1:45688 - "GET /api/v1/nodes/test-node-2/fs/cat?path=case9_latency_7e626973.txt&session_id=session-32-1511b094 HTTP/1.1" 200 OK -[📁🗑️] AI Remove: case9_latency_7e626973.txt (Session: session-32-1511b094) -> Dispatching to 2 nodes -INFO: 127.0.0.1:45688 - "POST /api/v1/nodes/test-node-1/fs/rm HTTP/1.1" 200 OK - [📁] Sync Status from test-node-1: Deleted case9_latency_7e626973.txt - [📁] Sync Status from test-node-2: Deleted case9_latency_7e626973.txt - [📁📥] Received Manifest from test-node-1 for session-32-1511b094 - [📁📥] Received Manifest from test-node-2 for session-32-1511b094 - [📁⚡] Fast Sync Complete: case1_c501c142.txt - [📁📢] Broadcasting case1_c501c142.txt from test-node-1 to: test-node-2 - [📁⚡] Fast Sync Complete: case1_c501c142.txt - [📁📢] Broadcasting case1_c501c142.txt from test-node-2 to: test-node-1 - [📁] Sync Status from test-node-2: File case1_c501c142.txt synced - [📁] Sync Status from test-node-1: File case1_c501c142.txt synced -INFO: 127.0.0.1:45688 - "GET /api/v1/nodes/test-node-1/fs/cat?path=case9_latency_7e626973.txt&session_id=session-32-1511b094 HTTP/1.1" 404 Not Found -INFO: 127.0.0.1:45688 - "GET /api/v1/nodes/test-node-2/fs/cat?path=case9_latency_7e626973.txt&session_id=session-32-1511b094 HTTP/1.1" 404 Not Found -[📁✏️] AI Write: case11_hub_e51e3d64.txt (Session: session-32-1511b094) -> Dispatching to 2 nodes -INFO: 127.0.0.1:45688 - "POST /api/v1/nodes/hub/fs/touch HTTP/1.1" 200 OK - [📁] Sync Status from test-node-1: File written - [📁] Sync Status from test-node-2: File written - [📁📥] Received Manifest from test-node-1 for session-32-1511b094 - [📁📥] Received Manifest from test-node-2 for session-32-1511b094 -INFO: 127.0.0.1:45688 - "GET /api/v1/nodes/hub/fs/cat?path=case11_hub_e51e3d64.txt&session_id=session-32-1511b094 HTTP/1.1" 200 OK -[📁✏️] AI Write: case10_resync_22602b38.txt (Session: session-32-1511b094) -> Dispatching to 2 nodes -INFO: 127.0.0.1:45688 - "POST /api/v1/nodes/test-node-1/fs/touch HTTP/1.1" 200 OK - [📁] Sync Status from test-node-1: File written - [📁] Sync Status from test-node-2: File written - [📁📥] Received Manifest from test-node-1 for session-32-1511b094 - [📁📥] Received Manifest from test-node-2 for session-32-1511b094 -INFO: 127.0.0.1:45688 - "GET /api/v1/nodes/test-node-2/fs/cat?path=case10_resync_22602b38.txt&session_id=session-32-1511b094 HTTP/1.1" 200 OK - [📁⚡] Fast Sync Complete: case2_4e925763.txt - [📁📢] Broadcasting case2_4e925763.txt from test-node-2 to: test-node-1 - [📁⚡] Fast Sync Complete: case2_4e925763.txt - [📁📢] Broadcasting case2_4e925763.txt from test-node-1 to: test-node-2 - [📁] Sync Status from test-node-1: File case2_4e925763.txt synced - [📁] Sync Status from test-node-2: File case2_4e925763.txt synced - [📁⚡] Fast Sync Complete: case11_hub_e51e3d64.txt - [📁📢] Broadcasting case11_hub_e51e3d64.txt from test-node-2 to: test-node-1 - [📁⚡] Fast Sync Complete: case11_hub_e51e3d64.txt - [📁📢] Broadcasting case11_hub_e51e3d64.txt from test-node-1 to: test-node-2 - [📁] Sync Status from test-node-1: File case11_hub_e51e3d64.txt synced - [📁] Sync Status from test-node-2: File case11_hub_e51e3d64.txt synced - -================================================== -📡 CORTEX MESH DASHBOARD | 2 Nodes Online --------------------------------------------------- - 🟢 test-node-1 | Workers: 0 | Running: 0 tasks - Capabilities: {'has_sudo': 'true', 'has_display': 'false', 'os_release': '6.10.11-linuxkit', 'arch': 'aarch64', 'os': 'linux', 'gpu': 'none', 'local_ip': '172.27.0.2', 'is_root': 'false', 'shell': 'v1'} - 🟢 test-node-2 | Workers: 0 | Running: 0 tasks - Capabilities: {'has_sudo': 'true', 'has_display': 'false', 'os_release': '6.10.11-linuxkit', 'arch': 'aarch64', 'os': 'linux', 'gpu': 'none', 'local_ip': '172.27.0.2', 'is_root': 'false', 'shell': 'v1'} -================================================== - - [📁⚡] Fast Sync Complete: case10_resync_22602b38.txt - [📁📢] Broadcasting case10_resync_22602b38.txt from test-node-1 to: test-node-2 - [📁⚡] Fast Sync Complete: case10_resync_22602b38.txt - [📁📢] Broadcasting case10_resync_22602b38.txt from test-node-2 to: test-node-1 - [📁] Sync Status from test-node-2: File case10_resync_22602b38.txt synced - [📁] Sync Status from test-node-1: File case10_resync_22602b38.txt synced -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/cat?path=case10_resync_22602b38.txt&session_id=session-32-1511b094 HTTP/1.1" 200 OK -[📁✏️] AI Write: case5_large_dc713ede.txt (Session: session-32-1511b094) -> Dispatching to 2 nodes -INFO: 127.0.0.1:60888 - "POST /api/v1/nodes/test-node-1/fs/touch HTTP/1.1" 200 OK - [📁] Sync Status from test-node-1: File written - [📁📥] Received Manifest from test-node-1 for session-32-1511b094 - [📁] Sync Status from test-node-2: File written - [📁📥] Received Manifest from test-node-2 for session-32-1511b094 -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/cat?path=case5_large_dc713ede.txt&session_id=session-32-1511b094 HTTP/1.1" 200 OK -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-1/fs/cat?path=case5_large_dc713ede.txt&session_id=session-32-1511b094 HTTP/1.1" 200 OK -[📁✏️] AI Write: case6_large_a32dad68.txt (Session: session-32-1511b094) -> Dispatching to 2 nodes -INFO: 127.0.0.1:60888 - "POST /api/v1/nodes/test-node-1/fs/touch HTTP/1.1" 200 OK - [📁] Sync Status from test-node-1: File written - [📁📥] Received Manifest from test-node-1 for session-32-1511b094 - [📁] Sync Status from test-node-2: File written - [📁📥] Received Manifest from test-node-2 for session-32-1511b094 -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/cat?path=case6_large_a32dad68.txt&session_id=session-32-1511b094 HTTP/1.1" 200 OK -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-1/fs/cat?path=case6_large_a32dad68.txt&session_id=session-32-1511b094 HTTP/1.1" 200 OK -[📁✏️] AI Write: case7_large_de27d02d.txt (Session: session-32-1511b094) -> Dispatching to 2 nodes -INFO: 127.0.0.1:60888 - "POST /api/v1/nodes/test-node-1/fs/touch HTTP/1.1" 200 OK - [📁] Sync Status from test-node-1: File written - [📁📥] Received Manifest from test-node-1 for session-32-1511b094 - [📁] Sync Status from test-node-2: File written - [📁📥] Received Manifest from test-node-2 for session-32-1511b094 -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/cat?path=case7_large_de27d02d.txt&session_id=session-32-1511b094 HTTP/1.1" 200 OK -[📁🗑️] AI Remove: case7_large_de27d02d.txt (Session: session-32-1511b094) -> Dispatching to 2 nodes -INFO: 127.0.0.1:60888 - "POST /api/v1/nodes/test-node-1/fs/rm HTTP/1.1" 200 OK - [📁] Sync Status from test-node-1: Deleted case7_large_de27d02d.txt - [📁📥] Received Manifest from test-node-1 for session-32-1511b094 - [📁] Sync Status from test-node-2: Deleted case7_large_de27d02d.txt - [📁📥] Received Manifest from test-node-2 for session-32-1511b094 -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/cat?path=case7_large_de27d02d.txt&session_id=session-32-1511b094 HTTP/1.1" 404 Not Found -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-1/fs/cat?path=case7_large_de27d02d.txt&session_id=session-32-1511b094 HTTP/1.1" 404 Not Found - [📁📥] Sync Starting: case5_large_dc713ede.txt (20971520 bytes, 5 chunks) -[📁✏️] AI Write: case8_large_4cf00f89.txt (Session: session-32-1511b094) -> Dispatching to 2 nodes -INFO: 127.0.0.1:60888 - "POST /api/v1/nodes/test-node-1/fs/touch HTTP/1.1" 200 OK - [📁📢] Broadcasting case5_large_dc713ede.txt from test-node-2 to: test-node-1 - [📁📥] Sync Starting: case5_large_dc713ede.txt (20971520 bytes, 5 chunks) - [📁📢] Broadcasting case5_large_dc713ede.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting case5_large_dc713ede.txt from test-node-2 to: test-node-1 - [📁📢] Broadcasting case5_large_dc713ede.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting case5_large_dc713ede.txt from test-node-2 to: test-node-1 - [📁📢] Broadcasting case5_large_dc713ede.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting case5_large_dc713ede.txt from test-node-2 to: test-node-1 - [📁📢] Broadcasting case5_large_dc713ede.txt from test-node-1 to: test-node-2 -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/cat?path=case8_large_4cf00f89.txt&session_id=session-32-1511b094 HTTP/1.1" 200 OK -[📁🗑️] AI Remove: case8_large_4cf00f89.txt (Session: session-32-1511b094) -> Dispatching to 2 nodes -INFO: 127.0.0.1:60888 - "POST /api/v1/nodes/test-node-2/fs/rm HTTP/1.1" 200 OK -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-1/fs/cat?path=case8_large_4cf00f89.txt&session_id=session-32-1511b094 HTTP/1.1" 404 Not Found -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-1/fs/cat?path=case8_large_4cf00f89.txt&session_id=session-32-1511b094 HTTP/1.1" 404 Not Found -INFO: 127.0.0.1:60888 - "POST /api/v1/nodes/test-node-1/dispatch?user_id=915a44b3-7ab9-4670-bb86-cb5ae31304bc HTTP/1.1" 200 OK -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK - [📁✅] Sync Complete: case5_large_dc713ede.txt (Swapped and verified) - [📁📢] Broadcasting case5_large_dc713ede.txt from test-node-2 to: test-node-1 - [📁] Sync Status from test-node-2: File written - [📁📥] Received Manifest from test-node-2 for session-32-1511b094 - [📁] Sync Status from test-node-2: Deleted case8_large_4cf00f89.txt - [📁📥] Received Manifest from test-node-2 for session-32-1511b094 -[⚠️] Hash Mismatch for /app/ai-hub/data/mirrors/session-32-1511b094/case5_large_dc713ede.txt.cortex_tmp: expected 6fa6fa165fcf0e31a3bd47070ef118b90c75026a185e2b9f8ca4a1c2876d3c8d, got f267cbcf29a3502174cde6937a0a3e9844a1954823397068eeaed716902871b0 - [📁📢] Broadcasting case5_large_dc713ede.txt from test-node-1 to: test-node-2 - [📁] Sync Status from test-node-1: File written - [📁📥] Received Manifest from test-node-1 for session-32-1511b094 - [📁] Sync Status from test-node-1: Deleted case8_large_4cf00f89.txt - [📁📥] Received Manifest from test-node-1 for session-32-1511b094 - [📁] Sync Status from test-node-1: File case5_large_dc713ede.txt synced - [📁] Sync Status from test-node-2: File case5_large_dc713ede.txt synced -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK - [📁📥] Sync Starting: case6_large_a32dad68.txt (20971520 bytes, 5 chunks) - [📁📢] Broadcasting case6_large_a32dad68.txt from test-node-1 to: test-node-2 - [📁📥] Sync Starting: case6_large_a32dad68.txt (20971520 bytes, 5 chunks) - [📁📢] Broadcasting case6_large_a32dad68.txt from test-node-2 to: test-node-1 - [📁📢] Broadcasting case6_large_a32dad68.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting case6_large_a32dad68.txt from test-node-2 to: test-node-1 - [📁📢] Broadcasting case6_large_a32dad68.txt from test-node-1 to: test-node-2 -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK - [📁📢] Broadcasting case6_large_a32dad68.txt from test-node-2 to: test-node-1 - [📁📢] Broadcasting case6_large_a32dad68.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting case6_large_a32dad68.txt from test-node-2 to: test-node-1 - [📁✅] Sync Complete: case6_large_a32dad68.txt (Swapped and verified) - [📁📢] Broadcasting case6_large_a32dad68.txt from test-node-1 to: test-node-2 -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK -[⚠️] Hash Mismatch for /app/ai-hub/data/mirrors/session-32-1511b094/case6_large_a32dad68.txt.cortex_tmp: expected 6fa6fa165fcf0e31a3bd47070ef118b90c75026a185e2b9f8ca4a1c2876d3c8d, got f267cbcf29a3502174cde6937a0a3e9844a1954823397068eeaed716902871b0 - [📁📢] Broadcasting case6_large_a32dad68.txt from test-node-2 to: test-node-1 - [📁] Sync Status from test-node-2: File case6_large_a32dad68.txt synced - [📁] Sync Status from test-node-1: File case6_large_a32dad68.txt synced -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK - [📁📥] Sync Starting: gigabyte_05438911.txt (1048576000 bytes, 250 chunks) - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - -================================================== -📡 CORTEX MESH DASHBOARD | 2 Nodes Online --------------------------------------------------- - 🟢 test-node-1 | Workers: 0 | Running: 0 tasks - Capabilities: {'has_sudo': 'true', 'has_display': 'false', 'os_release': '6.10.11-linuxkit', 'arch': 'aarch64', 'os': 'linux', 'gpu': 'none', 'local_ip': '172.27.0.2', 'is_root': 'false', 'shell': 'v1'} - 🟢 test-node-2 | Workers: 0 | Running: 0 tasks - Capabilities: {'has_sudo': 'true', 'has_display': 'false', 'os_release': '6.10.11-linuxkit', 'arch': 'aarch64', 'os': 'linux', 'gpu': 'none', 'local_ip': '172.27.0.2', 'is_root': 'false', 'shell': 'v1'} -================================================== - -17:41:45 - LiteLLM:INFO: utils.py:3895 - +INFO: 127.0.0.1:44218 - "POST /api/v1/stt/transcribe HTTP/1.1" 200 OK +INFO: 127.0.0.1:44222 - "GET /api/v1/nodes/test-node-1/status HTTP/1.1" 200 OK +INFO: 127.0.0.1:44222 - "POST /api/v1/sessions/ HTTP/1.1" 500 Internal Server Error +INFO: 127.0.0.1:44222 - "POST /api/v1/sessions/ HTTP/1.1" 500 Internal Server Error +INFO: 127.0.0.1:38894 - "POST /api/v1/sessions/ HTTP/1.1" 500 Internal Server Error +INFO: 127.0.0.1:38898 - "POST /api/v1/users/login/local HTTP/1.1" 200 OK +INFO: 127.0.0.1:38900 - "POST /api/v1/users/login/local HTTP/1.1" 401 Unauthorized +INFO: 127.0.0.1:38910 - "POST /api/v1/users/login/local HTTP/1.1" 401 Unauthorized +05:00:57 - LiteLLM:INFO: utils.py:3895 - LiteLLM completion() model= gemini-3-flash-preview; provider = gemini INFO:LiteLLM: LiteLLM completion() model= gemini-3-flash-preview; provider = gemini - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK -17:41:46 - LiteLLM:INFO: utils.py:3895 - -LiteLLM completion() model= deepseek-chat; provider = deepseek -INFO:LiteLLM: -LiteLLM completion() model= deepseek-chat; provider = deepseek - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK -INFO:app.app:[Health Check] System LLM statuses updated. - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - -================================================== -📡 CORTEX MESH DASHBOARD | 2 Nodes Online --------------------------------------------------- - 🟢 test-node-1 | Workers: 0 | Running: 0 tasks - Capabilities: {'has_sudo': 'true', 'has_display': 'false', 'os_release': '6.10.11-linuxkit', 'arch': 'aarch64', 'os': 'linux', 'gpu': 'none', 'local_ip': '172.27.0.2', 'is_root': 'false', 'shell': 'v1'} - 🟢 test-node-2 | Workers: 0 | Running: 0 tasks - Capabilities: {'has_sudo': 'true', 'has_display': 'false', 'os_release': '6.10.11-linuxkit', 'arch': 'aarch64', 'os': 'linux', 'gpu': 'none', 'local_ip': '172.27.0.2', 'is_root': 'false', 'shell': 'v1'} -================================================== - - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK - -================================================== -📡 CORTEX MESH DASHBOARD | 2 Nodes Online --------------------------------------------------- - 🟢 test-node-1 | Workers: 0 | Running: 0 tasks - Capabilities: {'has_sudo': 'true', 'has_display': 'false', 'os_release': '6.10.11-linuxkit', 'arch': 'aarch64', 'os': 'linux', 'gpu': 'none', 'local_ip': '172.27.0.2', 'is_root': 'false', 'shell': 'v1'} - 🟢 test-node-2 | Workers: 0 | Running: 0 tasks - Capabilities: {'has_sudo': 'true', 'has_display': 'false', 'os_release': '6.10.11-linuxkit', 'arch': 'aarch64', 'os': 'linux', 'gpu': 'none', 'local_ip': '172.27.0.2', 'is_root': 'false', 'shell': 'v1'} -================================================== - -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK - [📁✅] Sync Complete: gigabyte_05438911.txt (Swapped and verified) - [📁📢] Broadcasting gigabyte_05438911.txt from test-node-1 to: test-node-2 -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-1/fs/ls?path=.&session_id=session-32-1511b094 HTTP/1.1" 200 OK -[📁🗑️] AI Remove: gigabyte_05438911.txt (Session: session-32-1511b094) -> Dispatching to 2 nodes -INFO: 127.0.0.1:60888 - "POST /api/v1/nodes/test-node-1/fs/rm HTTP/1.1" 200 OK - [📁] Sync Status from test-node-2: File not found -INFO: 127.0.0.1:60888 - "POST /api/v1/sessions/ HTTP/1.1" 200 OK -[📁📤] Initiating Workspace Push for Session session-33-6641a37a to test-node-1 -[📁📤] Initiating Workspace Push for Session session-33-6641a37a to test-node-2 - [📁] Sync Status from test-node-2: Synchronized - [📁] Sync Status from test-node-1: Deleted gigabyte_05438911.txt - [📁📥] Received Manifest from test-node-1 for session-32-1511b094 - [📁] Sync Status from test-node-1: Synchronized -INFO: 127.0.0.1:60888 - "POST /api/v1/sessions/33/nodes HTTP/1.1" 200 OK - [📁] Sync Status from test-node-2: File gigabyte_05438911.txt synced -[📁✏️] AI Write: autopurge_7eee7d91.txt (Session: session-33-6641a37a) -> Dispatching to 2 nodes -INFO: 127.0.0.1:60888 - "POST /api/v1/nodes/test-node-1/fs/touch HTTP/1.1" 200 OK - [📁] Sync Status from test-node-2: File written - [📁] Sync Status from test-node-1: File written - [📁📥] Received Manifest from test-node-1 for session-33-6641a37a - [📁📥] Received Manifest from test-node-2 for session-33-6641a37a -INFO: 127.0.0.1:60888 - "GET /api/v1/nodes/test-node-2/fs/cat?path=autopurge_7eee7d91.txt&session_id=session-33-6641a37a HTTP/1.1" 200 OK - [📁⚡] Fast Sync Complete: autopurge_7eee7d91.txt - [📁📢] Broadcasting autopurge_7eee7d91.txt from test-node-2 to: test-node-1 - [📁⚡] Fast Sync Complete: autopurge_7eee7d91.txt - [📁📢] Broadcasting autopurge_7eee7d91.txt from test-node-1 to: test-node-2 - [📁] Sync Status from test-node-1: File autopurge_7eee7d91.txt synced - [📁] Sync Status from test-node-2: File autopurge_7eee7d91.txt synced -INFO: 127.0.0.1:60888 - "DELETE /api/v1/sessions/33 HTTP/1.1" 200 OK - -================================================== -📡 CORTEX MESH DASHBOARD | 2 Nodes Online --------------------------------------------------- - 🟢 test-node-1 | Workers: 0 | Running: 0 tasks - Capabilities: {'has_sudo': 'true', 'has_display': 'false', 'os_release': '6.10.11-linuxkit', 'arch': 'aarch64', 'os': 'linux', 'gpu': 'none', 'local_ip': '172.27.0.2', 'is_root': 'false', 'shell': 'v1'} - 🟢 test-node-2 | Workers: 0 | Running: 0 tasks - Capabilities: {'has_sudo': 'true', 'has_display': 'false', 'os_release': '6.10.11-linuxkit', 'arch': 'aarch64', 'os': 'linux', 'gpu': 'none', 'local_ip': '172.27.0.2', 'is_root': 'false', 'shell': 'v1'} -================================================== - -INFO: 127.0.0.1:60888 - "POST /api/v1/nodes/test-node-1/dispatch?user_id=915a44b3-7ab9-4670-bb86-cb5ae31304bc HTTP/1.1" 200 OK -INFO: 127.0.0.1:60888 - "POST /api/v1/nodes/test-node-2/dispatch?user_id=915a44b3-7ab9-4670-bb86-cb5ae31304bc HTTP/1.1" 200 OK -INFO: 127.0.0.1:60888 - "DELETE /api/v1/sessions/32 HTTP/1.1" 200 OK -INFO: 127.0.0.1:35984 - "POST /api/v1/sessions/ HTTP/1.1" 200 OK -INFO: 127.0.0.1:35984 - "POST /api/v1/sessions/34/chat HTTP/1.1" 200 OK -INFO:app.core.services.rag:[RAG] Mesh Context gathered. Length: 0 chars. -WARNING:app.core.services.prompt:Prompt with slug 'rag-pipeline' not found. -INFO:root:[Architect] Starting autonomous loop (Turn 1). Prompt Size: 4133 chars across 2 messages. -INFO:root:[Architect] Msg 0 (system): You are the Cortex AI Assistant, the **Master-Architect** of a decentralized agent mesh. - -## 🏗️ Orchestration Strategy (The Master-Worker Pattern): -- **Action-First**: You are an action-oriented agent. If you possess a tool to perform a task, you MUST invoke it in the current turn. Do not wait for a second confirmation. -- **Large Data Rule**: For files larger than 100KB, DO NOT use `mesh_file_explorer`'s `write` action. Instead, use `mesh_terminal_control` with native commands like `dd`, `head`... -INFO:root:[Architect] Msg 1 (user): What is the capital of France? Please respond with just the city name.... -17:42:17 - LiteLLM:INFO: utils.py:3895 - -LiteLLM completion() model= gemini-3-flash-preview; provider = gemini -INFO:LiteLLM: -LiteLLM completion() model= gemini-3-flash-preview; provider = gemini -INFO:root:[Architect] First chunk received after 0.75s -INFO: 127.0.0.1:35988 - "POST /api/v1/users/login/local HTTP/1.1" 200 OK -INFO: 127.0.0.1:35992 - "POST /api/v1/users/login/local HTTP/1.1" 401 Unauthorized -INFO: 127.0.0.1:35996 - "POST /api/v1/users/login/local HTTP/1.1" 401 Unauthorized -17:42:18 - LiteLLM:INFO: utils.py:3895 - -LiteLLM completion() model= gemini-3-flash-preview; provider = gemini -INFO:LiteLLM: -LiteLLM completion() model= gemini-3-flash-preview; provider = gemini -INFO: 127.0.0.1:36002 - "POST /api/v1/users/me/config/verify_llm HTTP/1.1" 200 OK -17:42:20 - LiteLLM:INFO: utils.py:3895 - +INFO: 127.0.0.1:38920 - "POST /api/v1/users/me/config/verify_llm HTTP/1.1" 200 OK +05:00:57 - LiteLLM:INFO: utils.py:3895 - LiteLLM completion() model= gemini-3-flash-preview; provider = gemini INFO:LiteLLM: LiteLLM completion() model= gemini-3-flash-preview; provider = gemini @@ -829,16 +306,16 @@ Give Feedback / Get Help: https://github.com/BerriAI/litellm/issues/new LiteLLM.Info: If you need to debug this error, use `litellm._turn_on_debug()'. -INFO: 127.0.0.1:36016 - "POST /api/v1/users/me/config/verify_llm HTTP/1.1" 200 OK +INFO: 127.0.0.1:38932 - "POST /api/v1/users/me/config/verify_llm HTTP/1.1" 200 OK INFO:app.core.services.preference:Saving updated global preferences via admin 915a44b3-7ab9-4670-bb86-cb5ae31304bc 🏠 Configuration synchronized to app/config.yaml -INFO: 127.0.0.1:36024 - "PUT /api/v1/users/me/config HTTP/1.1" 200 OK -INFO: 127.0.0.1:36024 - "GET /api/v1/users/me/config HTTP/1.1" 200 OK -17:42:20 - LiteLLM:INFO: utils.py:3895 - +INFO: 127.0.0.1:38940 - "PUT /api/v1/users/me/config HTTP/1.1" 200 OK +INFO: 127.0.0.1:38940 - "GET /api/v1/users/me/config HTTP/1.1" 200 OK +05:00:58 - LiteLLM:INFO: utils.py:3895 - LiteLLM completion() model= gemini-3-flash-preview; provider = gemini INFO:LiteLLM: LiteLLM completion() model= gemini-3-flash-preview; provider = gemini -INFO: 127.0.0.1:36032 - "POST /api/v1/users/me/config/verify_llm HTTP/1.1" 200 OK +INFO: 127.0.0.1:38954 - "POST /api/v1/users/me/config/verify_llm HTTP/1.1" 200 OK ERROR:app.api.routes.user:LLM Verification failed for non_existent_provider_xyz (None): LiteLLM Error (non_existent_provider_xyz/unknown_model): litellm.BadRequestError: LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model=non_existent_provider_xyz/unknown_model Pass model as E.g. For 'Huggingface' inference endpoints pass in `completion(model='huggingface/starcoder',..)` Learn more: https://docs.litellm.ai/docs/providers @@ -847,7 +324,7 @@ Provider List: https://docs.litellm.ai/docs/providers -INFO: 127.0.0.1:36042 - "POST /api/v1/users/me/config/verify_llm HTTP/1.1" 200 OK +INFO: 127.0.0.1:38970 - "POST /api/v1/users/me/config/verify_llm HTTP/1.1" 200 OK Provider List: https://docs.litellm.ai/docs/providers @@ -902,15 +379,336 @@ Provider List: https://docs.litellm.ai/docs/providers -INFO: 127.0.0.1:36048 - "GET /api/v1/users/me/config/models?provider_name=gemini HTTP/1.1" 200 OK +INFO: 127.0.0.1:38978 - "GET /api/v1/users/me/config/models?provider_name=gemini HTTP/1.1" 200 OK WARNING:app.core.grpc.services.grpc_server:Results listener closed for test-node-1: WARNING:app.core.grpc.services.grpc_server:Results listener closed for test-node-2: -WARNING:app.core.grpc.services.grpc_server:[📶] gRPC Stream TERMINATED for test-node-2. Cleaning up. -INFO:app.core.services.node_registry:[📋] NodeRegistry: Deregistered test-node-2 WARNING:app.core.grpc.services.grpc_server:[📶] gRPC Stream TERMINATED for test-node-1. Cleaning up. +WARNING:app.core.grpc.services.grpc_server:[📶] gRPC Stream TERMINATED for test-node-2. Cleaning up. INFO:app.core.services.node_registry:[📋] NodeRegistry: Deregistered test-node-1 -INFO: Shutting down -INFO: Waiting for application shutdown. -INFO:app.app:[M6] Stopping gRPC server... -INFO: Application shutdown complete. -INFO: Finished server process [6031] +INFO:app.core.services.node_registry:[📋] NodeRegistry: Deregistered test-node-2 + +================================================== +📡 CORTEX MESH DASHBOARD | 0 Nodes Online +-------------------------------------------------- + No nodes currently connected. +================================================== + + +================================================== +📡 CORTEX MESH DASHBOARD | 0 Nodes Online +-------------------------------------------------- + No nodes currently connected. +================================================== + + +================================================== +📡 CORTEX MESH DASHBOARD | 0 Nodes Online +-------------------------------------------------- + No nodes currently connected. +================================================== + + +================================================== +📡 CORTEX MESH DASHBOARD | 0 Nodes Online +-------------------------------------------------- + No nodes currently connected. +================================================== + + +================================================== +📡 CORTEX MESH DASHBOARD | 0 Nodes Online +-------------------------------------------------- + No nodes currently connected. +================================================== + +ERROR:app.core.orchestration.scheduler:[Scheduler] Zombie Sweeper iteration failed: (sqlite3.OperationalError) no such column: agent_instances.total_runs +[SQL: SELECT agent_instances.id AS agent_instances_id, agent_instances.template_id AS agent_instances_template_id, agent_instances.session_id AS agent_instances_session_id, agent_instances.mesh_node_id AS agent_instances_mesh_node_id, agent_instances.status AS agent_instances_status, agent_instances.current_workspace_jail AS agent_instances_current_workspace_jail, agent_instances.last_heartbeat AS agent_instances_last_heartbeat, agent_instances.total_runs AS agent_instances_total_runs, agent_instances.successful_runs AS agent_instances_successful_runs, agent_instances.total_tokens_accumulated AS agent_instances_total_tokens_accumulated, agent_instances.total_input_tokens AS agent_instances_total_input_tokens, agent_instances.total_output_tokens AS agent_instances_total_output_tokens, agent_instances.total_running_time_seconds AS agent_instances_total_running_time_seconds, agent_instances.tool_call_counts AS agent_instances_tool_call_counts +FROM agent_instances +WHERE agent_instances.status = ? AND agent_instances.last_heartbeat < ?] +[parameters: ('active', '2026-03-24 04:58:40.135427')] +(Background on this error at: https://sqlalche.me/e/20/e3q8) +05:01:40 - LiteLLM:INFO: utils.py:3895 - +LiteLLM completion() model= gemini-3-flash-preview; provider = gemini +INFO:LiteLLM: +LiteLLM completion() model= gemini-3-flash-preview; provider = gemini +05:01:40 - LiteLLM:INFO: utils.py:3895 - +LiteLLM completion() model= deepseek-chat; provider = deepseek +INFO:LiteLLM: +LiteLLM completion() model= deepseek-chat; provider = deepseek +INFO:app.app:[Health Check] System LLM statuses updated. + +================================================== +📡 CORTEX MESH DASHBOARD | 0 Nodes Online +-------------------------------------------------- + No nodes currently connected. +================================================== + + +================================================== +📡 CORTEX MESH DASHBOARD | 0 Nodes Online +-------------------------------------------------- + No nodes currently connected. +================================================== + + +================================================== +📡 CORTEX MESH DASHBOARD | 0 Nodes Online +-------------------------------------------------- + No nodes currently connected. +================================================== + + +================================================== +📡 CORTEX MESH DASHBOARD | 0 Nodes Online +-------------------------------------------------- + No nodes currently connected. +================================================== + + +================================================== +📡 CORTEX MESH DASHBOARD | 0 Nodes Online +-------------------------------------------------- + No nodes currently connected. +================================================== + + +================================================== +📡 CORTEX MESH DASHBOARD | 0 Nodes Online +-------------------------------------------------- + No nodes currently connected. +================================================== + +ERROR:app.core.orchestration.scheduler:[Scheduler] Zombie Sweeper iteration failed: (sqlite3.OperationalError) no such column: agent_instances.total_runs +[SQL: SELECT agent_instances.id AS agent_instances_id, agent_instances.template_id AS agent_instances_template_id, agent_instances.session_id AS agent_instances_session_id, agent_instances.mesh_node_id AS agent_instances_mesh_node_id, agent_instances.status AS agent_instances_status, agent_instances.current_workspace_jail AS agent_instances_current_workspace_jail, agent_instances.last_heartbeat AS agent_instances_last_heartbeat, agent_instances.total_runs AS agent_instances_total_runs, agent_instances.successful_runs AS agent_instances_successful_runs, agent_instances.total_tokens_accumulated AS agent_instances_total_tokens_accumulated, agent_instances.total_input_tokens AS agent_instances_total_input_tokens, agent_instances.total_output_tokens AS agent_instances_total_output_tokens, agent_instances.total_running_time_seconds AS agent_instances_total_running_time_seconds, agent_instances.tool_call_counts AS agent_instances_tool_call_counts +FROM agent_instances +WHERE agent_instances.status = ? AND agent_instances.last_heartbeat < ?] +[parameters: ('active', '2026-03-24 04:59:40.149991')] +(Background on this error at: https://sqlalche.me/e/20/e3q8) + +================================================== +📡 CORTEX MESH DASHBOARD | 0 Nodes Online +-------------------------------------------------- + No nodes currently connected. +================================================== + + +================================================== +📡 CORTEX MESH DASHBOARD | 0 Nodes Online +-------------------------------------------------- + No nodes currently connected. +================================================== + + +================================================== +📡 CORTEX MESH DASHBOARD | 0 Nodes Online +-------------------------------------------------- + No nodes currently connected. +================================================== + + +================================================== +📡 CORTEX MESH DASHBOARD | 0 Nodes Online +-------------------------------------------------- + No nodes currently connected. +================================================== + + +================================================== +📡 CORTEX MESH DASHBOARD | 0 Nodes Online +-------------------------------------------------- + No nodes currently connected. +================================================== + + +================================================== +📡 CORTEX MESH DASHBOARD | 0 Nodes Online +-------------------------------------------------- + No nodes currently connected. +================================================== + +ERROR:app.core.orchestration.scheduler:[Scheduler] Zombie Sweeper iteration failed: (sqlite3.OperationalError) no such column: agent_instances.total_runs +[SQL: SELECT agent_instances.id AS agent_instances_id, agent_instances.template_id AS agent_instances_template_id, agent_instances.session_id AS agent_instances_session_id, agent_instances.mesh_node_id AS agent_instances_mesh_node_id, agent_instances.status AS agent_instances_status, agent_instances.current_workspace_jail AS agent_instances_current_workspace_jail, agent_instances.last_heartbeat AS agent_instances_last_heartbeat, agent_instances.total_runs AS agent_instances_total_runs, agent_instances.successful_runs AS agent_instances_successful_runs, agent_instances.total_tokens_accumulated AS agent_instances_total_tokens_accumulated, agent_instances.total_input_tokens AS agent_instances_total_input_tokens, agent_instances.total_output_tokens AS agent_instances_total_output_tokens, agent_instances.total_running_time_seconds AS agent_instances_total_running_time_seconds, agent_instances.tool_call_counts AS agent_instances_tool_call_counts +FROM agent_instances +WHERE agent_instances.status = ? AND agent_instances.last_heartbeat < ?] +[parameters: ('active', '2026-03-24 05:00:40.159972')] +(Background on this error at: https://sqlalche.me/e/20/e3q8) + +================================================== +📡 CORTEX MESH DASHBOARD | 0 Nodes Online +-------------------------------------------------- + No nodes currently connected. +================================================== + + +================================================== +📡 CORTEX MESH DASHBOARD | 0 Nodes Online +-------------------------------------------------- + No nodes currently connected. +================================================== + + +================================================== +📡 CORTEX MESH DASHBOARD | 0 Nodes Online +-------------------------------------------------- + No nodes currently connected. +================================================== + + +================================================== +📡 CORTEX MESH DASHBOARD | 0 Nodes Online +-------------------------------------------------- + No nodes currently connected. +================================================== + + +================================================== +📡 CORTEX MESH DASHBOARD | 0 Nodes Online +-------------------------------------------------- + No nodes currently connected. +================================================== + + +================================================== +📡 CORTEX MESH DASHBOARD | 0 Nodes Online +-------------------------------------------------- + No nodes currently connected. +================================================== + +ERROR:app.core.orchestration.scheduler:[Scheduler] Zombie Sweeper iteration failed: (sqlite3.OperationalError) no such column: agent_instances.total_runs +[SQL: SELECT agent_instances.id AS agent_instances_id, agent_instances.template_id AS agent_instances_template_id, agent_instances.session_id AS agent_instances_session_id, agent_instances.mesh_node_id AS agent_instances_mesh_node_id, agent_instances.status AS agent_instances_status, agent_instances.current_workspace_jail AS agent_instances_current_workspace_jail, agent_instances.last_heartbeat AS agent_instances_last_heartbeat, agent_instances.total_runs AS agent_instances_total_runs, agent_instances.successful_runs AS agent_instances_successful_runs, agent_instances.total_tokens_accumulated AS agent_instances_total_tokens_accumulated, agent_instances.total_input_tokens AS agent_instances_total_input_tokens, agent_instances.total_output_tokens AS agent_instances_total_output_tokens, agent_instances.total_running_time_seconds AS agent_instances_total_running_time_seconds, agent_instances.tool_call_counts AS agent_instances_tool_call_counts +FROM agent_instances +WHERE agent_instances.status = ? AND agent_instances.last_heartbeat < ?] +[parameters: ('active', '2026-03-24 05:01:40.168100')] +(Background on this error at: https://sqlalche.me/e/20/e3q8) + +================================================== +📡 CORTEX MESH DASHBOARD | 0 Nodes Online +-------------------------------------------------- + No nodes currently connected. +================================================== + + +================================================== +📡 CORTEX MESH DASHBOARD | 0 Nodes Online +-------------------------------------------------- + No nodes currently connected. +================================================== + + +================================================== +📡 CORTEX MESH DASHBOARD | 0 Nodes Online +-------------------------------------------------- + No nodes currently connected. +================================================== + + +================================================== +📡 CORTEX MESH DASHBOARD | 0 Nodes Online +-------------------------------------------------- + No nodes currently connected. +================================================== + + +================================================== +📡 CORTEX MESH DASHBOARD | 0 Nodes Online +-------------------------------------------------- + No nodes currently connected. +================================================== + + +================================================== +📡 CORTEX MESH DASHBOARD | 0 Nodes Online +-------------------------------------------------- + No nodes currently connected. +================================================== + +ERROR:app.core.orchestration.scheduler:[Scheduler] Zombie Sweeper iteration failed: (sqlite3.OperationalError) no such column: agent_instances.total_runs +[SQL: SELECT agent_instances.id AS agent_instances_id, agent_instances.template_id AS agent_instances_template_id, agent_instances.session_id AS agent_instances_session_id, agent_instances.mesh_node_id AS agent_instances_mesh_node_id, agent_instances.status AS agent_instances_status, agent_instances.current_workspace_jail AS agent_instances_current_workspace_jail, agent_instances.last_heartbeat AS agent_instances_last_heartbeat, agent_instances.total_runs AS agent_instances_total_runs, agent_instances.successful_runs AS agent_instances_successful_runs, agent_instances.total_tokens_accumulated AS agent_instances_total_tokens_accumulated, agent_instances.total_input_tokens AS agent_instances_total_input_tokens, agent_instances.total_output_tokens AS agent_instances_total_output_tokens, agent_instances.total_running_time_seconds AS agent_instances_total_running_time_seconds, agent_instances.tool_call_counts AS agent_instances_tool_call_counts +FROM agent_instances +WHERE agent_instances.status = ? AND agent_instances.last_heartbeat < ?] +[parameters: ('active', '2026-03-24 05:02:40.181130')] +(Background on this error at: https://sqlalche.me/e/20/e3q8) + +================================================== +📡 CORTEX MESH DASHBOARD | 0 Nodes Online +-------------------------------------------------- + No nodes currently connected. +================================================== + + +================================================== +📡 CORTEX MESH DASHBOARD | 0 Nodes Online +-------------------------------------------------- + No nodes currently connected. +================================================== + + +================================================== +📡 CORTEX MESH DASHBOARD | 0 Nodes Online +-------------------------------------------------- + No nodes currently connected. +================================================== + + +================================================== +📡 CORTEX MESH DASHBOARD | 0 Nodes Online +-------------------------------------------------- + No nodes currently connected. +================================================== + + +================================================== +📡 CORTEX MESH DASHBOARD | 0 Nodes Online +-------------------------------------------------- + No nodes currently connected. +================================================== + + +================================================== +📡 CORTEX MESH DASHBOARD | 0 Nodes Online +-------------------------------------------------- + No nodes currently connected. +================================================== + +ERROR:app.core.orchestration.scheduler:[Scheduler] Zombie Sweeper iteration failed: (sqlite3.OperationalError) no such column: agent_instances.total_runs +[SQL: SELECT agent_instances.id AS agent_instances_id, agent_instances.template_id AS agent_instances_template_id, agent_instances.session_id AS agent_instances_session_id, agent_instances.mesh_node_id AS agent_instances_mesh_node_id, agent_instances.status AS agent_instances_status, agent_instances.current_workspace_jail AS agent_instances_current_workspace_jail, agent_instances.last_heartbeat AS agent_instances_last_heartbeat, agent_instances.total_runs AS agent_instances_total_runs, agent_instances.successful_runs AS agent_instances_successful_runs, agent_instances.total_tokens_accumulated AS agent_instances_total_tokens_accumulated, agent_instances.total_input_tokens AS agent_instances_total_input_tokens, agent_instances.total_output_tokens AS agent_instances_total_output_tokens, agent_instances.total_running_time_seconds AS agent_instances_total_running_time_seconds, agent_instances.tool_call_counts AS agent_instances_tool_call_counts +FROM agent_instances +WHERE agent_instances.status = ? AND agent_instances.last_heartbeat < ?] +[parameters: ('active', '2026-03-24 05:03:40.211583')] +(Background on this error at: https://sqlalche.me/e/20/e3q8) +05:06:43 - LiteLLM:INFO: utils.py:3895 - +LiteLLM completion() model= gemini-3-flash-preview; provider = gemini +INFO:LiteLLM: +LiteLLM completion() model= gemini-3-flash-preview; provider = gemini +05:06:44 - LiteLLM:INFO: utils.py:3895 - +LiteLLM completion() model= deepseek-chat; provider = deepseek +INFO:LiteLLM: +LiteLLM completion() model= deepseek-chat; provider = deepseek + +================================================== +📡 CORTEX MESH DASHBOARD | 0 Nodes Online +-------------------------------------------------- + No nodes currently connected. +================================================== + + +================================================== +📡 CORTEX MESH DASHBOARD | 0 Nodes Online +-------------------------------------------------- + No nodes currently connected. +================================================== + + +================================================== +📡 CORTEX MESH DASHBOARD | 0 Nodes Online +-------------------------------------------------- + No nodes currently connected. +================================================== + + +================================================== +📡 CORTEX MESH DASHBOARD | 0 Nodes Online +-------------------------------------------------- + No nodes currently connected. +================================================== + diff --git a/ai-hub/test.db-shm b/ai-hub/test.db-shm index 812cb9e..d69d10c 100644 --- a/ai-hub/test.db-shm +++ b/ai-hub/test.db-shm Binary files differ diff --git a/ai-hub/test.db-wal b/ai-hub/test.db-wal index 223af2a..724b98d 100644 --- a/ai-hub/test.db-wal +++ b/ai-hub/test.db-wal Binary files differ