Newer
Older
cortex-hub / poc-grpc-agent / test_recovery.py
import time
import subprocess
import os
import shutil

def run_recovery_test():
    print("[🚀] Starting Ghost Mirror Recovery Test...")
    
    # 0. Prep Mirror (Server side)
    mirror_dir = "/app/data/mirrors/recovery-session"
    os.makedirs(mirror_dir, exist_ok=True)
    with open(os.path.join(mirror_dir, "app.py"), "w") as f:
        f.write("print('v1')")

    # 1. Start Orchestrator
    print("[🛡️] Orchestrator: Starting...")
    orchestrator = subprocess.Popen(
        ["python3", "-m", "orchestrator.app"],
        cwd="/app/poc-grpc-agent",
        stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1
    )
    time.sleep(3)

    # 2. Start Node Alpha
    print("[🤖] Node Alpha: Starting...")
    node_alpha_sync = "/tmp/cortex-sync-recovery"
    if os.path.exists(node_alpha_sync): shutil.rmtree(node_alpha_sync)
    
    node1 = subprocess.Popen(
        ["python3", "-m", "agent_node.main"],
        cwd="/app/poc-grpc-agent",
        env={**os.environ, "AGENT_NODE_ID": "node-alpha", "CORTEX_SYNC_DIR": node_alpha_sync},
        stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1
    )
    
    time.sleep(10) # Wait for initial sync
    print("[✅] Initial Sync should be done.")
    
    # 3. Stop Node Alpha
    print("[🛑] Stopping Node Alpha...")
    node1.terminate()
    node1.wait()

    # 4. Modify Server Mirror (Simulate updates while node is offline)
    print("[📝] Updating Server Mirror to v2...")
    with open(os.path.join(mirror_dir, "app.py"), "w") as f:
        f.write("print('v2')")

    time.sleep(2)

    # 5. Restart Node Alpha
    print("[🤖] Node Alpha: Restarting...")
    node1_v2 = subprocess.Popen(
        ["python3", "-m", "agent_node.main"],
        cwd="/app/poc-grpc-agent",
        env={**os.environ, "AGENT_NODE_ID": "node-alpha", "CORTEX_SYNC_DIR": node_alpha_sync},
        stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1
    )

    # 6. Wait and Check
    time.sleep(15)
    print("\n[📊] Checking Results...")
    node_file = os.path.join(node_alpha_sync, "recovery-session", "app.py")
    if os.path.exists(node_file):
        with open(node_file, "r") as f:
            content = f.read()
            print(f"Node Content: {content}")
            if content == "print('v2')":
                print("[🏆] RECOVERY SUCCESSFUL!")
            else:
                print("[❌] RECOVERY FAILED - Content mismatch")
    else:
        print("[❌] RECOVERY FAILED - File not found")

    orchestrator.terminate()
    node1_v2.terminate()
    print("[✅] Done.")

if __name__ == "__main__":
    run_recovery_test()