diff --git a/ai-hub/app/api/routes/skills.py b/ai-hub/app/api/routes/skills.py index 8020b3e..810bf09 100644 --- a/ai-hub/app/api/routes/skills.py +++ b/ai-hub/app/api/routes/skills.py @@ -1,145 +1,351 @@ +import os +import shutil +import json +import logging +import yaml +from datetime import datetime from fastapi import APIRouter, Depends, HTTPException, Query from sqlalchemy.orm import Session -from typing import List, Optional +from typing import List, Optional, Any from app.db import models from app.api import schemas from app.api.dependencies import ServiceContainer, get_current_user, get_db +from app.config import settings + +logger = logging.getLogger(__name__) def create_skills_router(services: ServiceContainer) -> APIRouter: + os.makedirs(settings.SKILLS_DIR, exist_ok=True) router = APIRouter(prefix="/skills", tags=["Skills"]) + + # We load the FS loader from our new backend system + from app.core.skills.fs_loader import fs_loader - @router.get("/", response_model=List[schemas.SkillResponse]) + @router.get("/", response_model=List[Any]) def list_skills( db: Session = Depends(get_db), current_user: models.User = Depends(get_current_user), - feature: Optional[str] = Query(None, description="Filter skills by feature (e.g., 'chat', 'voice')") + feature: Optional[str] = Query(None, description="Filter skills by feature (e.g., 'swarm_control', 'voice_chat')") ): - """List all skills accessible to the user.""" - # Start queries - system_query = db.query(models.Skill).filter(models.Skill.is_system == True) - - if current_user.role == 'admin': - # Admins see ALL skills (system + user-owned for management) - user_query = db.query(models.Skill).filter(models.Skill.is_system == False) - else: - user_query = db.query(models.Skill).filter( - models.Skill.owner_id == current_user.id, - models.Skill.is_system == False - ) - - # Policy: Only show enabled skills to non-admins - if current_user.role != 'admin': - system_query = system_query.filter(models.Skill.is_enabled == True) - user_query = user_query.filter(models.Skill.is_enabled == True) - - # Target feature filtering - if feature: - system_skills = [s for s in system_query.all() if feature in (s.features or [])] - user_skills = [s for s in user_query.all() if feature in (s.features or [])] - else: - system_skills = system_query.all() - user_skills = user_query.all() - - # Skills shared with the user's group via Group Policy - group_skills = [] - if current_user.role != 'admin' and current_user.group and current_user.group.policy: - group_skill_names = current_user.group.policy.get("skills", []) - if group_skill_names: - g_query = db.query(models.Skill).filter( - models.Skill.name.in_(group_skill_names), - models.Skill.owner_id != current_user.id, - models.Skill.is_system == False - ) - if current_user.role != 'admin': - g_query = g_query.filter(models.Skill.is_enabled == True) - - if feature: - group_skills = [s for s in g_query.all() if feature in (s.features or [])] - else: - group_skills = g_query.all() - - return system_skills + user_skills + group_skills + """List all skills accessible to the user (via File System).""" + all_skills = fs_loader.get_all_skills() + filtered = [] - @router.post("/", response_model=schemas.SkillResponse) + is_admin = current_user.role == 'admin' + + for s in all_skills: + # Enforce RBAC Metadata visibility + s_owner = s.get("owner_id") + s_sys = s.get("is_system", False) + + # Admins see everything. Users see their own and system skills. + if not is_admin: + if s_owner != current_user.id and not s_sys: + continue + # If group policies apply, we could add them here in the future + + # Filter by Feature Folder + if feature and feature not in s.get("features", []): + continue + + filtered.append(s) + + return filtered + + @router.post("/", response_model=Any) def create_skill( skill: schemas.SkillCreate, db: Session = Depends(get_db), current_user: models.User = Depends(get_current_user) ): - """Create a new skill.""" - existing = db.query(models.Skill).filter(models.Skill.name == skill.name).first() - if existing: - raise HTTPException(status_code=400, detail="Skill with this name already exists") + """Create a new skill folder on the physical filesystem.""" + feature = skill.features[0] if skill.features else "swarm_control" + + # We sanitize the name for the folder to prevent path traversal + folder_name = "".join([c for c in skill.name if c.isalnum() or c in (" ", "-", "_")]).strip().replace(" ", "_").lower() + if not folder_name: + raise HTTPException(status_code=400, detail="Invalid skill name.") - db_skill = models.Skill( - name=skill.name, - description=skill.description, - skill_type=skill.skill_type, - config=skill.config, - system_prompt=skill.system_prompt, - is_enabled=skill.is_enabled, - features=skill.features, - owner_id=current_user.id, - is_system=skill.is_system if current_user.role == 'admin' else False - ) - db.add(db_skill) - db.commit() - db.refresh(db_skill) - return db_skill + skill_path = os.path.join(settings.SKILLS_DIR, feature, folder_name) + + if os.path.exists(skill_path): + raise HTTPException(status_code=400, detail="A skill with this ID/folder already exists in this feature sector.") + + try: + os.makedirs(skill_path, exist_ok=True) + + # Generate the hidden metadata file + meta = { + "owner_id": current_user.id, + "is_system": skill.is_system if current_user.role == 'admin' else False, + "extra_metadata": skill.extra_metadata or {"emoji": "🛠️"} + } + with open(os.path.join(skill_path, ".metadata.json"), "w") as f: + json.dump(meta, f, indent=4) + + # Create a boilerplate SKILL.md + content = f"---\nname: {skill.name}\ndescription: {skill.description}\n---\n\n### Execution Logic (Bash)\n```bash\necho \"Executing {skill.name}...\"\n```\n" + with open(os.path.join(skill_path, "SKILL.md"), "w") as f: + f.write(content) + + except Exception as e: + logger.error(f"Failed to create FS skill: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + return { + "id": f"fs-{folder_name}", + "name": skill.name, + "description": skill.description, + "skill_type": "local", + "is_enabled": True, + "features": [feature], + "owner_id": current_user.id, + "is_system": meta["is_system"], + "extra_metadata": meta["extra_metadata"], + "created_at": datetime.utcnow().isoformat() + } - @router.put("/{skill_id}", response_model=schemas.SkillResponse) + @router.put("/{skill_id}", response_model=Any) def update_skill( - skill_id: int, + skill_id: str, skill_update: schemas.SkillUpdate, db: Session = Depends(get_db), current_user: models.User = Depends(get_current_user) ): - """Update an existing skill. User must be admin or the owner.""" - db_skill = db.query(models.Skill).filter(models.Skill.id == skill_id).first() - if not db_skill: - raise HTTPException(status_code=404, detail="Skill not found") + """Update top level metadata of an FS skill (emoji and system flag).""" + # skill_id usually follows 'fs-folder_name' from the frontend + folder_name = skill_id.replace("fs-", "", 1) if skill_id.startswith("fs-") else skill_id - # Block modification of system skills - if db_skill.is_system: - raise HTTPException(status_code=403, detail="System skills cannot be modified.") - - if db_skill.owner_id != current_user.id and current_user.role != 'admin': - raise HTTPException(status_code=403, detail="Not authorized to update this skill") - - if skill_update.name is not None and skill_update.name != db_skill.name: - existing = db.query(models.Skill).filter(models.Skill.name == skill_update.name).first() - if existing: - raise HTTPException(status_code=400, detail="Skill with this name already exists") + # Scan to find which feature it belongs to, since ID alone doesn't contain feature path + matched_path = None + for feature in os.listdir(settings.SKILLS_DIR): + f_path = os.path.join(settings.SKILLS_DIR, feature, folder_name) + if os.path.exists(f_path): + matched_path = f_path + break - for key, value in skill_update.model_dump(exclude_unset=True).items(): - if key == 'is_system' and current_user.role != 'admin': - continue - setattr(db_skill, key, value) + if not matched_path: + raise HTTPException(status_code=404, detail="Skill folder not found.") - db.commit() - db.refresh(db_skill) - return db_skill + meta_path = os.path.join(matched_path, ".metadata.json") + try: + with open(meta_path, "r") as f: + meta = json.load(f) + except: + meta = fs_loader._ensure_metadata(matched_path, folder_name) + + if meta.get("is_system") and current_user.role != 'admin': + raise HTTPException(status_code=403, detail="System skills cannot be modified.") + if meta.get("owner_id") != current_user.id and current_user.role != 'admin': + raise HTTPException(status_code=403, detail="Not authorized.") + + # Update allowed metadata + if skill_update.extra_metadata is not None: + meta["extra_metadata"] = skill_update.extra_metadata + if skill_update.is_system is not None and current_user.role == 'admin': + meta["is_system"] = skill_update.is_system + + # We don't support moving features or renaming folders via API easily right now + # The user can just recreate it to keep it simple, or edit SKILL.md. + + with open(meta_path, "w") as f: + json.dump(meta, f, indent=4) + + # Read the skill model back via fs_loader logic (slow but perfect sync) + skills = fs_loader.get_all_skills() + for s in skills: + if getattr(s, "id", "") == skill_id or s.get("id") == skill_id: + return s + return {"id": skill_id, "updated": True} @router.delete("/{skill_id}") def delete_skill( - skill_id: int, + skill_id: str, db: Session = Depends(get_db), current_user: models.User = Depends(get_current_user) ): - """Delete a skill.""" - db_skill = db.query(models.Skill).filter(models.Skill.id == skill_id).first() - if not db_skill: + """Delete a skill permanently via shutil.""" + folder_name = skill_id.replace("fs-", "", 1) if skill_id.startswith("fs-") else skill_id + matched_path = None + for feature in os.listdir(settings.SKILLS_DIR): + f_path = os.path.join(settings.SKILLS_DIR, feature, folder_name) + if os.path.exists(f_path): + matched_path = f_path + break + + if not matched_path: + raise HTTPException(status_code=404, detail="Skill folder not found.") + + meta_path = os.path.join(matched_path, ".metadata.json") + try: + with open(meta_path, "r") as f: + meta = json.load(f) + except: + meta = {} + + if meta.get("is_system") and current_user.role != 'admin': + raise HTTPException(status_code=403, detail="System skills cannot be deleted.") + if meta.get("owner_id") != current_user.id and current_user.role != 'admin': + raise HTTPException(status_code=403, detail="Not authorized.") + + try: + shutil.rmtree(matched_path) + # Send reload signal + return {"message": "Skill directory successfully deleted."} + except Exception as e: + raise HTTPException(status_code=500, detail=f"Filesystem deletion error: {e}") + + + # --- Skill File System (VFS) Endpoints --- + + @router.get("/{skill_id}/files", response_model=List[Any]) + def list_skill_files( + skill_id: str, + db: Session = Depends(get_db), + current_user: models.User = Depends(get_current_user) + ): + """Get file tree for a skill.""" + folder_name = skill_id.replace("fs-", "", 1) if skill_id.startswith("fs-") else skill_id + matched_path = None + for feature in os.listdir(settings.SKILLS_DIR): + f_path = os.path.join(settings.SKILLS_DIR, feature, folder_name) + if os.path.exists(f_path): + matched_path = f_path + break + + if not matched_path: raise HTTPException(status_code=404, detail="Skill not found") - # Block deletion of system skills - if db_skill.is_system: - raise HTTPException(status_code=403, detail="System skills cannot be deleted.") + files = [] + for root, _, filenames in os.walk(matched_path): + for file in filenames: + if file == ".metadata.json" or file.startswith('.'): continue + file_abs = os.path.join(root, file) + file_rel = os.path.relpath(file_abs, matched_path).replace('\\', '/') + files.append({"path": file_rel}) + + return files - if db_skill.owner_id != current_user.id and current_user.role != 'admin': - raise HTTPException(status_code=403, detail="Not authorized to delete this skill") + @router.get("/{skill_id}/files/{path:path}", response_model=Any) + def read_skill_file( + skill_id: str, + path: str, + db: Session = Depends(get_db), + current_user: models.User = Depends(get_current_user) + ): + """Read a specific skill file.""" + folder_name = skill_id.replace("fs-", "", 1) if skill_id.startswith("fs-") else skill_id + matched_path = None + for feature in os.listdir(settings.SKILLS_DIR): + f_path = os.path.join(settings.SKILLS_DIR, feature, folder_name) + if os.path.exists(f_path): + matched_path = f_path + break + + if not matched_path: + raise HTTPException(status_code=404, detail="Skill not found") - db.delete(db_skill) - db.commit() - return {"message": "Skill deleted"} + file_abs = os.path.join(matched_path, path) + if not os.path.exists(file_abs): + raise HTTPException(status_code=404, detail="File not found on disk") + + try: + with open(file_abs, "r", encoding="utf-8") as f: + content = f.read() + return {"file_path": path, "content": content} + except Exception as e: + raise HTTPException(status_code=500, detail="Cannot read physical file.") + + + @router.post("/{skill_id}/files/{path:path}", response_model=Any) + def create_or_update_skill_file( + skill_id: str, + path: str, + file_update: schemas.SkillFileUpdate, + db: Session = Depends(get_db), + current_user: models.User = Depends(get_current_user) + ): + """Create or update a skill file.""" + folder_name = skill_id.replace("fs-", "", 1) if skill_id.startswith("fs-") else skill_id + matched_path = None + for feature in os.listdir(settings.SKILLS_DIR): + f_path = os.path.join(settings.SKILLS_DIR, feature, folder_name) + if os.path.exists(f_path): + matched_path = f_path + break + + if not matched_path: + raise HTTPException(status_code=404, detail="Skill not found") + + meta_path = os.path.join(matched_path, ".metadata.json") + try: + with open(meta_path, "r") as f: + meta = json.load(f) + except: + meta = {} + + # Security Guard + if meta.get("is_system") and current_user.role != 'admin': + raise HTTPException(status_code=403, detail="System skills cannot be modified.") + if meta.get("owner_id") != current_user.id and current_user.role != 'admin': + raise HTTPException(status_code=403, detail="Not authorized to edit skill files") + + # Path Traversal Check + file_abs = os.path.abspath(os.path.join(matched_path, path)) + if not file_abs.startswith(os.path.abspath(matched_path)): + raise HTTPException(status_code=400, detail="Path traversal not allowed.") + + os.makedirs(os.path.dirname(file_abs), exist_ok=True) + + try: + with open(file_abs, "w", encoding="utf-8") as f: + f.write(file_update.content) + return {"file_path": path, "content": file_update.content} + except Exception as e: + raise HTTPException(status_code=500, detail=f"Failed to write physical file: {e}") + + @router.delete("/{skill_id}/files/{path:path}") + def delete_skill_file( + skill_id: str, + path: str, + db: Session = Depends(get_db), + current_user: models.User = Depends(get_current_user) + ): + """Delete a skill file literally from disk.""" + folder_name = skill_id.replace("fs-", "", 1) if skill_id.startswith("fs-") else skill_id + matched_path = None + for feature in os.listdir(settings.SKILLS_DIR): + f_path = os.path.join(settings.SKILLS_DIR, feature, folder_name) + if os.path.exists(f_path): + matched_path = f_path + break + + if not matched_path: + raise HTTPException(status_code=404, detail="Skill not found") + + meta_path = os.path.join(matched_path, ".metadata.json") + try: + with open(meta_path, "r") as f: + meta = json.load(f) + except: + meta = {} + + if meta.get("is_system") and current_user.role != 'admin': + raise HTTPException(status_code=403, detail="System skills cannot be modified.") + if meta.get("owner_id") != current_user.id and current_user.role != 'admin': + raise HTTPException(status_code=403, detail="Not authorized to edit skill files") + + file_abs = os.path.abspath(os.path.join(matched_path, path)) + if not file_abs.startswith(os.path.abspath(matched_path)): + raise HTTPException(status_code=400, detail="Path traversal not allowed.") + + if not os.path.exists(file_abs): + raise HTTPException(status_code=404, detail="File not found") + + try: + os.remove(file_abs) + return {"message": "File removed physically"} + except Exception as e: + raise HTTPException(status_code=500, detail=f"Could not delete: {e}") return router diff --git a/ai-hub/app/api/schemas.py b/ai-hub/app/api/schemas.py index 7927ccf..a82a82d 100644 --- a/ai-hub/app/api/schemas.py +++ b/ai-hub/app/api/schemas.py @@ -1,5 +1,5 @@ from pydantic import BaseModel, Field, ConfigDict -from typing import List, Literal, Optional +from typing import List, Literal, Optional, Union from datetime import datetime # --- User Schemas --- @@ -112,13 +112,10 @@ name: str description: Optional[str] = None skill_type: str = "local" # local, remote_grpc, mcp - config: dict = Field(default_factory=dict) - system_prompt: Optional[str] = None is_enabled: bool = True features: List[str] = Field(default_factory=lambda: ["chat"]) is_system: bool = False extra_metadata: dict = Field(default_factory=dict) - preview_markdown: Optional[str] = None class SkillCreate(SkillBase): pass @@ -127,20 +124,34 @@ name: Optional[str] = None description: Optional[str] = None skill_type: Optional[str] = None - config: Optional[dict] = None - system_prompt: Optional[str] = None is_enabled: Optional[bool] = None features: Optional[List[str]] = None is_system: Optional[bool] = None extra_metadata: Optional[dict] = None - preview_markdown: Optional[str] = None class SkillResponse(SkillBase): - id: int + id: Union[int, str] owner_id: str created_at: datetime model_config = ConfigDict(from_attributes=True) +class SkillFileBase(BaseModel): + file_path: str + content: Optional[str] = None + +class SkillFileCreate(SkillFileBase): + pass + +class SkillFileUpdate(BaseModel): + content: str + +class SkillFileResponse(SkillFileBase): + id: Union[int, str] + skill_id: Union[int, str] + created_at: Optional[datetime] = None + updated_at: Optional[datetime] = None + model_config = ConfigDict(from_attributes=True) + # --- Chat Schemas --- class ChatRequest(BaseModel): """Defines the shape of a request to the /chat endpoint.""" diff --git a/ai-hub/app/config.py b/ai-hub/app/config.py index 04d4216..8e81bda 100644 --- a/ai-hub/app/config.py +++ b/ai-hub/app/config.py @@ -147,6 +147,12 @@ get_from_yaml(["application", "secret_key"]) or \ self.OIDC_CLIENT_SECRET or "dev-secret-key-1337" + # --- Directory Settings --- + self.DATA_DIR: str = os.getenv("DATA_DIR") or \ + get_from_yaml(["application", "data_dir"]) or \ + "/app/data" # Hardcoded default for production Docker deployment + self.SKILLS_DIR: str = os.path.join(self.DATA_DIR, "skills") + # --- Database Settings --- self.DB_MODE: str = os.getenv("DB_MODE") or \ get_from_yaml(["database", "mode"]) or \ diff --git a/ai-hub/app/core/providers/factory.py b/ai-hub/app/core/providers/factory.py index b304abb..c8deaed 100644 --- a/ai-hub/app/core/providers/factory.py +++ b/ai-hub/app/core/providers/factory.py @@ -196,9 +196,30 @@ info = litellm.get_model_info(full_model) if info: # Prefer max_input_tokens as it represents the context window - # If not present, max_tokens likely represents the full window or the output window - return info.get("max_input_tokens") or info.get("max_tokens") or 100000 + input_tokens = info.get("max_input_tokens") + + # If litellm gave us an empty value or a suspiciously low value like 8192 + # (which is often the max_output_tokens, not the context window), override it + if not input_tokens or input_tokens <= 32000: + if "gemini" in full_model.lower(): + input_tokens = 1048576 # Gemini 1.5 1M context + elif "deepseek" in full_model.lower(): + input_tokens = 128000 + elif "gpt-4o" in full_model.lower(): + input_tokens = 128000 + elif "claude" in full_model.lower(): + input_tokens = 200000 + else: + input_tokens = info.get("max_tokens") or 100000 + + return input_tokens except: pass + # Final default behavior if completely unknown + if "gemini" in full_model.lower(): + return 1048576 + elif "deepseek" in full_model.lower(): + return 128000 + return 100000 \ No newline at end of file diff --git a/ai-hub/app/core/services/tool.py b/ai-hub/app/core/services/tool.py index ecc6aa2..f714390 100644 --- a/ai-hub/app/core/services/tool.py +++ b/ai-hub/app/core/services/tool.py @@ -33,37 +33,150 @@ tools = [s.to_tool_definition() for s in local_skills] - # 2. Add DB-defined skills (System skills or user-owned) - query = db.query(models.Skill).filter( - (models.Skill.is_system == True) | - (models.Skill.owner_id == user_id) - ).filter(models.Skill.is_enabled == True) + # 2. Add FS-defined skills (System skills or user-owned) + from app.core.skills.fs_loader import fs_loader + all_fs_skills = fs_loader.get_all_skills() - if feature: - # SQLAlchemy JSON containment check (SQLite specific or generic enough) - # For simplicity, we filter in Python if the DB driver is tricky - db_skills = query.all() - db_skills = [ds for ds in db_skills if feature in (ds.features or [])] - else: - db_skills = query.all() + class _DictObj: + def __init__(self, d): + for k, v in d.items(): + setattr(self, k, v) + + db_skills = [] + for fs_skill in all_fs_skills: + if fs_skill.get("is_enabled", True) and (fs_skill.get("is_system") or fs_skill.get("owner_id") == user_id): + if feature and feature not in fs_skill.get("features", ["chat"]): + continue + # Map virtual files array to object arrays for the legacy parsing logic + fs_skill["files"] = [_DictObj(f) for f in fs_skill.get("files", [])] + db_skills.append(_DictObj(fs_skill)) + + import litellm + max_md_len = 1000 + try: + # Attempt to resolve the active user's model configuration dynamically to get exact context sizes + user = db.query(models.User).filter(models.User.id == user_id).first() if db else None + m_name = "gemini-2.5-pro" + if user and user.preferences: + m_name = user.preferences.get("llm_model", m_name) + + model_info = litellm.get_model_info(m_name) + if model_info: + max_tokens = model_info.get("max_input_tokens", 8192) + # Cap a single skill's instruction block at 5% of the total context window to leave room + # for chat history and other plugins, with an absolute roof of 40k chars. (1 token ~= 4 chars) + max_md_len = max(min(int(max_tokens * 4 * 0.05), 40000), 1000) + except Exception as e: + logger.warning(f"Dynamic tool schema truncation failed to query model size: {e}") + for ds in db_skills: # Prevent duplicates if name overlaps with local if any(t["function"]["name"] == ds.name for t in tools): continue - # M3: Use the public description, but append internal AI instructions if available - # This makes the "system prompt" invisible to end users but fully visible to the Orchestrator. + # --- Lazy-Loading VFS Pattern (Phase 3 - Skills as Folders) --- + # Extract parameters from SKILL.md frontmatter instead of legacy DB config column description = ds.description or "" - if ds.system_prompt: - description += f"\n\nInternal Intelligence Protocol:\n{ds.system_prompt}" + parameters = {} + skill_md_file = next((f for f in ds.files if f.file_path == "SKILL.md"), None) if ds.files else None + + if skill_md_file and skill_md_file.content: + # If the skill text fits within the model's dynamic safe boundary, provide it directly! + if len(skill_md_file.content) > max_md_len: + excerpt = skill_md_file.content[:max_md_len].strip() + "..." + description += ( + f"\n\n[VFS Skill] Full instructions available. Call `read_skill_artifact` with " + f"skill_name='{ds.name}', file_path='SKILL.md' to load them before using this tool.\n" + f"Summary: {excerpt}" + ) + else: + description += f"\n\n[Instructions]\n{skill_md_file.content}" + + # Parse YAML frontmatter to get the tool schema parameters + if skill_md_file.content.startswith("---"): + try: + import yaml + parts = skill_md_file.content.split("---", 2) + if len(parts) >= 3: + fm = yaml.safe_load(parts[1]) + parameters = fm.get("config", {}).get("parameters", {}) + except Exception as e: + import logging + logger = logging.getLogger(__name__) + logger.warning(f"Error parsing SKILL.md frontmatter for {ds.name}: {e}") + + # If no parameters found in frontmatter, try parsing markdown directly + if not parameters: + try: + import re + # Parse legacy migrated json configs + mig_match = re.search(r"### Tool Config JSON\s+```(?:yaml|json)\s+(.+?)\s+```", skill_md_file.content, re.DOTALL | re.IGNORECASE) + if mig_match: + try: + import json + parameters = json.loads(mig_match.group(1).strip()) + except: + pass + + if not parameters: + # Parse Description override (optional) + desc_match = re.search(r"\*\*Description:\*\*\s*(.*?)(?=\n\n|\n#|$)", skill_md_file.content, re.DOTALL | re.IGNORECASE) + if desc_match: + extracted_desc = desc_match.group(1).strip() + # Keep the same logic as above when updating the description if parameters failed + if len(skill_md_file.content) > max_md_len: + excerpt = skill_md_file.content[:max_md_len].strip() + "..." + description = ( + f"{extracted_desc}\n\n[VFS Skill] Full instructions available. Call `read_skill_artifact` with " + f"skill_name='{ds.name}', file_path='SKILL.md' to load them before using this tool.\n" + f"Summary: {excerpt}" + ) + else: + description = f"{extracted_desc}\n\n[Instructions]\n{skill_md_file.content}" + + # Parse Parameters Table + table_pattern = r"\|\s*Name\s*\|\s*Type\s*\|\s*Description\s*\|\s*Required\s*\|\n(?:\|[-:\s]+\|[-:\s]+\|[-:\s]+\|[-:\s]+\|\n)(.*?)(?=\n\n|\n#|$)" + param_table_match = re.search(table_pattern, skill_md_file.content, re.DOTALL | re.IGNORECASE) + if param_table_match: + parameters = {"type": "object", "properties": {}, "required": []} + rows = param_table_match.group(1).strip().split('\n') + for row in rows: + if not row.strip() or '|' not in row: continue + cols = [c.strip() for c in row.split('|')][1:-1] + if len(cols) >= 4: + p_name = cols[0].replace('`', '').strip() + p_type = cols[1].strip() + p_desc = cols[2].strip() + p_req = cols[3].strip().lower() in ['yes', 'true', '1', 'y'] + + parameters["properties"][p_name] = { + "type": p_type, + "description": p_desc + } + if p_req: + parameters["required"].append(p_name) + except Exception as e: + import logging + logger = logging.getLogger(__name__) + logger.warning(f"Error parsing SKILL.md markdown for {ds.name}: {e}") + + # Automatically inject logical node parameters into the schema for all tools + if not parameters: parameters = {"type": "object", "properties": {}, "required": []} + if "properties" not in parameters: parameters["properties"] = {} + if "node_id" not in parameters["properties"]: + parameters["properties"]["node_id"] = { + "type": "string", + "description": "Optional specific mesh node ID to execute this on. Leave empty to auto-use the session's first attached node." + } + tools.append({ "type": "function", "function": { "name": ds.name, "description": description, - "parameters": ds.config.get("parameters", {}) + "parameters": parameters } }) @@ -79,17 +192,54 @@ result = await skill.execute(**arguments) return result.dict() - # 2. Handle System / DB Skills - if db: - db_skill = db.query(models.Skill).filter(models.Skill.name == tool_name).first() - if db_skill and db_skill.is_system: - return await self._execute_system_skill(db_skill, arguments, user_id=user_id, db=db, session_id=session_id, session_db_id=session_db_id, on_event=on_event) + # 2. Check the tool_registry for lightweight plugins (e.g. read_skill_artifact) + # These are system agents that DO NOT need a full SubAgent loop — their prepare_task + # returns a synchronous function we can call directly. + plugin = tool_registry.get_plugin(tool_name) + if plugin: + orchestrator = getattr(self._services, "orchestrator", None) + context = { + "db": db, + "user_id": user_id, + "session_id": session_id, + "node_id": arguments.get("node_id"), + "node_ids": arguments.get("node_ids"), + "services": self._services, + "orchestrator": orchestrator, + "assistant": orchestrator.assistant if orchestrator else None, + "on_event": on_event + } + task_fn, task_args = plugin.prepare_task(arguments, context) + if not task_fn: + return task_args # error dict + try: + import asyncio + if asyncio.iscoroutinefunction(task_fn): + return await task_fn(**task_args) + else: + return task_fn(**task_args) + except Exception as e: + logger.exception(f"Plugin '{tool_name}' execution failed: {e}") + return {"success": False, "error": str(e)} + + # 3. Handle System / FS Skills (full SubAgent or Bash) + from app.core.skills.fs_loader import fs_loader + all_fs_skills = fs_loader.get_all_skills() + for fs_skill in all_fs_skills: + if fs_skill.get("name") == tool_name: + class _DictObj: + def __init__(self, d): + for k, v in d.items(): + setattr(self, k, v) + fs_skill["files"] = [_DictObj(f) for f in fs_skill.get("files", [])] + db_skill_mock = _DictObj(fs_skill) + return await self._execute_system_skill(db_skill_mock, arguments, user_id=user_id, db=db, session_id=session_id, session_db_id=session_db_id, on_event=on_event) logger.error(f"Tool '{tool_name}' not found or handled yet.") return {"success": False, "error": "Tool not found"} - async def _execute_system_skill(self, skill: models.Skill, args: Dict[str, Any], user_id: str = None, db: Session = None, session_id: str = None, session_db_id: int = None, on_event = None) -> Any: - """Routes system skill execution to a stateful SubAgent.""" + async def _execute_system_skill(self, skill: Any, args: Dict[str, Any], user_id: str = None, db: Session = None, session_id: str = None, session_db_id: int = None, on_event = None) -> Any: + """Routes FS skill execution to a stateful SubAgent or Dynamic Plugin.""" from app.core.services.sub_agent import SubAgent from app.core.providers.factory import get_llm_provider @@ -104,6 +254,11 @@ if session: attached = session.attached_node_ids or [] + # Implicit fallback to first attached node if no target was specified + if not node_id and not node_ids and attached: + node_id = attached[0] + args["node_id"] = node_id + # Allow virtual node IDs for system maintenance allowed_ids = attached + ["hub", "server", "local"] @@ -172,7 +327,55 @@ plugin = tool_registry.get_plugin(skill.name) if not plugin: - return {"success": False, "error": f"Tool implementation '{skill.name}' not found in registry"} + # Check if this is a Dynamic Bash Skill + bash_logic = None + skill_md_file = next((f for f in skill.files if f.file_path == "SKILL.md"), None) if getattr(skill, "files", None) else None + if skill_md_file and skill_md_file.content: + import re + # Broadened regex: Allows 3-6 hashes, any title containing "Execution Logic", and handles files ending without newlines. + bash_match = re.search(r"#{3,6}\s*Execution Logic.*?```bash\s*\n(.*?)(?:```|\Z)", skill_md_file.content, re.DOTALL | re.IGNORECASE) + if bash_match: + bash_logic = bash_match.group(1).strip() + + if bash_logic: + class DynamicBashPlugin: + name = skill.name + retries = 0 + def prepare_task(self, invoke_args, invoke_context): + cmd = bash_logic + for k, v in invoke_args.items(): + cmd = cmd.replace(f"${{{k}}}", str(v)) + + timeout = int(invoke_args.get("timeout", 60)) + node_id = invoke_context.get("node_id") + node_ids = invoke_context.get("node_ids") + resolved_sid = invoke_context.get("session_id") + assistant = invoke_context.get("assistant") + services = invoke_context.get("services") + + if node_id in ["hub", "server", "local"] or (node_ids and any(nid in ["hub", "server", "local"] for nid in node_ids)): + def _hub_command(**kwargs): + import subprocess, os + cwd = os.getcwd() + if kwargs.get("resolved_sid") and getattr(services, "orchestrator", None): + try: cwd = services.orchestrator.mirror.get_workspace_path(kwargs.get("resolved_sid")) + except: pass + try: + proc = subprocess.run(kwargs.get("cmd"), shell=True, capture_output=True, text=True, timeout=kwargs.get("timeout"), cwd=cwd) + return {"status": "SUCCESS" if proc.returncode == 0 else "FAILED", "stdout": proc.stdout, "stderr": proc.stderr, "exit_code": proc.returncode, "node_id": "hub"} + except subprocess.TimeoutExpired as e: + return {"status": "TIMEOUT", "stdout": e.stdout or "", "stderr": e.stderr or "", "error": "Command timed out"} + except Exception as e: + return {"status": "ERROR", "error": str(e)} + return _hub_command, {"cmd": cmd, "timeout": timeout, "resolved_sid": resolved_sid} + elif node_ids and isinstance(node_ids, list): + return assistant.dispatch_swarm, {"node_ids": node_ids, "cmd": cmd, "timeout": timeout, "session_id": resolved_sid, "no_abort": False} + elif node_id: + return assistant.dispatch_single, {"node_id": node_id, "cmd": cmd, "timeout": timeout, "session_id": resolved_sid, "no_abort": False} + return None, {"success": False, "error": "target node_id or node_ids is required for Bash execution"} + plugin = DynamicBashPlugin() + else: + return {"success": False, "error": f"Tool implementation '{skill.name}' not found in registry and no Bash logic found in SKILL.md"} context = { "db": db, @@ -211,59 +414,60 @@ return {"success": False, "error": res["error"], "sub_agent_status": sub_agent.status} # M6: Post-processing for Binary Artifacts (Screenshots, etc.) - if skill.name == "browser_automation_agent" and isinstance(res, dict): - # Organise browser data by session for better UX - if resolved_sid and resolved_sid != "__fs_explorer__": - try: - abs_workspace = assistant.mirror.get_workspace_path(resolved_sid) - # M6: Use .browser_data (ignored from node sync) - base_dir = os.path.join(abs_workspace, ".browser_data") - os.makedirs(base_dir, exist_ok=True) - - timestamp = int(time.time()) - action = args.get("action", "unknown").lower() - - # Clean filename for the image: {timestamp}_{action}.png - # This allows better "next/prev" sorting in the parent gallery - ss_filename = f"{timestamp}_{action}.png" + if isinstance(res, dict): + # Unconditionally prevent binary data from leaking into the JSON serializer + screenshot_bits = res.pop("_screenshot_bytes", None) + + if skill.name == "browser_automation_agent": + # Organise browser data by session for better UX + if resolved_sid and resolved_sid != "__fs_explorer__": + try: + abs_workspace = assistant.mirror.get_workspace_path(resolved_sid) + # M6: Use .browser_data (ignored from node sync) + base_dir = os.path.join(abs_workspace, ".browser_data") + os.makedirs(base_dir, exist_ok=True) + + timestamp = int(time.time()) + action = args.get("action", "unknown").lower() + + # Clean filename for the image: {timestamp}_{action}.png + ss_filename = f"{timestamp}_{action}.png" - # Save Screenshot if available - if "_screenshot_bytes" in res: - bits = res.pop("_screenshot_bytes") - if bits: + # Save Screenshot if available + if screenshot_bits: ss_path = os.path.join(base_dir, ss_filename) with open(ss_path, "wb") as f: - f.write(bits) + f.write(screenshot_bits) res["screenshot_url"] = f"/.browser_data/{resolved_sid}/{ss_filename}" res["_visual_feedback"] = f"Action screenshot captured: {res['screenshot_url']}" - # Save Metadata/A11y into a hidden or specific sub-folder if needed, - # but keep images in the root of the session for quick gallery view. - action_dir = os.path.join(base_dir, ".metadata", f"{timestamp}_{action}") - os.makedirs(action_dir, exist_ok=True) + # Save Metadata/A11y into a hidden or specific sub-folder if needed, + # but keep images in the root of the session for quick gallery view. + action_dir = os.path.join(base_dir, ".metadata", f"{timestamp}_{action}") + os.makedirs(action_dir, exist_ok=True) - # Save Metadata/Result for easy debugging in file explorer - import json - meta = { - "timestamp": timestamp, - "action": action, - "url": res.get("url"), - "title": res.get("title"), - "success": res.get("success"), - "error": res.get("error"), - "eval_result": res.get("eval_result") - } - with open(os.path.join(action_dir, "metadata.json"), "w") as f: - json.dump(meta, f, indent=2) - - # Optional: Save A11y summary for quick viewing - if "a11y_summary" in res: - with open(os.path.join(action_dir, "a11y_summary.txt"), "w") as f: - f.write(res["a11y_summary"]) + # Save Metadata/Result for easy debugging in file explorer + import json + meta = { + "timestamp": timestamp, + "action": action, + "url": res.get("url"), + "title": res.get("title"), + "success": res.get("success"), + "error": res.get("error"), + "eval_result": res.get("eval_result") + } + with open(os.path.join(action_dir, "metadata.json"), "w") as f: + json.dump(meta, f, indent=2) + + # Optional: Save A11y summary for quick viewing + if "a11y_summary" in res: + with open(os.path.join(action_dir, "a11y_summary.txt"), "w") as f: + f.write(res["a11y_summary"]) - logger.info(f"[ToolService] Browser artifacts saved to: {action_dir}") - except Exception as sse: - logger.warning(f"Failed to persist browser data to workspace: {sse}") + logger.info(f"[ToolService] Browser artifacts saved to: {action_dir}") + except Exception as sse: + logger.warning(f"Failed to persist browser data to workspace: {sse}") logger.info(f"[ToolService] System skill '{skill.name}' completed (Status: {sub_agent.status}).") return res diff --git a/ai-hub/app/core/skills/bootstrap.py b/ai-hub/app/core/skills/bootstrap.py index 7112500..7e276ca 100644 --- a/ai-hub/app/core/skills/bootstrap.py +++ b/ai-hub/app/core/skills/bootstrap.py @@ -29,29 +29,43 @@ logger.info(f"Syncing system skill: {skill_def['name']}") existing.description = skill_def.get("description") existing.skill_type = skill_def.get("skill_type") - existing.config = skill_def.get("config") existing.is_enabled = skill_def.get("is_enabled", True) existing.features = skill_def.get("features", ["swarm_control"]) existing.extra_metadata = skill_def.get("extra_metadata", {}) - existing.preview_markdown = skill_def.get("preview_markdown") - existing.system_prompt = skill_def.get("system_prompt") existing.is_system = True existing.owner_id = admin.id + db.commit() # Commit to get ID for VFS files + target_skill = existing else: logger.info(f"Creating new system skill: {skill_def['name']}") - new_skill = models.Skill( + target_skill = models.Skill( name=skill_def["name"], description=skill_def.get("description"), skill_type=skill_def.get("skill_type"), - config=skill_def.get("config", {}), is_enabled=skill_def.get("is_enabled", True), features=skill_def.get("features", ["swarm_control"]), extra_metadata=skill_def.get("extra_metadata", {}), - preview_markdown=skill_def.get("preview_markdown"), is_system=True, owner_id=admin.id ) - db.add(new_skill) + db.add(target_skill) + db.commit() # Commit to get ID + + # --- Update VFS --- + vfs_files = skill_def.get("_vfs_files", []) + existing_files = db.query(models.SkillFile).filter(models.SkillFile.skill_id == target_skill.id).all() + existing_file_map = {f.file_path: f for f in existing_files} + + for vf in vfs_files: + if vf["path"] in existing_file_map: + existing_file_map[vf["path"]].content = vf["content"] + del existing_file_map[vf["path"]] + else: + db.add(models.SkillFile(skill_id=target_skill.id, file_path=vf["path"], content=vf["content"])) + + # Delete any files that were removed from the filesystem + for removed_file in existing_file_map.values(): + db.delete(removed_file) try: db.commit() diff --git a/ai-hub/app/core/skills/fs_loader.py b/ai-hub/app/core/skills/fs_loader.py new file mode 100644 index 0000000..84a6024 --- /dev/null +++ b/ai-hub/app/core/skills/fs_loader.py @@ -0,0 +1,120 @@ +import os +import yaml +import json +import logging +from typing import List, Dict, Any +from app.config import settings + +logger = logging.getLogger(__name__) + +class FileSystemSkillLoader: + def __init__(self, base_dir: str): + self.base_dir = base_dir + + def _ensure_metadata(self, folder_path: str, skill_name: str, is_system: bool = False): + """ + Auto-generates a .metadata.json file if it's missing. + Assigns 'admin' as default owner if it's auto-generated during boot. + """ + meta_path = os.path.join(folder_path, ".metadata.json") + if not os.path.exists(meta_path): + default_metadata = { + "owner_id": "admin", + "is_system": is_system, + "extra_metadata": {"emoji": "🛠️"} + } + try: + with open(meta_path, "w") as f: + json.dump(default_metadata, f, indent=4) + logger.info(f"Generated default .metadata.json for skill '{skill_name}'") + except Exception as e: + logger.error(f"Failed to generate metadata for {skill_name}: {e}") + return default_metadata + + try: + with open(meta_path, "r") as f: + return json.load(f) + except Exception as e: + logger.error(f"Error reading {meta_path}: {e}") + return {} + + def get_all_skills(self) -> List[Dict[str, Any]]: + """ + Recursively walks the base direction and parses all SKILL.md rules + into the common dictionary schema used by the orchestrator. + """ + skills = [] + if not os.path.exists(self.base_dir): + return skills + + # Level 1 directories define Features (e.g., /data/skills/swarm_control) + for feature in os.listdir(self.base_dir): + feature_path = os.path.join(self.base_dir, feature) + if not os.path.isdir(feature_path): + continue + + # Level 2 directories define the specific Skills + for skill_id in os.listdir(feature_path): + skill_path = os.path.join(feature_path, skill_id) + if not os.path.isdir(skill_path): + continue + + skill_md_path = os.path.join(skill_path, "SKILL.md") + if not os.path.exists(skill_md_path): + continue + + try: + with open(skill_md_path, "r", encoding='utf-8') as f: + skill_content = f.read() + + # Create virtual file schema + vfs_files = [] + for root, _, files in os.walk(skill_path): + for file in files: + # Hide .metadata.json and macOS .DS_Store from the VFS visible to LLMs + if file == ".metadata.json" or file.startswith('.'): + continue + file_abs = os.path.join(root, file) + file_rel = os.path.relpath(file_abs, skill_path).replace('\\', '/') + try: + with open(file_abs, "r", encoding='utf-8') as ff: + vfs_files.append({"file_path": file_rel, "content": ff.read()}) + except Exception: + pass # skip binary or unreadable + + # Extract or Generate Metadata + metadata = self._ensure_metadata(skill_path, skill_id, is_system=False) + + # Extract Description directly from SKILL.md (Fallback or yaml frontmatter) + skill_name = skill_id + skill_desc = "" + + if skill_content.startswith("---"): + parts = skill_content.split("---", 2) + if len(parts) >= 3: + frontmatter = yaml.safe_load(parts[1]) or {} + skill_name = frontmatter.get("name", skill_name) + skill_desc = frontmatter.get("description", "") + + # Generate the legacy internal dict schema + skill_def = { + "id": f"fs-{skill_id}", + "name": skill_name, + "description": skill_desc or f"Skill loaded dynamically from {skill_id}", + "skill_type": "local", + "is_enabled": True, + "features": [feature], # Sourced directly from folder hierarchy + "is_system": metadata.get("is_system", False), + "owner_id": metadata.get("owner_id", "admin"), + "extra_metadata": metadata.get("extra_metadata", {"emoji": "🛠️"}), + "files": vfs_files + } + skills.append(skill_def) + + except Exception as e: + logger.error(f"Error parsing FS Skill at {skill_path}: {e}") + + return skills + +# Create a global instance initialized with the data directory config +fs_loader = FileSystemSkillLoader(base_dir=os.path.join(settings.DATA_DIR, "skills")) diff --git a/ai-hub/app/core/skills/loader.py b/ai-hub/app/core/skills/loader.py index 10e403d..4cbea57 100644 --- a/ai-hub/app/core/skills/loader.py +++ b/ai-hub/app/core/skills/loader.py @@ -25,21 +25,19 @@ skill_def = frontmatter - # Split body into Documentation and AI Instructions - # Convention: Use common headers or tags to denote the start of AI instructions - human_docs = body - ai_instructions = "" + vfs_files = [] + for root, _, files in os.walk(item_path): + for file in files: + if file.startswith('.'): continue + file_abs = os.path.join(root, file) + file_rel = os.path.relpath(file_abs, item_path).replace('\\', '/') + try: + with open(file_abs, "r") as ff: + vfs_files.append({"path": file_rel, "content": ff.read()}) + except Exception: + pass - separators = ["# AI Instructions", "# Intelligence Protocol", ""] - for sep in separators: - if sep in body: - split_parts = body.split(sep, 1) - human_docs = split_parts[0].strip() - ai_instructions = split_parts[1].strip() - break - - skill_def["preview_markdown"] = human_docs - skill_def["system_prompt"] = ai_instructions + skill_def["_vfs_files"] = vfs_files # Ensure metadata exists if "extra_metadata" not in skill_def: diff --git a/ai-hub/app/core/tools/definitions/read_skill_artifact.py b/ai-hub/app/core/tools/definitions/read_skill_artifact.py new file mode 100644 index 0000000..f6b7e11 --- /dev/null +++ b/ai-hub/app/core/tools/definitions/read_skill_artifact.py @@ -0,0 +1,64 @@ +from typing import Dict, Any, Tuple, Callable, Optional +from app.core.tools.base import BaseToolPlugin +import logging + +logger = logging.getLogger(__name__) + + +class ReadSkillArtifactTool(BaseToolPlugin): + """ + Lazy-loading tool that allows the AI agent to read any file stored in + the Skill Virtual File System (VFS) on demand. + + Instead of injecting the entire skill system_prompt or 1000-line scripts + into the initial context window, the agent is given only a brief summary + and can call this tool to read the relevant SKILL.md or any script in + the /scripts/ or /artifacts/ directories when it needs them. + """ + + @property + def name(self) -> str: + return "read_skill_artifact" + + @property + def retries(self) -> int: + return 0 # No retries needed — it's a simple DB read + + def prepare_task(self, args: Dict[str, Any], context: Dict[str, Any]) -> Tuple[Optional[Callable], Dict[str, Any]]: + skill_name = args.get("skill_name", "").strip() + file_path = args.get("file_path", "SKILL.md").strip() + db = context.get("db") + + if not skill_name: + return None, {"success": False, "error": "skill_name is required"} + if not db: + return None, {"success": False, "error": "Database context not available"} + + def _read_artifact(skill_name: str, file_path: str) -> dict: + from app.db import models as m + + skill = db.query(m.Skill).filter(m.Skill.name == skill_name).first() + if not skill: + return {"success": False, "error": f"Skill '{skill_name}' not found."} + + skill_file = ( + db.query(m.SkillFile) + .filter(m.SkillFile.skill_id == skill.id, m.SkillFile.file_path == file_path) + .first() + ) + if not skill_file: + return { + "success": False, + "error": f"File '{file_path}' not found in skill '{skill_name}'.", + "hint": "Use list_skill_files to discover available files." + } + + logger.info(f"[ReadSkillArtifact] Lazy-loaded '{file_path}' from skill '{skill_name}' ({len(skill_file.content or '')} chars)") + return { + "success": True, + "skill_name": skill_name, + "file_path": file_path, + "content": skill_file.content or "" + } + + return _read_artifact, {"skill_name": skill_name, "file_path": file_path} diff --git a/ai-hub/app/db/migrate.py b/ai-hub/app/db/migrate.py index 4abf9f6..ac6fb85 100644 --- a/ai-hub/app/db/migrate.py +++ b/ai-hub/app/db/migrate.py @@ -160,17 +160,37 @@ except Exception as e: logger.error(f"Failed to create 'skill_group_access': {e}") - # --- Skill table migrations --- + # Create skill_files table if it doesn't exist + if not inspector.has_table("skill_files"): + logger.info("Creating table 'skill_files'...") + try: + conn.execute(text(""" + CREATE TABLE IF NOT EXISTS skill_files ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + skill_id INTEGER NOT NULL REFERENCES skills(id), + file_path TEXT NOT NULL, + content TEXT, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP + ) + """)) + + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_skill_files_skill_id ON skill_files(skill_id)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_skill_files_file_path ON skill_files(file_path)")) + + conn.commit() + logger.info("Table 'skill_files' created.") + except Exception as e: + logger.error(f"Failed to create 'skill_files': {e}") + if inspector.has_table("skills"): skill_columns = [c["name"] for c in inspector.get_columns("skills")] skill_required_columns = [ - ("system_prompt", "TEXT"), ("is_enabled", "INTEGER DEFAULT 1"), ("features", "TEXT DEFAULT '[\"chat\"]'"), ("is_system", "INTEGER DEFAULT 0"), ("skill_type", "TEXT DEFAULT 'local'"), - ("extra_metadata", "TEXT DEFAULT '{}'"), - ("preview_markdown", "TEXT"), + ("extra_metadata", "TEXT DEFAULT '{}'") ] for col_name, col_type in skill_required_columns: if col_name not in skill_columns: @@ -181,6 +201,18 @@ logger.info(f"Successfully added '{col_name}' to 'skills'.") except Exception as e: logger.error(f"Failed to add column '{col_name}' to 'skills': {e}") + + # Remove deprecated columns + deprecated_columns = ["system_prompt", "preview_markdown", "config"] + for col_name in deprecated_columns: + if col_name in skill_columns: + logger.info(f"Dropping deprecated column '{col_name}' from 'skills' table...") + try: + conn.execute(text(f"ALTER TABLE skills DROP COLUMN {col_name}")) + conn.commit() + logger.info(f"Successfully dropped '{col_name}' from 'skills'.") + except Exception as e: + logger.error(f"Failed to drop column '{col_name}' from 'skills'. SQLite might not support drop column: {e}") logger.info("Database migrations complete.") diff --git a/ai-hub/app/db/models/__init__.py b/ai-hub/app/db/models/__init__.py index 0e1d233..9850d37 100644 --- a/ai-hub/app/db/models/__init__.py +++ b/ai-hub/app/db/models/__init__.py @@ -1,13 +1,13 @@ from .user import User, Group from .session import Session, Message from .document import Document, VectorMetadata -from .asset import PromptTemplate, Skill, SkillGroupAccess, MCPServer, AssetPermission +from .asset import PromptTemplate, Skill, SkillFile, SkillGroupAccess, MCPServer, AssetPermission from .node import AgentNode, NodeGroupAccess __all__ = [ "User", "Group", "Session", "Message", "Document", "VectorMetadata", - "PromptTemplate", "Skill", "SkillGroupAccess", "MCPServer", "AssetPermission", + "PromptTemplate", "Skill", "SkillFile", "SkillGroupAccess", "MCPServer", "AssetPermission", "AgentNode", "NodeGroupAccess" ] diff --git a/ai-hub/app/db/models/asset.py b/ai-hub/app/db/models/asset.py index 142ed8b..af17e56 100644 --- a/ai-hub/app/db/models/asset.py +++ b/ai-hub/app/db/models/asset.py @@ -1,5 +1,5 @@ from datetime import datetime -from sqlalchemy import Column, Integer, String, Text, DateTime, ForeignKey, Boolean, JSON +from sqlalchemy import Column, Integer, String, Text, DateTime, ForeignKey, Boolean, JSON, LargeBinary from sqlalchemy.orm import relationship from ..database import Base @@ -32,21 +32,19 @@ name = Column(String, unique=True, index=True, nullable=False) description = Column(String, nullable=True) skill_type = Column(String, default="local", nullable=False) - config = Column(JSON, default={}, nullable=True) - system_prompt = Column(Text, nullable=True) is_enabled = Column(Boolean, default=True) features = Column(JSON, default=["chat"], nullable=True) owner_id = Column(String, ForeignKey('users.id'), nullable=False) is_system = Column(Boolean, default=False) - preview_markdown = Column(Text, nullable=True) extra_metadata = Column(JSON, default={}, nullable=True) created_at = Column(DateTime, default=datetime.utcnow) owner = relationship("User") + files = relationship("SkillFile", back_populates="skill", cascade="all, delete-orphan") def __repr__(self): return f"" @@ -64,6 +62,22 @@ skill = relationship("Skill") group = relationship("Group") +class SkillFile(Base): + __tablename__ = 'skill_files' + + id = Column(Integer, primary_key=True, index=True) + skill_id = Column(Integer, ForeignKey('skills.id'), nullable=False) + file_path = Column(String, index=True, nullable=False) + content = Column(Text, nullable=True) + + created_at = Column(DateTime, default=datetime.utcnow) + updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) + + skill = relationship("Skill", back_populates="files") + + def __repr__(self): + return f"" + class MCPServer(Base): __tablename__ = 'mcp_servers' diff --git a/ai-hub/docs/api_reference/models.md b/ai-hub/docs/api_reference/models.md index fccfdfe..75c310b 100644 --- a/ai-hub/docs/api_reference/models.md +++ b/ai-hub/docs/api_reference/models.md @@ -381,6 +381,7 @@ |----------|------|-------------| | `source` | `string` | 'empty' | 'server' | 'node_local' | | `path` | `anyOf` | | +| `source_node_id` | `anyOf` | | ## `NodeDispatchRequest` @@ -581,33 +582,18 @@ | `name*` | `string` | | | `description` | `anyOf` | | | `skill_type` | `string` | | -| `config` | `object` | | -| `system_prompt` | `anyOf` | | | `is_enabled` | `boolean` | | | `features` | `array` | | | `is_system` | `boolean` | | | `extra_metadata` | `object` | | -| `preview_markdown` | `anyOf` | | -## `SkillResponse` +## `SkillFileUpdate` **Type:** `object` | Property | Type | Description | |----------|------|-------------| -| `name*` | `string` | | -| `description` | `anyOf` | | -| `skill_type` | `string` | | -| `config` | `object` | | -| `system_prompt` | `anyOf` | | -| `is_enabled` | `boolean` | | -| `features` | `array` | | -| `is_system` | `boolean` | | -| `extra_metadata` | `object` | | -| `preview_markdown` | `anyOf` | | -| `id*` | `integer` | | -| `owner_id*` | `string` | | -| `created_at*` | `string` | | +| `content*` | `string` | | ## `SkillUpdate` @@ -618,13 +604,10 @@ | `name` | `anyOf` | | | `description` | `anyOf` | | | `skill_type` | `anyOf` | | -| `config` | `anyOf` | | -| `system_prompt` | `anyOf` | | | `is_enabled` | `anyOf` | | | `features` | `anyOf` | | | `is_system` | `anyOf` | | | `extra_metadata` | `anyOf` | | -| `preview_markdown` | `anyOf` | | ## `SpeechRequest` diff --git a/ai-hub/docs/api_reference/skills.md b/ai-hub/docs/api_reference/skills.md index 9b38855..5bd5862 100644 --- a/ai-hub/docs/api_reference/skills.md +++ b/ai-hub/docs/api_reference/skills.md @@ -4,13 +4,13 @@ **Summary:** List Skills -**Description:** List all skills accessible to the user. +**Description:** List all skills accessible to the user (via File System). #### Parameters | Name | In | Required | Type | Description | |------|----|----------|------|-------------| -| `feature` | query | No | anyOf | Filter skills by feature (e.g., 'chat', 'voice') | +| `feature` | query | No | anyOf | Filter skills by feature (e.g., 'swarm_control', 'voice_chat') | | `x-user-id` | header | No | anyOf | | #### Responses @@ -35,7 +35,7 @@ **Summary:** Create Skill -**Description:** Create a new skill. +**Description:** Create a new skill folder on the physical filesystem. #### Parameters @@ -74,13 +74,13 @@ **Summary:** Update Skill -**Description:** Update an existing skill. User must be admin or the owner. +**Description:** Update top level metadata of an FS skill (emoji and system flag). #### Parameters | Name | In | Required | Type | Description | |------|----|----------|------|-------------| -| `skill_id` | path | Yes | integer | | +| `skill_id` | path | Yes | string | | | `x-user-id` | header | No | anyOf | | #### Request Body @@ -114,13 +114,13 @@ **Summary:** Delete Skill -**Description:** Delete a skill. +**Description:** Delete a skill permanently via shutil. #### Parameters | Name | In | Required | Type | Description | |------|----|----------|------|-------------| -| `skill_id` | path | Yes | integer | | +| `skill_id` | path | Yes | string | | | `x-user-id` | header | No | anyOf | | #### Responses @@ -141,3 +141,139 @@ --- +## GET `/skills/{skill_id}/files` + +**Summary:** List Skill Files + +**Description:** Get file tree for a skill. + +#### Parameters + +| Name | In | Required | Type | Description | +|------|----|----------|------|-------------| +| `skill_id` | path | Yes | string | | +| `x-user-id` | header | No | anyOf | | + +#### Responses + +| Status Code | Description | +|-------------|-------------| +| `200` | Successful Response | +| `422` | Validation Error | + +#### Example Usage + +```bash +curl -X 'GET' \ + 'http://localhost:8000/api/v1/skills/{skill_id}/files' \ + -H 'accept: application/json' \ + -H 'X-User-ID: ' +``` + +--- + +## GET `/skills/{skill_id}/files/{path}` + +**Summary:** Read Skill File + +**Description:** Read a specific skill file. + +#### Parameters + +| Name | In | Required | Type | Description | +|------|----|----------|------|-------------| +| `skill_id` | path | Yes | string | | +| `path` | path | Yes | string | | +| `x-user-id` | header | No | anyOf | | + +#### Responses + +| Status Code | Description | +|-------------|-------------| +| `200` | Successful Response | +| `422` | Validation Error | + +#### Example Usage + +```bash +curl -X 'GET' \ + 'http://localhost:8000/api/v1/skills/{skill_id}/files/{path}' \ + -H 'accept: application/json' \ + -H 'X-User-ID: ' +``` + +--- + +## POST `/skills/{skill_id}/files/{path}` + +**Summary:** Create Or Update Skill File + +**Description:** Create or update a skill file. + +#### Parameters + +| Name | In | Required | Type | Description | +|------|----|----------|------|-------------| +| `skill_id` | path | Yes | string | | +| `path` | path | Yes | string | | +| `x-user-id` | header | No | anyOf | | + +#### Request Body + +**Required:** Yes + +- **Media Type:** `application/json` +- **Schema:** `SkillFileUpdate` (Define in Models) + +#### Responses + +| Status Code | Description | +|-------------|-------------| +| `200` | Successful Response | +| `422` | Validation Error | + +#### Example Usage + +```bash +curl -X 'POST' \ + 'http://localhost:8000/api/v1/skills/{skill_id}/files/{path}' \ + -H 'accept: application/json' \ + -H 'X-User-ID: ' \ + -H 'Content-Type: application/json' \ + -d '{}' +``` + +--- + +## DELETE `/skills/{skill_id}/files/{path}` + +**Summary:** Delete Skill File + +**Description:** Delete a skill file literally from disk. + +#### Parameters + +| Name | In | Required | Type | Description | +|------|----|----------|------|-------------| +| `skill_id` | path | Yes | string | | +| `path` | path | Yes | string | | +| `x-user-id` | header | No | anyOf | | + +#### Responses + +| Status Code | Description | +|-------------|-------------| +| `200` | Successful Response | +| `422` | Validation Error | + +#### Example Usage + +```bash +curl -X 'DELETE' \ + 'http://localhost:8000/api/v1/skills/{skill_id}/files/{path}' \ + -H 'accept: application/json' \ + -H 'X-User-ID: ' +``` + +--- + diff --git a/ai_snippet b/ai_snippet new file mode 160000 index 0000000..0d6a119 --- /dev/null +++ b/ai_snippet @@ -0,0 +1 @@ +Subproject commit 0d6a1191fa1ac69d9c8328056047d2472355298f diff --git a/docs/features/skills_page.md b/docs/features/skills_page.md new file mode 100644 index 0000000..fcf0e02 --- /dev/null +++ b/docs/features/skills_page.md @@ -0,0 +1,63 @@ +# Cortex Skills Feature Architecture + +## Overview +The "Cortex Skills" feature serves as the central orchestration and management interface for AI capabilities within the Cortex Hub ecosystem. It provides a robust full-stack solution for viewing, creating, sharing, and managing modular AI skills (native, MCP, or gRPC protocols). + +## Frontend Architecture + +Located primarily in `/app/frontend/src/features/skills`, the frontend follows a clean Container/Presenter pattern to separate state management from UI rendering. + +### Components +1. **`SkillsPage` (Container)** + - **Location:** `/app/frontend/src/features/skills/pages/SkillsPage.js` + - Manages state, data fetching (CRUD operations using `apiService`), and filtering logic. + - Holds complex states such as advanced "Engineering Mode" toggles, modal visibility, and contextual payload editing. + +2. **`SkillsPageContent` (Presenter)** + - **Location:** `/app/frontend/src/features/skills/components/SkillsPageContent.js` + - Renders the UI and consumes state via a shared `context` property. + - Designed with modern UI paradigms: dynamic grids, blurring (`backdrop-blur`), visual cues, micro-animations, and a responsive sidebar. + +### Frontend Features +- **Sidebar & Filtering System**: Four distinct visual filters: + - *All Libraries*: Every skill available to the user. + - *System Core*: Built-in immutable skills. + - *My Creations*: Skills created and owned by the logged-in user. + - *Shared Works*: Skills inherited via group policy. +- **Universal Search & System Toggling**: Fast text-based searching (checking names and descriptions) paired with a toggle to hide system skills to reduce visual noise. +- **Skill Grid**: Visually distinct skill cards utilizing `extra_metadata.emoji` for quick identification. Displays disabled skills with an `opacity-60 grayscale` filter. +- **Docs Viewer Modal**: Integrated `ReactMarkdown` rendering for `preview_markdown`, allowing users to see public instructions vs the raw "Intelligence Blueprint". +- **Engineering Mode (Editor Modal)**: Contains both basic fields (Name, Emoji, Manifesto, Protocol Type) and advanced configuration states (Intelligence Protocol System Prompt, Network Config JSON validator, Markdown definitions). + +## Backend Architecture + +The backend REST API is built in FastAPI under `/app/ai-hub/app/api/routes/skills.py`. It is responsible for advanced access control, system-level safety, and complex querying. + +### Endpoints +- `GET /skills/`: Lists skills, enforcing multi-level Access Control Logic (ACL). +- `POST /skills/`: Creates a given skill configuration. +- `PUT /skills/{skill_id}`: Overwrites skill configurations. +- `DELETE /skills/{skill_id}`: Purges a skill from the ecosystem. + +### Access Control & Policy Logic +1. **System Protection**: + - `is_system` skills are **immutable**. They cannot be modified or deleted via the API, not even by administrators. This protects the core platform loops. + - Only admins can create new `is_system` skills. +2. **Read/Visibility Rules**: + - **Administrators**: Complete visibility of all skills, regardless of `is_enabled` status or ownership. + - **Normal Users**: View system skills (if enabled), their own skills (where `owner_id` == `user.id` and enabled), and **Group Skills**. Group skills are dynamically matched by checking `current_user.group.policy.get("skills", [])`. +3. **Filtering**: + - The API supports a `feature` query parameter (e.g., `?feature=swarm_control` or `?feature=voice`), filtering the query arrays to only return relevant capabilities for specific frontend sub-systems. +4. **Validations**: + - Automatic uniqueness checks for the `name` field on creation and modification. + - Strict owner-only (or admin) checks injected into PUT and DELETE policies to prevent arbitrary modification. + +## Data Model Structure (Key Fields) +- `name` (String): Unique string identifier (e.g. `browser_eval`). +- `skill_type` (String): Network protocol format - Native (`local`), MCP (`mcp`), or remote (`remote_grpc`). +- `config` (JSON): Configuration network JSON payload. +- `system_prompt` (String): Internal LLM instructions invisible to end-users unless explicitly provided. +- `features` (Array): Target application bindings mapping where the skill should be active (e.g., `['swarm_control', 'voice_chat', 'workflow']`). +- `is_system` & `is_enabled` (Boolean): Core booleans managing lifecycle state and system protection. +- `extra_metadata` (JSON): Contains UI metadata and arbitrary frontend parameters (e.g., UI `emoji`). +- `preview_markdown` (String): The operational documentation presented to the user. diff --git a/docs/refactors/skill_filesystem_refactor.md b/docs/refactors/skill_filesystem_refactor.md new file mode 100644 index 0000000..f49238d --- /dev/null +++ b/docs/refactors/skill_filesystem_refactor.md @@ -0,0 +1,52 @@ +# Skill File-System Refactor Implementation Plan + +## Overview +This document outlines the transition fully away from database-driven skill storage towards a pure File-System-Based Architecture. Skills will become directories on the server grouped by features (e.g., `swarm_control`, `voice_chat`), simplifying the user interface and maximizing flexibility (enabling git versioning, native file inclusion, etc.). + +## Phase 1: Backend Architecture & Tool Loader Reform +### 1.1 Data Structure Updates +- Define a base directory configuration `settings.SKILLS_DIR` (e.g., `/app/data/skills`). +- Deprecate existing SQLAlchemy models for `Skill` and `SkillFile` (do not delete db tables yet for backup). + +### 1.2 Hot-Reload Engine & File Watcher (`app.core.skills.loader`) +- Create a `FileSystemSkillLoader` script. +- The loader will `os.walk()` through `settings.SKILLS_DIR` on boot, or use a background File System Watcher (e.g., `watchdog`) to dynamically update capabilities. + - Level 1 directories define the "Feature" (e.g., `swarm_control`, `voice_chat`). + - Level 2 directories define the "Skill Unique ID" (e.g., `get_weather`). +- Reads `SKILL.md` from these folders to mount Markdown schemas and execute scripts. + +### 1.3 Invisible RBAC Metadata (`.metadata.json`) +- If a skill folder is newly created (either manually on the server or via API) or if pre-existing files are detected, the loader will check for a hidden `.metadata.json` file. +- **Server Boot Initialization**: When the server boots and recursively scans the directory structure, it will auto-generate `.metadata.json` for any existing folder that lacks one, assigning ownership defaults (e.g., admin). +- **Backend API Creation**: If the user creates the skill from the frontend UI, the backend implicitly writes a `.metadata.json` assigning `owner_id` to that user. +- **Manual Server Manipulation**: If an administrator copies a folder directly to the server via terminal, the File Watcher detects the missing `.metadata.json` and auto-generates it, defaulting the `owner_id` to the cluster admin. +- The frontend UI will explicitly hide `.metadata.json` from the Visual Explorer so developers aren't bothered by system files. Permission modifications in the future will interact exclusively with this file. + +### 1.4 Update `tool.py` +- Modify `_execute_system_skill` and `get_tools` inside `app/core/services/tool.py` to route through the new `FileSystemSkillLoader` cache instead of invoking the database sessions. The `.metadata.json` overrides will dictate authorization blocks. + +## Phase 2: Refactoring the REST API (`app.api.routes.skills`) +### 2.1 Decouple from SQLAlchemy +- Rewrite the `GET /api/v1/skills/` endpoint to return JSON payloads mapping the physical directories (e.g., reading directories, returning the tree object). +- Replace `POST`, `PUT`, `DELETE` operations with standard Python `os` and `shutil` commands: + - Creating a skill creates a folder. + - Adding a file creates a physical file inside that folder. + - Deleting a skill calls `shutil.rmtree()`. +- Expose an endpoint to recursively read the tree structure of a skill (`/api/v1/skills/{feature}/{skill_id}/tree`). + +## Phase 3: Frontend Simplification Experience (`ai_unified_frontend`) +### 3.1 Strip Declarative Database Forms +- Eliminate modal logic dealing with `name`, `description`, `skill_type`, and `is_system` checkboxes from `SkillsPage.js`. +- Provide a pure UI File Explorer that does not use forms to "generate" a skill. Instead, users simply right-click to "New Folder", typing the exact directory name they want (e.g., `get_weather`). +- There is no backend conversion or "slugifying" of names. If a user types illegal characters for a folder name, the underlying OS rejection is simply passed back to the UI. The human-readable title and emojis will be parsed directly from the `SKILL.md` content (e.g., `### Skill Name: Get Weather 🌤️`). + +### 3.2 Direct VSCode-Like IDE Integration +- Turn the `SkillsPage` layout into a two-panel IDE layout: + - **Left Sidebar**: File Explorer detailing the tree (`Feature > Folder Name > Files`). + - **Right Panel**: A large `Monaco Editor` or `CodeMirror` instance loading the physical file content. +- Support creation of arbitrary code assets (Python, textual configs) alongside the `SKILL.md` via UI "Right Click -> New File". + +## Considerations +1. **Migration Script**: Before deleting DB dependencies, we optionally need a Python script to sweep existing skills out of PostgreSQL and physically write them into `/app/data/skills/`. +2. **Access Security**: If multi-tenancy is still important, `settings.SKILLS_DIR` could be parameterized per user (`/app/data/skills/{user_id}/...`) or we accept skills as global components for the Hub. +3. **Execution Sandbox**: Existing nodes and bash execution sandboxes will remain perfectly intact. The only change is how the definitions arrive at the orchestrator. diff --git a/docs/refactors/skill_folder_framework.md b/docs/refactors/skill_folder_framework.md new file mode 100644 index 0000000..d067a67 --- /dev/null +++ b/docs/refactors/skill_folder_framework.md @@ -0,0 +1,62 @@ +# Skill Framework Refactoring Proposal (Folder-based Architecture) + +## 1. Problem Statement: The Monolithic String Anti-Pattern +Currently, the Cortex skills system stores AI capabilities as single database rows (e.g., `system_prompt`, `preview_markdown`, `config`). The UI consists of large text areas where users paste monolithic prompts containing all custom scripts and reference data. + +**Shortcomings:** +- **Context Window Bloat:** Putting 1,000+ line scripts directly into `system_prompt` exhausts the LLM’s context window limit and degrades reasoning capabilities. +- **Static Functionality:** Current skills lack the ability to encapsulate executable code securely without cluttering the prompt. +- **Divergence from State-of-the-Art:** Modern AI orchestration frameworks define tools/skills as discrete file structures or sandboxed resources, not string values in a relational database. + +### Industry Validation +Our research confirms that the industry standard has moved away from monolithic prompting: +1. **OpenHands (formerly OpenDevin)**: Operates using a Runtime Sandbox (Docker container). It grants agents `execute` and `execute_ipython_shell` actions. Global skills and repository guidelines are stored as markdown files (like `AGENTS.md`) and python scripts within the workspace, which are read/executed *only when triggered*, rather than injected into the system prompt upfront. +2. **OpenAI Assistants API**: Utilizes a Sandboxed Code Interpreter. Instead of pasting data and scripts into the system instructions, developers upload files (Python scripts, CSVs) which are mounted to `/mnt/data/` within the sandbox. The LLM writes small wrapper scripts to execute or read these files dynamically. +3. **Anthropic Model Context Protocol (MCP)**: Separates "Resources" (lazily loaded file URIs) from "Tools" (executables). The agent decides when to read a resource URI rather than having the server push the entire file context into the conversation automatically. + +## 2. The Solution: "Skills as Folders" (Lazy Loading Architecture) + +The skill definition paradigm must shift from **database forms** to **file trees**. A skill should represent a containerized environment of rules, references, and executable assets. + +### Proposed Structure of a Skill: +A given skill (e.g., `mesh-file-explorer`) would be managed just like a Git repository folder containing: + +```text +/skills/mesh-file-explorer/ +├── SKILL.md # Core instructions & meta-rationale (What the LLM reads first) +├── scripts/ # Executable runtimes to lazy load (e.g., node.js CLI tools, Python scrapers) +│ ├── run_explorer.py +│ └── helper.sh +├── examples/ # Example usages or inputs (few-shot prompting material) +│ └── successful_logs.txt +└── artifacts/ # Binary plugins or reference files +``` + +### The "Lazy Loading" Advantage +The primary benefit of this folder structure is **Lazy Context Injection**. +1. **The LLM starts only with the metadata:** The agent is given a brief summary of the skill via `SKILL.md` or a standard system tool describing the folder's purpose. +2. **On-Demand Context:** The agent has a subset tool like `view_skill_artifact` or `execute_plugin_script`. If the LLM determines it needs to run a web scraper, it calls `scripts/run_scraper.py`. +3. **Reduction in Tokens:** The 1,000+ line Python scraper is **never** loaded into the conversation prompt. Only its execution results or help output are printed to the agent context. + +## 3. Implementation Roadmap + +### Phase 1: Storage and Backend Overhaul +- **File System Virtualization:** Transition from storing huge SQL Strings (`system_prompt`) to a virtualized file system mapping. Skills can either be saved to a network drive, synced through the agent-node mesh, or abstracted behind an Object Storage system (S3/minio) or a Virtual File System DB design. +- **REST APIs (Virtual File Explorer):** + Replace the flat `/skills` CRUD with a hierarchy: + - `GET /skills/:id/tree` (Fetch folder hierarchy) + - `GET /skills/:id/files/:path` (Read asset contents) + - `POST /skills/:id/files/:path` (Upload/Create code scripts inside a skill) + +### Phase 2: Frontend "Skill Studio" (IDE-like UI) +The current UI requires replacing the simple forms ("Engineering Mode") with a "Skill Editor Workspace" modeled after basic web-IDEs (like VSCode web or Git repository interfaces). +- **Left Panel:** File tree showing `SKILL.md`, `scripts/`, `artifacts/`. +- **Center Canvas:** Code editor (e.g. Monaco / CodeMirror) to edit the currently selected file. +- **Asset Uploads:** Support for drag-and-dropping Python code, shell scripts, or CSV reference files straight into the skill. + +### Phase 3: Agentic API (Tool Adaptation) +- **New Standard Tools for the Agent:** Inject a system tool to let the agent explore available folders and execute skill artifacts. +- When an agent equips a "Skill", the system mounts that specific skill's `/scripts` directory directly into the Agent's sandbox `PATH` environment variable, making tool invocation native and seamless in bash. + +## 4. Summary of Value +Through this refactoring, skills graduate from **"Large Prompts"** to **"Software Packages"**. This creates an ecosystem where developers can drop in a complex Docker network or Python repository into a skill folder, and the Cortex LLM can dynamically research and execute those resources as needed without breaking context sizes. diff --git a/docs/refactors/skill_implementation_plan.md b/docs/refactors/skill_implementation_plan.md new file mode 100644 index 0000000..5fa4ed3 --- /dev/null +++ b/docs/refactors/skill_implementation_plan.md @@ -0,0 +1,94 @@ +# Skill Framework Migration: Step-by-Step AI Implementation Plan + +This document serves as a self-contained, step-by-step guide for an AI agent to execute the migration from the flat, database-backed skill system to the new "Skills as Folders" (Lazy-Loading) architecture. + +**Prerequisites to check before starting:** +- Backend uses FastAPI (`/app/ai-hub/app/api/`). +- Frontend uses React (`/app/frontend/src/features/skills/`). +- Database uses SQLAlchemy (`/app/db/models.py`). + +--- + +## Phase 1: Database & Model Virtualization (Backend) +**Objective:** Evolve the `Skill` model to support a Virtual File System (VFS) instead of monolithic text fields. + +- [x] **Step 1.1: Create the `SkillFile` SQLAlchemy Model** + - **Location:** `/app/ai-hub/app/db/models.py` + - **Action:** Define a new `SkillFile` model with: + - `id` (Integer, Primary Key) + - `skill_id` (Integer, Foreign Key to `skills.id`) + - `file_path` (String, e.g., `SKILL.md`, `scripts/run.py`, `artifacts/data.json`) + - `content` (Text or LargeBinary, to hold file data) + - `created_at` / `updated_at` (DateTime) + - **Action:** Add a `files = relationship("SkillFile", back_populates="skill", cascade="all, delete-orphan")` to the main `Skill` model. + +- [x] **Step 1.2: Deprecate Legacy Fields in the `Skill` Model** + - **Location:** `/app/ai-hub/app/db/models.py` + - **Action:** Mark `system_prompt`, `preview_markdown`, and `config` as nullable/deprecated. The core logic should now expect a `SKILL.md` file in the `SkillFile` table to replace `system_prompt` and `preview_markdown`. + +- [x] **Step 1.3: Generate & Run Alembic Migration** + - **Action:** Run the command to generate an alembic migration script for the new `SkillFile` table and apply it to the database. + +--- + +## Phase 2: REST API Overhaul (Backend) +**Objective:** Expose endpoints for standard IDE operations (CRUD for files/folders). + +- [x] **Step 2.1: Implement File Tree API** + - **Location:** `/app/ai-hub/app/api/routes/skills.py` + - **Action:** Create `GET /skills/{skill_id}/files`. + - **Logic:** Return a hierarchical JSON representation of the files associated with the skill (e.g., simulating folders by parsing the `file_path` strings). + +- [x] **Step 2.2: Implement File Content CRUD Operations** + - **Location:** `/app/ai-hub/app/api/routes/skills.py` + - **Action:** Create `GET /skills/{skill_id}/files/{path:path}` to read a specific file's text/binary content. + - **Action:** Create `POST /skills/{skill_id}/files/{path:path}` to create or update file contents. + - **Action:** Create `DELETE /skills/{skill_id}/files/{path:path}` to remove a file. + +- [x] **Step 2.3: Modify Skill Creation Logic** + - **Location:** `/app/ai-hub/app/api/routes/skills.py` (POST `/skills/`) + - **Action:** Upon creating a new skill, automatically seed the `SkillFile` database with a default `SKILL.md` file containing basic boilerplate instructions. + +--- + +## Phase 3: "Lazy Loading" Agent Logic Integration (Backend/Worker) +**Objective:** Teach the AI agent how to interact with this new folder structure instead of relying on the giant `system_prompt`. + +- [x] **Step 3.1: Create the `read_skill_artifact` Tool** + - **Location:** Core agent tooling directory (e.g., `/app/ai-hub/app/core/agent/tools.py` or similar). + - **Action:** Program a standard tool allowing the LLM to provide a `skill_name` and `file_path` to read content dynamically during a session. + +- [x] **Step 3.2: Update Agent Initialization Context** + - **Location:** Node/Agent execution wrapper (`sessions.py` or agent orchestrator). + - **Action:** When loading a skill for an agent, inject *only* the contents of `SKILL.md` into the initial system prompt. Provide the agent with the `read_skill_artifact` tool to discover and read anything in the `/scripts/` or `/artifacts/` folders when necessary. + +--- + +## Phase 4: The "Skill Studio" IDE Upgrade (Frontend) +**Objective:** Replace the basic textarea forms with a true workspace IDE UI. + +- [ ] **Step 4.1: Abstract File APIs in Frontend Service** + - **Location:** `/app/frontend/src/services/apiService.js` + - **Action:** Add helper functions: `getSkillFiles(id)`, `getSkillFileContent(id, path)`, `saveSkillFile(id, path, content)`, `deleteSkillFile(id, path)`. + +- [ ] **Step 4.2: Build the File Tree Sidebar Component** + - **Location:** `/app/frontend/src/features/skills/components/SkillFileTree.js` (New Component) + - **Action:** Implement a clickable folder/file tree UI. Selecting a file triggers a state update for the active open code editor. + +- [ ] **Step 4.3: Implement the Code Editor Component** + - **Location:** `/app/frontend/src/features/skills/components/SkillEditor.js` (New Component) + - **Action:** Drop in a code editor (like Monaco Editor, or a high-quality textarea with syntax highlighting) to edit the active file fetched in Step 4.2. + - **Action:** Add "Save" shortcuts (Ctrl+S) hooked to the `saveSkillFile` API. + +- [ ] **Step 4.4: Refactor `SkillsPageContent.js`** + - **Location:** `/app/frontend/src/features/skills/components/SkillsPageContent.js` + - **Action:** Remove the legacy "Engineering Mode" and monolithic textareas (`formData.system_prompt`, `formData.preview_markdown`, `formData.config`). + - **Action:** Replace the modal body with the newly created Workspace Layout: Side-by-side split view of `SkillFileTree` (Left) and `SkillEditor` (Right). + +--- + +## Final Validation Checklist +1. Create a new skill; verify `SKILL.md` is automatically created. +2. Add a new file `scripts/test.py` via the Frontend UI. +3. Observe the AI Agent reading `SKILL.md` by default, but executing a tool to read `scripts/test.py` only when requested. +4. Verify token counts drop significantly compared to the legacy monolithic system prompt. diff --git a/docs/refactors/skill_symlink_plan.md b/docs/refactors/skill_symlink_plan.md new file mode 100644 index 0000000..beb18e1 --- /dev/null +++ b/docs/refactors/skill_symlink_plan.md @@ -0,0 +1,52 @@ +# Phase 4: Native Skill Symlinking + +## Observation +Currently, the LLM reads the contents of bash scripts from `SKILL.md` entirely into context, and executes them remotely using generic commands. This breaks down if a skill involves complex file directories (like 1000 lines of python configuration, custom shell scripts, etc.). +Since Cortex utilizes a powerful bi-directional gRPC file synchronizer that maps a specific server-side Session directory (`/tmp/cortex-sync/{session_id}/`) directly down to the Client Node Workers, we can dynamically expose tools directly to the Node's active filesystem by **symlinking** the Skill's folder. + +## Objective +When a session is active, dynamically mount any active File-System Skills (e.g. `weather_api`) straight into the session workspace directory (`.skills/`). Because of the background `mesh_file_explorer` file syncing loop, any symlinked `.skills` folder on the Server will automatically be evaluated and populated down to the running Node Worker. +This allows the AI to execute `bash .skills/weather_api/run.sh` natively without loading any code into its context! + +## Step-by-Step Implementation Plan + +### 1. Identify Workspace Initialization Hook +The AI Server initializes the file sync workspace folder whenever a session starts or connects: +- Files involved: `app/ai-hub/app/core/services/session.py` or wherever session folders are mapped (`/tmp/cortex-sync/{session_id}`). +- **Goal:** During startup of the worker container node, or when the Agent loops starts, we must run a Symlink sync process. + +### 2. Map the Linked Folders +- The central `DATA_DIR/skills/` directory holds all physical skills. +- The Session Workspace directory is located at `/tmp/cortex-sync/{session_id}/`. +- Inside the workspace, create a hidden orchestrator directory `.skills`. +- Loop through all active tools loaded by `ToolService.get_available_tools(...)` for the given User/Session. +- For every active tool found on the File System, create a relative symlink from `DATA_DIR/skills/{feature}/{skill_id}` to `/tmp/cortex-sync/{session_id}/.skills/{skill_id}`. + +### 3. Automatically Ignore `.skills` in Git/History tracking (Optional) +- Ensure the bi-directional sync does NOT push changes from `.skills/` BACK up to the original physical folder if the AI ruins them. This is critical for security. +- Wait, symlinks in Python (`os.symlink`) point to the read-only or original folder. If the Node modifies it, it modifies the original tool! +- **Alternative:** Hard copy the scripts, OR use actual read-only Docker Volume mounts to the nodes (wait, the nodes are remote distributed workers!). If they are remote, the File Sync daemon using Python `os.walk` will follow symlinks and sync the physical files down to the remote Node. +- The remote Node will treat them as raw downloaded files! It modifies *its* localized copies, not the Server's source! +- However, if the Node tries to upload the changed files back, the Server's `file_sync` daemon will write the changes back to the symlink, modifying the global tool! +- **Mitigation:** We must add `.skills/` into the `ignored_paths` constant within `app/core/grpc/shared_core/ignore.py` ON THE UPSTREAM route (Node -> Server) so that changes aren't persisted backward. + +### 4. Inject `.skills/` Execution Logic into the System Prompt +- For each injected skill, the Tool System currently parses `Summary: ...` or the `| Parameters |`. +- Modify `tool.py` so that instead of saying "Call read_skill_artifact to see instructions", the system prompt explicitly tells the AI: + ```text + This skill is natively mapped to your workspace. + You can execute it directly on the node via: `bash .skills/{skill_id}/{executable}`. + ``` + +### 5. AI Self-Improvement/Evolution Capabilities +Since the `.skills/` directory is bi-directionally synced between the true File System and the Node Workspace: +1. The AI can natively use `mesh_file_explorer` to `read` any script located inside `.skills/`. +2. The AI can use terminal sed, python ast or file tools to modify/debug its own skill source code automatically if it fails. +3. Because the directory is physically synced, the server overwrites the permanent `/app/data/skills/` folder seamlessly. The AI becomes capable of hot-fixing its own execution scripts permanently for all future sessions! + +### 6. Finalizing Skill Definitions +- Users can create a `run.sh` or `main.py` directly alongside `SKILL.md` in the VSCode IDE UI. +- The AI LLM gets instructions to just call that file directly via `mesh_terminal_control`. + +## Summary +By symlinking `DATA_DIR/skills/` -> `/tmp/cortex-sync/{session_id}/.skills/`, the gRPC network will sync the scripts as raw text files across the internet directly into the Client Node's OS container. The Agent gets zero context bloat, executing vast tools effortlessly, and gains the absolute power to spontaneously view, trace, and self-improve its own capabilities across sessions! diff --git a/frontend/src/features/skills/components/SkillEditor.js b/frontend/src/features/skills/components/SkillEditor.js new file mode 100644 index 0000000..4597819 --- /dev/null +++ b/frontend/src/features/skills/components/SkillEditor.js @@ -0,0 +1,70 @@ +import React, { useState, useEffect } from 'react'; + +export default function SkillEditor({ activeFile, content, onChange, onSave, isSaving }) { + const [localContent, setLocalContent] = useState(content || ''); + + useEffect(() => { + setLocalContent(content || ''); + }, [content, activeFile]); + + const handleKeyDown = (e) => { + if ((e.ctrlKey || e.metaKey) && e.key === 's') { + e.preventDefault(); + onSave(localContent); + return; + } + + // Handle tab indentation + if (e.key === 'Tab' && !e.shiftKey) { + e.preventDefault(); + const start = e.target.selectionStart; + const end = e.target.selectionEnd; + const newContent = localContent.substring(0, start) + " " + localContent.substring(end); + setLocalContent(newContent); + onChange(newContent); + // Move cursor + setTimeout(() => { + e.target.selectionStart = e.target.selectionEnd = start + 4; + }, 0); + } + }; + + const handleChange = (e) => { + setLocalContent(e.target.value); + onChange(e.target.value); + }; + + if (!activeFile) { + return ( +
+

Select a file from the tree to edit

+
+ ); + } + + return ( +
+
+
+ {activeFile} + {localContent !== content && } +
+ +
+