feat: dynamic agent persistence via MinIO + PostgreSQL

pyproject.toml

@@ -26,6 +26,8 @@ dependencies = [
     "pyyaml>=6.0.3",
     "sse-starlette>=3.2.0",
     "uvicorn[standard]>=0.40.0",
+    "asyncpg>=0.30.0",
+    "miniopy-async>=1.21.0",
 ]
 
 [project.optional-dependencies]

src/application/routes/agents.py

@@ -1,39 +1,97 @@
 import logging
-from pathlib import Path
+import re
 from typing import Annotated
 
-from fastapi import APIRouter, Depends
+from fastapi import APIRouter, Depends, File, Form, HTTPException, UploadFile, status
 
-from src.application.use_cases.load_agent_config import LoadAgentConfigUseCase
-from src.dependencies import get_agents_dir, get_load_agent_config_use_case
+from src.application.use_cases.create_agent_config import CreateAgentConfigUseCase
+from src.application.use_cases.delete_agent_config import DeleteAgentConfigUseCase
+from src.application.use_cases.get_agent_config import GetAgentConfigUseCase
+from src.application.use_cases.list_agent_configs import ListAgentConfigsUseCase
+from src.application.use_cases.update_agent_config import UpdateAgentConfigUseCase
+from src.dependencies import (
+    get_create_agent_config_use_case,
+    get_delete_agent_config_use_case,
+    get_get_agent_config_use_case,
+    get_list_agent_configs_use_case,
+    get_update_agent_config_use_case,
+)
 from src.domain.entities.agent_config import AgentConfig
+from src.domain.entities.agent_config_metadata import AgentConfigMetadata
 
 logger = logging.getLogger("composable-agents")
 
 router = APIRouter(prefix="/api/v1/agents", tags=["agents"])
 
+AGENT_NAME_PATTERN = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9._-]{0,98}[a-zA-Z0-9]$")
+MAX_UPLOAD_SIZE = 1024 * 1024  # 1 MB
 
-@router.get("")
+
+def _validate_agent_name(name: str) -> None:
+    if not AGENT_NAME_PATTERN.match(name):
+        raise HTTPException(
+            status_code=400,
+            detail=f"Invalid agent name '{name}'. Must match pattern: alphanumeric, dots, hyphens, underscores, 2-100 chars.",
+        )
+
+
+async def _read_yaml_upload(file: UploadFile) -> str:
+    data = await file.read()
+    if len(data) > MAX_UPLOAD_SIZE:
+        raise HTTPException(status_code=400, detail=f"File too large. Maximum size is {MAX_UPLOAD_SIZE} bytes.")
+    try:
+        return data.decode("utf-8")
+    except UnicodeDecodeError as e:
+        raise HTTPException(status_code=400, detail="File must be valid UTF-8 encoded YAML.") from e
+
+
+@router.get("", response_model=list[AgentConfigMetadata])
 async def list_agents(
-    use_case: Annotated[LoadAgentConfigUseCase, Depends(get_load_agent_config_use_case)],
-    agents_dir: Annotated[str, Depends(get_agents_dir)],
-) -> list[AgentConfig]:
-    agents_path = Path(agents_dir)
-    configs: list[AgentConfig] = []
-    if agents_path.exists():
-        for yaml_file in sorted(agents_path.glob("*.yaml")):
-            config = use_case.execute(yaml_file)
-            configs.append(config)
-    logger.info("Listed %d agents from %s", len(configs), agents_dir)
-    return configs
-
-
-@router.get("/{agent_name}")
+    use_case: Annotated[ListAgentConfigsUseCase, Depends(get_list_agent_configs_use_case)],
+) -> list[AgentConfigMetadata]:
+    """List all agent configuration metadata."""
+    return await use_case.execute()
+
+
+@router.get("/{agent_name}", response_model=AgentConfig)
 async def get_agent(
     agent_name: str,
-    use_case: Annotated[LoadAgentConfigUseCase, Depends(get_load_agent_config_use_case)],
-    agents_dir: Annotated[str, Depends(get_agents_dir)],
+    use_case: Annotated[GetAgentConfigUseCase, Depends(get_get_agent_config_use_case)],
 ) -> AgentConfig:
-    logger.debug("Loading agent config: %s", agent_name)
-    config_path = Path(agents_dir) / f"{agent_name}.yaml"
-    return use_case.execute(config_path)
+    """Retrieve a single agent configuration by name."""
+    _validate_agent_name(agent_name)
+    return await use_case.execute(name=agent_name)
+
+
+@router.post("", response_model=AgentConfig, status_code=status.HTTP_201_CREATED)
+async def create_agent(
+    use_case: Annotated[CreateAgentConfigUseCase, Depends(get_create_agent_config_use_case)],
+    agent_name: str = Form(...),
+    file: UploadFile = File(...),
+) -> AgentConfig:
+    """Create a new agent configuration from an uploaded YAML file."""
+    _validate_agent_name(agent_name)
+    yaml_content = await _read_yaml_upload(file)
+    return await use_case.execute(name=agent_name, yaml_content=yaml_content)
+
+
+@router.put("/{agent_name}", response_model=AgentConfig)
+async def update_agent(
+    agent_name: str,
+    use_case: Annotated[UpdateAgentConfigUseCase, Depends(get_update_agent_config_use_case)],
+    file: UploadFile = File(...),
+) -> AgentConfig:
+    """Update an existing agent configuration from an uploaded YAML file."""
+    _validate_agent_name(agent_name)
+    yaml_content = await _read_yaml_upload(file)
+    return await use_case.execute(name=agent_name, yaml_content=yaml_content)
+
+
+@router.delete("/{agent_name}", status_code=status.HTTP_204_NO_CONTENT)
+async def delete_agent(
+    agent_name: str,
+    use_case: Annotated[DeleteAgentConfigUseCase, Depends(get_delete_agent_config_use_case)],
+) -> None:
+    """Delete an agent configuration."""
+    _validate_agent_name(agent_name)
+    await use_case.execute(name=agent_name)

src/application/use_cases/create_agent_config.py

@@ -0,0 +1,63 @@
+import logging
+from datetime import UTC, datetime
+
+from src.domain.entities.agent_config import AgentConfig
+from src.domain.entities.agent_config_metadata import AgentConfigMetadata
+from src.domain.exceptions import AgentConfigAlreadyExistsError, ConfigError
+from src.domain.ports.agent_config_loader import AgentConfigLoader
+from src.domain.ports.agent_config_repository import AgentConfigRepository
+from src.domain.ports.agent_config_store import AgentConfigStore
+
+logger = logging.getLogger("composable-agents")
+
+
+class CreateAgentConfigUseCase:
+    """Create a new agent configuration in persistent storage."""
+
+    def __init__(
+        self,
+        config_loader: AgentConfigLoader,
+        config_store: AgentConfigStore,
+        config_repository: AgentConfigRepository,
+    ) -> None:
+        self._config_loader = config_loader
+        self._config_store = config_store
+        self._config_repository = config_repository
+
+    async def execute(self, name: str, yaml_content: str) -> AgentConfig:
+        """Parse YAML, validate, store in MinIO, save metadata in PostgreSQL.
+
+        Args:
+            name: Agent name.
+            yaml_content: Raw YAML configuration string.
+
+        Returns:
+            Validated AgentConfig.
+
+        Raises:
+            AgentConfigAlreadyExistsError: If an agent with this name already exists.
+            ConfigError: If the YAML is invalid.
+        """
+        config = self._config_loader.load_from_string(yaml_content)
+
+        if config.name != name:
+            raise ConfigError(f"Agent name in YAML '{config.name}' does not match provided name '{name}'")
+
+        if await self._config_repository.exists(name):
+            raise AgentConfigAlreadyExistsError(f"Agent config already exists: {name}")
+
+        await self._config_store.put(name, yaml_content)
+
+        now = datetime.now(UTC)
+        metadata = AgentConfigMetadata(
+            name=name,
+            model=config.model,
+            minio_path=f"{name}.yaml",
+            is_builtin=False,
+            created_at=now,
+            updated_at=now,
+        )
+        await self._config_repository.save(metadata)
+
+        logger.info("Created agent config '%s'", name)
+        return config

src/application/use_cases/delete_agent_config.py

@@ -0,0 +1,43 @@
+import logging
+
+from src.domain.exceptions import ConfigError
+from src.domain.ports.agent_config_repository import AgentConfigRepository
+from src.domain.ports.agent_config_store import AgentConfigStore
+from src.domain.ports.agent_registry import AgentRegistry
+
+logger = logging.getLogger("composable-agents")
+
+
+class DeleteAgentConfigUseCase:
+    """Delete an agent configuration from persistent storage."""
+
+    def __init__(
+        self,
+        config_store: AgentConfigStore,
+        config_repository: AgentConfigRepository,
+        agent_registry: AgentRegistry,
+    ) -> None:
+        self._config_store = config_store
+        self._config_repository = config_repository
+        self._agent_registry = agent_registry
+
+    async def execute(self, name: str) -> None:
+        """Delete from MinIO and PostgreSQL, invalidate cache.
+
+        Args:
+            name: Agent name to delete.
+
+        Raises:
+            AgentNotFoundError: If no agent with this name exists.
+            ConfigError: If the agent is built-in.
+        """
+        metadata = await self._config_repository.get(name)
+
+        if metadata.is_builtin:
+            raise ConfigError(f"Cannot delete built-in agent: {name}")
+
+        await self._config_store.delete(name)
+        await self._config_repository.delete(name)
+        await self._agent_registry.invalidate(name)
+
+        logger.info("Deleted agent config '%s'", name)

src/application/use_cases/get_agent_config.py

@@ -0,0 +1,37 @@
+import logging
+
+from src.domain.entities.agent_config import AgentConfig
+from src.domain.ports.agent_config_loader import AgentConfigLoader
+from src.domain.ports.agent_config_store import AgentConfigStore
+
+logger = logging.getLogger("composable-agents")
+
+
+class GetAgentConfigUseCase:
+    """Retrieve a single agent configuration from persistent storage."""
+
+    def __init__(
+        self,
+        config_loader: AgentConfigLoader,
+        config_store: AgentConfigStore,
+    ) -> None:
+        self._config_loader = config_loader
+        self._config_store = config_store
+
+    async def execute(self, name: str) -> AgentConfig:
+        """Fetch YAML from MinIO and parse into AgentConfig.
+
+        Args:
+            name: Agent name.
+
+        Returns:
+            Validated AgentConfig.
+
+        Raises:
+            AgentNotFoundError: If no YAML exists for this agent.
+            ConfigError: If the YAML is invalid.
+        """
+        yaml_content = await self._config_store.get(name)
+        config = self._config_loader.load_from_string(yaml_content)
+        logger.debug("Loaded agent config '%s' from store", name)
+        return config

src/application/use_cases/list_agent_configs.py

@@ -0,0 +1,23 @@
+import logging
+
+from src.domain.entities.agent_config_metadata import AgentConfigMetadata
+from src.domain.ports.agent_config_repository import AgentConfigRepository
+
+logger = logging.getLogger("composable-agents")
+
+
+class ListAgentConfigsUseCase:
+    """List all agent configuration metadata from persistent storage."""
+
+    def __init__(self, config_repository: AgentConfigRepository) -> None:
+        self._config_repository = config_repository
+
+    async def execute(self) -> list[AgentConfigMetadata]:
+        """Return all agent config metadata from the repository.
+
+        Returns:
+            List of AgentConfigMetadata.
+        """
+        result = await self._config_repository.list_all()
+        logger.debug("Listed %d agent configs from repository", len(result))
+        return result

src/application/use_cases/seed_agents.py

@@ -0,0 +1,69 @@
+import logging
+from datetime import UTC, datetime
+from pathlib import Path
+
+import yaml
+
+from src.domain.entities.agent_config_metadata import AgentConfigMetadata
+from src.domain.ports.agent_config_loader import AgentConfigLoader
+from src.domain.ports.agent_config_repository import AgentConfigRepository
+from src.domain.ports.agent_config_store import AgentConfigStore
+
+logger = logging.getLogger("composable-agents")
+
+
+class SeedAgentsUseCase:
+    """Seed built-in agent configurations from a local directory into persistent storage."""
+
+    def __init__(
+        self,
+        config_loader: AgentConfigLoader,
+        config_store: AgentConfigStore,
+        config_repository: AgentConfigRepository,
+    ) -> None:
+        self._config_loader = config_loader
+        self._config_store = config_store
+        self._config_repository = config_repository
+
+    async def execute(self, agents_dir: Path) -> None:
+        """For each YAML file in agents_dir, upload to MinIO and save metadata if not already present.
+
+        If a YAML references system_prompt_file, the prompt is read from disk and inlined
+        before uploading so that the stored YAML is self-contained.
+
+        Args:
+            agents_dir: Path to the directory containing seed agent YAML files.
+        """
+        if not agents_dir.exists():
+            logger.warning("Agents directory does not exist: %s", agents_dir)
+            return
+
+        for yaml_file in sorted(agents_dir.glob("*.yaml")):
+            agent_name = yaml_file.stem
+
+            if await self._config_repository.exists(agent_name):
+                logger.debug("Agent '%s' already seeded, skipping", agent_name)
+                continue
+
+            config = self._config_loader.load(yaml_file)
+
+            raw = yaml.safe_load(yaml_file.read_text(encoding="utf-8"))
+            if raw.get("system_prompt_file"):
+                raw.pop("system_prompt_file")
+                raw["system_prompt"] = config.system_prompt
+            yaml_content = yaml.dump(raw, default_flow_style=False, allow_unicode=True)
+
+            await self._config_store.put(agent_name, yaml_content)
+
+            now = datetime.now(UTC)
+            metadata = AgentConfigMetadata(
+                name=agent_name,
+                model=config.model,
+                minio_path=f"{agent_name}.yaml",
+                is_builtin=True,
+                created_at=now,
+                updated_at=now,
+            )
+            await self._config_repository.save(metadata)
+
+            logger.info("Seeded built-in agent '%s'", agent_name)

src/application/use_cases/send_message.py

@@ -25,7 +25,13 @@ async def execute(self, thread_id: str, request: ChatRequest) -> Message:
             await self._threads.add_message(thread_id, human_msg)
             response = await runner.invoke(thread_id, request.message)
         else:
-            logger.info("[thread=%s][agent=%s] HITL action=%s tool_call_id=%s", thread_id, thread.agent_name, request.action, request.tool_call_id)
+            logger.info(
+                "[thread=%s][agent=%s] HITL action=%s tool_call_id=%s",
+                thread_id,
+                thread.agent_name,
+                request.action,
+                request.tool_call_id,
+            )
             match request.action:
                 case "approve":
                     response = await runner.approve_hitl(thread_id, request.tool_call_id)
@@ -35,5 +41,11 @@ async def execute(self, thread_id: str, request: ChatRequest) -> Message:
                     response = await runner.edit_hitl(thread_id, request.tool_call_id, request.edits)
 
         await self._threads.add_message(thread_id, response)
-        logger.info("[thread=%s][agent=%s] Response received, status=%s len=%d", thread_id, thread.agent_name, response.status, len(response.content or ""))
+        logger.info(
+            "[thread=%s][agent=%s] Response received, status=%s len=%d",
+            thread_id,
+            thread.agent_name,
+            response.status,
+            len(response.content or ""),
+        )
         return response