fix(audit): code audit, health check, and performance improvements

alembic/versions/039_app_settings_id_to_uuid.py

@@ -0,0 +1,47 @@
+"""Convert app_settings.id from VARCHAR(36) to native UUID.
+
+Migration 026 created the column as String(36), but the ORM UUIDMixin
+declares it as UUID(as_uuid=False).  The type mismatch causes asyncpg
+to emit ``WHERE app_settings.id = $1::UUID`` which Postgres rejects
+with "operator does not exist: character varying = uuid".
+
+Revision ID: 039_app_settings_id_to_uuid
+Revises: 038_fix_proposalstatus_enum_case
+Create Date: 2026-03-14
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+revision: str = "039_app_settings_id_to_uuid"
+down_revision: str | None = "038_fix_proposalstatus_enum_case"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    op.alter_column(
+        "app_settings",
+        "id",
+        type_=postgresql.UUID(as_uuid=False),
+        existing_type=sa.String(36),
+        postgresql_using="id::uuid",
+    )
+
+
+def downgrade() -> None:
+    op.alter_column(
+        "app_settings",
+        "id",
+        type_=sa.String(36),
+        existing_type=postgresql.UUID(as_uuid=False),
+        postgresql_using="id::text",
+    )

alembic/versions/040_normalize_insightstatus_data.py

@@ -0,0 +1,109 @@
+"""Normalize insight enum values to match ORM entity definitions.
+
+The insightstatus PG enum contains UPPERCASE labels (PENDING, REVIEWED,
+ACTIONED, DISMISSED) and some lowercase duplicates (pending, actioned),
+but the ORM expects the .value side of the Python enum: generated,
+reviewed, acted_upon, dismissed.
+
+Similarly, insighttype has UPPERCASE labels (ENERGY_OPTIMIZATION, etc.)
+but the ORM expects lowercase .value strings (energy_optimization, etc.).
+
+This migration:
+  1. Adds missing canonical enum labels
+  2. Converts all existing row data to canonical lowercase values
+  3. Leaves orphan enum labels in place (PG can't drop enum values)
+
+Revision ID: 040_normalize_insightstatus_data
+Revises: 039_app_settings_id_to_uuid
+Create Date: 2026-03-15
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from alembic import op
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+revision: str = "040_normalize_insightstatus_data"
+down_revision: str | None = "039_app_settings_id_to_uuid"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+# ── insightstatus ──────────────────────────────────────────────────────
+# ORM values: generated, reviewed, acted_upon, dismissed
+_STATUS_LABELS_TO_ADD = ["generated", "reviewed", "acted_upon", "dismissed"]
+_STATUS_DATA_MAP = {
+    "PENDING": "generated",
+    "pending": "generated",
+    "REVIEWED": "reviewed",
+    "ACTIONED": "acted_upon",
+    "actioned": "acted_upon",
+    "DISMISSED": "dismissed",
+}
+
+# ── insighttype ────────────────────────────────────────────────────────
+# ORM values (from InsightType.value): energy_optimization, anomaly,
+# pattern, recommendation, maintenance_prediction, automation_gap, etc.
+_TYPE_LABELS_TO_ADD = [
+    "energy_optimization",
+    "anomaly",
+    "pattern",
+    "recommendation",
+    "maintenance_prediction",
+]
+_TYPE_DATA_MAP = {
+    "ENERGY_OPTIMIZATION": "energy_optimization",
+    "ANOMALY_DETECTION": "anomaly",
+    "USAGE_PATTERN": "pattern",
+    "COST_SAVING": "recommendation",
+    "MAINTENANCE_PREDICTION": "maintenance_prediction",
+}
+
+
+def upgrade() -> None:
+    # 1. Add missing canonical labels to the PG enums
+    for label in _STATUS_LABELS_TO_ADD:
+        op.execute(f"ALTER TYPE insightstatus ADD VALUE IF NOT EXISTS '{label}'")
+
+    for label in _TYPE_LABELS_TO_ADD:
+        op.execute(f"ALTER TYPE insighttype ADD VALUE IF NOT EXISTS '{label}'")
+
+    # ADD VALUE must commit before the values can be used in DML,
+    # so we need a separate transaction for the UPDATEs.
+    # Alembic runs each migration in its own transaction by default.
+    # We force a commit here, then run the UPDATEs.
+    op.execute("COMMIT")
+
+    # 2. Convert stale data rows to canonical values
+    for old, new in _STATUS_DATA_MAP.items():
+        op.execute(f"UPDATE insights SET status = '{new}' WHERE status = '{old}'")
+
+    for old, new in _TYPE_DATA_MAP.items():
+        op.execute(f"UPDATE insights SET type = '{new}' WHERE type = '{old}'")
+
+    # Re-open a transaction for Alembic's version-table update
+    op.execute("BEGIN")
+
+
+def downgrade() -> None:
+    # Best-effort reverse: map canonical values back to UPPERCASE
+    _STATUS_REVERSE = {
+        "generated": "PENDING",
+        "reviewed": "REVIEWED",
+        "acted_upon": "ACTIONED",
+        "dismissed": "DISMISSED",
+    }
+    _TYPE_REVERSE = {
+        "energy_optimization": "ENERGY_OPTIMIZATION",
+        "anomaly": "ANOMALY_DETECTION",
+        "pattern": "USAGE_PATTERN",
+        "recommendation": "COST_SAVING",
+        "maintenance_prediction": "MAINTENANCE_PREDICTION",
+    }
+    for old, new in _STATUS_REVERSE.items():
+        op.execute(f"UPDATE insights SET status = '{new}' WHERE status = '{old}'")
+    for old, new in _TYPE_REVERSE.items():
+        op.execute(f"UPDATE insights SET type = '{new}' WHERE type = '{old}'")

alembic/versions/041_schema_alignment.py

@@ -0,0 +1,132 @@
+"""Align database column types with ORM entity definitions.
+
+Fixes schema drift accumulated across early migrations where column
+types diverged from the ORM:
+
+Part A — Convert VARCHAR(36) primary keys to native UUID:
+  - insights.id
+  - workflow_definition.id
+  - tool_group.id
+
+Part B — Fix insights column types:
+  - impact:        varchar(50)  -> insightimpact enum
+  - evidence:      json         -> jsonb
+  - entities:      json         -> jsonb (HA entity ID strings, not UUIDs)
+  - script_output: json         -> jsonb
+
+Part C — Fix conversation.status:
+  - varchar(20) -> conversationstatus enum
+
+Revision ID: 041_schema_alignment
+Revises: 040_normalize_insightstatus_data
+Create Date: 2026-03-15
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+revision: str = "041_schema_alignment"
+down_revision: str | None = "040_normalize_insightstatus_data"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    # ── Part A: VARCHAR(36) PKs → UUID ────────────────────────────────
+    for table in ("insights", "workflow_definition", "tool_group"):
+        op.alter_column(
+            table,
+            "id",
+            type_=postgresql.UUID(as_uuid=False),
+            existing_type=sa.String(36),
+            postgresql_using="id::uuid",
+        )
+
+    # ── Part B: insights column types ─────────────────────────────────
+    # impact: varchar(50) → insightimpact enum
+    op.execute(
+        "ALTER TABLE insights ALTER COLUMN impact TYPE insightimpact USING impact::insightimpact"
+    )
+
+    # evidence: json → jsonb
+    op.alter_column(
+        "insights",
+        "evidence",
+        type_=postgresql.JSONB(),
+        existing_type=sa.JSON(),
+        postgresql_using="evidence::jsonb",
+    )
+
+    # entities: json → jsonb (stores HA entity ID strings, not UUIDs)
+    op.alter_column(
+        "insights",
+        "entities",
+        type_=postgresql.JSONB(),
+        existing_type=sa.JSON(),
+        postgresql_using="entities::jsonb",
+        server_default=sa.text("'[]'::jsonb"),
+    )
+
+    # script_output: json → jsonb
+    op.alter_column(
+        "insights",
+        "script_output",
+        type_=postgresql.JSONB(),
+        existing_type=sa.JSON(),
+        postgresql_using="script_output::jsonb",
+    )
+
+    # ── Part C: conversation.status → conversationstatus enum ─────────
+    op.execute(
+        "ALTER TABLE conversation "
+        "ALTER COLUMN status TYPE conversationstatus "
+        "USING status::conversationstatus"
+    )
+
+
+def downgrade() -> None:
+    # ── Part C reverse ────────────────────────────────────────────────
+    op.execute("ALTER TABLE conversation ALTER COLUMN status TYPE varchar(20) USING status::text")
+
+    # ── Part B reverse ────────────────────────────────────────────────
+    op.alter_column(
+        "insights",
+        "script_output",
+        type_=sa.JSON(),
+        existing_type=postgresql.JSONB(),
+    )
+
+    op.alter_column(
+        "insights",
+        "entities",
+        type_=sa.JSON(),
+        existing_type=postgresql.JSONB(),
+        server_default=None,
+    )
+
+    op.alter_column(
+        "insights",
+        "evidence",
+        type_=sa.JSON(),
+        existing_type=postgresql.JSONB(),
+    )
+
+    op.execute("ALTER TABLE insights ALTER COLUMN impact TYPE varchar(50) USING impact::text")
+
+    # ── Part A reverse ────────────────────────────────────────────────
+    for table in ("insights", "workflow_definition", "tool_group"):
+        op.alter_column(
+            table,
+            "id",
+            type_=sa.String(36),
+            existing_type=postgresql.UUID(as_uuid=False),
+            postgresql_using="id::text",
+        )

specs/001-project-aether/features/41-code-audit/plan.md

@@ -0,0 +1,48 @@
+# Feature 41: Code Audit — Implementation Plan
+
+## Phase 1: Quick Wins (P0 — this session)
+
+### T1. Fix mutable default in proposals route
+- File: `src/api/routes/proposals.py:444`
+- Change: Replace `body: dict[str, Any] = {}` with `body: dict[str, Any] = Body(default={})`
+
+### T2. Narrow error handling in tool functions
+- Files: `src/tools/agent_tools.py`, `src/tools/diagnostic_tools.py`
+- Change: Replace `except Exception as e: return f"..."` with specific exceptions
+  and structured error responses that preserve diagnostic information.
+
+### T3. Fix N+1 query in list_proposals
+- File: `src/api/routes/proposals.py:116-120`
+- Change: Replace per-status loop with single `repo.list_all()` query
+
+### T4. Extract model_context boilerplate
+- File: `src/tools/agent_tools.py`
+- Change: Create reusable `_run_with_model_context()` helper to eliminate 3× duplication
+
+## Phase 2: Performance (P1 — this session)
+
+### T5. Use fast model for orchestrator classification
+- File: `src/agents/orchestrator.py`
+- Change: Override model selection in `_get_classification_llm()` to use a fast/cheap
+  model regardless of user-selected model
+
+### T6. Concurrent automation config fetches in discovery sync
+- File: `src/dal/sync.py:_sync_automation_entities()`
+- Change: Use `asyncio.gather()` with semaphore for concurrent config fetches
+
+### T7. Fix orchestrator session management
+- File: `src/agents/orchestrator.py:173-189`
+- Change: Use `async with get_session()` context manager
+
+## Phase 3: Modularity (P2 — future session)
+
+### T8. Split proposals.py into subpackage
+### T9. Split handlers.py streaming logic
+### T10. Add public method for HA entity state lookup
+### T11. Split dal/agents.py into per-repository files
+
+## Phase 4: Polish (P3 — future session)
+
+### T12. Remove redundant logging imports
+### T13. Split checkpoints.py model/saver
+### T14. Extract scheduler job definitions