Ask-the-Archive v2 Implementation Plan

-- +goose Up
-- Server-side Ask-the-Archive: per-user conversations + their messages, plus admin AI
-- settings (retention/budget) and daily usage metering. All owner-scoped in the app.
CREATE TABLE ai_conversations (
    id         uuid PRIMARY KEY,
    user_id    uuid NOT NULL,
    title      text NOT NULL,
    saved      boolean NOT NULL DEFAULT false,
    created_at timestamptz NOT NULL DEFAULT now(),
    updated_at timestamptz NOT NULL DEFAULT now()
);
CREATE INDEX ai_conversations_user ON ai_conversations (user_id, updated_at DESC);

CREATE TABLE ai_messages (
    id              uuid PRIMARY KEY,
    conversation_id uuid NOT NULL REFERENCES ai_conversations(id) ON DELETE CASCADE,
    role            text NOT NULL,
    content         text NOT NULL,
    no_results      boolean NOT NULL DEFAULT false,
    model_provider  text NOT NULL DEFAULT '',
    model_name      text NOT NULL DEFAULT '',
    sources         jsonb NOT NULL DEFAULT '[]',
    created_at      timestamptz NOT NULL DEFAULT now()
);
CREATE INDEX ai_messages_conv ON ai_messages (conversation_id, created_at ASC);

-- Single-row admin settings (id is pinned to 1). Seeded here so GetSettings always finds it.
CREATE TABLE ai_settings (
    id                  int PRIMARY KEY CHECK (id = 1),
    chat_retention_days int NOT NULL DEFAULT 30,
    daily_token_budget  bigint NOT NULL DEFAULT 0,
    updated_at          timestamptz NOT NULL DEFAULT now()
);
INSERT INTO ai_settings (id) VALUES (1);

-- Per-day, per-feature usage metering (upserted after each generation call).
CREATE TABLE ai_usage (
    day        date NOT NULL,
    feature    text NOT NULL,
    requests   int NOT NULL DEFAULT 0,
    tokens_in  bigint NOT NULL DEFAULT 0,
    tokens_out bigint NOT NULL DEFAULT 0,
    PRIMARY KEY (day, feature)
);

-- +goose Down
DROP TABLE IF EXISTS ai_usage;
DROP TABLE IF EXISTS ai_settings;
DROP TABLE IF EXISTS ai_messages;
DROP TABLE IF EXISTS ai_conversations;

package app

import (
    "context"
    "encoding/json"
    "time"
)

// Conversation is one stored chat (owner-scoped). MessageCount is populated only by List.
type Conversation struct {
    ID           string
    UserID       string
    Title        string
    Saved        bool
    CreatedAt    time.Time
    UpdatedAt    time.Time
    MessageCount int
}

// StoredMessage is one persisted turn. Sources is the opaque citation-cards JSON
// ([{ref,doc_id,title,score,snippet}]) as stored in the jsonb column.
type StoredMessage struct {
    ID             string
    ConversationID string
    Role           Role
    Content        string
    NoResults      bool
    ModelProvider  string
    ModelName      string
    Sources        json.RawMessage
    CreatedAt      time.Time
}

// Settings are the admin-managed AI knobs (single row).
type Settings struct {
    ChatRetentionDays int
    DailyTokenBudget  int64
}

// UsageRow is one (day, feature) metering bucket.
type UsageRow struct {
    Day        string // YYYY-MM-DD
    Feature    string
    Requests   int
    TokensIn   int64
    TokensOut  int64
}

// Repository is the ai context's Postgres port: conversations, messages, settings, usage.
// Every conversation/message method is OWNER-SCOPED — a userID that does not own the row
// yields kernel.ErrNotFound (no existence disclosure).
type Repository interface {
    CreateConversation(ctx context.Context, c Conversation) error
    ListConversations(ctx context.Context, userID string, limit int) ([]Conversation, error)
    GetConversation(ctx context.Context, userID, id string) (Conversation, error)
    ConversationMessages(ctx context.Context, userID, id string) ([]StoredMessage, error)
    RenameConversation(ctx context.Context, userID, id, title string) error
    SetConversationSaved(ctx context.Context, userID, id string, saved bool) error
    DeleteConversation(ctx context.Context, userID, id string) error
    DeleteAllConversations(ctx context.Context, userID string) error
    // AppendMessages inserts msgs (owner-checked against convID) and bumps updated_at, atomically.
    AppendMessages(ctx context.Context, userID, convID string, msgs []StoredMessage) error
    // DeleteLastAssistant removes the newest assistant message of convID (owner-checked); no-op if none.
    DeleteLastAssistant(ctx context.Context, userID, convID string) error
    // LastTurns returns the last n messages of convID (owner-checked), oldest-first.
    LastTurns(ctx context.Context, userID, convID string, n int) ([]StoredMessage, error)
    // PurgeStaleConversations deletes NOT saved conversations older than retentionDays; returns count.
    PurgeStaleConversations(ctx context.Context, retentionDays int) (int, error)

    GetSettings(ctx context.Context) (Settings, error)
    PutSettings(ctx context.Context, s Settings) error
    // UsageToday returns today's total input+output tokens across all features.
    UsageToday(ctx context.Context) (int64, error)
    // RecordUsage upserts one (today, feature) bucket, adding requests=1 + the token deltas.
    RecordUsage(ctx context.Context, feature string, tokensIn, tokensOut int64) error
    UsageWindow(ctx context.Context, days int) ([]UsageRow, error)
}

// Package adapters implements the ai context's ports: the ChatProvider backends and
// (here) the Postgres Repository for conversations, messages, settings and usage.
package adapters

import (
    "context"
    "encoding/json"
    "errors"
    "fmt"
    "time"

    "github.com/jackc/pgx/v5"

    "github.com/Virtue-Digital-Indonesia/obscura/internal/ai/app"
    "github.com/Virtue-Digital-Indonesia/obscura/internal/kernel"
    "github.com/Virtue-Digital-Indonesia/obscura/internal/platform/db"
)

// Store implements app.Repository over Postgres (house pattern: one *db.DB, every
// statement runs through s.db.Exec(ctx) so it joins an ambient transaction when open).
type Store struct{ db *db.DB }

// NewStore constructs the ai repository.
func NewStore(d *db.DB) *Store { return &Store{db: d} }

func notFound() error {
    return &kernel.Error{Kind: kernel.ErrNotFound, Code: "ai.conversation.not_found", Message: "conversation not found"}
}

func (s *Store) CreateConversation(ctx context.Context, c app.Conversation) error {
    _, err := s.db.Exec(ctx).Exec(ctx,
        `INSERT INTO ai_conversations (id, user_id, title, saved, created_at, updated_at)
         VALUES ($1,$2,$3,$4,$5,$5)`, c.ID, c.UserID, c.Title, c.Saved, c.CreatedAt)
    if err != nil {
        return fmt.Errorf("ai create conversation: %w", err)
    }
    return nil
}

func (s *Store) ListConversations(ctx context.Context, userID string, limit int) ([]app.Conversation, error) {
    rows, err := s.db.Exec(ctx).Query(ctx,
        `SELECT c.id, c.user_id, c.title, c.saved, c.created_at, c.updated_at,
                (SELECT count(*) FROM ai_messages m WHERE m.conversation_id = c.id)
           FROM ai_conversations c WHERE c.user_id = $1 ORDER BY c.updated_at DESC LIMIT $2`, userID, limit)
    if err != nil {
        return nil, fmt.Errorf("ai list conversations: %w", err)
    }
    defer rows.Close()
    var out []app.Conversation
    for rows.Next() {
        var c app.Conversation
        if err := rows.Scan(&c.ID, &c.UserID, &c.Title, &c.Saved, &c.CreatedAt, &c.UpdatedAt, &c.MessageCount); err != nil {
            return nil, err
        }
        out = append(out, c)
    }
    return out, rows.Err()
}

func (s *Store) GetConversation(ctx context.Context, userID, id string) (app.Conversation, error) {
    var c app.Conversation
    err := s.db.Exec(ctx).QueryRow(ctx,
        `SELECT id, user_id, title, saved, created_at, updated_at FROM ai_conversations
           WHERE id = $1 AND user_id = $2`, id, userID).
        Scan(&c.ID, &c.UserID, &c.Title, &c.Saved, &c.CreatedAt, &c.UpdatedAt)
    if errors.Is(err, pgx.ErrNoRows) {
        return app.Conversation{}, notFound()
    }
    if err != nil {
        return app.Conversation{}, fmt.Errorf("ai get conversation: %w", err)
    }
    return c, nil
}

func (s *Store) ConversationMessages(ctx context.Context, userID, id string) ([]app.StoredMessage, error) {
    if _, err := s.GetConversation(ctx, userID, id); err != nil {
        return nil, err // owner check → 404 for a foreign/unknown id
    }
    rows, err := s.db.Exec(ctx).Query(ctx,
        `SELECT id, conversation_id, role, content, no_results, model_provider, model_name, sources, created_at
           FROM ai_messages WHERE conversation_id = $1 ORDER BY created_at ASC`, id)
    if err != nil {
        return nil, fmt.Errorf("ai conversation messages: %w", err)
    }
    defer rows.Close()
    return scanMessages(rows)
}

func scanMessages(rows pgx.Rows) ([]app.StoredMessage, error) {
    var out []app.StoredMessage
    for rows.Next() {
        var m app.StoredMessage
        var role string
        var src []byte
        if err := rows.Scan(&m.ID, &m.ConversationID, &role, &m.Content, &m.NoResults,
            &m.ModelProvider, &m.ModelName, &src, &m.CreatedAt); err != nil {
            return nil, err
        }
        m.Role = app.Role(role)
        m.Sources = json.RawMessage(src)
        out = append(out, m)
    }
    return out, rows.Err()
}

func (s *Store) RenameConversation(ctx context.Context, userID, id, title string) error {
    tag, err := s.db.Exec(ctx).Exec(ctx,
        `UPDATE ai_conversations SET title = $3, updated_at = now() WHERE id = $1 AND user_id = $2`, id, userID, title)
    if err != nil {
        return fmt.Errorf("ai rename conversation: %w", err)
    }
    if tag.RowsAffected() == 0 {
        return notFound()
    }
    return nil
}

func (s *Store) SetConversationSaved(ctx context.Context, userID, id string, saved bool) error {
    tag, err := s.db.Exec(ctx).Exec(ctx,
        `UPDATE ai_conversations SET saved = $3, updated_at = now() WHERE id = $1 AND user_id = $2`, id, userID, saved)
    if err != nil {
        return fmt.Errorf("ai set saved: %w", err)
    }
    if tag.RowsAffected() == 0 {
        return notFound()
    }
    return nil
}

func (s *Store) DeleteConversation(ctx context.Context, userID, id string) error {
    // Idempotent: a foreign/absent id simply affects no rows (cascade removes messages).
    if _, err := s.db.Exec(ctx).Exec(ctx,
        `DELETE FROM ai_conversations WHERE id = $1 AND user_id = $2`, id, userID); err != nil {
        return fmt.Errorf("ai delete conversation: %w", err)
    }
    return nil
}

func (s *Store) DeleteAllConversations(ctx context.Context, userID string) error {
    if _, err := s.db.Exec(ctx).Exec(ctx, `DELETE FROM ai_conversations WHERE user_id = $1`, userID); err != nil {
        return fmt.Errorf("ai delete all conversations: %w", err)
    }
    return nil
}

func (s *Store) AppendMessages(ctx context.Context, userID, convID string, msgs []app.StoredMessage) error {
    return s.db.Do(ctx, func(ctx context.Context) error {
        if _, err := s.GetConversation(ctx, userID, convID); err != nil {
            return err // owner check inside the tx
        }
        for _, m := range msgs {
            src := m.Sources
            if len(src) == 0 {
                src = json.RawMessage("[]")
            }
            if _, err := s.db.Exec(ctx).Exec(ctx,
                `INSERT INTO ai_messages (id, conversation_id, role, content, no_results, model_provider, model_name, sources, created_at)
                 VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)`,
                m.ID, convID, string(m.Role), m.Content, m.NoResults, m.ModelProvider, m.ModelName, []byte(src), m.CreatedAt); err != nil {
                return fmt.Errorf("ai append message: %w", err)
            }
        }
        if _, err := s.db.Exec(ctx).Exec(ctx, `UPDATE ai_conversations SET updated_at = now() WHERE id = $1`, convID); err != nil {
            return fmt.Errorf("ai bump conversation: %w", err)
        }
        return nil
    })
}

func (s *Store) DeleteLastAssistant(ctx context.Context, userID, convID string) error {
    if _, err := s.GetConversation(ctx, userID, convID); err != nil {
        return err
    }
    _, err := s.db.Exec(ctx).Exec(ctx,
        `DELETE FROM ai_messages WHERE id = (
            SELECT id FROM ai_messages WHERE conversation_id = $1 AND role = 'assistant'
            ORDER BY created_at DESC LIMIT 1)`, convID)
    if err != nil {
        return fmt.Errorf("ai delete last assistant: %w", err)
    }
    return nil
}

func (s *Store) LastTurns(ctx context.Context, userID, convID string, n int) ([]app.StoredMessage, error) {
    if _, err := s.GetConversation(ctx, userID, convID); err != nil {
        return nil, err
    }
    rows, err := s.db.Exec(ctx).Query(ctx,
        `SELECT id, conversation_id, role, content, no_results, model_provider, model_name, sources, created_at FROM (
            SELECT * FROM ai_messages WHERE conversation_id = $1 ORDER BY created_at DESC LIMIT $2
         ) t ORDER BY created_at ASC`, convID, n)
    if err != nil {
        return nil, fmt.Errorf("ai last turns: %w", err)
    }
    defer rows.Close()
    return scanMessages(rows)
}

func (s *Store) PurgeStaleConversations(ctx context.Context, retentionDays int) (int, error) {
    if retentionDays <= 0 {
        return 0, nil
    }
    tag, err := s.db.Exec(ctx).Exec(ctx,
        `DELETE FROM ai_conversations WHERE NOT saved AND updated_at < now() - make_interval(days => $1)`, retentionDays)
    if err != nil {
        return 0, fmt.Errorf("ai purge stale: %w", err)
    }
    return int(tag.RowsAffected()), nil
}

func (s *Store) GetSettings(ctx context.Context) (app.Settings, error) {
    var st app.Settings
    err := s.db.Exec(ctx).QueryRow(ctx,
        `SELECT chat_retention_days, daily_token_budget FROM ai_settings WHERE id = 1`).
        Scan(&st.ChatRetentionDays, &st.DailyTokenBudget)
    if err != nil {
        return app.Settings{}, fmt.Errorf("ai get settings: %w", err)
    }
    return st, nil
}

func (s *Store) PutSettings(ctx context.Context, st app.Settings) error {
    _, err := s.db.Exec(ctx).Exec(ctx,
        `UPDATE ai_settings SET chat_retention_days = $1, daily_token_budget = $2, updated_at = now() WHERE id = 1`,
        st.ChatRetentionDays, st.DailyTokenBudget)
    if err != nil {
        return fmt.Errorf("ai put settings: %w", err)
    }
    return nil
}

func (s *Store) UsageToday(ctx context.Context) (int64, error) {
    var total int64
    err := s.db.Exec(ctx).QueryRow(ctx,
        `SELECT coalesce(sum(tokens_in + tokens_out), 0) FROM ai_usage WHERE day = current_date`).Scan(&total)
    if err != nil {
        return 0, fmt.Errorf("ai usage today: %w", err)
    }
    return total, nil
}

func (s *Store) RecordUsage(ctx context.Context, feature string, tokensIn, tokensOut int64) error {
    _, err := s.db.Exec(ctx).Exec(ctx,
        `INSERT INTO ai_usage (day, feature, requests, tokens_in, tokens_out)
         VALUES (current_date, $1, 1, $2, $3)
         ON CONFLICT (day, feature) DO UPDATE
           SET requests = ai_usage.requests + 1,
               tokens_in = ai_usage.tokens_in + EXCLUDED.tokens_in,
               tokens_out = ai_usage.tokens_out + EXCLUDED.tokens_out`, feature, tokensIn, tokensOut)
    if err != nil {
        return fmt.Errorf("ai record usage: %w", err)
    }
    return nil
}

func (s *Store) UsageWindow(ctx context.Context, days int) ([]app.UsageRow, error) {
    rows, err := s.db.Exec(ctx).Query(ctx,
        `SELECT to_char(day, 'YYYY-MM-DD'), feature, requests, tokens_in, tokens_out FROM ai_usage
           WHERE day >= current_date - make_interval(days => $1) ORDER BY day DESC, feature ASC`, days)
    if err != nil {
        return nil, fmt.Errorf("ai usage window: %w", err)
    }
    defer rows.Close()
    var out []app.UsageRow
    for rows.Next() {
        var u app.UsageRow
        if err := rows.Scan(&u.Day, &u.Feature, &u.Requests, &u.TokensIn, &u.TokensOut); err != nil {
            return nil, err
        }
        out = append(out, u)
    }
    return out, rows.Err()
}

var _ app.Repository = (*Store)(nil)

// (struct) add field:  repo Repository
func NewService(chat ChatProvider, repo Repository) *Service {
    return &Service{chat: chat, repo: repo}
}

// --- conversations (owner-scoped; store enforces ownership → kernel.ErrNotFound) ---
func (s *Service) ListConversations(ctx context.Context, userID string, limit int) ([]Conversation, error) {
    return s.repo.ListConversations(ctx, userID, limit)
}
func (s *Service) ConversationMessages(ctx context.Context, userID, id string) (Conversation, []StoredMessage, error) {
    c, err := s.repo.GetConversation(ctx, userID, id)
    if err != nil {
        return Conversation{}, nil, err
    }
    msgs, err := s.repo.ConversationMessages(ctx, userID, id)
    return c, msgs, err
}
func (s *Service) RenameConversation(ctx context.Context, userID, id, title string) error {
    return s.repo.RenameConversation(ctx, userID, id, title)
}
func (s *Service) SetConversationSaved(ctx context.Context, userID, id string, saved bool) error {
    return s.repo.SetConversationSaved(ctx, userID, id, saved)
}
func (s *Service) DeleteConversation(ctx context.Context, userID, id string) error {
    return s.repo.DeleteConversation(ctx, userID, id)
}
func (s *Service) DeleteAllConversations(ctx context.Context, userID string) error {
    return s.repo.DeleteAllConversations(ctx, userID)
}
// --- admin settings + usage ---
func (s *Service) GetSettings(ctx context.Context) (Settings, error) { return s.repo.GetSettings(ctx) }
func (s *Service) PutSettings(ctx context.Context, st Settings) error { return s.repo.PutSettings(ctx, st) }
func (s *Service) UsageWindow(ctx context.Context, days int) ([]UsageRow, error) { return s.repo.UsageWindow(ctx, days) }
// --- retention (called by the scheduler job) ---
func (s *Service) PurgeStaleChats(ctx context.Context) (int, error) {
    st, err := s.repo.GetSettings(ctx)
    if err != nil {
        return 0, err
    }
    return s.repo.PurgeStaleConversations(ctx, st.ChatRetentionDays)
}

    aiSvc := aiapp.NewService(
        aiadapters.SelectProvider(cfg.AI.Provider, cfg.AI.BaseURL, cfg.AI.APIKey, cfg.AI.Model),
        aiadapters.NewStore(database),
    )

    if err := rbacStore.CreatePermission(ctx, "ai.manage", "manage AI settings (chat retention, token budget) and view usage"); err != nil {
        logger.Warn("seed ai permission", "err", err)
    }

package httpapi

import (
    "encoding/json"
    "net/http"
    "strings"

    "github.com/go-chi/chi/v5"

    aiapp "github.com/Virtue-Digital-Indonesia/obscura/internal/ai/app"
    "github.com/Virtue-Digital-Indonesia/obscura/internal/kernel"
)

type convDTO struct {
    ID           string `json:"id"`
    Title        string `json:"title"`
    Saved        bool   `json:"saved"`
    UpdatedAt    string `json:"updated_at"`
    MessageCount int    `json:"message_count"`
}

// ListConversations returns the caller's conversations, newest first.
func (s *Server) ListConversations(w http.ResponseWriter, r *http.Request) {
    p, _ := PrincipalFrom(r.Context())
    cs, err := s.ai.ListConversations(r.Context(), string(p.UserID), 100)
    if err != nil {
        writeProblem(w, err)
        return
    }
    out := make([]convDTO, 0, len(cs))
    for _, c := range cs {
        out = append(out, convDTO{ID: c.ID, Title: c.Title, Saved: c.Saved, UpdatedAt: c.UpdatedAt.UTC().Format("2006-01-02T15:04:05Z07:00"), MessageCount: c.MessageCount})
    }
    writeJSON(w, http.StatusOK, map[string]any{"conversations": out})
}

// GetConversation returns one conversation's messages (with stored sources), owner-scoped.
func (s *Server) GetConversation(w http.ResponseWriter, r *http.Request) {
    p, _ := PrincipalFrom(r.Context())
    c, msgs, err := s.ai.ConversationMessages(r.Context(), string(p.UserID), chi.URLParam(r, "id"))
    if err != nil {
        writeProblem(w, err)
        return
    }
    type msgDTO struct {
        Role          string          `json:"role"`
        Content       string          `json:"content"`
        NoResults     bool            `json:"no_results"`
        ModelProvider string          `json:"model_provider"`
        ModelName     string          `json:"model_name"`
        Sources       json.RawMessage `json:"sources"`
    }
    out := make([]msgDTO, 0, len(msgs))
    for _, m := range msgs {
        out = append(out, msgDTO{Role: string(m.Role), Content: m.Content, NoResults: m.NoResults, ModelProvider: m.ModelProvider, ModelName: m.ModelName, Sources: m.Sources})
    }
    writeJSON(w, http.StatusOK, map[string]any{
        "id": c.ID, "title": c.Title, "saved": c.Saved, "messages": out,
    })
}

// PatchConversation renames (title) and/or toggles saved.
func (s *Server) PatchConversation(w http.ResponseWriter, r *http.Request) {
    p, _ := PrincipalFrom(r.Context())
    id := chi.URLParam(r, "id")
    var body struct {
        Title *string `json:"title"`
        Saved *bool   `json:"saved"`
    }
    if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
        writeProblem(w, &kernel.Error{Kind: kernel.ErrValidation, Code: "request.invalid_json", Message: "invalid request body"})
        return
    }
    if body.Title != nil {
        title := strings.TrimSpace(*body.Title)
        if title == "" || len([]rune(title)) > 120 {
            writeProblem(w, &kernel.Error{Kind: kernel.ErrValidation, Code: "ai.conversation.bad_title", Message: "title must be 1..120 characters"})
            return
        }
        if err := s.ai.RenameConversation(r.Context(), string(p.UserID), id, title); err != nil {
            writeProblem(w, err)
            return
        }
    }
    if body.Saved != nil {
        if err := s.ai.SetConversationSaved(r.Context(), string(p.UserID), id, *body.Saved); err != nil {
            writeProblem(w, err)
            return
        }
    }
    w.WriteHeader(http.StatusNoContent)
}

// DeleteConversation removes one conversation (idempotent → 204).
func (s *Server) DeleteConversation(w http.ResponseWriter, r *http.Request) {
    p, _ := PrincipalFrom(r.Context())
    if err := s.ai.DeleteConversation(r.Context(), string(p.UserID), chi.URLParam(r, "id")); err != nil {
        writeProblem(w, err)
        return
    }
    w.WriteHeader(http.StatusNoContent)
}

// DeleteAllConversations clears the caller's history.
func (s *Server) DeleteAllConversations(w http.ResponseWriter, r *http.Request) {
    p, _ := PrincipalFrom(r.Context())
    if err := s.ai.DeleteAllConversations(r.Context(), string(p.UserID)); err != nil {
        writeProblem(w, err)
        return
    }
    w.WriteHeader(http.StatusNoContent)
}

var _ = aiapp.Conversation{} // keep the aiapp import if unused after edits (remove if gofmt/vet flags)

            r.With(s.requireModule("ai"), s.requireModule("semantic")).Get("/ai/conversations", s.ListConversations)
            r.With(s.requireModule("ai"), s.requireModule("semantic")).Delete("/ai/conversations", s.DeleteAllConversations)
            r.With(s.requireModule("ai"), s.requireModule("semantic")).Get("/ai/conversations/{id}", s.GetConversation)
            r.With(s.requireModule("ai"), s.requireModule("semantic")).Patch("/ai/conversations/{id}", s.PatchConversation)
            r.With(s.requireModule("ai"), s.requireModule("semantic")).Delete("/ai/conversations/{id}", s.DeleteConversation)

// ChatResponse is the model's reply plus best-effort token accounting (exact when the
// provider reports usage; a chars/4 estimate otherwise).
type ChatResponse struct {
    Content   string
    TokensIn  int
    TokensOut int
}

    var out struct {
        Choices []struct {
            Message struct {
                Content string `json:"content"`
            } `json:"message"`
        } `json:"choices"`
        Usage struct {
            PromptTokens     int `json:"prompt_tokens"`
            CompletionTokens int `json:"completion_tokens"`
        } `json:"usage"`
    }
    // ... existing unmarshal + no-choices guard ...
    return app.ChatResponse{
        Content:   out.Choices[0].Message.Content,
        TokensIn:  out.Usage.PromptTokens,
        TokensOut: out.Usage.CompletionTokens,
    }, nil

    var out struct {
        Content []struct {
            Type string `json:"type"`
            Text string `json:"text"`
        } `json:"content"`
        Usage struct {
            InputTokens  int `json:"input_tokens"`
            OutputTokens int `json:"output_tokens"`
        } `json:"usage"`
    }
    // ... existing loop building sb ...
    return app.ChatResponse{Content: sb.String(), TokensIn: out.Usage.InputTokens, TokensOut: out.Usage.OutputTokens}, nil

// budgetExhausted is the 429 returned when the admin daily token budget is spent.
func budgetExhausted() error {
    return &kernel.Error{Kind: kernel.ErrRateLimited, Code: "ai.budget_exhausted",
        Message: "the daily AI budget has been reached — try again tomorrow or raise it in Admin → AI"}
}

// meter guards a generation call against the daily token budget, runs it, and records
// usage (best-effort; metering never fails a successful call). estIn is a fallback input
// estimate used only when the provider reports zero tokens.
func (s *Service) meter(ctx context.Context, feature string, estIn int, call func() (ChatResponse, error)) (ChatResponse, error) {
    if st, err := s.repo.GetSettings(ctx); err == nil && st.DailyTokenBudget > 0 {
        if used, uerr := s.repo.UsageToday(ctx); uerr == nil && used >= st.DailyTokenBudget {
            return ChatResponse{}, budgetExhausted()
        }
    }
    resp, err := call()
    if err != nil {
        return resp, err
    }
    in, out := resp.TokensIn, resp.TokensOut
    if in == 0 {
        in = estIn
    }
    if out == 0 {
        out = len(resp.Content) / 4
    }
    _ = s.repo.RecordUsage(ctx, feature, int64(in), int64(out)) // best-effort
    return resp, nil
}

// estTokens is the chars/4 fallback for the prompt size.
func estTokens(system string, msgs []Message) int {
    total := len(system)
    for _, m := range msgs {
        total += len(m.Content)
    }
    return total / 4
}

func (s *Service) AnswerFromSources(ctx context.Context, question string, history []Message, sources []Source) (ChatResponse, error) {
    // ... existing prompt build (b, msgs, System) ...
    req := ChatRequest{System: /*…the existing system string…*/, Messages: msgs, MaxTokens: 1024}
    return s.meter(ctx, "ask", estTokens(req.System, msgs), func() (ChatResponse, error) {
        return s.chat.Complete(ctx, req)
    })
}

// StartConversation creates a conversation titled from the first question (clipped) and
// returns its id. now is the caller's clock (kernel time).
func (s *Service) StartConversation(ctx context.Context, userID, firstQuestion string, now time.Time) (string, error) {
    title := strings.TrimSpace(firstQuestion)
    if r := []rune(title); len(r) > 60 {
        title = strings.TrimSpace(string(r[:60])) + "…"
    }
    if title == "" {
        title = "New conversation"
    }
    id := kernel.NewID()
    if err := s.repo.CreateConversation(ctx, Conversation{ID: id, UserID: userID, Title: title, CreatedAt: now}); err != nil {
        return "", err
    }
    return id, nil
}

// HistoryForAsk returns the last n turns of a conversation as chat Messages (owner-checked).
func (s *Service) HistoryForAsk(ctx context.Context, userID, convID string, n int) ([]Message, error) {
    stored, err := s.repo.LastTurns(ctx, userID, convID, n)
    if err != nil {
        return nil, err
    }
    out := make([]Message, 0, len(stored))
    for _, m := range stored {
        out = append(out, Message{Role: m.Role, Content: m.Content})
    }
    return out, nil
}

// PersistTurn appends the user question + the assistant answer (with its sources jsonb)
// to a conversation, owner-checked and atomic. sourcesJSON is the exact [{ref,…}] array.
func (s *Service) PersistTurn(ctx context.Context, userID, convID, question, answer string, noResults bool, model ModelInfo, sourcesJSON []byte, now time.Time) error {
    return s.repo.AppendMessages(ctx, userID, convID, []StoredMessage{
        {ID: kernel.NewID(), Role: RoleUser, Content: question, CreatedAt: now},
        {ID: kernel.NewID(), Role: RoleAssistant, Content: answer, NoResults: noResults,
            ModelProvider: model.Provider, ModelName: model.Model, Sources: sourcesJSON, CreatedAt: now.Add(time.Millisecond)},
    })
}

// DropLastAssistant removes a conversation's most recent assistant message (for retry).
func (s *Service) DropLastAssistant(ctx context.Context, userID, convID string) error {
    return s.repo.DeleteLastAssistant(ctx, userID, convID)
}

    if body.Retry && body.Incognito {
        writeProblem(w, &kernel.Error{Kind: kernel.ErrValidation, Code: "ai.retry_incognito", Message: "retry is not available in incognito"})
        return
    }
    convID := strings.TrimSpace(body.ConversationID)
    // Server-authoritative history for a stored conversation; client history only for incognito.
    if !body.Incognito && convID != "" {
        if body.Retry {
            if derr := s.ai.DropLastAssistant(r.Context(), string(p.UserID), convID); derr != nil {
                writeProblem(w, derr)
                return
            }
        }
        hist, herr := s.ai.HistoryForAsk(r.Context(), string(p.UserID), convID, askArchiveMaxTurns)
        if herr != nil {
            writeProblem(w, herr) // foreign/unknown id → 404
            return
        }
        history = hist // REPLACES the client-provided history block for stored chats
    }

    now := time.Now().UTC()
    // persist writes the turn (creating the conversation on first ask) and returns the id
    // to echo to the client. No-op (returns "") in incognito or retrieval-only.
    persist := func(answer string, noResults bool, sourcesJSON []byte) (string, error) {
        if body.Incognito || body.RetrievalOnly {
            return "", nil
        }
        id := convID
        if id == "" {
            nid, err := s.ai.StartConversation(r.Context(), string(p.UserID), question, now)
            if err != nil {
                return "", err
            }
            id = nid
        }
        if err := s.ai.PersistTurn(r.Context(), string(p.UserID), id, question, answer, noResults, s.ai.Info(), sourcesJSON, now); err != nil {
            return "", err
        }
        return id, nil
    }

    if len(srcs) == 0 {
        logAsk("no_results")
        cid, perr := persist("", true, []byte("[]"))
        if perr != nil { writeProblem(w, perr); return }
        writeJSON(w, http.StatusOK, map[string]any{"answer": "", "sources": []sourceOut{}, "no_results": true, "conversation_id": cid})
        return
    }

    logAsk("answered")
    srcJSON, _ := json.Marshal(out) // out is []sourceOut — the exact citation cards
    cid, perr := persist(resp.Content, false, srcJSON)
    if perr != nil { writeProblem(w, perr); return }
    writeJSON(w, http.StatusOK, map[string]any{"answer": resp.Content, "sources": out, "no_results": false, "model": s.ai.Info(), "conversation_id": cid})