Backup System — Implementation Plan

-- +goose Up
-- Admin-tunable backup schedule/retention (singleton row id=1). Read by the obscura-backup sidecar
-- each poll (via psql) AND by the app's Admin -> Backups view. Seeded with safe defaults so a fresh
-- deploy has a sane schedule the moment the `backup` profile is brought up; until then the sidecar
-- isn't running, so enabled=true is a no-op. run_requested_at is the "Run backup now" flag: the app
-- sets it, the sidecar runs one backup on its next poll and clears it. Anti-footgun floors
-- (interval_hours>=1, retention_count>=1) are enforced in app validation (a runaway 0-hour schedule
-- would fill the disk; a 0 retention would prune every backup away).
CREATE TABLE backup_settings (
    id               int PRIMARY KEY DEFAULT 1 CHECK (id = 1),
    enabled          bool NOT NULL DEFAULT true,
    interval_hours   int  NOT NULL DEFAULT 24,
    retention_count  int  NOT NULL DEFAULT 7,
    run_requested_at timestamptz,
    updated_at       timestamptz NOT NULL DEFAULT now()
);
INSERT INTO backup_settings (id) VALUES (1);

-- +goose Down
DROP TABLE backup_settings;

package domain

import "fmt"

// Settings is the admin-tunable backup schedule/retention (the singleton backup_settings row). The
// obscura-backup sidecar reads these via psql each poll cycle; the app reads/writes them for the
// Admin -> Backups view. run_requested_at is NOT modeled here (it is a write-only flag owned by the
// SettingsStore.RequestRun trigger + the sidecar's clear).
type Settings struct {
    Enabled        bool
    IntervalHours  int
    RetentionCount int
}

// Anti-footgun floors (mirrors the rate-limit floor pattern): interval >= 1 (never a runaway
// 0-hour loop that fills the disk), retention >= 1 (never prune to nothing).
const (
    FloorIntervalHours  = 1
    FloorRetentionCount = 1
)

// Defaults returns the seeded settings (matches migration 00087): enabled, daily, keep 7.
func Defaults() Settings {
    return Settings{Enabled: true, IntervalHours: 24, RetentionCount: 7}
}

// Validate enforces the safety floors. A rejected save returns a clear, admin-facing error.
func (s Settings) Validate() error {
    if s.IntervalHours < FloorIntervalHours {
        return fmt.Errorf("interval_hours must be at least %d (safety floor to prevent runaway backups)", FloorIntervalHours)
    }
    if s.RetentionCount < FloorRetentionCount {
        return fmt.Errorf("retention_count must be at least %d (safety floor to keep at least one backup)", FloorRetentionCount)
    }
    return nil
}

// SetStatus is one backup set's status.json (metadata only — never the db.dump/blob bytes).
// Written by the sidecar's backup.sh; parsed best-effort by the FSCatalog. The JSON tags match the
// sidecar's on-disk shape AND the API response.
type SetStatus struct {
    TS              string `json:"ts"`
    Result          string `json:"result"` // "ok" | "error"
    DBBytes         int64  `json:"db_bytes"`
    BlobBytes       int64  `json:"blob_bytes"`
    BlobCount       int64  `json:"blob_count"`
    DurationSeconds int64  `json:"duration_seconds"`
    Error           string `json:"error,omitempty"`
}

// CatalogView is the app-side, read-only view of the mounted backup dir: the newest-first set
// history + the latest-run summary. Enabled is filesystem-derived (dir present AND >= 1 completed
// set), distinct from Settings.Enabled (the auto-backup toggle). Sets is always non-nil so the API
// serializes [] not null.
type CatalogView struct {
    Enabled   bool        `json:"enabled"`
    BackupDir string      `json:"backup_dir"`
    LastRun   *SetStatus  `json:"last_run"`
    Sets      []SetStatus `json:"sets"`
}

package adapters

import (
    "context"
    "errors"
    "fmt"

    "github.com/jackc/pgx/v5"

    "github.com/Virtue-Digital-Indonesia/obscura/internal/backup/domain"
    "github.com/Virtue-Digital-Indonesia/obscura/internal/platform/db"
)

// SettingsStore reads/writes the singleton backup_settings row (id=1). The obscura-backup sidecar
// reads the SAME row directly via psql — the app never RPCs the sidecar; the DB row is the only
// coupling. Mirrors the ratelimit adapters.Store fallback discipline.
type SettingsStore struct{ db *db.DB }

// NewSettingsStore builds the backup-settings store.
func NewSettingsStore(d *db.DB) *SettingsStore { return &SettingsStore{db: d} }

// Load reads the singleton settings, falling back to seeded defaults when the row is missing or
// out-of-range (a corrupt row must never brick the Admin view).
func (s *SettingsStore) Load(ctx context.Context) (domain.Settings, error) {
    var st domain.Settings
    err := s.db.Exec(ctx).QueryRow(ctx,
        `SELECT enabled, interval_hours, retention_count FROM backup_settings WHERE id = 1`).
        Scan(&st.Enabled, &st.IntervalHours, &st.RetentionCount)
    if errors.Is(err, pgx.ErrNoRows) {
        return domain.Defaults(), nil
    }
    if err != nil {
        return domain.Defaults(), fmt.Errorf("backup settings load: %w", err)
    }
    if st.Validate() != nil {
        return domain.Defaults(), nil
    }
    return st, nil
}

// Save writes the singleton settings and bumps updated_at. It does NOT touch run_requested_at (that
// flag is owned by RequestRun + the sidecar's clear).
func (s *SettingsStore) Save(ctx context.Context, st domain.Settings) error {
    if _, err := s.db.Exec(ctx).Exec(ctx,
        `UPDATE backup_settings
            SET enabled = $1, interval_hours = $2, retention_count = $3, updated_at = now()
          WHERE id = 1`,
        st.Enabled, st.IntervalHours, st.RetentionCount); err != nil {
        return fmt.Errorf("backup settings save: %w", err)
    }
    return nil
}

// RequestRun sets run_requested_at = now() (the "Run backup now" trigger). COALESCE keeps an already
// pending request from being restacked (the earliest pending timestamp wins). The sidecar clears it
// after running one backup on its next poll.
func (s *SettingsStore) RequestRun(ctx context.Context) error {
    if _, err := s.db.Exec(ctx).Exec(ctx,
        `UPDATE backup_settings SET run_requested_at = COALESCE(run_requested_at, now()) WHERE id = 1`); err != nil {
        return fmt.Errorf("backup settings request run: %w", err)
    }
    return nil
}

package adapters

import (
    "encoding/json"
    "os"
    "path/filepath"
    "sort"
    "strings"
    "time"

    "github.com/Virtue-Digital-Indonesia/obscura/internal/backup/domain"
)

// tsLayout is the sidecar's compact UTC set-name format (date -u +%Y%m%dT%H%M%SZ). Because it is
// zero-padded and big-endian, a plain reverse string sort is chronological newest-first.
const tsLayout = "20060102T150405Z"

// FSCatalog is the app's read-only view of the mounted backup directory ($BACKUP_DIR, bind-mounted
// :ro). It reads metadata ONLY (each set's status.json + latest.json) — it NEVER opens db.dump or
// blob bytes, and never writes. Tolerant by construction: a missing/unreadable dir -> Enabled=false;
// a set with a missing/malformed status.json is skipped; nothing here can 500 the status endpoint.
type FSCatalog struct{ dir string }

// NewFSCatalog builds the catalog over a mounted backup dir (cfg.BackupDir, default /backups).
func NewFSCatalog(dir string) *FSCatalog { return &FSCatalog{dir: dir} }

// Read scans the backup dir. A completed set is a subdir named <TS> (NOT <TS>.partial) containing a
// status.json. Sets are returned newest-first; LastRun comes from latest.json (written on success).
func (c *FSCatalog) Read() domain.CatalogView {
    out := domain.CatalogView{Enabled: false, BackupDir: c.dir, Sets: []domain.SetStatus{}}
    entries, err := os.ReadDir(c.dir)
    if err != nil {
        return out // dir absent/unreadable -> disabled (profile not enabled / nothing mounted)
    }
    names := make([]string, 0, len(entries))
    for _, e := range entries {
        if !e.IsDir() || strings.HasSuffix(e.Name(), ".partial") {
            continue
        }
        names = append(names, e.Name())
    }
    sort.Sort(sort.Reverse(sort.StringSlice(names))) // newest-first
    for _, name := range names {
        if st, ok := readStatus(filepath.Join(c.dir, name, "status.json")); ok {
            out.Sets = append(out.Sets, st)
        }
    }
    if len(out.Sets) > 0 {
        out.Enabled = true
    }
    if last, ok := readStatus(filepath.Join(c.dir, "latest.json")); ok {
        out.LastRun = &last
    }
    return out
}

// LatestSuccessUnix returns the unix seconds of the most recent SUCCESSFUL backup (from latest.json),
// or (0,false) when there is none. Feeds the obscura_backup_last_success_timestamp_seconds gauge.
func (c *FSCatalog) LatestSuccessUnix() (float64, bool) {
    last, ok := readStatus(filepath.Join(c.dir, "latest.json"))
    if !ok || last.Result != "ok" || last.TS == "" {
        return 0, false
    }
    t, err := time.Parse(tsLayout, last.TS)
    if err != nil {
        return 0, false
    }
    return float64(t.Unix()), true
}

func readStatus(path string) (domain.SetStatus, bool) {
    b, err := os.ReadFile(path)
    if err != nil {
        return domain.SetStatus{}, false
    }
    var st domain.SetStatus
    if err := json.Unmarshal(b, &st); err != nil {
        return domain.SetStatus{}, false
    }
    return st, true
}

    // BackupDir is the mounted read-only backup directory the app inspects for the Admin -> Backups
    // view + the stale-backup metric (the obscura-backup sidecar writes it; the app only reads it).
    // When the path does not exist the feature reports disabled. Default matches the compose mount.
    BackupDir string `env:"BACKUP_DIR" envDefault:"/backups"`

git add go/migrations/00087_backup_settings.sql go/internal/backup/domain/settings.go go/internal/backup/adapters/settings_pg.go go/internal/backup/adapters/fscatalog.go go/internal/platform/config/config.go
git commit -m "feat(backup): migration 00087 backup_settings + settings domain, SettingsStore, FSCatalog, BACKUP_DIR"

# syntax=docker/dockerfile:1
# Disaster-recovery backup sidecar. Real pg_dump/pg_restore (version-matched to the
# pgvector/pgvector:pg17 server) + the MinIO client `mc`, plus the backup/entrypoint/restore
# scripts. OFF by default: only runs under the `backup` compose profile, never in the demo stack.
# Built from the REPO ROOT context (context: ..) because the repo-root .dockerignore excludes
# deploy/ — so the scripts live under scripts/ (not ignored) and are COPYed from there.
FROM postgres:17

# MinIO client (static amd64 binary). Air-gapped installs can pre-place /usr/local/bin/mc and
# override MC_URL to a file:// or internal mirror. ADD fetches at build time (connected build host).
ARG MC_URL=https://dl.min.io/client/mc/release/linux-amd64/mc
ADD ${MC_URL} /usr/local/bin/mc
RUN chmod +x /usr/local/bin/mc

COPY scripts/backup.sh /backup.sh
COPY scripts/entrypoint.sh /entrypoint.sh
COPY scripts/restore.sh /restore.sh
RUN chmod +x /backup.sh /entrypoint.sh /restore.sh

# Runs as root (the postgres image's default before its own entrypoint steps down — which we
# replace): needed to write the host-mounted ./backups bind volume regardless of host ownership.
ENTRYPOINT ["/entrypoint.sh"]

#!/bin/bash
# One backup set: pg_dump -Fc + mc mirror of the MinIO bucket into $BACKUP_DIR/<TS>/, built in a
# <TS>.partial dir and atomically renamed to <TS>/ ONLY on full success (a crashed run never looks
# complete). Writes status.json (ok|error), overwrites latest.json on success, and prunes completed
# sets to RETENTION_COUNT (never deleting the newest). Any failure -> status.json result:error in
# the .partial dir, prior good sets untouched, non-zero exit (visible in `docker logs`).
set -uo pipefail

BACKUP_DIR="${BACKUP_DIR:-/backups}"
RETENTION_COUNT="${RETENTION_COUNT:-7}"
BUCKET="${S3_BUCKET:-obscura}"

TS="$(date -u +%Y%m%dT%H%M%SZ)"
PARTIAL="$BACKUP_DIR/$TS.partial"
FINAL="$BACKUP_DIR/$TS"
start=$(date -u +%s)
mkdir -p "$PARTIAL/blobs"

fail() {
  local msg="$1" dur
  dur=$(( $(date -u +%s) - start ))
  cat > "$PARTIAL/status.json" <<EOF
{"ts":"$TS","result":"error","db_bytes":0,"blob_count":0,"blob_bytes":0,"duration_seconds":$dur,"error":"$msg"}
EOF
  echo "obscura-backup: FAILED: $msg" >&2
  exit 1
}

# 1) Postgres custom-format dump (compressed, selective restore).
pg_dump -Fc -d "$DATABASE_URL" -f "$PARTIAL/db.dump" || fail "pg_dump failed"
db_bytes=$(stat -c%s "$PARTIAL/db.dump" 2>/dev/null || echo 0)
[ "$db_bytes" -gt 0 ] || fail "db.dump is empty"

# 2) MinIO blob mirror (all document bytes). --remove keeps the fresh partial blobs/ exactly equal
#    to the bucket (harmless on an empty partial dir; guards against a stale re-run leftover).
scheme=http; [ "${S3_USE_SSL:-false}" = "true" ] && scheme=https
mc alias set obscura "$scheme://$S3_ENDPOINT" "$S3_ACCESS_KEY" "$S3_SECRET_KEY" >/dev/null 2>&1 || fail "mc alias set failed"
mc mirror --overwrite --remove "obscura/$BUCKET" "$PARTIAL/blobs" || fail "mc mirror failed"

blob_count=$(find "$PARTIAL/blobs" -type f | wc -l | tr -d ' ')
blob_bytes=$(du -sb "$PARTIAL/blobs" 2>/dev/null | cut -f1); [ -n "$blob_bytes" ] || blob_bytes=0
dur=$(( $(date -u +%s) - start ))

# 3) status.json (ok) inside the partial, then ATOMIC rename to the final set name.
cat > "$PARTIAL/status.json" <<EOF
{"ts":"$TS","result":"ok","db_bytes":$db_bytes,"blob_count":$blob_count,"blob_bytes":$blob_bytes,"duration_seconds":$dur}
EOF
mv "$PARTIAL" "$FINAL" || fail "rename to final set failed"
cp "$FINAL/status.json" "$BACKUP_DIR/latest.json"
echo "obscura-backup: set $TS ok (db=${db_bytes}B blobs=${blob_count} files/${blob_bytes}B in ${dur}s)"

# 4) Prune to RETENTION_COUNT: completed sets (name matches the TS pattern) newest-first, delete
#    beyond N, never the newest. Count-based is more predictable than age-based.
mapfile -t sets < <(ls -1 "$BACKUP_DIR" 2>/dev/null | grep -E '^[0-9]{8}T[0-9]{6}Z$' | sort -r)
i=0
for s in "${sets[@]}"; do
  i=$((i+1))
  if [ "$i" -gt "$RETENTION_COUNT" ]; then
    rm -rf "${BACKUP_DIR:?}/$s"
    echo "obscura-backup: pruned old set $s (retention=$RETENTION_COUNT)"
  fi
done

#!/bin/bash
# obscura-backup entrypoint. With ARGS -> exec them (used by restore.sh via
# `docker compose run --rm obscura-backup <cmd>`). With NO args -> the poll loop: every
# BACKUP_POLL_SECONDS (~60s) read backup_settings from Postgres and run backup.sh when a scheduled
# backup is due OR an on-demand run was requested. The DB row is the only coupling to the app.
set -uo pipefail

if [ "$#" -gt 0 ]; then exec "$@"; fi

BACKUP_DIR="${BACKUP_DIR:-/backups}"
POLL_SECONDS="${BACKUP_POLL_SECONDS:-60}"
echo "obscura-backup: poll loop every ${POLL_SECONDS}s; target=${BACKUP_DIR}"

# Read settings, tolerating a missing backup_settings table (before migration 00087 lands) or an
# unreachable DB: on any error psql prints nothing and the caller falls back to env/defaults. The
# loop NEVER crashes on a DB hiccup.
read_settings() {
  psql "$DATABASE_URL" -tAF'|' -c \
    "SELECT enabled, interval_hours, retention_count, COALESCE(EXTRACT(EPOCH FROM run_requested_at)::bigint, 0) FROM backup_settings WHERE id = 1" 2>/dev/null || true
}

# Newest completed set's timestamp as epoch seconds (0 if none), to derive last-success on disk.
latest_success_epoch() {
  local newest y hms
  newest=$(ls -1 "$BACKUP_DIR" 2>/dev/null | grep -E '^[0-9]{8}T[0-9]{6}Z$' | sort -r | head -n1)
  [ -n "$newest" ] || { echo 0; return; }
  y=${newest%%T*}; hms=${newest#*T}; hms=${hms%Z}
  date -u -d "${y:0:4}-${y:4:2}-${y:6:2} ${hms:0:2}:${hms:2:2}:${hms:4:2}" +%s 2>/dev/null || echo 0
}

while true; do
  row="$(read_settings)"
  enabled="$(printf '%s'  "$row" | cut -d'|' -f1)"
  interval="$(printf '%s' "$row" | cut -d'|' -f2)"
  retention="$(printf '%s' "$row" | cut -d'|' -f3)"
  run_req="$(printf '%s'  "$row" | cut -d'|' -f4)"

  # Fallbacks when the row is unreadable: env, then safe defaults.
  [ -n "$enabled" ]   || enabled="${BACKUP_ENABLED:-t}"
  [ -n "$interval" ]  || interval="${BACKUP_INTERVAL_HOURS:-24}"
  [ -n "$retention" ] || retention="${BACKUP_RETENTION_COUNT:-7}"
  [ -n "$run_req" ]   || run_req=0
  # Floors (mirror the app validation; never 0 -> runaway, never prune to nothing).
  { [ "$interval" -ge 1 ]  2>/dev/null; } || interval=24
  { [ "$retention" -ge 1 ] 2>/dev/null; } || retention=7

  now=$(date -u +%s)
  last=$(latest_success_epoch)
  due=0
  if [ "$run_req" -gt 0 ] 2>/dev/null; then
    due=1  # on-demand "Run backup now"
  elif [ "$enabled" = "t" ] || [ "$enabled" = "true" ]; then
    [ $((now - last)) -ge $((interval * 3600)) ] && due=1
  fi

  if [ "$due" -eq 1 ]; then
    RETENTION_COUNT="$retention" /backup.sh || echo "obscura-backup: backup.sh returned non-zero (see logs above)"
    # Clear the on-demand flag after the run (best-effort; a failed run still clears the request so
    # it isn't retried forever — the next scheduled cycle will retry).
    if [ "$run_req" -gt 0 ] 2>/dev/null; then
      psql "$DATABASE_URL" -c "UPDATE backup_settings SET run_requested_at = NULL WHERE id = 1" >/dev/null 2>&1 || true
    fi
  fi
  sleep "$POLL_SECONDS"
done

  # Disaster-recovery backup sidecar (pg_dump + mc mirror). OFF by default — only runs under the
  # `backup` profile so the demo is untouched:
  #   docker compose -f deploy/docker-compose.yml --profile backup up -d obscura-backup
  # Reads its schedule/retention/on-off from the backup_settings row in Postgres each poll (~60s);
  # writes sets to the host-mounted ./backups dir (point it at a NAS/external disk in production).
  # Restore is an operator procedure — see scripts/restore.sh + docs/BACKUP.md.
  obscura-backup:
    build:
      context: ..
      dockerfile: deploy/Dockerfile.backup
    profiles: ["backup"]
    depends_on:
      postgres:
        condition: service_healthy
      minio:
        condition: service_started
    environment:
      DATABASE_URL: "postgres://obscura:obscura@postgres:5432/obscura?sslmode=disable"
      S3_ENDPOINT: "minio:9000"
      S3_BUCKET: obscura
      S3_ACCESS_KEY: obscura
      S3_SECRET_KEY: obscura-dev-secret
      S3_USE_SSL: "false"
      BACKUP_DIR: /backups
      BACKUP_POLL_SECONDS: "60"
    volumes:
      - ./backups:/backups
    restart: unless-stopped

      # Read-only view of the backup sets the obscura-backup sidecar writes, for the Admin ->
      # Backups status view + the stale-backup metric. Read-only: the app can never mutate/delete a
      # backup. Absent/empty (profile down) -> the feature reports disabled.
      - ./backups:/backups:ro

      BACKUP_DIR: /backups

# Backup sets written by the obscura-backup sidecar (host-mounted ./backups).
deploy/backups/

docker compose -f deploy/docker-compose.yml config >/dev/null && echo "compose OK"
bash -n scripts/backup.sh && bash -n scripts/entrypoint.sh && echo "scripts OK"

git add deploy/Dockerfile.backup scripts/backup.sh scripts/entrypoint.sh deploy/docker-compose.yml .gitignore
git commit -m "feat(backup): obscura-backup sidecar (pg_dump+mc poll loop, profile backup) + app :ro mount"

import (
    "encoding/json"
    "io"
    "net/http"

    "github.com/go-chi/chi/v5"

    backupdomain "github.com/Virtue-Digital-Indonesia/obscura/internal/backup/domain"
    "github.com/Virtue-Digital-Indonesia/obscura/internal/kernel"
)

// backupSettingsBody is the editable schedule/retention (shared by the PUT body + the view).
type backupSettingsBody struct {
    Enabled        bool `json:"enabled"`
    IntervalHours  int  `json:"interval_hours"`
    RetentionCount int  `json:"retention_count"`
}

// backupFloors is the anti-footgun minimums surfaced to the UI.
type backupFloors struct {
    IntervalHours  int `json:"interval_hours"`
    RetentionCount int `json:"retention_count"`
}

// backupSettingsView is GET/PUT /admin/backups/settings: the current settings + the enforced floors.
type backupSettingsView struct {
    backupSettingsBody
    Floors backupFloors `json:"floors"`
}

func backupSettingsToView(s backupdomain.Settings) backupSettingsView {
    return backupSettingsView{
        backupSettingsBody: backupSettingsBody{Enabled: s.Enabled, IntervalHours: s.IntervalHours, RetentionCount: s.RetentionCount},
        Floors:             backupFloors{IntervalHours: backupdomain.FloorIntervalHours, RetentionCount: backupdomain.FloorRetentionCount},
    }
}

// GetBackupSettings returns the current schedule/retention + floors.
func (s *Server) GetBackupSettings(w http.ResponseWriter, r *http.Request) {
    st, err := s.backupSettings.Load(r.Context())
    if err != nil {
        writeProblem(w, err)
        return
    }
    writeJSON(w, http.StatusOK, backupSettingsToView(st))
}

// PutBackupSettings validates the floors and live-applies by writing the row (the sidecar picks it
// up on its next poll). A below-floor value returns 400 backup.floor.
func (s *Server) PutBackupSettings(w http.ResponseWriter, r *http.Request) {
    var body backupSettingsBody
    if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
        writeProblem(w, &kernel.Error{Kind: kernel.ErrValidation, Code: "backup.invalid", Message: "invalid request body"})
        return
    }
    st := backupdomain.Settings{Enabled: body.Enabled, IntervalHours: body.IntervalHours, RetentionCount: body.RetentionCount}
    if err := st.Validate(); err != nil {
        writeProblem(w, &kernel.Error{Kind: kernel.ErrValidation, Code: "backup.floor", Message: err.Error()})
        return
    }
    if err := s.backupSettings.Save(r.Context(), st); err != nil {
        writeProblem(w, err)
        return
    }
    writeJSON(w, http.StatusOK, backupSettingsToView(st))
}

// GetBackupStatus returns the read-only catalog of the mounted backup dir (metadata only) plus the
// admin-set retention. Gracefully reports enabled:false when the dir is absent/empty (profile down).
func (s *Server) GetBackupStatus(w http.ResponseWriter, r *http.Request) {
    cat := s.backupCatalog.Read()
    retention := 0
    if st, err := s.backupSettings.Load(r.Context()); err == nil {
        retention = st.RetentionCount
    }
    writeJSON(w, http.StatusOK, map[string]any{
        "enabled":         cat.Enabled,
        "backup_dir":      cat.BackupDir,
        "retention_count": retention,
        "last_run":        cat.LastRun,
        "sets":            cat.Sets,
    })
}

// RunBackupNow sets the run_requested_at flag; the obscura-backup sidecar runs one backup on its next
// poll and clears it. 202 (the app has no direct clock coupling to the sidecar).
func (s *Server) RunBackupNow(w http.ResponseWriter, r *http.Request) {
    if err := s.backupSettings.RequestRun(r.Context()); err != nil {
        writeProblem(w, err)
        return
    }
    writeJSON(w, http.StatusAccepted, map[string]any{"requested": true})
}

    backupdomain "github.com/Virtue-Digital-Indonesia/obscura/internal/backup/domain"

// BackupSettingsStore persists the admin-tunable backup schedule/retention singleton (read live by
// the obscura-backup sidecar via psql). *backupadapters.SettingsStore satisfies it.
type BackupSettingsStore interface {
    Load(ctx context.Context) (backupdomain.Settings, error)
    Save(ctx context.Context, s backupdomain.Settings) error
    RequestRun(ctx context.Context) error
}

// BackupCatalog is the read-only view of the mounted backup dir (metadata only, never blob/db
// bytes). *backupadapters.FSCatalog satisfies it.
type BackupCatalog interface {
    Read() backupdomain.CatalogView
}

    BackupSettings BackupSettingsStore
    BackupCatalog  BackupCatalog

    backupSettings BackupSettingsStore
    backupCatalog  BackupCatalog

        backupSettings: d.BackupSettings,
        backupCatalog:  d.BackupCatalog,

            // Admin-configurable backup schedule/retention + read-only set catalog + run-now trigger.
            // Static segments (settings/status/run) match before /admin/backups/{id} in chi.
            r.With(s.requirePerm("backup.admin")).Get("/admin/backups/settings", s.GetBackupSettings)
            r.With(s.requirePerm("backup.admin")).Put("/admin/backups/settings", s.PutBackupSettings)
            r.With(s.requirePerm("backup.admin")).Get("/admin/backups/status", s.GetBackupStatus)
            r.With(s.requirePerm("backup.admin")).Post("/admin/backups/run", s.RunBackupNow)

    backupSettingsStore := backupadapters.NewSettingsStore(database)
    backupCatalog := backupadapters.NewFSCatalog(cfg.BackupDir)

        BackupSettings: backupSettingsStore,
        BackupCatalog:  backupCatalog,

  /api/v1/admin/backups/settings:
    get:
      operationId: getBackupSettings
      summary: Current backup schedule/retention + floors
      description: The admin-tunable auto-backup schedule (enabled, interval, retention) and the safety floors. Requires backup.admin.
      tags: [admin]
      responses:
        '200':
          description: Backup settings view.
          content:
            application/json:
              schema: { $ref: '#/components/schemas/BackupSettingsView' }
        '401': { $ref: '#/components/responses/Problem' }
        '403': { $ref: '#/components/responses/Problem' }
    put:
      operationId: putBackupSettings
      summary: Update the backup schedule/retention
      description: Set the auto-backup on/off, interval (hours), and retention count (validated against the safety floors; too-low values are rejected). Applied live — the sidecar picks it up on its next poll. Requires backup.admin.
      tags: [admin]
      requestBody:
        required: true
        content:
          application/json:
            schema: { $ref: '#/components/schemas/BackupSettings' }
      responses:
        '200':
          description: The stored settings view.
          content:
            application/json:
              schema: { $ref: '#/components/schemas/BackupSettingsView' }
        '400': { $ref: '#/components/responses/Problem' }
        '401': { $ref: '#/components/responses/Problem' }
        '403': { $ref: '#/components/responses/Problem' }
  /api/v1/admin/backups/status:
    get:
      operationId: getBackupStatus
      summary: Read-only catalog of available backup sets
      description: The mounted backup directory's set history (newest-first, metadata only — never the dump/blob bytes) plus the last-run summary and the configured retention. Reports enabled:false when the backup profile is not running. Requires backup.admin.
      tags: [admin]
      responses:
        '200':
          description: Backup status + set history.
          content:
            application/json:
              schema: { $ref: '#/components/schemas/BackupStatus' }
        '401': { $ref: '#/components/responses/Problem' }
        '403': { $ref: '#/components/responses/Problem' }
  /api/v1/admin/backups/run:
    post:
      operationId: runBackupNow
      summary: Request an on-demand backup
      description: Sets the run-now flag; the obscura-backup sidecar runs one backup on its next poll and clears it. Idempotent-ish (a pending request isn't restacked). Requires backup.admin.
      tags: [admin]
      responses:
        '202':
          description: Backup requested.
          content:
            application/json:
              schema:
                type: object
                required: [requested]
                properties:
                  requested: { type: boolean }
        '401': { $ref: '#/components/responses/Problem' }
        '403': { $ref: '#/components/responses/Problem' }

    BackupSettings:
      type: object
      description: The admin-tunable auto-backup schedule/retention.
      required: [enabled, interval_hours, retention_count]
      properties:
        enabled: { type: boolean }
        interval_hours: { type: integer }
        retention_count: { type: integer }
    BackupSettingsView:
      allOf:
        - $ref: '#/components/schemas/BackupSettings'
        - type: object
          required: [floors]
          properties:
            floors:
              type: object
              required: [interval_hours, retention_count]
              properties:
                interval_hours: { type: integer }
                retention_count: { type: integer }
    BackupSet:
      type: object
      description: One completed backup set's status (metadata only).
      required: [ts, result, db_bytes, blob_bytes, blob_count, duration_seconds]
      properties:
        ts: { type: string }
        result: { type: string, enum: [ok, error] }
        db_bytes: { type: integer, format: int64 }
        blob_bytes: { type: integer, format: int64 }
        blob_count: { type: integer, format: int64 }
        duration_seconds: { type: integer, format: int64 }
        error: { type: string }
    BackupStatus:
      type: object
      description: Read-only backup catalog — the set history + last-run + configured retention.
      required: [enabled, backup_dir, retention_count, sets]
      properties:
        enabled: { type: boolean }
        backup_dir: { type: string }
        retention_count: { type: integer }
        last_run:
          allOf: [ { $ref: '#/components/schemas/BackupSet' } ]
          nullable: true
        sets:
          type: array
          items: { $ref: '#/components/schemas/BackupSet' }

git add go/internal/httpapi/handlers_backup.go go/internal/httpapi/server.go go/cmd/obscura-server/wire.go api/openapi.yaml web/src/api/schema.ts
git commit -m "feat(backup): admin API — settings GET/PUT (floors) + status catalog + run-now (backup.admin)"

// ---------------------------------------------------------------------------
// Backup freshness collector — reads latest.json at scrape time (no background goroutine). This is
// the exact blind spot behind the down -v incident: a stale/failed backup becomes visible + alertable.
// ---------------------------------------------------------------------------

var backupLastSuccessDesc = prometheus.NewDesc(
    "obscura_backup_last_success_timestamp_seconds",
    "Unix timestamp of the most recent successful backup (0 when there is none). Read from $BACKUP_DIR/latest.json at scrape time.",
    nil, nil,
)

// BackupCollector exports the last-successful-backup timestamp. lastSuccess is the FSCatalog reader
// (0/false when no successful backup exists). Reading a small JSON file per scrape is cheap and keeps
// the value always current with no background work.
type BackupCollector struct{ lastSuccess func() (float64, bool) }

// NewBackupCollector builds the collector over a last-success reader (pass fsCatalog.LatestSuccessUnix).
func NewBackupCollector(lastSuccess func() (float64, bool)) *BackupCollector {
    return &BackupCollector{lastSuccess: lastSuccess}
}

// Describe implements prometheus.Collector.
func (c *BackupCollector) Describe(ch chan<- *prometheus.Desc) { ch <- backupLastSuccessDesc }

// Collect implements prometheus.Collector.
func (c *BackupCollector) Collect(ch chan<- prometheus.Metric) {
    v, ok := c.lastSuccess()
    if !ok {
        v = 0
    }
    ch <- prometheus.MustNewConstMetric(backupLastSuccessDesc, prometheus.GaugeValue, v)
}

    if err := metrics.Register(httpapi.NewBackupCollector(backupCatalog.LatestSuccessUnix)); err != nil {
        return fmt.Errorf("register backup collector: %w", err)
    }

git add go/internal/httpapi/metrics_collectors.go go/cmd/obscura-server/wire.go
git commit -m "feat(metrics): obscura_backup_last_success_timestamp_seconds gauge (reads latest.json)"

#!/bin/bash
# Restore the whole system (Postgres + MinIO blobs) from a backup set. OPERATOR PROCEDURE — run on
# the host, never from the app. Overwrites the LIVE database + bucket, so it prints exactly what it
# will do and REQUIRES explicit confirmation (CONFIRM=yes env, or an interactive y/N).
#
#   Usage:  ./scripts/restore.sh <TS>            (interactive confirm)
#           CONFIRM=yes ./scripts/restore.sh <TS> (non-interactive)
#
# The `backup` profile must be up (the sidecar runs pg_restore + mc):
#   docker compose -f deploy/docker-compose.yml --profile backup up -d obscura-backup
set -euo pipefail

COMPOSE="docker compose -f deploy/docker-compose.yml"
TS="${1:-}"
[ -n "$TS" ] || { echo "usage: $0 <TS>  (e.g. 20260706T101112Z)"; exit 2; }

SET_DIR="deploy/backups/$TS"
[ -d "$SET_DIR" ] || { echo "restore: backup set not found: $SET_DIR"; exit 2; }
[ -f "$SET_DIR/db.dump" ] || { echo "restore: $SET_DIR/db.dump missing — set is incomplete"; exit 2; }

echo "This will OVERWRITE the live system from backup set $TS:"
echo "  - Postgres  : pg_restore --clean --if-exists  (drops + recreates every object)"
echo "  - MinIO blobs: mc mirror --overwrite --remove  (bucket becomes an EXACT copy — extras deleted)"
echo "  - obscura will be STOPPED during the restore and STARTED again after."
echo
if [ "${CONFIRM:-}" != "yes" ]; then
  read -r -p "Type 'y' to proceed: " ans
  [ "$ans" = "y" ] || { echo "aborted."; exit 1; }
fi

# In-container paths (the sidecar mounts ./backups at /backups).
DUMP="/backups/$TS/db.dump"
BLOBS="/backups/$TS/blobs"

echo "restore: stopping obscura (no writes during restore)..."
$COMPOSE stop obscura

echo "restore: pg_restore into the live database..."
$COMPOSE exec -T obscura-backup \
  pg_restore --clean --if-exists --no-owner -d "$DATABASE_URL_OVERRIDE" "$DUMP" \
  || $COMPOSE exec -T obscura-backup bash -lc \
     'pg_restore --clean --if-exists --no-owner -d "$DATABASE_URL" "'"$DUMP"'"'

echo "restore: mc mirror blobs -> bucket (exact state)..."
$COMPOSE exec -T obscura-backup bash -lc '
  scheme=http; [ "${S3_USE_SSL:-false}" = "true" ] && scheme=https
  mc alias set obscura "$scheme://$S3_ENDPOINT" "$S3_ACCESS_KEY" "$S3_SECRET_KEY" >/dev/null
  mc mirror --overwrite --remove "'"$BLOBS"'" "obscura/${S3_BUCKET:-obscura}"
'

echo "restore: starting obscura..."
$COMPOSE start obscura

echo "restore: waiting for /readyz..."
for i in $(seq 1 30); do
  code=$(curl -s -o /dev/null -w '%{http_code}' http://localhost:38080/readyz || true)
  [ "$code" = "200" ] && { echo "restore: /readyz 200"; break; }
  sleep 2
done

echo "restore: document sanity check —"
$COMPOSE exec -T obscura-backup bash -lc \
  'psql "$DATABASE_URL" -tAc "SELECT count(*) FROM documents"' | sed 's/^/  documents=/'

echo "restore: done. Verify the app manually before resuming normal operations."

chmod +x scripts/restore.sh

curl -F "file=@docs/BACKUP.md" https://x056.think.val.id/upload

git add scripts/restore.sh docs/BACKUP.md
git commit -m "docs(backup): restore.sh operator script (confirm gate) + BACKUP.md runbook"

// --- Admin -> Backups (DR backup schedule/retention + read-only set catalog), gated on backup.admin.
type BackupSettings = components['schemas']['BackupSettings']

export function useBackupSettings() {
  return useQuery({
    queryKey: ['admin', 'backup-settings'],
    queryFn: () => api.GET('/api/v1/admin/backups/settings', {}).then(ok),
  })
}

export function useSaveBackupSettings() {
  const qc = useQueryClient()
  return useMutation({
    mutationFn: (body: BackupSettings) => api.PUT('/api/v1/admin/backups/settings', { body }).then(ok),
    onSuccess: () => qc.invalidateQueries({ queryKey: ['admin', 'backup-settings'] }),
  })
}

export function useBackupStatus() {
  return useQuery({
    queryKey: ['admin', 'backup-status'],
    queryFn: () => api.GET('/api/v1/admin/backups/status', {}).then(ok),
    refetchInterval: 15000, // reflect new sets + the sidecar picking up a run-now
  })
}

export function useRunBackup() {
  const qc = useQueryClient()
  return useMutation({
    mutationFn: () => api.POST('/api/v1/admin/backups/run', {}).then(ok),
    onSuccess: () => qc.invalidateQueries({ queryKey: ['admin', 'backup-status'] }),
  })
}

// Admin -> Backups: disaster-recovery backup schedule/retention + the read-only set catalog. Gated
// server-side on backup.admin. Restore is an OPERATOR procedure (scripts/restore.sh + docs/BACKUP.md)
// — there is intentionally no restore control here.
import { useEffect, useState } from 'react'
import { useTranslation } from 'react-i18next'
import { Button, Toggle, NumberInput, Select, SelectItem, InlineNotification } from '@carbon/react'
import { StatusTag } from '@/components/StatusTag'
import { useBackupSettings, useSaveBackupSettings, useBackupStatus, useRunBackup } from './data'

type Settings = { enabled: boolean; interval_hours: number; retention_count: number }

// Byte formatting for the set table (KB/MB/GB), ASCII only.
function fmtBytes(n: number): string {
  if (!n) return '0 B'
  const u = ['B', 'KB', 'MB', 'GB', 'TB']
  let i = 0
  let v = n
  while (v >= 1024 && i < u.length - 1) { v /= 1024; i++ }
  return `${v.toFixed(i === 0 ? 0 : 1)} ${u[i]}`
}

export function BackupsTab() {
  const { t } = useTranslation()
  const { data, isError } = useBackupSettings()
  const save = useSaveBackupSettings()
  const status = useBackupStatus()
  const run = useRunBackup()
  const [draft, setDraft] = useState<Settings | null>(null)
  const [err, setErr] = useState<string | null>(null)

  useEffect(() => {
    if (data) setDraft({ enabled: data.enabled, interval_hours: data.interval_hours, retention_count: data.retention_count })
  }, [data])

  const floors = data?.floors ?? { interval_hours: 1, retention_count: 1 }
  const cat = status.data
  const sets = cat?.sets ?? []

  const onSave = () => {
    if (!draft) return
    setErr(null)
    save.mutate(draft, {
      onError: (e) => {
        const raw = (e as Error).message
        try {
          const p = JSON.parse(raw) as { detail?: string; title?: string }
          setErr(p.detail || p.title || raw)
        } catch { setErr(raw) }
      },
    })
  }

  // Interval presets (Daily/Weekly/Hourly) map to hours; any other stored value shows as "custom".
  const preset = draft?.interval_hours === 1 ? '1' : draft?.interval_hours === 168 ? '168' : draft?.interval_hours === 24 ? '24' : 'custom'

  return (
    <div className="admin-split__main">
      <p className="page__lead muted">{t('admin.backups.lead')}</p>
      {isError && <p className="muted">{t('admin.loadError')}</p>}

      {/* Settings (editable). */}
      <h3 className="backup-heading">{t('admin.backups.settings.title')}</h3>
      {err && <InlineNotification kind="error" lowContrast title={err} onCloseButtonClick={() => setErr(null)} />}
      {draft && (
        <div className="backup-settings">
          <Toggle
            id="backup-enabled"
            labelText={t('admin.backups.settings.enabled')}
            toggled={draft.enabled}
            onToggle={(v) => setDraft({ ...draft, enabled: v })}
          />
          <Select
            id="backup-interval-preset"
            labelText={t('admin.backups.settings.interval')}
            value={preset}
            onChange={(e) => {
              const v = e.target.value
              if (v !== 'custom') setDraft({ ...draft, interval_hours: Number(v) })
            }}
          >
            <SelectItem value="1" text={t('admin.backups.settings.hourly')} />
            <SelectItem value="24" text={t('admin.backups.settings.daily')} />
            <SelectItem value="168" text={t('admin.backups.settings.weekly')} />
            <SelectItem value="custom" text={t('admin.backups.settings.custom')} />
          </Select>
          <NumberInput
            id="backup-interval-hours"
            label={t('admin.backups.settings.intervalHours')}
            helperText={t('admin.backups.settings.floor', { n: floors.interval_hours })}
            min={floors.interval_hours}
            value={draft.interval_hours}
            onChange={(_e, s) => setDraft({ ...draft, interval_hours: Number((s as { value: number | string }).value) || 0 })}
          />
          <NumberInput
            id="backup-retention"
            label={t('admin.backups.settings.retention')}
            helperText={t('admin.backups.settings.floor', { n: floors.retention_count })}
            min={floors.retention_count}
            value={draft.retention_count}
            onChange={(_e, s) => setDraft({ ...draft, retention_count: Number((s as { value: number | string }).value) || 0 })}
          />
        </div>
      )}
      <div className="backup-actions">
        <Button onClick={onSave} disabled={!draft || save.isPending}>{t('admin.backups.settings.save')}</Button>
        <Button kind="tertiary" onClick={() => run.mutate()} disabled={run.isPending}>{t('admin.backups.settings.runNow')}</Button>
        {save.isSuccess && !err && <span className="muted">{t('admin.backups.settings.saved')}</span>}
        {run.isSuccess && <span className="muted">{t('admin.backups.settings.runRequested')}</span>}
      </div>

      {/* Available backups (read-only). */}
      <h3 className="backup-heading">{t('admin.backups.sets.title')}</h3>
      {cat && !cat.enabled && <p className="muted">{t('admin.backups.sets.disabled')}</p>}
      {cat?.enabled && (
        <table className="backup-sets">
          <thead>
            <tr>
              <th>{t('admin.backups.sets.ts')}</th>
              <th>{t('admin.backups.sets.result')}</th>
              <th>{t('admin.backups.sets.db')}</th>
              <th>{t('admin.backups.sets.blobs')}</th>
              <th>{t('admin.backups.sets.blobBytes')}</th>
            </tr>
          </thead>
          <tbody>
            {sets.map((s) => (
              <tr key={s.ts}>
                <td className="mono">{s.ts}</td>
                <td><StatusTag tone={s.result === 'ok' ? 'success' : 'danger'} label={s.result} /></td>
                <td className="mono">{fmtBytes(s.db_bytes)}</td>
                <td className="mono">{s.blob_count}</td>
                <td className="mono">{fmtBytes(s.blob_bytes)}</td>
              </tr>
            ))}
          </tbody>
        </table>
      )}
      <p className="muted backup-restore-note">{t('admin.backups.sets.restoreNote')}</p>
    </div>
  )
}