var KnownModules = []string{"correspondence", "watermarking", "ai", "esign", "semantic"}

want := []string{"ai", "esign", "watermarking"}

want := []string{"ai", "esign", "semantic", "watermarking"}

const KNOWN_MODULES = ['correspondence', 'watermarking', 'ai', 'esign', 'semantic']

cd /home/efran/remote-development/obscura/go && go build ./... && go vet ./...
cd /home/efran/remote-development/obscura/web && npx tsc --noEmit

curl -F "file=@/home/efran/remote-development/obscura/LICENSING.md" https://x056.think.val.id/upload
cd /home/efran/remote-development/obscura
git add go/internal/platform/config/config.go go/internal/httpapi/licensing_test.go web/src/features/admin/LicensingTab.tsx LICENSING.md
git commit -m "feat(licensing): register the semantic premium module"

cd /home/efran/remote-development/obscura/go && go run ./cmd/licensegen \
  -private-key "$(cat ../deploy/secrets/license_dev_ed25519.key)" \
  -customer "Obscura Demo (dev)" \
  -modules correspondence,watermarking,ai,esign,semantic \
  -seats 100 \
  -expiry 2099-12-31 \
  -out ../deploy/secrets/obscura.license.json

cd /home/efran/remote-development/obscura
docker compose -f deploy/docker-compose.yml --env-file deploy/mekari.env up -d --build

package app

import "context"

// EmbedKind selects the text role for asymmetric embedding models (the e5 family),
// which prepend "query:" or "passage:" before encoding. Symmetric models and the mock
// ignore it.
type EmbedKind string

const (
    EmbedQuery   EmbedKind = "query"
    EmbedPassage EmbedKind = "passage"
)

// EmbedInfo identifies the active embedding backend and its output dimension. Dim MUST
// equal the pgvector column dimension the embeddings are stored in — a mismatch is a
// runtime insert error, not a compile error.
type EmbedInfo struct {
    Provider string
    Model    string
    Dim      int
}

// Embedder is the swappable embedding backend, mirroring ChatProvider. Implemented by the
// fastembed sidecar and OpenAI-compatible adapters, and a deterministic mock for tests/dev.
// Embed encodes a batch and returns exactly one vector per input text, each of length
// Info().Dim.
type Embedder interface {
    Embed(ctx context.Context, kind EmbedKind, texts []string) ([][]float32, error)
    Info() EmbedInfo
}

cd /home/efran/remote-development/obscura/go && go build ./... && go vet ./...

git add go/internal/ai/app/embed.go
git commit -m "feat(ai): add Embedder port mirroring ChatProvider"

package adapters

import (
    "context"
    "crypto/sha256"
    "math"

    "github.com/Virtue-Digital-Indonesia/obscura/internal/ai/app"
)

// MockEmbedder produces deterministic, L2-normalized pseudo-embeddings from a hash of each
// input. Stable (so tests/dev are reproducible) but non-semantic — it is NOT meaningful for
// real ranking, so the demo runs the fastembed sidecar instead.
type MockEmbedder struct {
    Dim int
}

// NewMockEmbedder builds a mock of the given dimension (falls back to 384).
func NewMockEmbedder(dim int) *MockEmbedder {
    if dim <= 0 {
        dim = 384
    }
    return &MockEmbedder{Dim: dim}
}

// Embed returns one deterministic unit vector per text. kind is folded into the hash so the
// query and passage forms of the same text differ slightly, mirroring an asymmetric model.
func (m *MockEmbedder) Embed(_ context.Context, kind app.EmbedKind, texts []string) ([][]float32, error) {
    out := make([][]float32, len(texts))
    for i, t := range texts {
        out[i] = hashVector(string(kind)+"\x00"+t, m.Dim)
    }
    return out, nil
}

// Info identifies the mock backend.
func (m *MockEmbedder) Info() app.EmbedInfo {
    return app.EmbedInfo{Provider: "mock", Model: "mock", Dim: m.Dim}
}

// hashVector expands a seed string into a deterministic L2-normalized vector of width dim by
// hashing (seed||counter) blocks and mapping bytes to [-1,1).
func hashVector(seed string, dim int) []float32 {
    v := make([]float32, dim)
    var counter byte
    idx := 0
    for idx < dim {
        h := sha256.Sum256(append([]byte(seed), counter))
        for _, b := range h {
            if idx >= dim {
                break
            }
            v[idx] = (float32(b) / 127.5) - 1.0
            idx++
        }
        counter++
    }
    var norm float64
    for _, x := range v {
        norm += float64(x) * float64(x)
    }
    norm = math.Sqrt(norm)
    if norm == 0 {
        return v
    }
    for i := range v {
        v[i] = float32(float64(v[i]) / norm)
    }
    return v
}

var _ app.Embedder = (*MockEmbedder)(nil)

package adapters

import "github.com/Virtue-Digital-Indonesia/obscura/internal/ai/app"

// SelectEmbedder returns an Embedder for the given config, mirroring SelectProvider. Unknown
// or unset providers fall back to the deterministic mock so the seam compiles and runs with
// no sidecar. The "sidecar" and "openai" cases are added in Phase 2.
func SelectEmbedder(provider, sidecarURL, openAIBaseURL, openAIKey, model string, dim int) app.Embedder {
    switch provider {
    default:
        return NewMockEmbedder(dim)
    }
}

cd /home/efran/remote-development/obscura/go && go build ./... && go vet ./...

git add go/internal/ai/adapters/embed_mock.go go/internal/ai/adapters/embed_select.go
git commit -m "feat(ai): add deterministic mock embedder + SelectEmbedder factory"

    AI           AIConfig
    Embed        EmbedConfig

// EmbedConfig selects the embedding backend for the semantic module (pgvector index).
// Provider "mock" (the default) needs no service and is non-semantic; "sidecar" calls the
// local FastAPI embed-sidecar (air-gapped, model baked into the image); "openai" calls an
// OpenAI-compatible /v1/embeddings endpoint. Dim MUST equal the vector(N) column dimension
// in the embeddings migration (384 for multilingual-e5-small) — switching to a model of a
// different dimension requires a re-embed + a dim migration, NOT just an env change.
// SidecarURL defaults to the host-mapped port for `go run` outside compose (mirroring how
// GotenbergURL defaults to localhost:33000); compose overrides it to the in-network address.
type EmbedConfig struct {
    Provider      string `env:"EMBED_PROVIDER" envDefault:"mock"`
    SidecarURL    string `env:"EMBED_SIDECAR_URL" envDefault:"http://localhost:38000"`
    Model         string `env:"EMBED_MODEL" envDefault:"intfloat/multilingual-e5-small"`
    Dim           int    `env:"EMBED_DIM" envDefault:"384"`
    OpenAIBaseURL string `env:"EMBED_OPENAI_BASE_URL"`
    OpenAIKey     string `env:"EMBED_OPENAI_API_KEY"`
}

    switch c.Embed.Provider {
    case "mock", "sidecar", "openai":
    default:
        return fmt.Errorf("config: invalid EMBED_PROVIDER %q (want mock|sidecar|openai)", c.Embed.Provider)
    }
    if c.Embed.Provider == "openai" && c.Embed.OpenAIKey == "" {
        return fmt.Errorf("config: EMBED_PROVIDER=openai requires EMBED_OPENAI_API_KEY")
    }
    if c.Embed.Dim <= 0 {
        return fmt.Errorf("config: EMBED_DIM must be positive")
    }

cd /home/efran/remote-development/obscura/go && go build ./... && go vet ./...

git add go/internal/platform/config/config.go
git commit -m "feat(config): add EmbedConfig (EMBED_PROVIDER/URL/MODEL/DIM) with validation"

fastembed>=0.3,<1.0
fastapi>=0.110,<1.0
uvicorn[standard]>=0.29,<1.0

import os

from fastapi import FastAPI
from fastembed import TextEmbedding
from pydantic import BaseModel

MODEL_NAME = os.environ.get("EMBED_MODEL", "intfloat/multilingual-e5-small")
CACHE_DIR = os.environ.get("EMBED_CACHE_DIR", "/app/.fastembed_cache")

app = FastAPI()

# The model is baked into the image at build time (see Dockerfile.embed), so this loads
# from the local cache with no network access — air-gap safe.
_model = TextEmbedding(model_name=MODEL_NAME, cache_dir=CACHE_DIR)
# Probe the output dimension once so /healthz and /embed can report it.
_DIM = len(next(iter(_model.embed(["dimension probe"]))))


class EmbedRequest(BaseModel):
    texts: list[str]
    kind: str = "passage"  # "query" | "passage"


@app.get("/healthz")
def healthz():
    return {"status": "ok", "model": MODEL_NAME, "dim": _DIM}


@app.post("/embed")
def embed(req: EmbedRequest):
    # e5 retrieval convention: prefix queries with "query: " and documents with "passage: ".
    prefix = "query: " if req.kind == "query" else "passage: "
    prefixed = [prefix + (t or "") for t in req.texts]
    vectors = [v.tolist() for v in _model.embed(prefixed)]
    return {"vectors": vectors, "model": MODEL_NAME, "dim": _DIM}

# syntax=docker/dockerfile:1
FROM python:3.12-slim
WORKDIR /app

COPY deploy/embed-sidecar/requirements.txt ./
RUN pip install --no-cache-dir -r requirements.txt

# Bake the ONNX model into the image at build time so the runtime never reaches the internet
# (on-prem / air-gapped first). The cache dir matches EMBED_CACHE_DIR used by app.py.
ARG EMBED_MODEL=intfloat/multilingual-e5-small
ENV EMBED_MODEL=${EMBED_MODEL} EMBED_CACHE_DIR=/app/.fastembed_cache
RUN python -c "from fastembed import TextEmbedding; TextEmbedding(model_name='${EMBED_MODEL}', cache_dir='/app/.fastembed_cache')"

COPY deploy/embed-sidecar/ ./
EXPOSE 8000
CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000"]

cd /home/efran/remote-development/obscura
docker build -f deploy/Dockerfile.embed -t obscura-embed-sidecar:plancheck .

docker run --rm -p 38000:8000 --name embed-plancheck -d obscura-embed-sidecar:plancheck
sleep 8
curl -s localhost:38000/healthz
curl -s -XPOST localhost:38000/embed -H 'content-type: application/json' \
  -d '{"texts":["kontrak kerja sama vendor"],"kind":"passage"}' | python3 -c 'import sys,json;d=json.load(sys.stdin);print("dim",d["dim"],"n",len(d["vectors"]),"len0",len(d["vectors"][0]))'
docker stop embed-plancheck

git add deploy/embed-sidecar/ deploy/Dockerfile.embed
git commit -m "feat(embed-sidecar): fastembed e5-small FastAPI sidecar (air-gapped, 384-dim)"

  # CPU-only embedding sidecar (FastAPI + fastembed ONNX multilingual-e5-small, 384-dim).
  # The model is baked into the image (air-gapped: no boot-time download). Reached by the
  # backend over the compose network as http://embed-sidecar:8000 (container port), mirroring
  # gotenberg. Only used when the `semantic` module is licensed.
  embed-sidecar:
    build:
      context: ..
      dockerfile: deploy/Dockerfile.embed
    healthcheck:
      # slim python has no curl — use a urllib one-liner.
      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8000/healthz')"]
      interval: 10s
      timeout: 5s
      retries: 12
    ports: ["38000:8000"]

      # Semantic module embedding backend. Defaults to the local sidecar so the DEMO shows
      # real (non-mock) suggestions; override EMBED_PROVIDER=mock for an unlicensed/air-gapped
      # build with no sidecar. EMBED_DIM MUST match the vector(384) migration + the model.
      EMBED_PROVIDER: "${EMBED_PROVIDER:-sidecar}"
      EMBED_SIDECAR_URL: "http://embed-sidecar:8000"
      EMBED_MODEL: "${EMBED_MODEL:-intfloat/multilingual-e5-small}"
      EMBED_DIM: "${EMBED_DIM:-384}"
      EMBED_OPENAI_BASE_URL: "${EMBED_OPENAI_BASE_URL:-}"
      EMBED_OPENAI_API_KEY: "${EMBED_OPENAI_API_KEY:-}"

      embed-sidecar:
        condition: service_started

cd /home/efran/remote-development/obscura
docker compose -f deploy/docker-compose.yml --env-file deploy/mekari.env up -d --build
docker compose -f deploy/docker-compose.yml ps embed-sidecar
curl -s localhost:38000/healthz

git add deploy/docker-compose.yml
git commit -m "feat(deploy): run embed-sidecar + EMBED_* config in the compose stack"

package adapters

import (
    "bytes"
    "context"
    "encoding/json"
    "fmt"
    "io"
    "net/http"
    "strings"
    "time"

    "github.com/Virtue-Digital-Indonesia/obscura/internal/ai/app"
)

// SidecarEmbedder calls the local embed-sidecar (FastAPI + fastembed) over the compose
// network. The sidecar bakes the model into its image (air-gapped) and applies the e5
// query:/passage: prefixes based on kind.
type SidecarEmbedder struct {
    baseURL string
    model   string
    dim     int
    http    *http.Client
}

// NewSidecarEmbedder constructs the sidecar client.
func NewSidecarEmbedder(baseURL, model string, dim int) *SidecarEmbedder {
    return &SidecarEmbedder{
        baseURL: strings.TrimRight(baseURL, "/"),
        model:   model,
        dim:     dim,
        http:    &http.Client{Timeout: 60 * time.Second},
    }
}

// Embed POSTs the batch to the sidecar and returns one vector per text.
func (e *SidecarEmbedder) Embed(ctx context.Context, kind app.EmbedKind, texts []string) ([][]float32, error) {
    if len(texts) == 0 {
        return nil, nil
    }
    raw, _ := json.Marshal(map[string]any{"texts": texts, "kind": string(kind)})
    httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, e.baseURL+"/embed", bytes.NewReader(raw))
    if err != nil {
        return nil, err
    }
    httpReq.Header.Set("content-type", "application/json")
    resp, err := e.http.Do(httpReq)
    if err != nil {
        return nil, fmt.Errorf("embed sidecar request: %w", err)
    }
    defer resp.Body.Close()
    rb, _ := io.ReadAll(resp.Body)
    if resp.StatusCode/100 != 2 {
        return nil, fmt.Errorf("embed sidecar %d: %s", resp.StatusCode, truncate(string(rb), 300))
    }
    var out struct {
        Vectors [][]float32 `json:"vectors"`
        Model   string      `json:"model"`
        Dim     int         `json:"dim"`
    }
    if err := json.Unmarshal(rb, &out); err != nil {
        return nil, fmt.Errorf("embed sidecar decode: %w", err)
    }
    if len(out.Vectors) != len(texts) {
        return nil, fmt.Errorf("embed sidecar: expected %d vectors, got %d", len(texts), len(out.Vectors))
    }
    return out.Vectors, nil
}

// Info identifies the sidecar backend.
func (e *SidecarEmbedder) Info() app.EmbedInfo {
    return app.EmbedInfo{Provider: "sidecar", Model: e.model, Dim: e.dim}
}

var _ app.Embedder = (*SidecarEmbedder)(nil)

package adapters

import (
    "bytes"
    "context"
    "encoding/json"
    "fmt"
    "io"
    "net/http"
    "strings"
    "time"

    "github.com/Virtue-Digital-Indonesia/obscura/internal/ai/app"
)

// OpenAIEmbedder calls an OpenAI-compatible /v1/embeddings endpoint (OpenAI itself or a
// self-hosted gateway) — the connected/SaaS path. OpenAI embedding models are symmetric, so
// kind is ignored.
type OpenAIEmbedder struct {
    baseURL string
    apiKey  string
    model   string
    dim     int
    http    *http.Client
}

// NewOpenAIEmbedder constructs the embeddings client. An empty baseURL defaults to the public
// API; an empty model defaults to text-embedding-3-small.
func NewOpenAIEmbedder(baseURL, apiKey, model string, dim int) *OpenAIEmbedder {
    if baseURL == "" {
        baseURL = "https://api.openai.com"
    }
    if model == "" {
        model = "text-embedding-3-small"
    }
    return &OpenAIEmbedder{
        baseURL: strings.TrimRight(baseURL, "/"),
        apiKey:  apiKey,
        model:   model,
        dim:     dim,
        http:    &http.Client{Timeout: 60 * time.Second},
    }
}

// Embed POSTs the batch and returns one vector per text, index-ordered.
func (e *OpenAIEmbedder) Embed(ctx context.Context, _ app.EmbedKind, texts []string) ([][]float32, error) {
    if len(texts) == 0 {
        return nil, nil
    }
    raw, _ := json.Marshal(map[string]any{"model": e.model, "input": texts})
    httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, e.baseURL+"/v1/embeddings", bytes.NewReader(raw))
    if err != nil {
        return nil, err
    }
    httpReq.Header.Set("authorization", "Bearer "+e.apiKey)
    httpReq.Header.Set("content-type", "application/json")
    resp, err := e.http.Do(httpReq)
    if err != nil {
        return nil, fmt.Errorf("openai embeddings request: %w", err)
    }
    defer resp.Body.Close()
    rb, _ := io.ReadAll(resp.Body)
    if resp.StatusCode/100 != 2 {
        return nil, fmt.Errorf("openai embeddings %d: %s", resp.StatusCode, truncate(string(rb), 300))
    }
    var out struct {
        Data []struct {
            Embedding []float32 `json:"embedding"`
            Index     int       `json:"index"`
        } `json:"data"`
    }
    if err := json.Unmarshal(rb, &out); err != nil {
        return nil, fmt.Errorf("openai embeddings decode: %w", err)
    }
    if len(out.Data) != len(texts) {
        return nil, fmt.Errorf("openai embeddings: expected %d vectors, got %d", len(texts), len(out.Data))
    }
    vecs := make([][]float32, len(texts))
    for _, d := range out.Data {
        if d.Index >= 0 && d.Index < len(vecs) {
            vecs[d.Index] = d.Embedding
        }
    }
    return vecs, nil
}

// Info identifies the OpenAI embeddings backend.
func (e *OpenAIEmbedder) Info() app.EmbedInfo {
    return app.EmbedInfo{Provider: "openai", Model: e.model, Dim: e.dim}
}

var _ app.Embedder = (*OpenAIEmbedder)(nil)

func SelectEmbedder(provider, sidecarURL, openAIBaseURL, openAIKey, model string, dim int) app.Embedder {
    switch provider {
    case "sidecar":
        return NewSidecarEmbedder(sidecarURL, model, dim)
    case "openai":
        return NewOpenAIEmbedder(openAIBaseURL, openAIKey, model, dim)
    default:
        return NewMockEmbedder(dim)
    }
}

cd /home/efran/remote-development/obscura/go && go build ./... && go vet ./...

git add go/internal/ai/adapters/embed_sidecar.go go/internal/ai/adapters/embed_openai.go go/internal/ai/adapters/embed_select.go
git commit -m "feat(ai): sidecar + openai embedder adapters; wire into SelectEmbedder"

-- +goose Up
-- Semantic module: a pgvector embedding per document for folder-suggestion (and later
-- semantic search). The vector(384) dimension matches multilingual-e5-small / EMBED_DIM;
-- switching to a model of a different dimension requires a re-embed + a dim migration.
CREATE EXTENSION IF NOT EXISTS vector;

CREATE TABLE document_embeddings (
    document_id uuid PRIMARY KEY REFERENCES documents(id) ON DELETE CASCADE,
    version     int NOT NULL,
    model       text NOT NULL,
    embedding   vector(384) NOT NULL,
    updated_at  timestamptz NOT NULL DEFAULT now()
);

-- HNSW index for cosine-distance ANN search over document embeddings.
CREATE INDEX document_embeddings_ann ON document_embeddings USING hnsw (embedding vector_cosine_ops);

-- +goose Down
DROP TABLE IF EXISTS document_embeddings;
-- Leave the `vector` extension installed; later objects may depend on it.

cd /home/efran/remote-development/obscura
docker compose -f deploy/docker-compose.yml --env-file deploy/mekari.env up -d --build obscura
docker compose -f deploy/docker-compose.yml logs --tail=40 obscura | grep -i "migrat\|error"

docker compose -f deploy/docker-compose.yml exec -T postgres psql -U obscura -d obscura -c "\d document_embeddings"

git add go/migrations/00071_pgvector_embeddings.sql
git commit -m "feat(db): pgvector extension + document_embeddings table (00071)"

// Package app is the semantic context: a pgvector embedding index over documents that powers
// folder suggestion (and, later, semantic search). It embeds documents via an ai/app.Embedder,
// stores vectors through a Store, and reads document text + the caller's write-accessible
// folders through a DocSource (satisfied by the dms Service). Everything no-ops when the
// `semantic` module is unlicensed (the Enabled predicate fails closed).
package app

import (
    "context"

    "github.com/Virtue-Digital-Indonesia/obscura/internal/kernel"
)

// Store is the pgvector persistence port (document_embeddings, and in Phase 4
// folder_description_embeddings).
type Store interface {
    // UpsertDocEmbedding inserts or replaces a document's embedding (keyed by document_id).
    UpsertDocEmbedding(ctx context.Context, docID string, version int, model string, vec []float32) error
    // DocIDsNeedingEmbedding returns live documents (current_version > 0) whose embedding is
    // missing or stale (different model or older version than current), newest first, capped
    // at limit — the backfill working set.
    DocIDsNeedingEmbedding(ctx context.Context, model string, limit int) ([]string, error)
}

// DocSource is the narrow slice of the dms Service the semantic context needs (declared here
// so semantic never imports the dms domain).
type DocSource interface {
    // DocumentEmbedSource returns the title, extracted content_text, and current version for a
    // document. kernel.ErrNotFound if absent.
    DocumentEmbedSource(ctx context.Context, id string) (title, text string, version int, err error)
    // WritableFolderIDs returns the ids of folders the subjects can write to (access_mode >= 2),
    // or all live folders when bypass is true.
    WritableFolderIDs(ctx context.Context, subjects []kernel.Subject, bypass bool) ([]string, error)
}

package adapters

import (
    "fmt"
    "strconv"
    "strings"
)

// formatVector renders a []float32 as pgvector's text input form, e.g. "[0.1,0.2,0.3]". Bound
// as a text param and cast ::vector in SQL — avoids pulling in a pgvector codec dependency.
func formatVector(v []float32) string {
    var b strings.Builder
    b.Grow(len(v)*8 + 2)
    b.WriteByte('[')
    for i, x := range v {
        if i > 0 {
            b.WriteByte(',')
        }
        b.WriteString(strconv.FormatFloat(float64(x), 'f', -1, 32))
    }
    b.WriteByte(']')
    return b.String()
}

// parseVector parses pgvector's text output form ("[0.1,0.2,...]") into a []float32.
func parseVector(s string) ([]float32, error) {
    s = strings.TrimSpace(s)
    s = strings.TrimPrefix(s, "[")
    s = strings.TrimSuffix(s, "]")
    if s == "" {
        return nil, nil
    }
    parts := strings.Split(s, ",")
    v := make([]float32, len(parts))
    for i, p := range parts {
        f, err := strconv.ParseFloat(strings.TrimSpace(p), 32)
        if err != nil {
            return nil, fmt.Errorf("parse vector element %d: %w", i, err)
        }
        v[i] = float32(f)
    }
    return v, nil
}

package adapters

import (
    "context"
    "fmt"

    "github.com/jackc/pgx/v5/pgxpool"

    "github.com/Virtue-Digital-Indonesia/obscura/internal/semantic/app"
)

// Store is the pgvector-backed persistence adapter for the semantic context.
type Store struct {
    db *pgxpool.Pool
}

// NewStore builds the Store over the shared pgx pool.
func NewStore(db *pgxpool.Pool) *Store { return &Store{db: db} }

// UpsertDocEmbedding inserts or replaces a document's embedding row.
func (s *Store) UpsertDocEmbedding(ctx context.Context, docID string, version int, model string, vec []float32) error {
    const q = `
INSERT INTO document_embeddings (document_id, version, model, embedding, updated_at)
VALUES ($1, $2, $3, $4::vector, now())
ON CONFLICT (document_id) DO UPDATE
SET version = EXCLUDED.version, model = EXCLUDED.model, embedding = EXCLUDED.embedding, updated_at = now()`
    if _, err := s.db.Exec(ctx, q, docID, version, model, formatVector(vec)); err != nil {
        return fmt.Errorf("upsert doc embedding: %w", err)
    }
    return nil
}

// DocIDsNeedingEmbedding returns live documents whose embedding is missing or stale.
func (s *Store) DocIDsNeedingEmbedding(ctx context.Context, model string, limit int) ([]string, error) {
    const q = `
SELECT d.id::text
FROM documents d
LEFT JOIN document_embeddings e ON e.document_id = d.id
WHERE d.deleted_at IS NULL
  AND d.current_version > 0
  AND (e.document_id IS NULL OR e.model <> $1 OR e.version <> d.current_version)
ORDER BY d.created_at DESC
LIMIT $2`
    rows, err := s.db.Query(ctx, q, model, limit)
    if err != nil {
        return nil, fmt.Errorf("list docs needing embedding: %w", err)
    }
    defer rows.Close()
    var ids []string
    for rows.Next() {
        var id string
        if err := rows.Scan(&id); err != nil {
            return nil, err
        }
        ids = append(ids, id)
    }
    return ids, rows.Err()
}

var _ app.Store = (*Store)(nil)

package app

import (
    "context"
    "errors"
    "log/slog"
    "sync"
    "time"

    aiapp "github.com/Virtue-Digital-Indonesia/obscura/internal/ai/app"
    "github.com/Virtue-Digital-Indonesia/obscura/internal/kernel"
)

// embedCharLimit bounds the text sent to the embedder per document. multilingual-e5-small has
// a ~512-token window (the sidecar truncates too); this caps the request payload.
const embedCharLimit = 2000

// Service owns the embedding pipeline + folder suggestion. enabled fails closed: until the
// composition root wires it to the live license, it reports false (no embeddings written).
type Service struct {
    store    Store
    embedder aiapp.Embedder
    docs     DocSource
    model    string
    dim      int
    logger   *slog.Logger

    mu      sync.RWMutex
    enabled func() bool
}

// NewService builds the semantic Service. The enabled predicate defaults to "always false"
// (fail-closed) until SetEnabled is called with the live license check.
func NewService(store Store, embedder aiapp.Embedder, docs DocSource, model string, dim int, logger *slog.Logger) *Service {
    if logger == nil {
        logger = slog.Default()
    }
    return &Service{
        store:    store,
        embedder: embedder,
        docs:     docs,
        model:    model,
        dim:      dim,
        logger:   logger,
        enabled:  func() bool { return false },
    }
}

// SetEnabled installs the live "is the semantic module licensed" predicate (read from the
// hot-swappable license). Called once by the composition root after the HTTP server exists.
func (s *Service) SetEnabled(fn func() bool) {
    s.mu.Lock()
    defer s.mu.Unlock()
    if fn != nil {
        s.enabled = fn
    }
}

// Enabled reports whether the semantic module is currently licensed.
func (s *Service) Enabled() bool {
    s.mu.RLock()
    defer s.mu.RUnlock()
    return s.enabled()
}

// OnContentText is the embed-on-upload hook the dms Service calls (best-effort) after a
// document's content_text lands. It detaches from the request and embeds asynchronously so it
// never blocks the upload; a no-op when the module is unlicensed.
func (s *Service) OnContentText(docID string) {
    if !s.Enabled() {
        return
    }
    go func() {
        ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
        defer cancel()
        if err := s.EmbedDocument(ctx, docID); err != nil {
            s.logger.Warn("semantic: embed-on-upload failed", "doc", docID, "err", err)
        }
    }()
}

// EmbedDocument embeds one document's (title + extracted text) and upserts the vector. Returns
// nil (skips) when the document has no embeddable text yet.
func (s *Service) EmbedDocument(ctx context.Context, docID string) error {
    title, text, version, err := s.docs.DocumentEmbedSource(ctx, docID)
    if err != nil {
        if errors.Is(err, kernel.ErrNotFound) {
            return nil
        }
        return err
    }
    body := buildEmbedText(title, text, "")
    if body == "" {
        return nil
    }
    vecs, err := s.embedder.Embed(ctx, aiapp.EmbedPassage, []string{body})
    if err != nil {
        return err
    }
    if len(vecs) != 1 || len(vecs[0]) != s.dim {
        s.logger.Warn("semantic: unexpected embedding shape; skipping", "doc", docID, "want_dim", s.dim)
        return nil
    }
    return s.store.UpsertDocEmbedding(ctx, docID, version, s.model, vecs[0])
}

// RunBackfill embeds up to limit documents whose embedding is missing or stale. Each document
// is best-effort (a per-doc error is logged, not fatal). Returns the number of candidates it
// processed. A no-op (0, nil) when unlicensed.
func (s *Service) RunBackfill(ctx context.Context, limit int) (int, error) {
    if !s.Enabled() {
        return 0, nil
    }
    ids, err := s.store.DocIDsNeedingEmbedding(ctx, s.model, limit)
    if err != nil {
        return 0, err
    }
    for _, id := range ids {
        if err := s.EmbedDocument(ctx, id); err != nil {
            s.logger.Warn("semantic: backfill embed failed", "doc", id, "err", err)
        }
    }
    return len(ids), nil
}

// buildEmbedText joins the signals into the text to embed, clipped to embedCharLimit.
func buildEmbedText(title, text, filename string) string {
    parts := make([]byte, 0, embedCharLimit+64)
    add := func(s string) {
        if s == "" {
            return
        }
        if len(parts) > 0 {
            parts = append(parts, '\n')
        }
        parts = append(parts, s...)
    }
    add(title)
    add(filename)
    add(text)
    if len(parts) > embedCharLimit {
        parts = parts[:embedCharLimit]
    }
    return string(parts)
}

cd /home/efran/remote-development/obscura/go && go build ./... && go vet ./...

git add go/internal/semantic/
git commit -m "feat(semantic): pgvector store + embed pipeline service (embed-on-upload + backfill)"

    // WritableFolderIDs returns the ids of folders any of subjects can write to (folder_acl_read
    // access_mode >= AccessReadWrite), excluding soft-deleted folders. bypass=true returns every
    // live folder id; empty subjects returns nil.
    WritableFolderIDs(ctx context.Context, subjects []kernel.Subject, bypass bool) ([]string, error)

// WritableFolderIDs returns the live folders any of subjects can write to (access_mode >= 2).
// bypass returns every live folder id (delegates to AllFolderIDs); empty subjects returns nil.
func (s *Store) WritableFolderIDs(ctx context.Context, subjects []kernel.Subject, bypass bool) ([]string, error) {
    if bypass {
        return s.AllFolderIDs(ctx)
    }
    if len(subjects) == 0 {
        return nil, nil
    }
    kinds := make([]string, len(subjects))
    ids := make([]string, len(subjects))
    for i, sub := range subjects {
        kinds[i] = string(sub.Kind)
        ids[i] = sub.ID
    }
    const q = `
SELECT DISTINCT r.folder_id::text
FROM folder_acl_read r
JOIN unnest($1::text[], $2::text[]) AS subj(kind, id)
  ON r.subject_kind = subj.kind AND r.subject_id = subj.id
JOIN folders f ON f.id = r.folder_id AND f.deleted_at IS NULL
WHERE r.access_mode >= 2`
    rows, err := s.db.Query(ctx, q, kinds, ids)
    if err != nil {
        return nil, fmt.Errorf("writable folder ids: %w", err)
    }
    defer rows.Close()
    var out []string
    for rows.Next() {
        var id string
        if err := rows.Scan(&id); err != nil {
            return nil, err
        }
        out = append(out, id)
    }
    return out, rows.Err()
}

// WritableFolderIDs returns the ids of folders the subjects can write to (Contributor+), or
// all live folders when bypass is true. Used by the semantic suggest flow to bound candidate
// folders to where the caller may actually file.
func (s *Service) WritableFolderIDs(ctx context.Context, subjects []kernel.Subject, bypass bool) ([]string, error) {
    return s.repo.WritableFolderIDs(ctx, subjects, bypass)
}

// DocumentEmbedSource returns the inputs the semantic embedding pipeline needs for a document
// — its title, extracted content_text, and current version — without exposing the dms domain
// type. kernel.ErrNotFound if the document is absent.
func (s *Service) DocumentEmbedSource(ctx context.Context, id string) (title, text string, version int, err error) {
    doc, err := s.repo.GetDocument(ctx, id)
    if err != nil {
        return "", "", 0, err
    }
    text, err = s.repo.GetContentText(ctx, id)
    if err != nil {
        return "", "", 0, err
    }
    return doc.Title, text, doc.CurrentVersion, nil
}

    // embedHook, when set, is invoked (best-effort, non-fatal) after content_text is
    // (re)written, so an out-of-context indexer (the semantic embedding pipeline) can react to
    // freshly-extracted text. nil = no hook (core build / unlicensed). The hook owns its own
    // goroutine + context.
    embedHook func(docID string)

// SetEmbedHook installs the post-content-text hook (the semantic embedding trigger). Idempotent;
// nil clears it.
func (s *Service) SetEmbedHook(fn func(docID string)) { s.embedHook = fn }

    return s.repo.SetContentText(ctx, docID, text)

    if err := s.repo.SetContentText(ctx, docID, text); err != nil {
        return err
    }
    if s.embedHook != nil {
        s.embedHook(docID)
    }
    return nil

cd /home/efran/remote-development/obscura/go && go build ./... && go vet ./...

git add go/internal/dms/app/service.go go/internal/dms/app/acl.go go/internal/dms/adapters/acl_pg.go
git commit -m "feat(dms): WritableFolderIDs, DocumentEmbedSource, and content-text embed hook"

// embedBackfillPageSize bounds how many documents one backfill sweep embeds; logged when hit so
// older un-embedded documents are never silently starved.
const embedBackfillPageSize = 200

// runEmbedBackfill embeds documents whose semantic embedding is missing or stale (the safety net
// behind the best-effort embed-on-upload hook, and the catch-up when `semantic` is first
// licensed). A no-op when the module is unlicensed. Idempotent; safe at every boot.
func runEmbedBackfill(ctx context.Context, semantic *semanticapp.Service, logger *slog.Logger) error {
    n, err := semantic.RunBackfill(ctx, embedBackfillPageSize)
    if err != nil {
        return err
    }
    if n == embedBackfillPageSize {
        logger.Warn("semantic embed backfill hit its page cap; older documents may be deferred this run", "cap", embedBackfillPageSize)
    }
    return nil
}

    semanticapp "github.com/Virtue-Digital-Indonesia/obscura/internal/semantic/app"

    embedder := aiadapters.SelectEmbedder(cfg.Embed.Provider, cfg.Embed.SidecarURL, cfg.Embed.OpenAIBaseURL, cfg.Embed.OpenAIKey, cfg.Embed.Model, cfg.Embed.Dim)
    logger.Info("embed provider", "provider", embedder.Info().Provider, "model", embedder.Info().Model, "dim", embedder.Info().Dim)
    semanticSvc := semanticapp.NewService(
        semanticadapters.NewStore(database),
        embedder,
        dmsSvc, // satisfies semanticapp.DocSource (DocumentEmbedSource + WritableFolderIDs)
        cfg.Embed.Model,
        cfg.Embed.Dim,
        logger,
    )

    semanticadapters "github.com/Virtue-Digital-Indonesia/obscura/internal/semantic/adapters"
    semanticapp "github.com/Virtue-Digital-Indonesia/obscura/internal/semantic/app"

        Semantic:       semanticSvc,

    // The semantic pipeline reads the LIVE (hot-swappable) license so a license up/downgrade
    // takes effect without a restart, mirroring esign's ModuleEnabled use. Fail-closed until set.
    semanticSvc.SetEnabled(func() bool { return api.ModuleEnabled("semantic") })
    // Embed each document when its extracted text lands (best-effort/async; no-op when unlicensed).
    dmsSvc.SetEmbedHook(semanticSvc.OnContentText)

    schedulerSvc.Register("semantic.embed_backfill", 1*time.Hour, func(ctx context.Context) error {
        return runEmbedBackfill(ctx, semanticSvc, logger)
    })
    if err := schedulerSvc.EnsureRegistered(ctx); err != nil {
        return fmt.Errorf("scheduler register (semantic): %w", err)
    }
    // Catch up immediately on a fresh deploy / first-time licensing (no-op when unlicensed).
    if err := schedulerSvc.TriggerNow(ctx, "semantic.embed_backfill"); err != nil {
        logger.Warn("could not trigger initial embed backfill", "err", err)
    }

cd /home/efran/remote-development/obscura/go && go build ./... && go vet ./...

git add go/cmd/obscura-server/wire.go go/cmd/obscura-server/jobs.go go/internal/httpapi/server.go
git commit -m "feat(semantic): wire embedder + service, license-gated embed hook + backfill sweep"

cd /home/efran/remote-development/obscura
docker compose -f deploy/docker-compose.yml --env-file deploy/mekari.env up -d --build

TOKEN=$(curl -s -XPOST localhost:38080/api/v1/auth/dev-login -H 'content-type: application/json' -d '{"email":"admin@obscura.local"}' | python3 -c 'import sys,json;print(json.load(sys.stdin)["token"])')
# create a document (root folder)
DOC=$(curl -s -XPOST localhost:38080/api/v1/documents -H "authorization: Bearer $TOKEN" -H 'content-type: application/json' -d '{"title":"PLAN-EMBED-TEST kontrak vendor"}' | python3 -c 'import sys,json;print(json.load(sys.stdin).get("id",""))')
echo "doc=$DOC"
# land extracted text (fires the embed hook)
curl -s -XPUT "localhost:38080/api/v1/documents/$DOC/content-text" -H "authorization: Bearer $TOKEN" -H 'content-type: application/json' -d '{"text":"Perjanjian kerja sama pengadaan barang dan jasa antara perusahaan dan vendor pihak ketiga."}' -o /dev/null -w "%{http_code}\n"
sleep 4
docker compose -f deploy/docker-compose.yml exec -T postgres psql -U obscura -d obscura -c "SELECT document_id, version, model, vector_dims(embedding) FROM document_embeddings WHERE document_id='$DOC';"

docker compose -f deploy/docker-compose.yml exec -T postgres psql -U obscura -d obscura -c "SELECT (SELECT count(*) FROM document_embeddings) AS embedded, (SELECT count(*) FROM documents WHERE deleted_at IS NULL AND current_version>0 AND content_text<>'') AS have_text;"

EMBED_PROVIDER=mock docker compose -f deploy/docker-compose.yml --env-file deploy/mekari.env up -d --build embed-sidecar obscura >/dev/null 2>&1 || true
# Temporarily install a core-only license to prove the gate (regenerate WITHOUT semantic),
# create a doc + set content-text, and assert NO new embedding row appears, then restore the
# 5-module demo license and redeploy.

// FolderCentroid is a folder's id, materialized path, and the centroid (mean) of its member
// documents' embeddings.
type FolderCentroid struct {
    FolderID string
    Path     string
    Centroid []float32
}

    // FolderCentroids returns avg(embedding) per folder over live member documents, restricted
    // to folderIDs. Folders in folderIDs with no embedded members are omitted.
    FolderCentroids(ctx context.Context, folderIDs []string) ([]FolderCentroid, error)

// FolderCentroids returns the mean embedding per folder over its live member documents,
// restricted to folderIDs. Computed on the fly (v1); materializing is the documented scale path.
func (s *Store) FolderCentroids(ctx context.Context, folderIDs []string) ([]app.FolderCentroid, error) {
    if len(folderIDs) == 0 {
        return nil, nil
    }
    const q = `
SELECT f.id::text, f.path, avg(e.embedding)::text
FROM document_embeddings e
JOIN documents d ON d.id = e.document_id AND d.deleted_at IS NULL
JOIN folders f ON f.id = d.folder_id AND f.deleted_at IS NULL
WHERE d.folder_id::text = ANY($1::text[])
GROUP BY f.id, f.path`
    rows, err := s.db.Query(ctx, q, folderIDs)
    if err != nil {
        return nil, fmt.Errorf("folder centroids: %w", err)
    }
    defer rows.Close()
    var out []app.FolderCentroid
    for rows.Next() {
        var id, path, vecText string
        if err := rows.Scan(&id, &path, &vecText); err != nil {
            return nil, err
        }
        vec, perr := parseVector(vecText)
        if perr != nil {
            return nil, perr
        }
        out = append(out, app.FolderCentroid{FolderID: id, Path: path, Centroid: vec})
    }
    return out, rows.Err()
}

// SuggestQuery is the input to a folder suggestion: any of title/text/filename (at least one).
type SuggestQuery struct {
    Title    string
    Text     string
    Filename string
}

// Suggestion is one ranked folder. Score is cosine similarity in [-1,1] (clamped to >= 0 for
// display); higher is better.
type Suggestion struct {
    FolderID string
    Path     string
    Score    float64
}

// SuggestFolders ranks the caller's write-accessible folders by cosine similarity of the query
// to each folder's member-document centroid, returning the top k (default 3). An empty index
// or no writable folders yields an empty slice (the UI then shows nothing). The route gates on
// requireModule("semantic"); this method assumes the caller is already authorized.
func (s *Service) SuggestFolders(ctx context.Context, subjects []kernel.Subject, bypass bool, q SuggestQuery, k int) ([]Suggestion, error) {
    if k <= 0 {
        k = 3
    }
    folderIDs, err := s.docs.WritableFolderIDs(ctx, subjects, bypass)
    if err != nil {
        return nil, err
    }
    if len(folderIDs) == 0 {
        return nil, nil
    }
    body := buildEmbedText(q.Title, q.Text, q.Filename)
    if body == "" {
        return nil, nil
    }
    vecs, err := s.embedder.Embed(ctx, aiapp.EmbedQuery, []string{body})
    if err != nil {
        return nil, err
    }
    if len(vecs) != 1 || len(vecs[0]) != s.dim {
        return nil, nil
    }
    query := vecs[0]
    centroids, err := s.store.FolderCentroids(ctx, folderIDs)
    if err != nil {
        return nil, err
    }
    out := make([]Suggestion, 0, len(centroids))
    for _, c := range centroids {
        score := cosine(query, c.Centroid)
        if score < 0 {
            score = 0
        }
        out = append(out, Suggestion{FolderID: c.FolderID, Path: c.Path, Score: score})
    }
    sort.SliceStable(out, func(i, j int) bool { return out[i].Score > out[j].Score })
    if len(out) > k {
        out = out[:k]
    }
    return out, nil
}

// cosine returns the cosine similarity of two equal-length vectors (0 if degenerate).
func cosine(a, b []float32) float64 {
    if len(a) != len(b) || len(a) == 0 {
        return 0
    }
    var dot, na, nb float64
    for i := range a {
        dot += float64(a[i]) * float64(b[i])
        na += float64(a[i]) * float64(a[i])
        nb += float64(b[i]) * float64(b[i])
    }
    if na == 0 || nb == 0 {
        return 0
    }
    return dot / (math.Sqrt(na) * math.Sqrt(nb))
}