TOKEN=$(curl -s -XPOST localhost:38080/api/v1/auth/dev-login -H 'content-type: application/json' -d '{"email":"admin@obscura.local"}' | python3 -c 'import sys,json;print(json.load(sys.stdin)["token"])')
curl -s localhost:38080/api/v1/me -H "authorization: Bearer $TOKEN" | python3 -c 'import sys,json;print(sorted(json.load(sys.stdin)["enabled_modules"]))'

fastapi>=0.110,<1.0
uvicorn[standard]>=0.29,<1.0
python-multipart>=0.0.9
pypdfium2>=4,<5
python-docx>=1.1,<2
openpyxl>=3.1,<4
python-pptx>=0.6,<2

import io
import re

from fastapi import FastAPI, File, Form, UploadFile

app = FastAPI()


def extract_pdf(data: bytes) -> str:
    import pypdfium2 as pdfium

    pdf = pdfium.PdfDocument(data)
    parts = []
    try:
        for i in range(len(pdf)):
            page = pdf[i]
            tp = page.get_textpage()
            parts.append(tp.get_text_range())
            tp.close()
            page.close()
    finally:
        pdf.close()
    return "\n".join(parts)


def extract_docx(data: bytes) -> str:
    import docx

    d = docx.Document(io.BytesIO(data))
    return "\n".join(p.text for p in d.paragraphs)


def extract_xlsx(data: bytes) -> str:
    import openpyxl

    wb = openpyxl.load_workbook(io.BytesIO(data), read_only=True, data_only=True)
    out = []
    try:
        for ws in wb.worksheets:
            for row in ws.iter_rows(values_only=True):
                cells = [str(c) for c in row if c is not None]
                if cells:
                    out.append(" ".join(cells))
    finally:
        wb.close()
    return "\n".join(out)


def extract_pptx(data: bytes) -> str:
    from pptx import Presentation

    prs = Presentation(io.BytesIO(data))
    out = []
    for slide in prs.slides:
        for shape in slide.shapes:
            if shape.has_text_frame:
                out.append(shape.text_frame.text)
    return "\n".join(out)


def strip_html(data: bytes) -> str:
    text = data.decode("utf-8", errors="replace")
    text = re.sub(r"<(script|style)[^>]*>.*?</\1>", " ", text, flags=re.S | re.I)
    text = re.sub(r"<[^>]+>", " ", text)
    return re.sub(r"\s+", " ", text).strip()


def dispatch(data: bytes, mime: str, filename: str) -> str:
    m = (mime or "").lower()
    name = (filename or "").lower()
    try:
        if "pdf" in m or name.endswith(".pdf"):
            return extract_pdf(data)
        if "wordprocessingml" in m or name.endswith(".docx"):
            return extract_docx(data)
        if "spreadsheetml" in m or name.endswith(".xlsx"):
            return extract_xlsx(data)
        if "presentationml" in m or name.endswith(".pptx"):
            return extract_pptx(data)
        if "html" in m or name.endswith((".html", ".htm")):
            return strip_html(data)
        if m.startswith("text/") or name.endswith((".txt", ".csv", ".md", ".markdown")):
            return data.decode("utf-8", errors="replace")
    except Exception:
        # Best-effort: a parse failure yields empty text, never an error.
        return ""
    return ""


@app.get("/healthz")
def healthz():
    return {"status": "ok", "formats": ["pdf", "docx", "xlsx", "pptx", "text", "html"]}


@app.post("/extract")
async def extract(file: UploadFile = File(...), mime: str = Form("")):
    data = await file.read()
    return {"text": dispatch(data, mime, file.filename or "")}

# syntax=docker/dockerfile:1
# Built with build context = this directory (deploy/extract-sidecar), NOT the repo root, because
# the repo-root .dockerignore excludes deploy/. Compose sets `context: extract-sidecar`.
FROM python:3.12-slim
WORKDIR /app
COPY requirements.txt ./
RUN pip install --no-cache-dir -r requirements.txt
COPY . ./
EXPOSE 8000
CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000"]

cd /home/efran/remote-development/obscura/deploy/extract-sidecar
docker build -t obscura-extract-sidecar:plancheck .
docker run --rm -p 38001:8000 --name extract-pc -d obscura-extract-sidecar:plancheck
sleep 4
curl -s localhost:38001/healthz
printf 'Hello extraction world. Kejaksaan Ceger.' > /tmp/t.txt
curl -s -XPOST localhost:38001/extract -F "file=@/tmp/t.txt;type=text/plain" -F "mime=text/plain" ; echo
docker stop extract-pc; rm -f /tmp/t.txt

cd /home/efran/remote-development/obscura
git add deploy/extract-sidecar/
git commit -m "feat(extract-sidecar): FastAPI text extractor (PDF/office/text, air-gapped)"

  # CPU-only text-extraction sidecar (FastAPI: pypdfium2 PDF + python-docx/openpyxl/python-pptx +
  # text/html). Pure-Python, air-gapped (deps baked in). Reached over the compose network as
  # http://extract-sidecar:8000. Core infra: populates content_text (keyword + semantic). Own build
  # context (repo-root .dockerignore excludes deploy/).
  extract-sidecar:
    build:
      context: extract-sidecar
      dockerfile: Dockerfile
    healthcheck:
      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8000/healthz')"]
      interval: 10s
      timeout: 5s
      retries: 12
    ports: ["38001:8000"]

      # Document text extraction (core infra; populates content_text → keyword + semantic).
      # Defaults to the sidecar so the demo extracts; set EXTRACT_PROVIDER=none to disable.
      EXTRACT_PROVIDER: "${EXTRACT_PROVIDER:-sidecar}"
      EXTRACT_SIDECAR_URL: "http://extract-sidecar:8000"

      extract-sidecar:
        condition: service_started

cd /home/efran/remote-development/obscura
docker compose -f deploy/docker-compose.yml --env-file deploy/mekari.env config >/dev/null && echo COMPOSE_VALID
git add deploy/docker-compose.yml
git commit -m "feat(deploy): run extract-sidecar + EXTRACT_* config in the compose stack"

    Embed        EmbedConfig
    Extract      ExtractConfig

// ExtractConfig selects the document text-extraction backend (core infra; populates content_text,
// which feeds the keyword search_tsv index and the embed pipeline). Provider "none" (the default)
// disables extraction (title-only); "sidecar" calls the local FastAPI extract-sidecar (air-gapped).
// SidecarURL defaults to the host-mapped port for `go run` outside compose; compose overrides it to
// the in-network address.
type ExtractConfig struct {
    Provider   string `env:"EXTRACT_PROVIDER" envDefault:"none"`
    SidecarURL string `env:"EXTRACT_SIDECAR_URL" envDefault:"http://localhost:38001"`
}

    switch c.Extract.Provider {
    case "none", "sidecar":
    default:
        return fmt.Errorf("config: invalid EXTRACT_PROVIDER %q (want none|sidecar)", c.Extract.Provider)
    }

cd /home/efran/remote-development/obscura/go && go build ./... && go vet ./...
cd /home/efran/remote-development/obscura && git add go/internal/platform/config/config.go
git commit -m "feat(config): add ExtractConfig (EXTRACT_PROVIDER/URL)"

// Package app is the extraction context: it turns a document version's bytes into content_text via
// a swappable Extractor, writing the result through the dms Service (which updates the keyword
// index and re-fires the embed hook). It reaches dms only through the narrow DocSource port, using
// primitive signatures so dms never imports this package.
package app

import "context"

// Extractor turns a file's bytes into plain text (the sidecar client). Never returns partial
// binary; unsupported input yields "".
type Extractor interface {
    Extract(ctx context.Context, data []byte, mimeType, filename string) (string, error)
}

// DocSource is the narrow slice of the dms Service the extraction context needs.
type DocSource interface {
    // VersionContentForExtract returns a version's raw bytes + MIME.
    VersionContentForExtract(ctx context.Context, docID string, version int) (data []byte, mimeType string, err error)
    // SetContentText writes the extracted text (bounded; fires the embed hook).
    SetContentText(ctx context.Context, docID, text string) error
    // DocsNeedingExtraction returns up to limit documents (parallel doc-id + current-version
    // slices) that have a file version but no content_text yet — the backfill working set.
    DocsNeedingExtraction(ctx context.Context, limit int) (docIDs []string, versions []int, err error)
}

package app

import (
    "context"
    "log/slog"
    "strings"
    "time"
)

// maxContentTextBytes mirrors the dms bound (the tsvector ~1MB hard limit); the extracted text is
// clipped below it before write, on a valid-UTF-8 boundary.
const maxContentTextBytes = 800 * 1024

// Service runs the extraction pipeline. A nil extractor (provider=none) makes every entry point a
// no-op, so extraction disables cleanly.
type Service struct {
    extractor Extractor
    docs      DocSource
    logger    *slog.Logger
}

// NewService builds the extraction Service. A nil extractor disables extraction.
func NewService(extractor Extractor, docs DocSource, logger *slog.Logger) *Service {
    if logger == nil {
        logger = slog.Default()
    }
    return &Service{extractor: extractor, docs: docs, logger: logger}
}

// OnVersionAdded is the AddVersion hook: extract the new version's text and store it (async,
// detached, best-effort, non-fatal). A no-op when extraction is disabled.
func (s *Service) OnVersionAdded(docID string, version int) {
    if s.extractor == nil {
        return
    }
    go func() {
        ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
        defer cancel()
        if err := s.extractOne(ctx, docID, version); err != nil {
            s.logger.Warn("extract: on-version-added failed", "doc", docID, "version", version, "err", err)
        }
    }()
}

// extractOne fetches a version's bytes, extracts the text, clips it, and writes content_text.
func (s *Service) extractOne(ctx context.Context, docID string, version int) error {
    data, mimeType, err := s.docs.VersionContentForExtract(ctx, docID, version)
    if err != nil {
        return err
    }
    text, err := s.extractor.Extract(ctx, data, mimeType, "")
    if err != nil {
        return err
    }
    return s.docs.SetContentText(ctx, docID, clipText(text))
}

// RunBackfill extracts up to limit documents that have a file version but no content_text yet.
// Best-effort per document; returns the count processed. A no-op when disabled.
func (s *Service) RunBackfill(ctx context.Context, limit int) (int, error) {
    if s.extractor == nil {
        return 0, nil
    }
    docIDs, versions, err := s.docs.DocsNeedingExtraction(ctx, limit)
    if err != nil {
        return 0, err
    }
    for i, id := range docIDs {
        if err := s.extractOne(ctx, id, versions[i]); err != nil {
            s.logger.Warn("extract: backfill failed", "doc", id, "err", err)
        }
    }
    return len(docIDs), nil
}

// clipText bounds text to maxContentTextBytes on a valid-UTF-8 boundary (SetContentText rejects
// oversized or invalid input).
func clipText(t string) string {
    if len(t) <= maxContentTextBytes {
        return t
    }
    return strings.ToValidUTF8(t[:maxContentTextBytes], "")
}

// Package adapters holds the extraction Extractor implementations: the HTTP sidecar client and the
// provider factory.
package adapters

import (
    "bytes"
    "context"
    "encoding/json"
    "fmt"
    "io"
    "mime/multipart"
    "net/http"
    "strings"
    "time"

    "github.com/Virtue-Digital-Indonesia/obscura/internal/extract/app"
)

// SidecarExtractor POSTs a file to the local extract-sidecar and returns the extracted text.
type SidecarExtractor struct {
    baseURL string
    http    *http.Client
}

// NewSidecarExtractor constructs the sidecar client.
func NewSidecarExtractor(baseURL string) *SidecarExtractor {
    return &SidecarExtractor{baseURL: strings.TrimRight(baseURL, "/"), http: &http.Client{Timeout: 60 * time.Second}}
}

// Extract sends the bytes + MIME as multipart form data and returns the text.
func (e *SidecarExtractor) Extract(ctx context.Context, data []byte, mimeType, filename string) (string, error) {
    if len(data) == 0 {
        return "", nil
    }
    var body bytes.Buffer
    w := multipart.NewWriter(&body)
    fw, err := w.CreateFormFile("file", nameOr(filename))
    if err != nil {
        return "", err
    }
    if _, err := fw.Write(data); err != nil {
        return "", err
    }
    if err := w.WriteField("mime", mimeType); err != nil {
        return "", err
    }
    if err := w.Close(); err != nil {
        return "", err
    }
    req, err := http.NewRequestWithContext(ctx, http.MethodPost, e.baseURL+"/extract", &body)
    if err != nil {
        return "", err
    }
    req.Header.Set("content-type", w.FormDataContentType())
    resp, err := e.http.Do(req)
    if err != nil {
        return "", fmt.Errorf("extract sidecar request: %w", err)
    }
    defer resp.Body.Close()
    rb, _ := io.ReadAll(resp.Body)
    if resp.StatusCode/100 != 2 {
        return "", fmt.Errorf("extract sidecar %d: %s", resp.StatusCode, truncate(string(rb), 300))
    }
    var out struct {
        Text string `json:"text"`
    }
    if err := json.Unmarshal(rb, &out); err != nil {
        return "", fmt.Errorf("extract sidecar decode: %w", err)
    }
    return out.Text, nil
}

func nameOr(n string) string {
    if n == "" {
        return "upload.bin"
    }
    return n
}

func truncate(s string, n int) string {
    if len(s) <= n {
        return s
    }
    return s[:n]
}

// SelectExtractor returns an Extractor for the config, or nil (extraction disabled) for any
// provider other than "sidecar".
func SelectExtractor(provider, sidecarURL string) app.Extractor {
    if provider == "sidecar" {
        return NewSidecarExtractor(sidecarURL)
    }
    return nil
}

var _ app.Extractor = (*SidecarExtractor)(nil)

cd /home/efran/remote-development/obscura && git add go/internal/extract
git commit -m "feat(extract): extraction context (service + sidecar adapter + ports)"

    // extractHook, when set, fires (best-effort) after a new version's bytes land, so the extraction
    // pipeline can populate content_text. nil = no extraction. The hook owns its own goroutine.
    extractHook func(docID string, version int)

// SetExtractHook installs the post-AddVersion hook (the extraction trigger). Idempotent; nil clears.
func (s *Service) SetExtractHook(fn func(docID string, version int)) { s.extractHook = fn }

    if err != nil {
        return 0, err
    }
    if s.extractHook != nil {
        s.extractHook(docID, newVersion)
    }
    return newVersion, nil

// VersionContentForExtract returns a version's raw bytes + MIME, for the extraction pipeline. It
// reads the whole blob into memory (bounded by the upload size).
func (s *Service) VersionContentForExtract(ctx context.Context, docID string, version int) ([]byte, string, error) {
    v, err := s.repo.GetVersion(ctx, docID, version)
    if err != nil {
        return nil, "", err
    }
    rc, err := s.blobs.Get(ctx, v.ContentHash)
    if err != nil {
        return nil, "", err
    }
    defer rc.Close()
    data, err := io.ReadAll(rc)
    if err != nil {
        return nil, "", err
    }
    return data, v.MIME, nil
}

    // DocsNeedingExtraction returns up to limit live documents that have a file version but no
    // content_text yet (newest first): parallel doc-id + current-version slices.
    DocsNeedingExtraction(ctx context.Context, limit int) (docIDs []string, versions []int, err error)

// DocsNeedingExtraction lists documents (doc id + current version) that have a file version but no
// extracted content_text yet — the extraction backfill working set.
func (s *Service) DocsNeedingExtraction(ctx context.Context, limit int) ([]string, []int, error) {
    return s.repo.DocsNeedingExtraction(ctx, limit)
}

// DocsNeedingExtraction returns live documents with a file version (current_version > 0) but empty
// content_text, newest first, capped at limit.
func (s *Store) DocsNeedingExtraction(ctx context.Context, limit int) ([]string, []int, error) {
    rows, err := s.db.Exec(ctx).Query(ctx,
        `SELECT id::text, current_version FROM documents
           WHERE deleted_at IS NULL AND current_version > 0 AND content_text = ''
           ORDER BY created_at DESC LIMIT $1`, limit)
    if err != nil {
        return nil, nil, fmt.Errorf("docs needing extraction: %w", err)
    }
    defer rows.Close()
    var ids []string
    var versions []int
    for rows.Next() {
        var id string
        var v int
        if err := rows.Scan(&id, &v); err != nil {
            return nil, nil, err
        }
        ids = append(ids, id)
        versions = append(versions, v)
    }
    return ids, versions, rows.Err()
}

cd /home/efran/remote-development/obscura/go && go build ./... && go vet ./...
cd /home/efran/remote-development/obscura && git add go/internal/dms
git commit -m "feat(dms): extract hook + version-bytes reader + docs-needing-extraction query"

// extractBackfillPageSize bounds how many documents one extraction sweep processes; logged when hit.
const extractBackfillPageSize = 200

// runExtractBackfill extracts content_text for documents that have a file version but none yet (the
// catch-up behind the best-effort AddVersion hook). A no-op when extraction is disabled.
func runExtractBackfill(ctx context.Context, extract *extractapp.Service, logger *slog.Logger) error {
    n, err := extract.RunBackfill(ctx, extractBackfillPageSize)
    if err != nil {
        return err
    }
    if n == extractBackfillPageSize {
        logger.Warn("extract backfill hit its page cap; older documents may be deferred this run", "cap", extractBackfillPageSize)
    }
    return nil
}

    extractapp "github.com/Virtue-Digital-Indonesia/obscura/internal/extract/app"

    extractSvc := extractapp.NewService(
        extractadapters.SelectExtractor(cfg.Extract.Provider, cfg.Extract.SidecarURL),
        dmsSvc, // satisfies extractapp.DocSource
        logger,
    )
    logger.Info("extract provider", "provider", cfg.Extract.Provider)

    extractadapters "github.com/Virtue-Digital-Indonesia/obscura/internal/extract/adapters"
    extractapp "github.com/Virtue-Digital-Indonesia/obscura/internal/extract/app"

    dmsSvc.SetExtractHook(extractSvc.OnVersionAdded)

    schedulerSvc.Register("dms.extract_backfill", 1*time.Hour, func(ctx context.Context) error {
        return runExtractBackfill(ctx, extractSvc, logger)
    })

    if err := schedulerSvc.TriggerNow(ctx, "dms.extract_backfill"); err != nil {
        logger.Warn("could not trigger initial extract backfill", "err", err)
    }

cd /home/efran/remote-development/obscura/go && go build ./... && go vet ./...
cd /home/efran/remote-development/obscura && git add go/cmd/obscura-server/wire.go go/cmd/obscura-server/jobs.go
git commit -m "feat(extract): wire extractor + AddVersion hook + extract backfill sweep"

cd /home/efran/remote-development/obscura
docker compose -f deploy/docker-compose.yml --env-file deploy/mekari.env up -d --build
docker compose -f deploy/docker-compose.yml ps extract-sidecar
curl -s localhost:38001/healthz

TOKEN=$(curl -s -XPOST localhost:38080/api/v1/auth/dev-login -H 'content-type: application/json' -d '{"email":"admin@obscura.local"}' | python3 -c 'import sys,json;print(json.load(sys.stdin)["token"])')
DOC=$(curl -s -XPOST localhost:38080/api/v1/documents -H "authorization: Bearer $TOKEN" -H 'content-type: application/json' -d '{"title":"PLAN-EXTRACT-TEST","classification":"none"}' | python3 -c 'import sys,json;print(json.load(sys.stdin).get("id",""))')
printf 'Perjanjian kerja sama pengadaan dengan Kejaksaan Ceger untuk layanan infrastruktur terkelola.' > /tmp/body.txt
curl -s -XPOST "localhost:38080/api/v1/documents/$DOC/versions" -H "authorization: Bearer $TOKEN" -F "file=@/tmp/body.txt;type=text/plain" -o /dev/null -w "upload: %{http_code}\n"
sleep 5
docker compose -f deploy/docker-compose.yml exec -T postgres psql -U obscura -d obscura -c "SELECT left(content_text,80) AS content, (SELECT vector_dims(embedding) FROM document_embeddings WHERE document_id='$DOC') AS emb_dims FROM documents WHERE id='$DOC';"
# Keyword search now matches a BODY word (not in the title):
curl -s -XPOST localhost:38080/api/v1/documents/search -H "authorization: Bearer $TOKEN" -H 'content-type: application/json' -d '{"Text":"pengadaan infrastruktur"}' | python3 -c 'import sys,json;d=json.load(sys.stdin);print("keyword hits:",[x["Title"] for x in (d.get("documents") or []) if x.get("Title","").startswith("PLAN-EXTRACT")])'
echo "CLEANUP_DOC=$DOC"; rm -f /tmp/body.txt

docker compose -f deploy/docker-compose.yml exec -T postgres psql -U obscura -d obscura -c "SELECT count(*) FILTER (WHERE content_text <> '') AS extracted, count(*) FILTER (WHERE content_text = '' AND current_version > 0) AS pending FROM documents WHERE deleted_at IS NULL AND current_version > 0;"