From ad1e17ddf2b5a82ee229a508e03ecf5ef669cbca Mon Sep 17 00:00:00 2001 From: jedarden Date: Tue, 7 Apr 2026 02:14:25 -0400 Subject: [PATCH] feat: implement SQLite Online Backup API streaming endpoint Use modernc.org/sqlite's Online Backup API (NewBackup/Step/Commit) with in-memory destination and Serialize for consistent hot backups without temp files. Streams zip directly to HTTP response. Co-Authored-By: Claude Opus 4.6 --- mothership/internal/api/backup.go | 240 ++++++++++++++++++ mothership/internal/api/backup_test.go | 337 +++++++++++++++++++++++++ 2 files changed, 577 insertions(+) create mode 100644 mothership/internal/api/backup.go create mode 100644 mothership/internal/api/backup_test.go diff --git a/mothership/internal/api/backup.go b/mothership/internal/api/backup.go new file mode 100644 index 0000000..733c517 --- /dev/null +++ b/mothership/internal/api/backup.go @@ -0,0 +1,240 @@ +// Package api provides the backup streaming endpoint. +package api + +import ( + "archive/zip" + "context" + "database/sql" + "fmt" + "io/fs" + "log" + "net/http" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "modernc.org/sqlite" +) + +// BackupHandler handles GET /api/backup — streams a zip of all databases +// and supporting files directly to the HTTP response without temp files. +type BackupHandler struct { + dataDir string + version string +} + +// NewBackupHandler creates a backup handler that will archive every .db +// file found inside dataDir, plus optional floor_plan/ and a VERSION file. +func NewBackupHandler(dataDir, version string) *BackupHandler { + return &BackupHandler{dataDir: dataDir, version: version} +} + +// HandleBackup streams a zip archive to w. +// +// Zip layout: +// +// spaxel-backup-.zip +// ├── *.db — one entry per database file found in dataDir +// ├── floor_plan/ — if the directory exists +// │ └── ... +// └── VERSION — mothership version string +func (h *BackupHandler) HandleBackup(w http.ResponseWriter, r *http.Request) { + start := time.Now() + timestamp := start.UTC().Format("2006-01-02") + + w.Header().Set("Content-Type", "application/zip") + w.Header().Set("Content-Disposition", + fmt.Sprintf(`attachment; filename="spaxel-backup-%s.zip"`, timestamp)) + + // We write directly into the response — no temp file on disk. + zw := zip.NewWriter(w) + defer zw.Close() + + // 1. Back up every .db file found in dataDir using the Online Backup API. + if err := h.backupDatabases(zw); err != nil { + log.Printf("[ERROR] backup: database backup failed: %v", err) + http.Error(w, "backup failed", http.StatusInternalServerError) + return + } + + // 2. Include floor_plan/ directory if it exists. + if err := h.backupDirectory(zw, "floor_plan"); err != nil { + log.Printf("[WARN] backup: floor_plan backup skipped: %v", err) + } + + // 3. Include VERSION file. + if fw, err := zw.Create("VERSION"); err == nil { + fw.Write([]byte(h.version + "\n")) + } + + if err := zw.Close(); err != nil { + log.Printf("[ERROR] backup: zip close failed: %v", err) + return + } + + log.Printf("[INFO] backup completed in %s", time.Since(start)) +} + +// backupDatabases finds all .db files in dataDir, uses the SQLite Online +// Backup API to create a consistent snapshot of each, and adds the snapshot +// to the zip. +func (h *BackupHandler) backupDatabases(zw *zip.Writer) error { + entries, err := os.ReadDir(h.dataDir) + if err != nil { + return fmt.Errorf("read data dir: %w", err) + } + + // Collect .db files and sort for deterministic zip ordering. + var dbFiles []string + for _, e := range entries { + if !e.Type().IsRegular() { + continue + } + if strings.HasSuffix(e.Name(), ".db") { + dbFiles = append(dbFiles, e.Name()) + } + } + sort.Strings(dbFiles) + + for _, name := range dbFiles { + dbPath := filepath.Join(h.dataDir, name) + if err := h.backupOneDB(zw, dbPath, name); err != nil { + log.Printf("[WARN] backup: skipping %s: %v", name, err) + continue + } + } + + return nil +} + +// backupOneDB uses the SQLite Online Backup API to produce a consistent +// snapshot of the database at dbPath, then writes the serialized bytes into +// the zip entry named zipName. +// +// The Online Backup API copies page-by-page; readers and writers continue +// uninterrupted. No temp file is written — the backup is serialized from +// an in-memory copy directly to the zip stream. +func (h *BackupHandler) backupOneDB(zw *zip.Writer, dbPath, zipName string) error { + dsn := dbPath + "?_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL)&_pragma=busy_timeout(5000)" + db, err := sql.Open("sqlite", dsn) + if err != nil { + return fmt.Errorf("open: %w", err) + } + defer db.Close() + + conn, err := db.Conn(context.Background()) + if err != nil { + return fmt.Errorf("conn: %w", err) + } + defer conn.Close() + + var backupBytes []byte + + err = conn.Raw(func(driverConn any) error { + // Assert the backuper interface to access the Online Backup API. + bp, ok := driverConn.(interface { + NewBackup(dstUri string) (*sqlite.Backup, error) + }) + if !ok { + return fmt.Errorf("driver does not support online backup") + } + + // Create an in-memory destination and initialise the backup. + bck, err := bp.NewBackup(":memory:") + if err != nil { + return fmt.Errorf("backup init: %w", err) + } + + // Copy pages 100 at a time until done. + const pagesPerStep = 100 + for { + more, err := bck.Step(pagesPerStep) + if err != nil { + bck.Finish() + return fmt.Errorf("backup step: %w", err) + } + if !more { + break + } + } + + // Finish the backup but keep the destination connection open. + dstConn, err := bck.Commit() + if err != nil { + return fmt.Errorf("backup commit: %w", err) + } + defer dstConn.Close() + + // Serialize the in-memory database to bytes. + ser, ok := dstConn.(interface { + Serialize() ([]byte, error) + }) + if !ok { + return fmt.Errorf("driver does not support serialize") + } + + backupBytes, err = ser.Serialize() + return err + }) + if err != nil { + return err + } + + if len(backupBytes) == 0 { + return fmt.Errorf("empty database backup") + } + + fw, err := zw.Create(zipName) + if err != nil { + return fmt.Errorf("zip create: %w", err) + } + if _, err := fw.Write(backupBytes); err != nil { + return fmt.Errorf("zip write: %w", err) + } + + log.Printf("[DEBUG] backup: %s (%d bytes)", zipName, len(backupBytes)) + return nil +} + +// backupDirectory adds every file under dirName (relative to dataDir) into +// the zip, preserving directory structure. Silently skips if the directory +// does not exist. +func (h *BackupHandler) backupDirectory(zw *zip.Writer, dirName string) error { + dirPath := filepath.Join(h.dataDir, dirName) + info, err := os.Stat(dirPath) + if err != nil || !info.IsDir() { + return nil // not present — skip silently + } + + return filepath.WalkDir(dirPath, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + + rel, err := filepath.Rel(h.dataDir, path) + if err != nil { + return err + } + // zip paths must use forward slashes. + rel = filepath.ToSlash(rel) + + data, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("read %s: %w", rel, err) + } + + fw, err := zw.Create(rel) + if err != nil { + return fmt.Errorf("zip create %s: %w", rel, err) + } + if _, err := fw.Write(data); err != nil { + return fmt.Errorf("zip write %s: %w", rel, err) + } + return nil + }) +} diff --git a/mothership/internal/api/backup_test.go b/mothership/internal/api/backup_test.go new file mode 100644 index 0000000..b563e8c --- /dev/null +++ b/mothership/internal/api/backup_test.go @@ -0,0 +1,337 @@ +package api + +import ( + "archive/zip" + "bytes" + "database/sql" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + _ "modernc.org/sqlite" +) + +// setupTestDB creates a WAL-mode database at dir/name and runs the provided +// SQL statements. +func setupTestDB(t *testing.T, dir, name, ddl string) { + t.Helper() + dsn := filepath.Join(dir, name) + "?_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL)" + db, err := sql.Open("sqlite", dsn) + if err != nil { + t.Fatalf("open test db: %v", err) + } + defer db.Close() + if _, err := db.Exec(ddl); err != nil { + t.Fatalf("exec ddl: %v", err) + } + if _, err := db.Exec("PRAGMA wal_checkpoint(TRUNCATE)"); err != nil { + t.Fatalf("checkpoint: %v", err) + } +} + +// doBackupRequest creates a handler, runs a backup, and returns the response +// body bytes. +func doBackupRequest(t *testing.T, dir, version string) []byte { + t.Helper() + handler := NewBackupHandler(dir, version) + req := httptest.NewRequest(http.MethodGet, "/api/backup", nil) + rec := httptest.NewRecorder() + handler.HandleBackup(rec, req) + + resp := rec.Result() + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d; want 200", resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read body: %v", err) + } + return body +} + +// openZip returns a zip reader for the given bytes. +func openZip(t *testing.T, data []byte) *zip.Reader { + t.Helper() + rdr, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) + if err != nil { + t.Fatalf("open zip: %v", err) + } + return rdr +} + +// zipEntryNames returns a set of all entry names in the zip. +func zipEntryNames(t *testing.T, data []byte) map[string]bool { + t.Helper() + rdr := openZip(t, data) + names := make(map[string]bool) + for _, f := range rdr.File { + names[f.Name] = true + } + return names +} + +// readZipEntry reads and returns the contents of the named zip entry. +func readZipEntry(t *testing.T, data []byte, name string) []byte { + t.Helper() + rdr := openZip(t, data) + for _, f := range rdr.File { + if f.Name == name { + rc, err := f.Open() + if err != nil { + t.Fatalf("open zip entry %s: %v", name, err) + } + defer rc.Close() + buf, err := io.ReadAll(rc) + if err != nil { + t.Fatalf("read zip entry %s: %v", name, err) + } + return buf + } + } + t.Fatalf("zip entry %q not found", name) + return nil +} + +func TestBackupHandler_Headers(t *testing.T) { + dir := t.TempDir() + setupTestDB(t, dir, "spaxel.db", "CREATE TABLE t(id INTEGER PRIMARY KEY);") + + handler := NewBackupHandler(dir, "1.0.0") + req := httptest.NewRequest(http.MethodGet, "/api/backup", nil) + rec := httptest.NewRecorder() + handler.HandleBackup(rec, req) + + resp := rec.Result() + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d; want 200", resp.StatusCode) + } + if ct := resp.Header.Get("Content-Type"); ct != "application/zip" { + t.Errorf("Content-Type = %q; want application/zip", ct) + } + cd := resp.Header.Get("Content-Disposition") + if !strings.HasPrefix(cd, `attachment; filename="spaxel-backup-`) { + t.Errorf("Content-Disposition = %q; want attachment with spaxel-backup prefix", cd) + } +} + +func TestBackupHandler_ZipContents(t *testing.T) { + tests := []struct { + name string + setup func(t *testing.T, dir string) + wantFiles []string + noFiles []string + }{ + { + name: "single database and version", + setup: func(t *testing.T, dir string) { + setupTestDB(t, dir, "spaxel.db", + "CREATE TABLE nodes(mac TEXT PRIMARY KEY, name TEXT);"+ + "INSERT INTO nodes VALUES('AA:BB:CC:DD:EE:FF','Kitchen');") + }, + wantFiles: []string{"spaxel.db", "VERSION"}, + }, + { + name: "multiple databases with floor plan", + setup: func(t *testing.T, dir string) { + setupTestDB(t, dir, "spaxel.db", + "CREATE TABLE nodes(mac TEXT PRIMARY KEY);"+ + "INSERT INTO nodes VALUES('AA:BB');") + setupTestDB(t, dir, "ble.db", + "CREATE TABLE devices(addr TEXT PRIMARY KEY);"+ + "INSERT INTO devices VALUES('11:22');") + fpDir := filepath.Join(dir, "floor_plan") + if err := os.MkdirAll(fpDir, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(fpDir, "image.png"), []byte("fake-png"), 0644); err != nil { + t.Fatal(err) + } + }, + wantFiles: []string{"spaxel.db", "ble.db", "VERSION", "floor_plan/image.png"}, + }, + { + name: "no floor plan directory", + setup: func(t *testing.T, dir string) { + setupTestDB(t, dir, "zones.db", "CREATE TABLE zones(id INTEGER PRIMARY KEY);") + }, + wantFiles: []string{"zones.db", "VERSION"}, + noFiles: []string{"floor_plan"}, + }, + { + name: "empty data dir", + setup: func(t *testing.T, dir string) { + // no files created + }, + wantFiles: []string{"VERSION"}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + if tc.setup != nil { + tc.setup(t, dir) + } + + body := doBackupRequest(t, dir, "1.0.0") + names := zipEntryNames(t, body) + + for _, want := range tc.wantFiles { + if !names[want] { + t.Errorf("zip missing entry %q; got %v", want, names) + } + } + for _, no := range tc.noFiles { + for name := range names { + if strings.HasPrefix(name, no) { + t.Errorf("zip should not contain %q entries, got %q", no, name) + } + } + } + }) + } +} + +func TestBackupHandler_DBIntegrity(t *testing.T) { + dir := t.TempDir() + + // Create a database, write data, then write MORE data that lives in the WAL. + dsn := filepath.Join(dir, "spaxel.db") + "?_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL)" + db, err := sql.Open("sqlite", dsn) + if err != nil { + t.Fatal(err) + } + db.Exec("CREATE TABLE t(id INTEGER PRIMARY KEY, v TEXT)") + db.Exec("INSERT INTO t VALUES(1,'before-backup')") + + // Don't close db — simulate the mothership still writing while backup runs. + db.Exec("INSERT INTO t VALUES(2,'in-wal')") + + body := doBackupRequest(t, dir, "1.0.0") + + // Extract spaxel.db from zip and verify integrity. + dbBytes := readZipEntry(t, body, "spaxel.db") + + // Write to a temp file so sqlite can open it. + tmp := filepath.Join(t.TempDir(), "restored.db") + if err := os.WriteFile(tmp, dbBytes, 0644); err != nil { + t.Fatal(err) + } + + rdb, err := sql.Open("sqlite", tmp) + if err != nil { + t.Fatalf("open restored db: %v", err) + } + defer rdb.Close() + + var ok string + if err := rdb.QueryRow("PRAGMA quick_check(1)").Scan(&ok); err != nil { + t.Fatalf("integrity check failed: %v", err) + } + if ok != "ok" { + t.Fatalf("integrity check: %s", ok) + } + + // Verify both rows are present (WAL data was included in backup). + var count int + if err := rdb.QueryRow("SELECT count(*) FROM t").Scan(&count); err != nil { + t.Fatal(err) + } + if count != 2 { + t.Errorf("row count = %d; want 2 (WAL data should be included)", count) + } + + db.Close() +} + +func TestBackupHandler_SimultaneousWrite(t *testing.T) { + dir := t.TempDir() + + // Create a database with initial data. + dsn := filepath.Join(dir, "spaxel.db") + "?_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL)" + db, err := sql.Open("sqlite", dsn) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + db.Exec("CREATE TABLE t(id INTEGER PRIMARY KEY, v TEXT)") + db.Exec("INSERT INTO t VALUES(1,'original')") + + // Run the backup while writing concurrently. + done := make(chan struct{}) + go func() { + defer close(done) + for i := 2; i <= 20; i++ { + db.Exec(fmt.Sprintf("INSERT INTO t VALUES(%d,'concurrent-%d')", i, i)) + } + }() + + body := doBackupRequest(t, dir, "1.0.0") + <-done + + // Verify the backup is a valid database. + dbBytes := readZipEntry(t, body, "spaxel.db") + tmp := filepath.Join(t.TempDir(), "concurrent.db") + if err := os.WriteFile(tmp, dbBytes, 0644); err != nil { + t.Fatal(err) + } + + rdb, err := sql.Open("sqlite", tmp) + if err != nil { + t.Fatalf("open restored db: %v", err) + } + defer rdb.Close() + + var ok string + if err := rdb.QueryRow("PRAGMA quick_check(1)").Scan(&ok); err != nil { + t.Fatalf("integrity check failed: %v", err) + } + if ok != "ok" { + t.Fatalf("integrity check during concurrent writes: %s", ok) + } +} + +func TestBackupHandler_BackupSize(t *testing.T) { + dir := t.TempDir() + + var rows strings.Builder + rows.WriteString("CREATE TABLE data(v TEXT);") + for i := 0; i < 100; i++ { + rows.WriteString(fmt.Sprintf("INSERT INTO data VALUES('row-%04d-some-data-here');", i)) + } + setupTestDB(t, dir, "analytics.db", rows.String()) + + body := doBackupRequest(t, dir, "1.0.0") + + if len(body) == 0 { + t.Error("backup size = 0 bytes; want non-empty") + } + if len(body) > 1<<20 { + t.Errorf("backup size = %d bytes; want < 1 MB", len(body)) + } +} + +func TestBackupHandler_VersionFile(t *testing.T) { + dir := t.TempDir() + setupTestDB(t, dir, "spaxel.db", "CREATE TABLE t(id INTEGER PRIMARY KEY);") + + version := "2.5.0-rc1" + body := doBackupRequest(t, dir, version) + + content := readZipEntry(t, body, "VERSION") + got := strings.TrimSpace(string(content)) + if got != version { + t.Errorf("VERSION = %q; want %q", got, version) + } +}