diff --git a/src/db/export/filter.ts b/src/db/export/filter.ts index 23876bd..45bb10d 100644 --- a/src/db/export/filter.ts +++ b/src/db/export/filter.ts @@ -1,12 +1,10 @@ -import type { ExportFilter, ExportKind } from "../types/ExportImport.js"; +import { + EXPORT_KIND_ORDER, + type ExportFilter, + type ExportKind, +} from "../types/ExportImport.js"; -const EXPORT_KINDS = new Set([ - "user", - "abode", - "resident", - "apikey", - "note", -]); +const EXPORT_KINDS = new Set(EXPORT_KIND_ORDER); export function isExportKind(x: unknown): x is ExportKind { return typeof x === "string" && EXPORT_KINDS.has(x as ExportKind); diff --git a/src/db/postgres/PostgresInterface.ts b/src/db/postgres/PostgresInterface.ts index 3be560c..3ed448f 100644 --- a/src/db/postgres/PostgresInterface.ts +++ b/src/db/postgres/PostgresInterface.ts @@ -45,13 +45,14 @@ import type { import type { WrappedPgClient } from "./pool.js"; import { Readable } from "node:stream"; import readline from "node:readline"; -import type { - Exportable, - ExportKind, - ExportOptions, - Importable, - ImportOptions, - ImportResult, +import { + EXPORT_KIND_ORDER, + type Exportable, + type ExportKind, + type ExportOptions, + type Importable, + type ImportOptions, + type ImportResult, } from "../types/ExportImport.js"; import { isExportKind, kindAllowed, recordAllowed } from "../export/filter.js"; @@ -527,18 +528,19 @@ export class PostgresInterface const db = this.#db; const source = this.name; - // `note` is omitted: the postgres backend has no note CRUD yet, so a pg - // database can hold none. Each `load()` is a single query, run lazily and - // skipped once the destination aborts. - const tables: [ - ExportKind, - () => Promise<{ uid?: string; aid?: string }[]>, - ][] = [ - ["user", () => selectClientUsers(db)], - ["abode", () => selectAbodes(db)], - ["resident", () => selectResidents(db)], - ["apikey", () => selectClientApikeys(db)], - ]; + // Emission follows the FK-safe EXPORT_KIND_ORDER (part of the wire + // contract; see ExportImport.ts). `note` has no loader: the postgres + // backend has no note CRUD yet, so a pg database can hold none. Each + // `load()` is a single query, run lazily and skipped once the destination + // aborts. + const loaders: Partial< + Record Promise<{ uid?: string; aid?: string }[]>> + > = { + user: () => selectClientUsers(db), + abode: () => selectAbodes(db), + resident: () => selectResidents(db), + apikey: () => selectClientApikeys(db), + }; async function* generate(): AsyncGenerator { if (signal?.aborted) return; @@ -552,8 +554,10 @@ export class PostgresInterface }, }) + "\n"; try { - for (const [kind, load] of tables) { + for (const kind of EXPORT_KIND_ORDER) { if (signal?.aborted) return; + const load = loaders[kind]; + if (!load) continue; if (!kindAllowed(filter, kind)) continue; for (const row of await load()) { if (signal?.aborted) return; diff --git a/src/db/sqlite/SqliteInterface.ts b/src/db/sqlite/SqliteInterface.ts index 03c96fa..c827a04 100644 --- a/src/db/sqlite/SqliteInterface.ts +++ b/src/db/sqlite/SqliteInterface.ts @@ -48,13 +48,14 @@ import type { import type { WrappedDb } from "./impl/types.js"; import { Readable } from "node:stream"; import readline from "node:readline"; -import type { - Exportable, - ExportKind, - ExportOptions, - Importable, - ImportOptions, - ImportResult, +import { + EXPORT_KIND_ORDER, + type Exportable, + type ExportKind, + type ExportOptions, + type Importable, + type ImportOptions, + type ImportResult, } from "../types/ExportImport.js"; import { isExportKind, kindAllowed, recordAllowed } from "../export/filter.js"; @@ -558,14 +559,16 @@ export class SqliteInterface // `load()` is exactly one `WrappedDb.all()`). Kept lazy so the first query // only fires once the destination starts pulling, and skipped entirely // once the signal is aborted — no further reads after the destination - // goes away. - const tables: [ExportKind, () => { uid?: string; aid?: string }[]][] = [ - ["user", () => selectClientUsers(db)], - ["abode", () => selectAbodes(db)], - ["resident", () => selectResidents(db)], - ["apikey", () => selectClientApikeys(db)], - ["note", () => selectNotes(db)], - ]; + // goes away. Emission follows the FK-safe EXPORT_KIND_ORDER (part of the + // wire contract; see ExportImport.ts). + const loaders: Record { uid?: string; aid?: string }[]> = + { + user: () => selectClientUsers(db), + abode: () => selectAbodes(db), + resident: () => selectResidents(db), + apikey: () => selectClientApikeys(db), + note: () => selectNotes(db), + }; async function* generate(): AsyncGenerator { if (signal?.aborted) return; @@ -579,10 +582,10 @@ export class SqliteInterface }, }) + "\n"; try { - for (const [kind, load] of tables) { + for (const kind of EXPORT_KIND_ORDER) { if (signal?.aborted) return; if (!kindAllowed(filter, kind)) continue; - for (const row of load()) { + for (const row of loaders[kind]()) { if (signal?.aborted) return; if (recordAllowed(filter, kind, row)) { yield JSON.stringify({ kind, data: row }) + "\n"; diff --git a/src/db/types/ExportImport.ts b/src/db/types/ExportImport.ts index 812defe..dbee43a 100644 --- a/src/db/types/ExportImport.ts +++ b/src/db/types/ExportImport.ts @@ -8,6 +8,34 @@ import type { DbInterface } from "./DbInterface.js"; */ export type ExportKind = "user" | "abode" | "resident" | "apikey" | "note"; +/** + * Canonical order in which record kinds are emitted into an export stream, and + * the order in which an importer may safely apply them. + * + * This ordering is **part of the wire contract**, not an implementation + * detail. It is FK-safe: every foreign key points only at a kind that appears + * earlier (or at the same kind, earlier in the stream), so an importer that + * inserts records one-by-one with referential integrity enforced never + * forward-references a row it hasn't inserted yet: + * + * - `abode.created_by`/`updated_by` → `user` + * - `resident.uid` → `user`, `resident.aid` → `abode` + * - `apikey.uid` → `user` + * - `note.aid` → `abode`, `note.created_by`/`updated_by` → `user` + * + * The sqlite backend can afford to relax this (it suspends FK enforcement for + * the load), but the postgres backend relies on it: it inserts sequentially + * with constraints live. Every {@link Exportable} MUST emit in this order, and + * reordering it is a breaking change to the format. + */ +export const EXPORT_KIND_ORDER = [ + "user", + "abode", + "resident", + "apikey", + "note", +] as const satisfies readonly ExportKind[]; + export type ExportFilter = { /** Include only these kinds; omit = all kinds. */ kinds?: ExportKind[]; @@ -55,8 +83,10 @@ export interface Importable { /** * The NDJSON envelope written/read for every line. The leading line is a - * `meta` record; a trailing `error` record may appear if the source failed - * after streaming had already begun. + * `meta` record; the record lines that follow are grouped by kind in + * {@link EXPORT_KIND_ORDER} (an FK-safe order importers may rely on); a + * trailing `error` record may appear if the source failed after streaming had + * already begun. */ export type ExportEnvelope = | { kind: "meta"; data: ExportMeta } diff --git a/test/backends/postgres/export-import.test.ts b/test/backends/postgres/export-import.test.ts index 3d1b789..214386f 100644 --- a/test/backends/postgres/export-import.test.ts +++ b/test/backends/postgres/export-import.test.ts @@ -4,6 +4,10 @@ import { Readable } from "node:stream"; import { PostgresInterface } from "../../../src/db/postgres/PostgresInterface.js"; import type { WrappedPgClient } from "../../../src/db/postgres/pool.js"; import type { SqlCode } from "../../../src/db/postgres/sql.js"; +import { + EXPORT_KIND_ORDER, + type ExportKind, +} from "../../../src/db/types/ExportImport.js"; // The postgres backend has no CI database, so these tests drive the real // PostgresInterface.export/import code paths against an in-memory fake client. @@ -158,6 +162,20 @@ describe("postgres export", () => { assert.ok(!kinds.has("note"), "note is unsupported on postgres"); }); + it("emits record kinds grouped in FK-safe EXPORT_KIND_ORDER", async () => { + const db = new PostgresInterface(new FakePg(seededRows())); + const lines = parseLines(await streamToString(db.export())); + assert.equal(lines[0]?.kind, "meta", "first line is meta"); + const rank = (k: string) => EXPORT_KIND_ORDER.indexOf(k as ExportKind); + let last = -1; + for (const { kind } of lines.slice(1)) { + const r = rank(kind); + assert.notEqual(r, -1, `unexpected kind ${kind}`); + assert.ok(r >= last, `kind ${kind} out of FK-safe order`); + last = r; + } + }); + it("applies the kinds filter", async () => { const db = new PostgresInterface(new FakePg(seededRows())); const lines = parseLines( diff --git a/test/tools/export-import.test.ts b/test/tools/export-import.test.ts index c8bdaca..aee0b91 100644 --- a/test/tools/export-import.test.ts +++ b/test/tools/export-import.test.ts @@ -16,6 +16,31 @@ import { import { inspectExportStream } from "../../src/db/export/inspect.js"; import { hashPassword } from "../../src/util/hash.js"; import type { ClientUser } from "../../src/db/types/User.js"; +import { + EXPORT_KIND_ORDER, + type ExportKind, +} from "../../src/db/types/ExportImport.js"; + +/** + * Assert the record lines of a parsed export are grouped in FK-safe + * EXPORT_KIND_ORDER: the leading line is `meta`, and every record kind's + * position in the order is non-decreasing down the stream. + */ +function assertFkSafeOrder(lines: { kind: string }[]): void { + assert.equal(lines[0]?.kind, "meta", "first line is meta"); + const rank = (k: string) => EXPORT_KIND_ORDER.indexOf(k as ExportKind); + let last = -1; + for (const { kind } of lines.slice(1)) { + if (kind === "error") continue; + const r = rank(kind); + assert.notEqual(r, -1, `unexpected kind ${kind}`); + assert.ok( + r >= last, + `kind ${kind} (order ${r}) appears after a later kind (order ${last})`, + ); + last = r; + } +} // --------------------------------------------------------------------------- // helpers @@ -144,6 +169,40 @@ async function seed(db: TestDb["db"]): Promise { }; } +// --------------------------------------------------------------------------- +// 0. wire-format ordering invariant +// --------------------------------------------------------------------------- + +describe("export ordering (FK-safe wire contract)", () => { + it("EXPORT_KIND_ORDER is the FK-safe dependency order", () => { + // Change-detector: reordering is a breaking change to the format and must + // keep every kind after the kinds it references (see ExportImport.ts). + assert.deepEqual(EXPORT_KIND_ORDER, [ + "user", + "abode", + "resident", + "apikey", + "note", + ]); + }); + + it("sqlite export emits record kinds grouped in EXPORT_KIND_ORDER", async () => { + const src = await createTestDb(); + try { + await seed(src.db); + const lines = parseLines(await streamToString(src.db.export())); + // every kind must be present so the ordering is actually exercised + const kinds = new Set(lines.map((l) => l.kind)); + for (const k of EXPORT_KIND_ORDER) { + assert.ok(kinds.has(k), `stream contains ${k}`); + } + assertFkSafeOrder(lines); + } finally { + src.close(); + } + }); +}); + // --------------------------------------------------------------------------- // 1. round-trip // ---------------------------------------------------------------------------