docs: codify the FK-safe stream ordering as a wire-format invariant
CI / lint (pull_request) Successful in 31s
CI / format (pull_request) Successful in 31s
CI / install-and-build (pull_request) Successful in 55s
CI / typecheck-tests (pull_request) Successful in 31s
CI / typecheck-source (pull_request) Successful in 31s
CI / test (pull_request) Successful in 42s
CI / lint (pull_request) Successful in 31s
CI / format (pull_request) Successful in 31s
CI / install-and-build (pull_request) Successful in 55s
CI / typecheck-tests (pull_request) Successful in 31s
CI / typecheck-source (pull_request) Successful in 31s
CI / test (pull_request) Successful in 42s
The postgres importer inserts records sequentially with FK enforcement live, so it depends on records arriving in dependency order. That requirement was implicit in each backend's export table list; make it explicit and enforced. - Add EXPORT_KIND_ORDER (user, abode, resident, apikey, note) as a documented single source of truth, with the FK dependency chain spelled out on its doc comment, and note the ordering guarantee on the ExportEnvelope wire-format doc. Derive the isExportKind set from it. - Both backends' export() now iterate EXPORT_KIND_ORDER via a loader map (postgres omits note by leaving it out of the map), so emission order is tied to the constant and can't drift. - Tests: a change-detector on EXPORT_KIND_ORDER, plus assertions that both the sqlite and postgres (fake) exports emit record kinds grouped in FK-safe order (kind rank non-decreasing down the stream, after the leading meta). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit was merged in pull request #13.
This commit is contained in:
@@ -1,12 +1,10 @@
|
||||
import type { ExportFilter, ExportKind } from "../types/ExportImport.js";
|
||||
import {
|
||||
EXPORT_KIND_ORDER,
|
||||
type ExportFilter,
|
||||
type ExportKind,
|
||||
} from "../types/ExportImport.js";
|
||||
|
||||
const EXPORT_KINDS = new Set<ExportKind>([
|
||||
"user",
|
||||
"abode",
|
||||
"resident",
|
||||
"apikey",
|
||||
"note",
|
||||
]);
|
||||
const EXPORT_KINDS = new Set<ExportKind>(EXPORT_KIND_ORDER);
|
||||
|
||||
export function isExportKind(x: unknown): x is ExportKind {
|
||||
return typeof x === "string" && EXPORT_KINDS.has(x as ExportKind);
|
||||
|
||||
@@ -45,13 +45,14 @@ import type {
|
||||
import type { WrappedPgClient } from "./pool.js";
|
||||
import { Readable } from "node:stream";
|
||||
import readline from "node:readline";
|
||||
import type {
|
||||
Exportable,
|
||||
ExportKind,
|
||||
ExportOptions,
|
||||
Importable,
|
||||
ImportOptions,
|
||||
ImportResult,
|
||||
import {
|
||||
EXPORT_KIND_ORDER,
|
||||
type Exportable,
|
||||
type ExportKind,
|
||||
type ExportOptions,
|
||||
type Importable,
|
||||
type ImportOptions,
|
||||
type ImportResult,
|
||||
} from "../types/ExportImport.js";
|
||||
import { isExportKind, kindAllowed, recordAllowed } from "../export/filter.js";
|
||||
|
||||
@@ -527,18 +528,19 @@ export class PostgresInterface
|
||||
const db = this.#db;
|
||||
const source = this.name;
|
||||
|
||||
// `note` is omitted: the postgres backend has no note CRUD yet, so a pg
|
||||
// database can hold none. Each `load()` is a single query, run lazily and
|
||||
// skipped once the destination aborts.
|
||||
const tables: [
|
||||
ExportKind,
|
||||
() => Promise<{ uid?: string; aid?: string }[]>,
|
||||
][] = [
|
||||
["user", () => selectClientUsers(db)],
|
||||
["abode", () => selectAbodes(db)],
|
||||
["resident", () => selectResidents(db)],
|
||||
["apikey", () => selectClientApikeys(db)],
|
||||
];
|
||||
// Emission follows the FK-safe EXPORT_KIND_ORDER (part of the wire
|
||||
// contract; see ExportImport.ts). `note` has no loader: the postgres
|
||||
// backend has no note CRUD yet, so a pg database can hold none. Each
|
||||
// `load()` is a single query, run lazily and skipped once the destination
|
||||
// aborts.
|
||||
const loaders: Partial<
|
||||
Record<ExportKind, () => Promise<{ uid?: string; aid?: string }[]>>
|
||||
> = {
|
||||
user: () => selectClientUsers(db),
|
||||
abode: () => selectAbodes(db),
|
||||
resident: () => selectResidents(db),
|
||||
apikey: () => selectClientApikeys(db),
|
||||
};
|
||||
|
||||
async function* generate(): AsyncGenerator<string> {
|
||||
if (signal?.aborted) return;
|
||||
@@ -552,8 +554,10 @@ export class PostgresInterface
|
||||
},
|
||||
}) + "\n";
|
||||
try {
|
||||
for (const [kind, load] of tables) {
|
||||
for (const kind of EXPORT_KIND_ORDER) {
|
||||
if (signal?.aborted) return;
|
||||
const load = loaders[kind];
|
||||
if (!load) continue;
|
||||
if (!kindAllowed(filter, kind)) continue;
|
||||
for (const row of await load()) {
|
||||
if (signal?.aborted) return;
|
||||
|
||||
@@ -48,13 +48,14 @@ import type {
|
||||
import type { WrappedDb } from "./impl/types.js";
|
||||
import { Readable } from "node:stream";
|
||||
import readline from "node:readline";
|
||||
import type {
|
||||
Exportable,
|
||||
ExportKind,
|
||||
ExportOptions,
|
||||
Importable,
|
||||
ImportOptions,
|
||||
ImportResult,
|
||||
import {
|
||||
EXPORT_KIND_ORDER,
|
||||
type Exportable,
|
||||
type ExportKind,
|
||||
type ExportOptions,
|
||||
type Importable,
|
||||
type ImportOptions,
|
||||
type ImportResult,
|
||||
} from "../types/ExportImport.js";
|
||||
import { isExportKind, kindAllowed, recordAllowed } from "../export/filter.js";
|
||||
|
||||
@@ -558,14 +559,16 @@ export class SqliteInterface
|
||||
// `load()` is exactly one `WrappedDb.all()`). Kept lazy so the first query
|
||||
// only fires once the destination starts pulling, and skipped entirely
|
||||
// once the signal is aborted — no further reads after the destination
|
||||
// goes away.
|
||||
const tables: [ExportKind, () => { uid?: string; aid?: string }[]][] = [
|
||||
["user", () => selectClientUsers(db)],
|
||||
["abode", () => selectAbodes(db)],
|
||||
["resident", () => selectResidents(db)],
|
||||
["apikey", () => selectClientApikeys(db)],
|
||||
["note", () => selectNotes(db)],
|
||||
];
|
||||
// goes away. Emission follows the FK-safe EXPORT_KIND_ORDER (part of the
|
||||
// wire contract; see ExportImport.ts).
|
||||
const loaders: Record<ExportKind, () => { uid?: string; aid?: string }[]> =
|
||||
{
|
||||
user: () => selectClientUsers(db),
|
||||
abode: () => selectAbodes(db),
|
||||
resident: () => selectResidents(db),
|
||||
apikey: () => selectClientApikeys(db),
|
||||
note: () => selectNotes(db),
|
||||
};
|
||||
|
||||
async function* generate(): AsyncGenerator<string> {
|
||||
if (signal?.aborted) return;
|
||||
@@ -579,10 +582,10 @@ export class SqliteInterface
|
||||
},
|
||||
}) + "\n";
|
||||
try {
|
||||
for (const [kind, load] of tables) {
|
||||
for (const kind of EXPORT_KIND_ORDER) {
|
||||
if (signal?.aborted) return;
|
||||
if (!kindAllowed(filter, kind)) continue;
|
||||
for (const row of load()) {
|
||||
for (const row of loaders[kind]()) {
|
||||
if (signal?.aborted) return;
|
||||
if (recordAllowed(filter, kind, row)) {
|
||||
yield JSON.stringify({ kind, data: row }) + "\n";
|
||||
|
||||
@@ -8,6 +8,34 @@ import type { DbInterface } from "./DbInterface.js";
|
||||
*/
|
||||
export type ExportKind = "user" | "abode" | "resident" | "apikey" | "note";
|
||||
|
||||
/**
|
||||
* Canonical order in which record kinds are emitted into an export stream, and
|
||||
* the order in which an importer may safely apply them.
|
||||
*
|
||||
* This ordering is **part of the wire contract**, not an implementation
|
||||
* detail. It is FK-safe: every foreign key points only at a kind that appears
|
||||
* earlier (or at the same kind, earlier in the stream), so an importer that
|
||||
* inserts records one-by-one with referential integrity enforced never
|
||||
* forward-references a row it hasn't inserted yet:
|
||||
*
|
||||
* - `abode.created_by`/`updated_by` → `user`
|
||||
* - `resident.uid` → `user`, `resident.aid` → `abode`
|
||||
* - `apikey.uid` → `user`
|
||||
* - `note.aid` → `abode`, `note.created_by`/`updated_by` → `user`
|
||||
*
|
||||
* The sqlite backend can afford to relax this (it suspends FK enforcement for
|
||||
* the load), but the postgres backend relies on it: it inserts sequentially
|
||||
* with constraints live. Every {@link Exportable} MUST emit in this order, and
|
||||
* reordering it is a breaking change to the format.
|
||||
*/
|
||||
export const EXPORT_KIND_ORDER = [
|
||||
"user",
|
||||
"abode",
|
||||
"resident",
|
||||
"apikey",
|
||||
"note",
|
||||
] as const satisfies readonly ExportKind[];
|
||||
|
||||
export type ExportFilter = {
|
||||
/** Include only these kinds; omit = all kinds. */
|
||||
kinds?: ExportKind[];
|
||||
@@ -55,8 +83,10 @@ export interface Importable {
|
||||
|
||||
/**
|
||||
* The NDJSON envelope written/read for every line. The leading line is a
|
||||
* `meta` record; a trailing `error` record may appear if the source failed
|
||||
* after streaming had already begun.
|
||||
* `meta` record; the record lines that follow are grouped by kind in
|
||||
* {@link EXPORT_KIND_ORDER} (an FK-safe order importers may rely on); a
|
||||
* trailing `error` record may appear if the source failed after streaming had
|
||||
* already begun.
|
||||
*/
|
||||
export type ExportEnvelope =
|
||||
| { kind: "meta"; data: ExportMeta }
|
||||
|
||||
@@ -4,6 +4,10 @@ import { Readable } from "node:stream";
|
||||
import { PostgresInterface } from "../../../src/db/postgres/PostgresInterface.js";
|
||||
import type { WrappedPgClient } from "../../../src/db/postgres/pool.js";
|
||||
import type { SqlCode } from "../../../src/db/postgres/sql.js";
|
||||
import {
|
||||
EXPORT_KIND_ORDER,
|
||||
type ExportKind,
|
||||
} from "../../../src/db/types/ExportImport.js";
|
||||
|
||||
// The postgres backend has no CI database, so these tests drive the real
|
||||
// PostgresInterface.export/import code paths against an in-memory fake client.
|
||||
@@ -158,6 +162,20 @@ describe("postgres export", () => {
|
||||
assert.ok(!kinds.has("note"), "note is unsupported on postgres");
|
||||
});
|
||||
|
||||
it("emits record kinds grouped in FK-safe EXPORT_KIND_ORDER", async () => {
|
||||
const db = new PostgresInterface(new FakePg(seededRows()));
|
||||
const lines = parseLines(await streamToString(db.export()));
|
||||
assert.equal(lines[0]?.kind, "meta", "first line is meta");
|
||||
const rank = (k: string) => EXPORT_KIND_ORDER.indexOf(k as ExportKind);
|
||||
let last = -1;
|
||||
for (const { kind } of lines.slice(1)) {
|
||||
const r = rank(kind);
|
||||
assert.notEqual(r, -1, `unexpected kind ${kind}`);
|
||||
assert.ok(r >= last, `kind ${kind} out of FK-safe order`);
|
||||
last = r;
|
||||
}
|
||||
});
|
||||
|
||||
it("applies the kinds filter", async () => {
|
||||
const db = new PostgresInterface(new FakePg(seededRows()));
|
||||
const lines = parseLines(
|
||||
|
||||
@@ -16,6 +16,31 @@ import {
|
||||
import { inspectExportStream } from "../../src/db/export/inspect.js";
|
||||
import { hashPassword } from "../../src/util/hash.js";
|
||||
import type { ClientUser } from "../../src/db/types/User.js";
|
||||
import {
|
||||
EXPORT_KIND_ORDER,
|
||||
type ExportKind,
|
||||
} from "../../src/db/types/ExportImport.js";
|
||||
|
||||
/**
|
||||
* Assert the record lines of a parsed export are grouped in FK-safe
|
||||
* EXPORT_KIND_ORDER: the leading line is `meta`, and every record kind's
|
||||
* position in the order is non-decreasing down the stream.
|
||||
*/
|
||||
function assertFkSafeOrder(lines: { kind: string }[]): void {
|
||||
assert.equal(lines[0]?.kind, "meta", "first line is meta");
|
||||
const rank = (k: string) => EXPORT_KIND_ORDER.indexOf(k as ExportKind);
|
||||
let last = -1;
|
||||
for (const { kind } of lines.slice(1)) {
|
||||
if (kind === "error") continue;
|
||||
const r = rank(kind);
|
||||
assert.notEqual(r, -1, `unexpected kind ${kind}`);
|
||||
assert.ok(
|
||||
r >= last,
|
||||
`kind ${kind} (order ${r}) appears after a later kind (order ${last})`,
|
||||
);
|
||||
last = r;
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// helpers
|
||||
@@ -144,6 +169,40 @@ async function seed(db: TestDb["db"]): Promise<Seed> {
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 0. wire-format ordering invariant
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("export ordering (FK-safe wire contract)", () => {
|
||||
it("EXPORT_KIND_ORDER is the FK-safe dependency order", () => {
|
||||
// Change-detector: reordering is a breaking change to the format and must
|
||||
// keep every kind after the kinds it references (see ExportImport.ts).
|
||||
assert.deepEqual(EXPORT_KIND_ORDER, [
|
||||
"user",
|
||||
"abode",
|
||||
"resident",
|
||||
"apikey",
|
||||
"note",
|
||||
]);
|
||||
});
|
||||
|
||||
it("sqlite export emits record kinds grouped in EXPORT_KIND_ORDER", async () => {
|
||||
const src = await createTestDb();
|
||||
try {
|
||||
await seed(src.db);
|
||||
const lines = parseLines(await streamToString(src.db.export()));
|
||||
// every kind must be present so the ordering is actually exercised
|
||||
const kinds = new Set(lines.map((l) => l.kind));
|
||||
for (const k of EXPORT_KIND_ORDER) {
|
||||
assert.ok(kinds.has(k), `stream contains ${k}`);
|
||||
}
|
||||
assertFkSafeOrder(lines);
|
||||
} finally {
|
||||
src.close();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 1. round-trip
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user