add export/import streaming to the pluggable backends #13
@@ -1,12 +1,10 @@
|
||||
import type { ExportFilter, ExportKind } from "../types/ExportImport.js";
|
||||
import {
|
||||
EXPORT_KIND_ORDER,
|
||||
type ExportFilter,
|
||||
type ExportKind,
|
||||
} from "../types/ExportImport.js";
|
||||
|
||||
const EXPORT_KINDS = new Set<ExportKind>([
|
||||
"user",
|
||||
"abode",
|
||||
"resident",
|
||||
"apikey",
|
||||
"note",
|
||||
]);
|
||||
const EXPORT_KINDS = new Set<ExportKind>(EXPORT_KIND_ORDER);
|
||||
|
||||
export function isExportKind(x: unknown): x is ExportKind {
|
||||
return typeof x === "string" && EXPORT_KINDS.has(x as ExportKind);
|
||||
|
||||
@@ -45,13 +45,14 @@ import type {
|
||||
import type { WrappedPgClient } from "./pool.js";
|
||||
import { Readable } from "node:stream";
|
||||
import readline from "node:readline";
|
||||
import type {
|
||||
Exportable,
|
||||
ExportKind,
|
||||
ExportOptions,
|
||||
Importable,
|
||||
ImportOptions,
|
||||
ImportResult,
|
||||
import {
|
||||
EXPORT_KIND_ORDER,
|
||||
type Exportable,
|
||||
type ExportKind,
|
||||
type ExportOptions,
|
||||
type Importable,
|
||||
type ImportOptions,
|
||||
type ImportResult,
|
||||
} from "../types/ExportImport.js";
|
||||
import { isExportKind, kindAllowed, recordAllowed } from "../export/filter.js";
|
||||
|
||||
@@ -527,18 +528,19 @@ export class PostgresInterface
|
||||
const db = this.#db;
|
||||
const source = this.name;
|
||||
|
||||
// `note` is omitted: the postgres backend has no note CRUD yet, so a pg
|
||||
// database can hold none. Each `load()` is a single query, run lazily and
|
||||
// skipped once the destination aborts.
|
||||
const tables: [
|
||||
ExportKind,
|
||||
() => Promise<{ uid?: string; aid?: string }[]>,
|
||||
][] = [
|
||||
["user", () => selectClientUsers(db)],
|
||||
["abode", () => selectAbodes(db)],
|
||||
["resident", () => selectResidents(db)],
|
||||
["apikey", () => selectClientApikeys(db)],
|
||||
];
|
||||
// Emission follows the FK-safe EXPORT_KIND_ORDER (part of the wire
|
||||
// contract; see ExportImport.ts). `note` has no loader: the postgres
|
||||
// backend has no note CRUD yet, so a pg database can hold none. Each
|
||||
// `load()` is a single query, run lazily and skipped once the destination
|
||||
// aborts.
|
||||
const loaders: Partial<
|
||||
Record<ExportKind, () => Promise<{ uid?: string; aid?: string }[]>>
|
||||
> = {
|
||||
user: () => selectClientUsers(db),
|
||||
abode: () => selectAbodes(db),
|
||||
resident: () => selectResidents(db),
|
||||
apikey: () => selectClientApikeys(db),
|
||||
};
|
||||
|
||||
async function* generate(): AsyncGenerator<string> {
|
||||
if (signal?.aborted) return;
|
||||
@@ -552,8 +554,10 @@ export class PostgresInterface
|
||||
},
|
||||
}) + "\n";
|
||||
try {
|
||||
for (const [kind, load] of tables) {
|
||||
for (const kind of EXPORT_KIND_ORDER) {
|
||||
if (signal?.aborted) return;
|
||||
const load = loaders[kind];
|
||||
if (!load) continue;
|
||||
if (!kindAllowed(filter, kind)) continue;
|
||||
for (const row of await load()) {
|
||||
if (signal?.aborted) return;
|
||||
|
||||
@@ -48,13 +48,14 @@ import type {
|
||||
import type { WrappedDb } from "./impl/types.js";
|
||||
import { Readable } from "node:stream";
|
||||
import readline from "node:readline";
|
||||
import type {
|
||||
Exportable,
|
||||
ExportKind,
|
||||
ExportOptions,
|
||||
Importable,
|
||||
ImportOptions,
|
||||
ImportResult,
|
||||
import {
|
||||
EXPORT_KIND_ORDER,
|
||||
type Exportable,
|
||||
type ExportKind,
|
||||
type ExportOptions,
|
||||
type Importable,
|
||||
type ImportOptions,
|
||||
type ImportResult,
|
||||
} from "../types/ExportImport.js";
|
||||
import { isExportKind, kindAllowed, recordAllowed } from "../export/filter.js";
|
||||
|
||||
@@ -558,14 +559,16 @@ export class SqliteInterface
|
||||
// `load()` is exactly one `WrappedDb.all()`). Kept lazy so the first query
|
||||
// only fires once the destination starts pulling, and skipped entirely
|
||||
// once the signal is aborted — no further reads after the destination
|
||||
// goes away.
|
||||
const tables: [ExportKind, () => { uid?: string; aid?: string }[]][] = [
|
||||
["user", () => selectClientUsers(db)],
|
||||
["abode", () => selectAbodes(db)],
|
||||
["resident", () => selectResidents(db)],
|
||||
["apikey", () => selectClientApikeys(db)],
|
||||
["note", () => selectNotes(db)],
|
||||
];
|
||||
// goes away. Emission follows the FK-safe EXPORT_KIND_ORDER (part of the
|
||||
// wire contract; see ExportImport.ts).
|
||||
const loaders: Record<ExportKind, () => { uid?: string; aid?: string }[]> =
|
||||
{
|
||||
user: () => selectClientUsers(db),
|
||||
abode: () => selectAbodes(db),
|
||||
resident: () => selectResidents(db),
|
||||
apikey: () => selectClientApikeys(db),
|
||||
note: () => selectNotes(db),
|
||||
};
|
||||
|
||||
async function* generate(): AsyncGenerator<string> {
|
||||
if (signal?.aborted) return;
|
||||
@@ -579,10 +582,10 @@ export class SqliteInterface
|
||||
},
|
||||
}) + "\n";
|
||||
try {
|
||||
for (const [kind, load] of tables) {
|
||||
for (const kind of EXPORT_KIND_ORDER) {
|
||||
if (signal?.aborted) return;
|
||||
if (!kindAllowed(filter, kind)) continue;
|
||||
for (const row of load()) {
|
||||
for (const row of loaders[kind]()) {
|
||||
if (signal?.aborted) return;
|
||||
if (recordAllowed(filter, kind, row)) {
|
||||
yield JSON.stringify({ kind, data: row }) + "\n";
|
||||
|
||||
@@ -8,6 +8,34 @@ import type { DbInterface } from "./DbInterface.js";
|
||||
*/
|
||||
export type ExportKind = "user" | "abode" | "resident" | "apikey" | "note";
|
||||
|
||||
/**
|
||||
* Canonical order in which record kinds are emitted into an export stream, and
|
||||
* the order in which an importer may safely apply them.
|
||||
*
|
||||
* This ordering is **part of the wire contract**, not an implementation
|
||||
* detail. It is FK-safe: every foreign key points only at a kind that appears
|
||||
* earlier (or at the same kind, earlier in the stream), so an importer that
|
||||
* inserts records one-by-one with referential integrity enforced never
|
||||
* forward-references a row it hasn't inserted yet:
|
||||
*
|
||||
* - `abode.created_by`/`updated_by` → `user`
|
||||
* - `resident.uid` → `user`, `resident.aid` → `abode`
|
||||
* - `apikey.uid` → `user`
|
||||
* - `note.aid` → `abode`, `note.created_by`/`updated_by` → `user`
|
||||
*
|
||||
* The sqlite backend can afford to relax this (it suspends FK enforcement for
|
||||
* the load), but the postgres backend relies on it: it inserts sequentially
|
||||
* with constraints live. Every {@link Exportable} MUST emit in this order, and
|
||||
* reordering it is a breaking change to the format.
|
||||
*/
|
||||
export const EXPORT_KIND_ORDER = [
|
||||
"user",
|
||||
"abode",
|
||||
"resident",
|
||||
"apikey",
|
||||
"note",
|
||||
] as const satisfies readonly ExportKind[];
|
||||
|
||||
export type ExportFilter = {
|
||||
/** Include only these kinds; omit = all kinds. */
|
||||
kinds?: ExportKind[];
|
||||
@@ -55,8 +83,10 @@ export interface Importable {
|
||||
|
||||
/**
|
||||
* The NDJSON envelope written/read for every line. The leading line is a
|
||||
* `meta` record; a trailing `error` record may appear if the source failed
|
||||
* after streaming had already begun.
|
||||
* `meta` record; the record lines that follow are grouped by kind in
|
||||
* {@link EXPORT_KIND_ORDER} (an FK-safe order importers may rely on); a
|
||||
* trailing `error` record may appear if the source failed after streaming had
|
||||
* already begun.
|
||||
*/
|
||||
export type ExportEnvelope =
|
||||
| { kind: "meta"; data: ExportMeta }
|
||||
|
||||
@@ -4,6 +4,10 @@ import { Readable } from "node:stream";
|
||||
import { PostgresInterface } from "../../../src/db/postgres/PostgresInterface.js";
|
||||
import type { WrappedPgClient } from "../../../src/db/postgres/pool.js";
|
||||
import type { SqlCode } from "../../../src/db/postgres/sql.js";
|
||||
import {
|
||||
EXPORT_KIND_ORDER,
|
||||
type ExportKind,
|
||||
} from "../../../src/db/types/ExportImport.js";
|
||||
|
||||
// The postgres backend has no CI database, so these tests drive the real
|
||||
// PostgresInterface.export/import code paths against an in-memory fake client.
|
||||
@@ -158,6 +162,20 @@ describe("postgres export", () => {
|
||||
assert.ok(!kinds.has("note"), "note is unsupported on postgres");
|
||||
});
|
||||
|
||||
it("emits record kinds grouped in FK-safe EXPORT_KIND_ORDER", async () => {
|
||||
const db = new PostgresInterface(new FakePg(seededRows()));
|
||||
const lines = parseLines(await streamToString(db.export()));
|
||||
assert.equal(lines[0]?.kind, "meta", "first line is meta");
|
||||
const rank = (k: string) => EXPORT_KIND_ORDER.indexOf(k as ExportKind);
|
||||
let last = -1;
|
||||
for (const { kind } of lines.slice(1)) {
|
||||
const r = rank(kind);
|
||||
assert.notEqual(r, -1, `unexpected kind ${kind}`);
|
||||
assert.ok(r >= last, `kind ${kind} out of FK-safe order`);
|
||||
last = r;
|
||||
}
|
||||
});
|
||||
|
||||
it("applies the kinds filter", async () => {
|
||||
const db = new PostgresInterface(new FakePg(seededRows()));
|
||||
const lines = parseLines(
|
||||
|
||||
@@ -16,6 +16,31 @@ import {
|
||||
import { inspectExportStream } from "../../src/db/export/inspect.js";
|
||||
import { hashPassword } from "../../src/util/hash.js";
|
||||
import type { ClientUser } from "../../src/db/types/User.js";
|
||||
import {
|
||||
EXPORT_KIND_ORDER,
|
||||
type ExportKind,
|
||||
} from "../../src/db/types/ExportImport.js";
|
||||
|
||||
/**
|
||||
* Assert the record lines of a parsed export are grouped in FK-safe
|
||||
* EXPORT_KIND_ORDER: the leading line is `meta`, and every record kind's
|
||||
* position in the order is non-decreasing down the stream.
|
||||
*/
|
||||
function assertFkSafeOrder(lines: { kind: string }[]): void {
|
||||
assert.equal(lines[0]?.kind, "meta", "first line is meta");
|
||||
const rank = (k: string) => EXPORT_KIND_ORDER.indexOf(k as ExportKind);
|
||||
let last = -1;
|
||||
for (const { kind } of lines.slice(1)) {
|
||||
if (kind === "error") continue;
|
||||
const r = rank(kind);
|
||||
assert.notEqual(r, -1, `unexpected kind ${kind}`);
|
||||
assert.ok(
|
||||
r >= last,
|
||||
`kind ${kind} (order ${r}) appears after a later kind (order ${last})`,
|
||||
);
|
||||
last = r;
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// helpers
|
||||
@@ -144,6 +169,40 @@ async function seed(db: TestDb["db"]): Promise<Seed> {
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 0. wire-format ordering invariant
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("export ordering (FK-safe wire contract)", () => {
|
||||
it("EXPORT_KIND_ORDER is the FK-safe dependency order", () => {
|
||||
// Change-detector: reordering is a breaking change to the format and must
|
||||
// keep every kind after the kinds it references (see ExportImport.ts).
|
||||
assert.deepEqual(EXPORT_KIND_ORDER, [
|
||||
"user",
|
||||
"abode",
|
||||
"resident",
|
||||
"apikey",
|
||||
"note",
|
||||
]);
|
||||
});
|
||||
|
||||
it("sqlite export emits record kinds grouped in EXPORT_KIND_ORDER", async () => {
|
||||
const src = await createTestDb();
|
||||
try {
|
||||
await seed(src.db);
|
||||
const lines = parseLines(await streamToString(src.db.export()));
|
||||
// every kind must be present so the ordering is actually exercised
|
||||
const kinds = new Set(lines.map((l) => l.kind));
|
||||
for (const k of EXPORT_KIND_ORDER) {
|
||||
assert.ok(kinds.has(k), `stream contains ${k}`);
|
||||
}
|
||||
assertFkSafeOrder(lines);
|
||||
} finally {
|
||||
src.close();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 1. round-trip
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user