add export/import streaming to the pluggable backends #13

Merged
codinget merged 4 commits from feat/export-import-streaming into master 2026-07-23 20:43:15 +02:00
17 changed files with 2251 additions and 8 deletions
+3
View File
@@ -25,6 +25,9 @@
"react-redux": "^9.2.0"
},
"bin": {
"abode-export": "dist/bin/abode-export.cjs",
"abode-import": "dist/bin/abode-import.cjs",
"abode-inspect": "dist/bin/abode-inspect.cjs",
"abode-migrate": "dist/bin/abode-migrate.cjs",
"abode-repl": "dist/bin/abode-repl.cjs",
"abode-tui": "dist/bin/abode-tui.cjs",
+7 -1
View File
@@ -9,7 +9,10 @@
"abode-migrate": "dist/bin/abode-migrate.cjs",
"abode-repl": "dist/bin/abode-repl.cjs",
"abode-web": "dist/bin/abode-web.cjs",
"abode-tui": "dist/bin/abode-tui.cjs"
"abode-tui": "dist/bin/abode-tui.cjs",
"abode-export": "dist/bin/abode-export.cjs",
"abode-import": "dist/bin/abode-import.cjs",
"abode-inspect": "dist/bin/abode-inspect.cjs"
},
"scripts": {
"repl": "tsx --import ./src/meta/dev/register.ts",
@@ -18,6 +21,9 @@
"abode-web": "tsx --import ./src/meta/dev/register.ts --import ./src/meta/dev/webhot.ts src/bin/abode-web.ts",
"abode-tui": "tsx --import ./src/meta/dev/register.ts --import ./src/meta/dev/silenthot.ts src/bin/abode-tui.ts",
"abode-sources": "tsx --import ./src/meta/dev/register.ts src/bin/abode-sources.ts",
"abode-export": "tsx --import ./src/meta/dev/register.ts src/bin/abode-export.ts",
"abode-import": "tsx --import ./src/meta/dev/register.ts src/bin/abode-import.ts",
"abode-inspect": "tsx --import ./src/meta/dev/register.ts src/bin/abode-inspect.ts",
"build": "NODE_ENV=production npm run build:impl",
"build:impl": "rm -rf dist && tsx node_modules/.bin/webpack && chmod +x dist/bin/* && chmod -x dist/bin/*.*",
"test": "node --import tsx/esm --import ./src/meta/dev/register.ts --test $(find test -name '*.test.ts' | sort)",
+97
View File
@@ -0,0 +1,97 @@
import { createWriteStream } from "node:fs";
import { pipeline } from "node:stream/promises";
import { getDbInterface } from "../db/index.js";
import { isExportable, type ExportFilter } from "../db/types/ExportImport.js";
import { isExportKind } from "../db/export/filter.js";
const args = process.argv.slice(2);
function printUsage(err: boolean | string = false): never {
const log = (err ? console.error : console.log).bind(console);
if (typeof err === "string") {
log(`Error: ${err}`);
log("");
}
log("Usage:");
log("\tabode-export --help");
log(
"\tabode-export <database-url> [--kinds=user,abode,...] [--exclude-kinds=...] \\",
);
log("\t [--abodes=aid,...] [--users=uid,...] [--out=file|-]");
process.exit(err ? 1 : 0);
}
if (["-h", "--help", "help"].some((x) => args.includes(x))) printUsage();
let url: string | undefined;
const flags = new Map<string, string>();
for (const arg of args) {
if (arg.startsWith("--")) {
const eq = arg.indexOf("=");
if (eq === -1) printUsage(`invalid flag: ${arg}`);
flags.set(arg.slice(2, eq), arg.slice(eq + 1));
} else if (url === undefined) {
url = arg;
} else {
printUsage("too many arguments");
}
}
if (!url) printUsage("missing <database-url>");
const knownFlags = ["kinds", "exclude-kinds", "abodes", "users", "out"];
for (const key of flags.keys()) {
if (!knownFlags.includes(key)) printUsage(`unknown flag: --${key}`);
}
function parseList(value: string | undefined): string[] | undefined {
if (value === undefined) return undefined;
return value.split(",").filter(Boolean);
}
function parseKinds(value: string | undefined) {
const list = parseList(value);
if (!list) return undefined;
const bad = list.filter((k) => !isExportKind(k));
if (bad.length) printUsage(`invalid kind(s): ${bad.join(", ")}`);
return list.filter(isExportKind);
}
const filter: ExportFilter = {};
const kinds = parseKinds(flags.get("kinds"));
if (kinds) filter.kinds = kinds;
const excludeKinds = parseKinds(flags.get("exclude-kinds"));
if (excludeKinds) filter.excludeKinds = excludeKinds;
const abodes = parseList(flags.get("abodes"));
if (abodes) filter.abodes = abodes;
const users = parseList(flags.get("users"));
if (users) filter.users = users;
const db = await getDbInterface(url);
if (!isExportable(db)) {
console.error(`Error: backend '${db.name}' does not support export`);
process.exit(1);
}
const out = flags.get("out") ?? "-";
const dest =
out === "-" ? process.stdout : createWriteStream(out, { encoding: "utf8" });
const ac = new AbortController();
const onSignal = () => ac.abort();
process.on("SIGINT", onSignal);
process.on("SIGTERM", onSignal);
try {
await pipeline(db.export({ filter, signal: ac.signal }), dest);
} catch (e) {
if (ac.signal.aborted) {
console.error("Export aborted");
process.exit(130);
}
throw e;
} finally {
process.off("SIGINT", onSignal);
process.off("SIGTERM", onSignal);
await db.close().catch(() => {});
}
process.exit(0);
+111
View File
@@ -0,0 +1,111 @@
import { createReadStream } from "node:fs";
import { getDbInterface } from "../db/index.js";
import { isExportKind } from "../db/export/filter.js";
import { isImportable, type ExportFilter } from "../db/types/ExportImport.js";
const args = process.argv.slice(2);
function printUsage(err: boolean | string = false): never {
const log = (err ? console.error : console.log).bind(console);
if (typeof err === "string") {
log(`Error: ${err}`);
log("");
}
log("Usage:");
log("\tabode-import --help");
log("\tabode-import <database-url> <input-file|-> [--kinds=...] \\");
log(
"\t [--exclude-kinds=...] [--abodes=aid,...] [--users=uid,...]",
);
log("");
log(
"The target must be a local backend (sqlite or postgres), already migrated",
);
log("(run abode-migrate first). Remote (api) targets are not importable.");
process.exit(err ? 1 : 0);
}
if (["-h", "--help", "help"].some((x) => args.includes(x))) printUsage();
const positional: string[] = [];
const flags = new Map<string, string>();
for (const arg of args) {
if (arg.startsWith("--")) {
const eq = arg.indexOf("=");
if (eq === -1) printUsage(`invalid flag: ${arg}`);
flags.set(arg.slice(2, eq), arg.slice(eq + 1));
} else {
positional.push(arg);
}
}
const url = positional[0];
const input = positional[1];
if (!url) printUsage("missing <database-url>");
if (!input) printUsage("missing <input-file|->");
if (positional.length > 2) printUsage("too many arguments");
const knownFlags = ["kinds", "exclude-kinds", "abodes", "users"];
for (const key of flags.keys()) {
if (!knownFlags.includes(key)) printUsage(`unknown flag: --${key}`);
}
function parseList(value: string | undefined): string[] | undefined {
if (value === undefined) return undefined;
return value.split(",").filter(Boolean);
}
function parseKinds(value: string | undefined) {
const list = parseList(value);
if (!list) return undefined;
const bad = list.filter((k) => !isExportKind(k));
if (bad.length) printUsage(`invalid kind(s): ${bad.join(", ")}`);
return list.filter(isExportKind);
}
const filter: ExportFilter = {};
const kinds = parseKinds(flags.get("kinds"));
if (kinds) filter.kinds = kinds;
const excludeKinds = parseKinds(flags.get("exclude-kinds"));
if (excludeKinds) filter.excludeKinds = excludeKinds;
const abodes = parseList(flags.get("abodes"));
if (abodes) filter.abodes = abodes;
const users = parseList(flags.get("users"));
if (users) filter.users = users;
// Resolve the backend generically. Import lives on the local backends (sqlite,
// postgres); the remote (api) interface has no `import`, so `isImportable`
// keeps it from ever running over HTTP.
const db = await getDbInterface(url);
if (!isImportable(db)) {
console.error(`Error: backend '${db.name}' does not support import`);
await db.close().catch(() => {});
process.exit(1);
}
const source = input === "-" ? process.stdin : createReadStream(input);
const ac = new AbortController();
const onSignal = () => ac.abort();
process.on("SIGINT", onSignal);
process.on("SIGTERM", onSignal);
try {
const result = await db.import(source, { filter, signal: ac.signal });
const total = Object.values(result.counts).reduce((a, b) => a + b, 0);
console.log(`Imported ${total} record(s):`);
for (const [kind, count] of Object.entries(result.counts)) {
console.log(`- ${kind}: ${count}`);
}
} catch (e) {
if (ac.signal.aborted) {
console.error("Import aborted; no changes committed");
process.exit(130);
}
throw e;
} finally {
process.off("SIGINT", onSignal);
process.off("SIGTERM", onSignal);
await db.close().catch(() => {});
}
process.exit(0);
+68
View File
@@ -0,0 +1,68 @@
import { createReadStream } from "node:fs";
import { inspectExportStream } from "../db/export/inspect.js";
import { isExportKind } from "../db/export/filter.js";
import type { ExportKind } from "../db/types/ExportImport.js";
const args = process.argv.slice(2);
function printUsage(err: boolean | string = false): never {
const log = (err ? console.error : console.log).bind(console);
if (typeof err === "string") {
log(`Error: ${err}`);
log("");
}
log("Usage:");
log("\tabode-inspect --help");
log("\tabode-inspect <input-file|-> [--stop-after=note,...]");
process.exit(err ? 1 : 0);
}
if (["-h", "--help", "help"].some((x) => args.includes(x))) printUsage();
let input: string | undefined;
const flags = new Map<string, string>();
for (const arg of args) {
if (arg.startsWith("--")) {
const eq = arg.indexOf("=");
if (eq === -1) printUsage(`invalid flag: ${arg}`);
flags.set(arg.slice(2, eq), arg.slice(eq + 1));
} else if (input === undefined) {
input = arg;
} else {
printUsage("too many arguments");
}
}
if (!input) printUsage("missing <input-file|->");
for (const key of flags.keys()) {
if (key !== "stop-after") printUsage(`unknown flag: --${key}`);
}
let stopAfterKinds: ExportKind[] | undefined;
const rawStop = flags.get("stop-after");
if (rawStop !== undefined) {
const list = rawStop.split(",").filter(Boolean);
const bad = list.filter((k) => !isExportKind(k));
if (bad.length) printUsage(`invalid kind(s): ${bad.join(", ")}`);
stopAfterKinds = list.filter(isExportKind);
}
const source = input === "-" ? process.stdin : createReadStream(input);
const { counts, meta } = await inspectExportStream(source, { stopAfterKinds });
if (meta) {
console.log("Meta:");
console.log(`- format version: ${meta.v ?? "?"}`);
console.log(`- exported at: ${meta.exportedAt ?? "?"}`);
console.log(`- source backend: ${meta.source ?? "?"}`);
if (meta.filter && Object.keys(meta.filter).length) {
console.log(`- effective filter: ${JSON.stringify(meta.filter)}`);
}
}
console.log("Counts:");
const entries = Object.entries(counts);
if (!entries.length) console.log("(none)");
for (const [kind, count] of entries) console.log(`- ${kind}: ${count}`);
process.exit(0);
+45 -1
View File
@@ -25,8 +25,11 @@ import type {
CreateUser,
UpdateUser,
} from "../types/User.js";
import { Readable } from "node:stream";
import type { ReadableStream as WebReadableStream } from "node:stream/web";
import type { Exportable, ExportOptions } from "../types/ExportImport.js";
export class ApiInterface implements DbInterface {
export class ApiInterface implements DbInterface, Exportable {
#root: string;
#headers: Record<string, string>;
#readonly: boolean;
@@ -296,4 +299,45 @@ export class ApiInterface implements DbInterface {
async listNotesByUserId(uid: string): Promise<PartialNote[]> {
return this.#call("GET", "/users/:uid/notes", { params: { uid } });
}
export(options: ExportOptions = {}): NodeJS.ReadableStream {
// Streaming NDJSON bypasses the JSON-only `#call` helper: the raw
// `fetch` gets the caller's `signal` directly, so aborting cancels the
// underlying HTTP request itself.
const { filter, signal } = options;
const sp = new URLSearchParams();
if (filter?.kinds) sp.set("kinds", filter.kinds.join(","));
if (filter?.excludeKinds)
sp.set("excludeKinds", filter.excludeKinds.join(","));
if (filter?.abodes) sp.set("abodes", filter.abodes.join(","));
if (filter?.users) sp.set("users", filter.users.join(","));
const query = sp.toString();
const url = this.#root + "/export" + (query ? `?${query}` : "");
const headers = { ...this.#headers };
async function* generate(): AsyncGenerator<Buffer | Uint8Array> {
const res = await fetch(url, { method: "GET", headers, signal });
if (!res.ok) {
const text = await res.text();
switch (res.status) {
case 400:
throw new InvalidAbodeError();
case 401:
throw new NotAuthorizedAbodeError();
case 403:
throw new ReadonlyAbodeError();
case 404:
throw new NotFoundAbodeError();
case 409:
throw new ConflictAbodeError();
default:
throw new Error(`${res.status} ${res.statusText} ${text}`);
}
}
if (!res.body) return;
yield* Readable.fromWeb(res.body as WebReadableStream<Uint8Array>);
}
return Readable.from(generate());
}
}
+82
View File
@@ -0,0 +1,82 @@
import {
EXPORT_KIND_ORDER,
type ExportFilter,
type ExportKind,
} from "../types/ExportImport.js";
const EXPORT_KINDS = new Set<ExportKind>(EXPORT_KIND_ORDER);
export function isExportKind(x: unknown): x is ExportKind {
return typeof x === "string" && EXPORT_KINDS.has(x as ExportKind);
}
/** Whether a `kind` survives a filter's `kinds`/`excludeKinds` rules. */
export function kindAllowed(
filter: ExportFilter | undefined,
kind: ExportKind,
): boolean {
if (!filter) return true;
if (filter.kinds && !filter.kinds.includes(kind)) return false;
if (filter.excludeKinds && filter.excludeKinds.includes(kind)) return false;
return true;
}
/**
* Whether an individual record passes a filter's `abodes`/`users`/`apikeys`
* allowlists. `abodes` scopes abode/resident/note (by aid); `users` scopes user
* (by uid); `apikey` records are scoped by `apikeys` when present, else by
* `users`. An absent allowlist means "unrestricted".
*/
export function recordAllowed(
filter: ExportFilter | undefined,
kind: ExportKind,
record: { uid?: string; aid?: string },
): boolean {
if (!filter) return true;
switch (kind) {
case "user":
return !filter.users || filter.users.includes(record.uid as string);
case "apikey": {
const allow = filter.apikeys ?? filter.users;
return !allow || allow.includes(record.uid as string);
}
case "abode":
case "resident":
case "note":
return !filter.abodes || filter.abodes.includes(record.aid as string);
}
}
function intersectList<T extends string>(a?: T[], b?: T[]): T[] | undefined {
if (!a) return b;
if (!b) return a;
const bs = new Set(b);
return a.filter((x) => bs.has(x));
}
function unionList<T extends string>(a?: T[], b?: T[]): T[] | undefined {
if (!a) return b;
if (!b) return a;
return [...new Set([...a, ...b])];
}
/**
* Combine two filters as a hard intersection: the result can never permit more
* than either input. An absent allowlist is treated as "unrestricted", so
* intersecting it with a present one yields the present one. `excludeKinds`
* are unioned (either exclusion still excludes).
*/
export function intersectExportFilters(
a: ExportFilter | null | undefined,
b: ExportFilter | null | undefined,
): ExportFilter {
if (!a) return b ?? {};
if (!b) return a;
return {
kinds: intersectList(a.kinds, b.kinds),
excludeKinds: unionList(a.excludeKinds, b.excludeKinds),
abodes: intersectList(a.abodes, b.abodes),
users: intersectList(a.users, b.users),
apikeys: intersectList(a.apikeys, b.apikeys),
};
}
+60
View File
@@ -0,0 +1,60 @@
import readline from "node:readline";
import type {
ExportFilter,
ExportKind,
ExportMeta,
} from "../types/ExportImport.js";
import { isExportKind } from "./filter.js";
export type InspectResult = {
counts: Partial<Record<ExportKind, number>>;
meta?: Partial<ExportMeta> & { filter?: ExportFilter };
};
/**
* Tally the kinds/counts contained in an NDJSON export stream without ever
* touching a database. Works on any stream — a file, {@link ApiInterface}'s
* export, or a pipe straight from {@link SqliteInterface}'s export.
*
* If `stopAfterKinds` is given, reading stops as soon as at least one record of
* every requested kind has been seen, rather than draining to EOF — useful for
* probing large dumps ("does this contain notes at all?").
*/
export async function inspectExportStream(
source: NodeJS.ReadableStream,
options: { signal?: AbortSignal; stopAfterKinds?: ExportKind[] } = {},
): Promise<InspectResult> {
const { signal, stopAfterKinds } = options;
const counts: Partial<Record<ExportKind, number>> = {};
let meta: InspectResult["meta"];
const rl = readline.createInterface({
input: source,
crlfDelay: Infinity,
signal,
});
try {
for await (const raw of rl) {
signal?.throwIfAborted();
const line = raw.trim();
if (!line) continue;
let parsed: { kind?: unknown; data?: unknown };
try {
parsed = JSON.parse(line);
} catch {
continue;
}
if (parsed.kind === "meta") {
meta = parsed.data as InspectResult["meta"];
continue;
}
if (!isExportKind(parsed.kind)) continue;
counts[parsed.kind] = (counts[parsed.kind] ?? 0) + 1;
if (stopAfterKinds && stopAfterKinds.every((k) => counts[k])) break;
}
} finally {
rl.close();
}
return { counts, meta };
}
+211 -1
View File
@@ -43,8 +43,22 @@ import type {
UpdateNote,
} from "../types/Note.js";
import type { WrappedPgClient } from "./pool.js";
import { Readable } from "node:stream";
import readline from "node:readline";
import {
EXPORT_KIND_ORDER,
type Exportable,
type ExportKind,
type ExportOptions,
type Importable,
type ImportOptions,
type ImportResult,
} from "../types/ExportImport.js";
import { isExportKind, kindAllowed, recordAllowed } from "../export/filter.js";
export class PostgresInterface implements BackendDbInterface {
export class PostgresInterface
implements BackendDbInterface, Exportable, Importable
{
#db: WrappedPgClient;
constructor(db: WrappedPgClient) {
@@ -508,4 +522,200 @@ export class PostgresInterface implements BackendDbInterface {
async listNotesByUserId(_uid: string): Promise<PartialNote[]> {
throw new Error("Unimplemented");
}
export(options: ExportOptions = {}): NodeJS.ReadableStream {
const { filter, signal } = options;
const db = this.#db;
const source = this.name;
// Emission follows the FK-safe EXPORT_KIND_ORDER (part of the wire
// contract; see ExportImport.ts). `note` has no loader: the postgres
// backend has no note CRUD yet, so a pg database can hold none. Each
// `load()` is a single query, run lazily and skipped once the destination
// aborts.
const loaders: Partial<
Record<ExportKind, () => Promise<{ uid?: string; aid?: string }[]>>
> = {
user: () => selectClientUsers(db),
abode: () => selectAbodes(db),
resident: () => selectResidents(db),
apikey: () => selectClientApikeys(db),
};
async function* generate(): AsyncGenerator<string> {
if (signal?.aborted) return;
yield JSON.stringify({
kind: "meta",
data: {
v: 1,
exportedAt: new Date().toISOString(),
source,
filter: filter ?? {},
},
}) + "\n";
try {
for (const kind of EXPORT_KIND_ORDER) {
if (signal?.aborted) return;
const load = loaders[kind];
if (!load) continue;
if (!kindAllowed(filter, kind)) continue;
for (const row of await load()) {
if (signal?.aborted) return;
if (recordAllowed(filter, kind, row)) {
yield JSON.stringify({ kind, data: row }) + "\n";
}
}
}
} catch (e) {
if (signal?.aborted) return;
yield JSON.stringify({
kind: "error",
data: {
message: e instanceof Error ? e.message : String(e),
code: e instanceof Error ? e.name : undefined,
},
}) + "\n";
}
}
return Readable.from(generate());
}
async import(
source: NodeJS.ReadableStream,
options: ImportOptions = {},
): Promise<ImportResult> {
this.#checkReadonly();
const { filter, signal } = options;
const counts: Partial<Record<ExportKind, number>> = {};
const rl = readline.createInterface({
input: source,
crlfDelay: Infinity,
signal,
});
// Postgres holds FK enforcement (unlike sqlite, which we toggle off): the
// FK-safe insertion order — user, abode, resident, apikey — keeps a full
// dump valid. `notes` are skipped entirely (unsupported on this backend).
// The transaction commits only if the whole stream is consumed cleanly; an
// error/abort rolls it back via `multi`.
try {
await this.#db.rethrow(() =>
this.#db.multi(async (tx) => {
for await (const raw of rl) {
signal?.throwIfAborted();
const line = raw.trim();
if (!line) continue;
let parsed: { kind?: unknown; data?: unknown };
try {
parsed = JSON.parse(line);
} catch {
throw new InvalidAbodeError();
}
if (parsed.kind === "meta") continue;
if (parsed.kind === "error") {
throw new Error(
`export stream reported an error: ${
(parsed.data as { message?: string })?.message ?? "unknown"
}`,
);
}
if (!isExportKind(parsed.kind)) continue;
if (parsed.kind === "note") continue; // unsupported on postgres
if (!kindAllowed(filter, parsed.kind)) continue;
const data = parsed.data as { uid?: string; aid?: string };
if (!recordAllowed(filter, parsed.kind, data)) continue;
await this.#importRecord(tx, parsed.kind, parsed.data);
counts[parsed.kind] = (counts[parsed.kind] ?? 0) + 1;
}
// An abort while blocked on the source closes readline without
// throwing, so re-check before the transaction commits.
signal?.throwIfAborted();
}),
);
} finally {
rl.close();
}
return { counts };
}
async #importRecord(
tx: WrappedPgClient,
kind: ExportKind,
data: unknown,
): Promise<void> {
switch (kind) {
case "user": {
const u = data as ClientUser;
// `password` is never exported; imported users land on the schema
// default ('#unset') and must reset before they can log in.
await tx.run(sql`
INSERT INTO "users"("uid", "email", "name", "flags", "created_at", "updated_at")
VALUES(
${{ uuid: u.uid }},
${{ text: u.email }},
${{ text: u.name }},
${{ jsonb: u.flags }},
${{ date: u.created_at }},
${{ date: u.updated_at }}
)
`);
break;
}
case "abode": {
const a = data as Abode;
await tx.run(sql`
INSERT INTO "abodes"("aid", "name", "created_at", "created_by", "updated_at", "updated_by")
VALUES(
${{ uuid: a.aid }},
${{ text: a.name }},
${{ date: a.created_at }},
${a.created_by ? { uuid: a.created_by } : { null: true }},
${{ date: a.updated_at }},
${a.updated_by ? { uuid: a.updated_by } : { null: true }}
)
`);
break;
}
case "resident": {
const r = data as Resident;
await tx.run(sql`
INSERT INTO "residents"("uid", "aid", "flags", "created_at", "created_by", "updated_at", "updated_by")
VALUES(
${{ uuid: r.uid }},
${{ uuid: r.aid }},
${{ jsonb: r.flags }},
${{ date: r.created_at }},
${r.created_by ? { uuid: r.created_by } : { null: true }},
${{ date: r.updated_at }},
${r.updated_by ? { uuid: r.updated_by } : { null: true }}
)
`);
break;
}
case "apikey": {
const k = data as ClientApikey;
// `token` is never exported; mint a fresh unique one so the record's
// metadata (kid/permissions/expiry) survives even though the original
// secret cannot.
await tx.run(sql`
INSERT INTO "apikeys"("uid", "kid", "token", "name", "permissions", "created_at", "expires_at")
VALUES(
${{ uuid: k.uid }},
${{ uuid: k.kid }},
${{ text: createApikeyToken() }},
${{ text: k.name }},
${{ jsonb: k.permissions }},
${{ date: k.created_at }},
${k.expires_at ? { date: k.expires_at } : { null: true }}
)
`);
break;
}
case "note":
break; // unsupported on postgres; skipped before reaching here
}
}
}
+2 -2
View File
@@ -130,10 +130,10 @@ export async function selectClientApikey(
}
export async function selectClientApikeys(
db: WrappedPgClient,
where: SqlCode,
where?: SqlCode,
): Promise<ClientApikey[]> {
const rows = await db.all<RawClientApikey>(
sql`${sqlClientApikey} WHERE ${where}`,
where ? sql`${sqlClientApikey} WHERE ${where}` : sqlClientApikey,
);
return rows.map(pgToClientApikey);
}
+233 -1
View File
@@ -34,6 +34,7 @@ import {
selectClientUser,
selectClientUsers,
selectNote,
selectNotes,
selectPartialNotes,
selectResident,
selectResidents,
@@ -45,8 +46,22 @@ import type {
UpdateNote,
} from "../types/Note.js";
import type { WrappedDb } from "./impl/types.js";
import { Readable } from "node:stream";
import readline from "node:readline";
import {
EXPORT_KIND_ORDER,
type Exportable,
type ExportKind,
type ExportOptions,
type Importable,
type ImportOptions,
type ImportResult,
} from "../types/ExportImport.js";
import { isExportKind, kindAllowed, recordAllowed } from "../export/filter.js";
export class SqliteInterface implements BackendDbInterface {
export class SqliteInterface
implements BackendDbInterface, Exportable, Importable
{
#db: WrappedDb;
constructor(db: WrappedDb) {
@@ -534,4 +549,221 @@ export class SqliteInterface implements BackendDbInterface {
async listNotesByUserId(uid: string): Promise<PartialNote[]> {
return selectPartialNotes(this.#db, sql`n."created_by" = ${{ uuid: uid }}`);
}
export(options: ExportOptions = {}): NodeJS.ReadableStream {
const { filter, signal } = options;
const db = this.#db;
const source = this.name;
// Per-table full materialization + JS-side per-record yielding (each
// `load()` is exactly one `WrappedDb.all()`). Kept lazy so the first query
// only fires once the destination starts pulling, and skipped entirely
// once the signal is aborted — no further reads after the destination
// goes away. Emission follows the FK-safe EXPORT_KIND_ORDER (part of the
// wire contract; see ExportImport.ts).
const loaders: Record<ExportKind, () => { uid?: string; aid?: string }[]> =
{
user: () => selectClientUsers(db),
abode: () => selectAbodes(db),
resident: () => selectResidents(db),
apikey: () => selectClientApikeys(db),
note: () => selectNotes(db),
};
async function* generate(): AsyncGenerator<string> {
if (signal?.aborted) return;
yield JSON.stringify({
kind: "meta",
data: {
v: 1,
exportedAt: new Date().toISOString(),
source,
filter: filter ?? {},
},
}) + "\n";
try {
for (const kind of EXPORT_KIND_ORDER) {
if (signal?.aborted) return;
if (!kindAllowed(filter, kind)) continue;
for (const row of loaders[kind]()) {
if (signal?.aborted) return;
if (recordAllowed(filter, kind, row)) {
yield JSON.stringify({ kind, data: row }) + "\n";
}
}
}
} catch (e) {
// Aborts unwind via early `return`, never here; a genuine mid-stream
// failure is surfaced as a trailing sentinel line (HTTP 200 headers
// are already flushed, so `convertError` can no longer apply).
if (signal?.aborted) return;
yield JSON.stringify({
kind: "error",
data: {
message: e instanceof Error ? e.message : String(e),
code: e instanceof Error ? e.name : undefined,
},
}) + "\n";
}
}
return Readable.from(generate());
}
async import(
source: NodeJS.ReadableStream,
options: ImportOptions = {},
): Promise<ImportResult> {
this.#checkReadonly();
const { filter, signal } = options;
const db = this.#db;
const counts: Partial<Record<ExportKind, number>> = {};
const rl = readline.createInterface({
input: source,
crlfDelay: Infinity,
signal,
});
// Bulk restore trusts the export's referential integrity, and a filtered
// dump may legitimately reference `created_by`/`updated_by` users outside
// its scope. Suppress FK enforcement for the duration (can only be toggled
// outside a transaction) and restore it in `finally`.
db.run(sql`PRAGMA foreign_keys = OFF`);
db.run(sql`BEGIN`);
try {
for await (const raw of rl) {
signal?.throwIfAborted();
const line = raw.trim();
if (!line) continue;
let parsed: { kind?: unknown; data?: unknown };
try {
parsed = JSON.parse(line);
} catch {
throw new InvalidAbodeError();
}
if (parsed.kind === "meta") continue;
if (parsed.kind === "error") {
throw new Error(
`export stream reported an error: ${
(parsed.data as { message?: string })?.message ?? "unknown"
}`,
);
}
if (!isExportKind(parsed.kind)) continue;
if (!kindAllowed(filter, parsed.kind)) continue;
const data = parsed.data as { uid?: string; aid?: string };
if (!recordAllowed(filter, parsed.kind, data)) continue;
this.#importRecord(parsed.kind, parsed.data);
counts[parsed.kind] = (counts[parsed.kind] ?? 0) + 1;
}
// An abort while blocked on the source closes readline without throwing,
// so re-check before committing to guarantee no partial commit.
signal?.throwIfAborted();
db.run(sql`COMMIT`);
} catch (e) {
try {
db.run(sql`ROLLBACK`);
} catch {
/* already rolled back */
}
throw e;
} finally {
rl.close();
db.run(sql`PRAGMA foreign_keys = ON`);
}
return { counts };
}
#importRecord(kind: ExportKind, data: unknown): void {
const db = this.#db;
switch (kind) {
case "user": {
const u = data as ClientUser;
// `password` is never exported; imported users land on the schema
// default ('#unset') and must reset before they can log in.
db.run(sql`
INSERT INTO "users"("uid", "email", "name", "flags", "created_at", "updated_at")
VALUES(
${{ uuid: u.uid }},
${{ text: u.email }},
${{ text: u.name }},
${{ jsonb: u.flags }},
${{ date: u.created_at }},
${{ date: u.updated_at }}
)
`);
break;
}
case "abode": {
const a = data as Abode;
db.run(sql`
INSERT INTO "abodes"("aid", "name", "created_at", "created_by", "updated_at", "updated_by")
VALUES(
${{ uuid: a.aid }},
${{ text: a.name }},
${{ date: a.created_at }},
${a.created_by ? { uuid: a.created_by } : { null: true }},
${{ date: a.updated_at }},
${a.updated_by ? { uuid: a.updated_by } : { null: true }}
)
`);
break;
}
case "resident": {
const r = data as Resident;
db.run(sql`
INSERT INTO "residents"("uid", "aid", "flags", "created_at", "created_by", "updated_at", "updated_by")
VALUES(
${{ uuid: r.uid }},
${{ uuid: r.aid }},
${{ jsonb: r.flags }},
${{ date: r.created_at }},
${r.created_by ? { uuid: r.created_by } : { null: true }},
${{ date: r.updated_at }},
${r.updated_by ? { uuid: r.updated_by } : { null: true }}
)
`);
break;
}
case "apikey": {
const k = data as ClientApikey;
// `token` is never exported; mint a fresh unique one so the record's
// metadata (kid/permissions/expiry) survives even though the original
// secret cannot.
db.run(sql`
INSERT INTO "apikeys"("uid", "kid", "token", "name", "permissions", "created_at", "expires_at")
VALUES(
${{ uuid: k.uid }},
${{ uuid: k.kid }},
${{ text: createApikeyToken() }},
${{ text: k.name }},
${{ jsonb: k.permissions }},
${{ date: k.created_at }},
${k.expires_at ? { date: k.expires_at } : { null: true }}
)
`);
break;
}
case "note": {
const n = data as Note;
db.run(sql`
INSERT INTO "notes"("nid", "aid", "name", "content", "properties", "created_at", "created_by", "updated_at", "updated_by")
VALUES(
${{ uuid: n.nid }},
${{ uuid: n.aid }},
${{ text: n.name }},
${{ text: n.content ?? "" }},
${{ jsonb: n.properties }},
${{ date: n.created_at }},
${n.created_by ? { uuid: n.created_by } : { null: true }},
${{ date: n.updated_at }},
${n.updated_by ? { uuid: n.updated_by } : { null: true }}
)
`);
break;
}
}
}
}
+2 -2
View File
@@ -118,10 +118,10 @@ export function selectClientApikey(
}
export function selectClientApikeys(
db: WrappedDb,
where: SqlCode,
where?: SqlCode,
): ClientApikey[] {
const rawApikeys = db.all<RawClientApikey>(
sql`${sqlClientApikey} WHERE ${where}`,
where ? sql`${sqlClientApikey} WHERE ${where}` : sqlClientApikey,
);
return rawApikeys.map(sqliteToClientApikey);
}
+110
View File
@@ -0,0 +1,110 @@
import type { DbInterface } from "./DbInterface.js";
/**
* The kinds of records that can travel through an export/import stream.
*
* `Session` is intentionally excluded: it has no CRUD/list surface in
* {@link DbInterface} and is ephemeral/non-portable between instances.
*/
export type ExportKind = "user" | "abode" | "resident" | "apikey" | "note";
/**
* Canonical order in which record kinds are emitted into an export stream, and
* the order in which an importer may safely apply them.
*
* This ordering is **part of the wire contract**, not an implementation
* detail. It is FK-safe: every foreign key points only at a kind that appears
* earlier (or at the same kind, earlier in the stream), so an importer that
* inserts records one-by-one with referential integrity enforced never
* forward-references a row it hasn't inserted yet:
*
* - `abode.created_by`/`updated_by` → `user`
* - `resident.uid` → `user`, `resident.aid` → `abode`
* - `apikey.uid` → `user`
* - `note.aid` → `abode`, `note.created_by`/`updated_by` → `user`
*
* The sqlite backend can afford to relax this (it suspends FK enforcement for
* the load), but the postgres backend relies on it: it inserts sequentially
* with constraints live. Every {@link Exportable} MUST emit in this order, and
* reordering it is a breaking change to the format.
*/
export const EXPORT_KIND_ORDER = [
"user",
"abode",
"resident",
"apikey",
"note",
] as const satisfies readonly ExportKind[];
export type ExportFilter = {
/** Include only these kinds; omit = all kinds. */
kinds?: ExportKind[];
/** Excluded after `kinds` is applied. */
excludeKinds?: ExportKind[];
/** aid allowlist — scopes abode/resident/note. */
abodes?: string[];
/** uid allowlist — scopes user (and apikey, unless `apikeys` is set). */
users?: string[];
/**
* uid allowlist scoping apikey records specifically. When set it takes
* precedence over `users` for the `apikey` kind — used to export a
* non-admin's own keys while still exporting co-residents' *user* records
* for referential integrity, without leaking their apikey metadata. Absent =
* fall back to `users`.
*/
apikeys?: string[];
};
export interface ExportOptions {
filter?: ExportFilter;
signal?: AbortSignal;
}
export interface Exportable {
/** Produce a stream of NDJSON lines (one JSON envelope per line). */
export(options?: ExportOptions): NodeJS.ReadableStream;
}
export interface ImportOptions {
filter?: ExportFilter;
signal?: AbortSignal;
}
export interface ImportResult {
counts: Partial<Record<ExportKind, number>>;
}
export interface Importable {
import(
source: NodeJS.ReadableStream,
options?: ImportOptions,
): Promise<ImportResult>;
}
/**
* The NDJSON envelope written/read for every line. The leading line is a
* `meta` record; the record lines that follow are grouped by kind in
* {@link EXPORT_KIND_ORDER} (an FK-safe order importers may rely on); a
* trailing `error` record may appear if the source failed after streaming had
* already begun.
*/
export type ExportEnvelope =
| { kind: "meta"; data: ExportMeta }
| { kind: ExportKind; data: unknown }
| { kind: "error"; data: { message: string; code?: string } };
export type ExportMeta = {
v: number;
exportedAt: string;
source: string;
/** The *effective* filter actually applied (may be narrower than requested). */
filter: ExportFilter;
};
export function isExportable(db: DbInterface): db is DbInterface & Exportable {
return typeof (db as Partial<Exportable>).export === "function";
}
export function isImportable(db: DbInterface): db is DbInterface & Importable {
return typeof (db as Partial<Importable>).import === "function";
}
+48
View File
@@ -16,6 +16,29 @@ import {
} from "../schema/validators.js";
import { authenticate } from "./middleware/authenticate.js";
import { InvalidAbodeError, NotFoundAbodeError } from "../db/types/errors.js";
import { isExportable } from "../db/types/ExportImport.js";
import type { ExportFilter, ExportKind } from "../db/types/ExportImport.js";
import { intersectExportFilters, isExportKind } from "../db/export/filter.js";
import { computeForcedExportFilter } from "./exportScope.js";
function parseExportFilter(query: Record<string, unknown>): ExportFilter {
const list = (v: unknown): string[] | undefined => {
if (typeof v !== "string" || !v) return undefined;
return v.split(",").filter(Boolean);
};
const kinds = (v: unknown): ExportKind[] | undefined =>
list(v)?.filter(isExportKind);
const filter: ExportFilter = {};
const k = kinds(query.kinds);
if (k) filter.kinds = k;
const ek = kinds(query.excludeKinds);
if (ek) filter.excludeKinds = ek;
const abodes = list(query.abodes);
if (abodes) filter.abodes = abodes;
const users = list(query.users);
if (users) filter.users = users;
return filter;
}
export function apirouter(db: BackendDbInterface): KoaRouter {
const router = new KoaRouter();
@@ -42,6 +65,31 @@ export function apirouter(db: BackendDbInterface): KoaRouter {
ctx.status = 204;
});
router.get("/export", authenticate(db), async (ctx) => {
// Everything that can throw a domain error runs *before* any byte is
// written, so `convertError` still applies. Once `ctx.body` is a stream,
// a mid-stream failure surfaces as a trailing `error` NDJSON line instead.
const forced = await computeForcedExportFilter(db, {
user: ctx.user!,
session: ctx.session!,
});
const effective = intersectExportFilters(
parseExportFilter(ctx.query),
forced,
);
if (!isExportable(db)) {
ctx.status = 501;
ctx.body = { ok: false, error: "export_unsupported" };
return;
}
const ac = new AbortController();
ctx.res.on("close", () => {
if (!ctx.res.writableEnded) ac.abort();
});
ctx.type = "application/x-ndjson";
ctx.body = db.export({ filter: effective, signal: ac.signal });
});
router.use("/users", authenticate(db));
router.get("/users", async (ctx) => {
ctx.body = await db.listUsers();
+64
View File
@@ -0,0 +1,64 @@
import type { Context } from "koa";
import type { BackendDbInterface } from "../db/types/DbInterface.js";
import type { ClientUser } from "../db/types/User.js";
import type { ExportFilter } from "../db/types/ExportImport.js";
/**
* Compute the export scope that must be *forced* on a caller, independent of
* anything they requested. Returns `null` when the caller is unrestricted (a
* global admin whose credential imposes no narrowing) — their own filter, if
* any, is then honored verbatim as a voluntary narrowing.
*
* Otherwise returns `{ abodes, users, apikeys }`: the abodes the caller resides
* in, the users needed to keep that data referentially whole (the caller plus
* every co-resident of those abodes), and — scoped tighter than `users` —
* apikeys limited to the caller alone, so a non-admin never exports another
* user's apikey metadata even though that user's record is included. This is
* the maximum a non-admin may export; the route intersects it with any
* caller-supplied filter (never a union).
*/
export async function computeForcedExportFilter(
db: BackendDbInterface,
ctx: { user: ClientUser; session: NonNullable<Context["session"]> },
): Promise<ExportFilter | null> {
const { user, session } = ctx;
if (user.flags.admin) {
if (session.source !== "apikey") return null;
const p = session.key.permissions;
const unrestricted =
!!p.admin &&
!!p.all &&
!p.restrict_users?.length &&
!p.restrict_abodes?.length;
if (unrestricted) return null;
}
const residencies = await db.listResidentsByUserId(user.uid);
const abodeSet = new Set(residencies.map((r) => r.aid));
const userSet = new Set<string>([user.uid]);
for (const aid of abodeSet) {
for (const u of await db.listUsersByAbodeId(aid)) userSet.add(u.uid);
}
let abodes = [...abodeSet];
let users = [...userSet];
// apikeys are self-only for non-admins, regardless of co-residency.
let apikeys = [user.uid];
// An apikey can only narrow what its owning user could otherwise export.
if (session.source === "apikey") {
const p = session.key.permissions;
if (p.restrict_abodes?.length) {
const allow = new Set(p.restrict_abodes);
abodes = abodes.filter((a) => allow.has(a));
}
if (p.restrict_users?.length) {
const allow = new Set(p.restrict_users);
users = users.filter((u) => allow.has(u));
apikeys = apikeys.filter((u) => allow.has(u));
}
}
return { abodes, users, apikeys };
}
@@ -0,0 +1,314 @@
import { describe, it } from "node:test";
import assert from "node:assert/strict";
import { Readable } from "node:stream";
import { PostgresInterface } from "../../../src/db/postgres/PostgresInterface.js";
import type { WrappedPgClient } from "../../../src/db/postgres/pool.js";
import type { SqlCode } from "../../../src/db/postgres/sql.js";
import {
EXPORT_KIND_ORDER,
type ExportKind,
} from "../../../src/db/types/ExportImport.js";
// The postgres backend has no CI database, so these tests drive the real
// PostgresInterface.export/import code paths against an in-memory fake client.
// They verify control flow (NDJSON shape, meta.source, filtering, note
// skipping, counts, insert dispatch, abort -> rollback); the SQL-arg forms are
// the same {uuid}/{text}/{jsonb}/{date} patterns the pg backend's own CRUD
// already exercises against real Postgres.
interface Rows {
users?: Record<string, unknown>[];
abodes?: Record<string, unknown>[];
residents?: Record<string, unknown>[];
apikeys?: Record<string, unknown>[];
}
class FakePg implements WrappedPgClient {
readonly = false;
inserts: { table: string; vars: unknown[] }[] = [];
committed = false;
rolledBack = false;
#rows: Rows;
constructor(rows: Rows = {}) {
this.#rows = rows;
}
async destroy(): Promise<void> {}
async all<R>(stmt: SqlCode): Promise<R[]> {
const s = stmt._sql;
if (s.includes('FROM "users"')) return (this.#rows.users ?? []) as R[];
if (s.includes('FROM "abodes"')) return (this.#rows.abodes ?? []) as R[];
if (s.includes('FROM "residents"'))
return (this.#rows.residents ?? []) as R[];
if (s.includes('FROM "apikeys"')) return (this.#rows.apikeys ?? []) as R[];
return [] as R[];
}
async get<R>(stmt: SqlCode): Promise<R | null> {
const rows = await this.all<R>(stmt);
return rows[0] ?? null;
}
async run(stmt: SqlCode): Promise<{ changes: number }> {
const table = stmt._sql.match(/INSERT INTO "(\w+)"/)?.[1] ?? "?";
this.inserts.push({ table, vars: stmt._vars });
return { changes: 1 };
}
async multi<R>(fn: (tx: WrappedPgClient) => Promise<R>): Promise<R> {
try {
const r = await fn(this);
this.committed = true;
return r;
} catch (e) {
this.rolledBack = true;
throw e;
}
}
async rethrow<R>(fn: () => Promise<R>): Promise<R> {
return fn();
}
}
const iso = "2026-01-02T03:04:05.000Z";
function seededRows(): Rows {
return {
users: [
{
uid: "11111111-1111-1111-1111-111111111111",
email: "u1@test.example",
name: "User One",
flags: {},
created_at: new Date(iso),
updated_at: new Date(iso),
},
],
abodes: [
{
aid: "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaa1",
name: "Abode One",
created_at: new Date(iso),
created_by: "11111111-1111-1111-1111-111111111111",
updated_at: new Date(iso),
updated_by: null,
},
{
aid: "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaa2",
name: "Abode Two",
created_at: new Date(iso),
created_by: null,
updated_at: new Date(iso),
updated_by: null,
},
],
residents: [
{
uid: "11111111-1111-1111-1111-111111111111",
aid: "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaa1",
flags: {},
created_at: new Date(iso),
created_by: null,
updated_at: new Date(iso),
updated_by: null,
},
],
apikeys: [
{
uid: "11111111-1111-1111-1111-111111111111",
kid: "kkkkkkkk-kkkk-kkkk-kkkk-kkkkkkkkkkk1",
name: "key one",
permissions: {},
created_at: new Date(iso),
expires_at: null,
},
],
};
}
async function streamToString(s: NodeJS.ReadableStream): Promise<string> {
let out = "";
for await (const chunk of s) out += chunk;
return out;
}
function parseLines(ndjson: string): { kind: string; data: any }[] {
return ndjson
.split("\n")
.filter(Boolean)
.map((l) => JSON.parse(l));
}
function line(kind: string, data: unknown): string {
return JSON.stringify({ kind, data }) + "\n";
}
describe("postgres export", () => {
it("streams meta + the four supported kinds, never note", async () => {
const db = new PostgresInterface(new FakePg(seededRows()));
const lines = parseLines(await streamToString(db.export()));
const meta = lines.find((l) => l.kind === "meta");
assert.ok(meta);
assert.equal(meta!.data.source, "postgres");
assert.equal(meta!.data.v, 1);
const kinds = new Set(lines.map((l) => l.kind));
assert.ok(kinds.has("user"));
assert.ok(kinds.has("abode"));
assert.ok(kinds.has("resident"));
assert.ok(kinds.has("apikey"));
assert.ok(!kinds.has("note"), "note is unsupported on postgres");
});
it("emits record kinds grouped in FK-safe EXPORT_KIND_ORDER", async () => {
const db = new PostgresInterface(new FakePg(seededRows()));
const lines = parseLines(await streamToString(db.export()));
assert.equal(lines[0]?.kind, "meta", "first line is meta");
const rank = (k: string) => EXPORT_KIND_ORDER.indexOf(k as ExportKind);
let last = -1;
for (const { kind } of lines.slice(1)) {
const r = rank(kind);
assert.notEqual(r, -1, `unexpected kind ${kind}`);
assert.ok(r >= last, `kind ${kind} out of FK-safe order`);
last = r;
}
});
it("applies the kinds filter", async () => {
const db = new PostgresInterface(new FakePg(seededRows()));
const lines = parseLines(
await streamToString(db.export({ filter: { kinds: ["abode"] } })),
);
const kinds = new Set(
lines.filter((l) => l.kind !== "meta").map((l) => l.kind),
);
assert.deepEqual(kinds, new Set(["abode"]));
});
it("applies the abodes allowlist to abode/resident records", async () => {
const db = new PostgresInterface(new FakePg(seededRows()));
const lines = parseLines(
await streamToString(
db.export({
filter: { abodes: ["aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaa1"] },
}),
),
);
const abodeAids = lines
.filter((l) => l.kind === "abode")
.map((l) => l.data.aid);
assert.deepEqual(abodeAids, ["aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaa1"]);
});
});
describe("postgres import", () => {
it("dispatches inserts per kind, skips note, and commits", async () => {
const fake = new FakePg();
const db = new PostgresInterface(fake);
const source = Readable.from([
line("meta", { v: 1 }),
line("user", {
uid: "11111111-1111-1111-1111-111111111111",
email: "u1@test.example",
name: "User One",
flags: {},
created_at: iso,
updated_at: iso,
}),
line("abode", {
aid: "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaa1",
name: "Abode One",
created_at: iso,
created_by: "11111111-1111-1111-1111-111111111111",
updated_at: iso,
updated_by: null,
}),
line("apikey", {
uid: "11111111-1111-1111-1111-111111111111",
kid: "kkkkkkkk-kkkk-kkkk-kkkk-kkkkkkkkkkk1",
name: "key one",
permissions: {},
created_at: iso,
expires_at: null,
}),
// note lines are silently skipped on postgres
line("note", {
nid: "nnnnnnnn-nnnn-nnnn-nnnn-nnnnnnnnnnn1",
aid: "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaa1",
name: "Note",
content: "x",
properties: {},
created_at: iso,
created_by: null,
updated_at: iso,
updated_by: null,
}),
]);
const result = await db.import(source);
assert.deepEqual(result.counts, { user: 1, abode: 1, apikey: 1 });
assert.ok(fake.committed);
assert.deepEqual(fake.inserts.map((i) => i.table).sort(), [
"abodes",
"apikeys",
"users",
]);
assert.ok(!fake.inserts.some((i) => i.table === "notes"), "no note insert");
// apikey gets a freshly-minted token (never exported)
const apikeyInsert = fake.inserts.find((i) => i.table === "apikeys")!;
assert.ok(
apikeyInsert.vars.some(
(v) => typeof v === "string" && v.startsWith("at_"),
),
"apikey insert carries a fresh token",
);
});
it("rejects and rolls back when the stream carries an error sentinel", async () => {
const fake = new FakePg();
const db = new PostgresInterface(fake);
const source = Readable.from([
line("meta", { v: 1 }),
line("user", {
uid: "11111111-1111-1111-1111-111111111111",
email: "u1@test.example",
name: "User One",
flags: {},
created_at: iso,
updated_at: iso,
}),
line("error", { message: "boom" }),
]);
await assert.rejects(() => db.import(source), /boom/);
assert.ok(fake.rolledBack);
assert.ok(!fake.committed);
});
it("rejects and rolls back on signal abort", async () => {
const fake = new FakePg();
const db = new PostgresInterface(fake);
const ac = new AbortController();
const source = Readable.from(
(async function* () {
yield line("meta", { v: 1 });
yield line("user", {
uid: "11111111-1111-1111-1111-111111111111",
email: "u1@test.example",
name: "User One",
flags: {},
created_at: iso,
updated_at: iso,
});
ac.abort();
await new Promise((r) => setTimeout(r, 1000));
})(),
);
await assert.rejects(() => db.import(source, { signal: ac.signal }));
assert.ok(fake.rolledBack);
assert.ok(!fake.committed);
});
});
+794
View File
@@ -0,0 +1,794 @@
import { describe, it, before, after } from "node:test";
import assert from "node:assert/strict";
import readline from "node:readline";
import { Readable } from "node:stream";
import { createServer } from "node:http";
import type { AddressInfo } from "node:net";
import Koa from "koa";
import { createTestDb, type TestDb } from "../helpers/sqlite.js";
import { apirouter } from "../../src/webapi/apirouter.js";
import { computeForcedExportFilter } from "../../src/webapi/exportScope.js";
import {
intersectExportFilters,
recordAllowed,
kindAllowed,
} from "../../src/db/export/filter.js";
import { inspectExportStream } from "../../src/db/export/inspect.js";
import { hashPassword } from "../../src/util/hash.js";
import type { ClientUser } from "../../src/db/types/User.js";
import {
EXPORT_KIND_ORDER,
type ExportKind,
} from "../../src/db/types/ExportImport.js";
/**
* Assert the record lines of a parsed export are grouped in FK-safe
* EXPORT_KIND_ORDER: the leading line is `meta`, and every record kind's
* position in the order is non-decreasing down the stream.
*/
function assertFkSafeOrder(lines: { kind: string }[]): void {
assert.equal(lines[0]?.kind, "meta", "first line is meta");
const rank = (k: string) => EXPORT_KIND_ORDER.indexOf(k as ExportKind);
let last = -1;
for (const { kind } of lines.slice(1)) {
if (kind === "error") continue;
const r = rank(kind);
assert.notEqual(r, -1, `unexpected kind ${kind}`);
assert.ok(
r >= last,
`kind ${kind} (order ${r}) appears after a later kind (order ${last})`,
);
last = r;
}
}
// ---------------------------------------------------------------------------
// helpers
// ---------------------------------------------------------------------------
async function streamToString(s: NodeJS.ReadableStream): Promise<string> {
let out = "";
for await (const chunk of s) out += chunk;
return out;
}
function parseLines(ndjson: string): { kind: string; data: any }[] {
return ndjson
.split("\n")
.filter(Boolean)
.map((l) => JSON.parse(l));
}
/** Normalise `*_at` fields to epoch ms so formatting never breaks equality. */
function norm(obj: Record<string, unknown>): Record<string, unknown> {
const out: Record<string, unknown> = {};
for (const [k, v] of Object.entries(obj)) {
out[k] = k.endsWith("_at")
? v == null
? null
: new Date(v as string).getTime()
: v;
}
return out;
}
function normSorted(
arr: Record<string, unknown>[],
key: (x: any) => string,
): Record<string, unknown>[] {
return arr.map(norm).sort((a, b) => key(a).localeCompare(key(b)));
}
interface Seed {
admin: ClientUser;
normal: ClientUser;
co: ClientUser;
aid1: string;
aid2: string;
nid1: string;
nid2: string;
}
async function seed(db: TestDb["db"]): Promise<Seed> {
const pw = await hashPassword("password");
const admin = await db.createUser({
email: "admin@test.example",
name: "Admin",
password: pw,
flags: { admin: true },
});
const normal = await db.createUser({
email: "normal@test.example",
name: "Normal",
password: pw,
flags: {},
});
const co = await db.createUser({
email: "co@test.example",
name: "Co Resident",
password: pw,
flags: {},
});
const abode1 = await db.createAbode(
{ name: "Abode One" },
{ uid: admin.uid },
);
const abode2 = await db.createAbode(
{ name: "Abode Two" },
{ uid: admin.uid },
);
await db.createResident(
{ uid: normal.uid, aid: abode1.aid, flags: {} },
{ uid: admin.uid },
);
await db.createResident(
{ uid: co.uid, aid: abode1.aid, flags: {} },
{ uid: admin.uid },
);
await db.createResident(
{ uid: admin.uid, aid: abode2.aid, flags: { admin: true } },
{ uid: admin.uid },
);
await db.createApikey({
uid: normal.uid,
name: "normal key",
permissions: {},
expires_at: null,
});
await db.createApikey({
uid: co.uid,
name: "co key",
permissions: {},
expires_at: null,
});
const note1 = await db.createNote(
{
aid: abode1.aid,
name: "Note One",
content: "hello",
properties: { type: "note" },
},
{ uid: normal.uid },
);
const note2 = await db.createNote(
{
aid: abode2.aid,
name: "Note Two",
content: "world",
properties: { type: "note" },
},
{ uid: admin.uid },
);
return {
admin,
normal,
co,
aid1: abode1.aid,
aid2: abode2.aid,
nid1: note1.nid,
nid2: note2.nid,
};
}
// ---------------------------------------------------------------------------
// 0. wire-format ordering invariant
// ---------------------------------------------------------------------------
describe("export ordering (FK-safe wire contract)", () => {
it("EXPORT_KIND_ORDER is the FK-safe dependency order", () => {
// Change-detector: reordering is a breaking change to the format and must
// keep every kind after the kinds it references (see ExportImport.ts).
assert.deepEqual(EXPORT_KIND_ORDER, [
"user",
"abode",
"resident",
"apikey",
"note",
]);
});
it("sqlite export emits record kinds grouped in EXPORT_KIND_ORDER", async () => {
const src = await createTestDb();
try {
await seed(src.db);
const lines = parseLines(await streamToString(src.db.export()));
// every kind must be present so the ordering is actually exercised
const kinds = new Set(lines.map((l) => l.kind));
for (const k of EXPORT_KIND_ORDER) {
assert.ok(kinds.has(k), `stream contains ${k}`);
}
assertFkSafeOrder(lines);
} finally {
src.close();
}
});
});
// ---------------------------------------------------------------------------
// 1. round-trip
// ---------------------------------------------------------------------------
describe("export/import: sqlite -> sqlite round-trip", () => {
it("reproduces users, abodes, residents, apikeys, notes", async () => {
const src = await createTestDb();
const dst = await createTestDb();
try {
const s = await seed(src.db);
const result = await dst.db.import(src.db.export());
assert.ok(result.counts.user && result.counts.user >= 3);
assert.ok(result.counts.note && result.counts.note >= 2);
assert.deepEqual(
normSorted(await dst.db.listUsers(), (u) => u.uid),
normSorted(await src.db.listUsers(), (u) => u.uid),
);
assert.deepEqual(
normSorted(await dst.db.listAbodes(), (a) => a.aid),
normSorted(await src.db.listAbodes(), (a) => a.aid),
);
assert.deepEqual(
normSorted(await dst.db.listResidents(), (r) => r.uid + r.aid),
normSorted(await src.db.listResidents(), (r) => r.uid + r.aid),
);
// apikeys: token is regenerated on import, so the ClientApikey view
// (which omits token) must still match exactly.
assert.deepEqual(
normSorted(await dst.db.listApikeysByUser(s.normal.uid), (k) => k.kid),
normSorted(await src.db.listApikeysByUser(s.normal.uid), (k) => k.kid),
);
// notes (full, with content)
const srcNote = await src.db.getNoteById(s.nid1);
const dstNote = await dst.db.getNoteById(s.nid1);
assert.deepEqual(norm(dstNote), norm(srcNote));
} finally {
src.close();
dst.close();
}
});
});
// ---------------------------------------------------------------------------
// 2. filter narrowing
// ---------------------------------------------------------------------------
describe("export/import: filter narrowing", () => {
it("scopes abode/resident/note to the requested abodes", async () => {
const src = await createTestDb();
try {
const s = await seed(src.db);
const ndjson = await streamToString(
src.db.export({ filter: { abodes: [s.aid1] } }),
);
const lines = parseLines(ndjson);
const abodeAids = lines
.filter((l) => l.kind === "abode")
.map((l) => l.data.aid);
assert.deepEqual(abodeAids, [s.aid1]);
const noteAids = new Set(
lines.filter((l) => l.kind === "note").map((l) => l.data.aid),
);
assert.ok(noteAids.has(s.aid1));
assert.ok(!noteAids.has(s.aid2));
const residentAids = new Set(
lines.filter((l) => l.kind === "resident").map((l) => l.data.aid),
);
assert.ok(!residentAids.has(s.aid2));
} finally {
src.close();
}
});
it("imports the narrowed dump into a fresh db with the same scope", async () => {
const src = await createTestDb();
const dst = await createTestDb();
try {
const s = await seed(src.db);
await dst.db.import(src.db.export({ filter: { abodes: [s.aid1] } }));
const abodes = await dst.db.listAbodes();
assert.deepEqual(
abodes.map((a) => a.aid),
[s.aid1],
);
const notes = await dst.db.listNotesByAbodeId(s.aid1);
assert.equal(notes.length, 1);
assert.equal((await dst.db.listNotesByAbodeId(s.aid2)).length, 0);
} finally {
src.close();
dst.close();
}
});
it("kinds filter selects only the requested kinds", async () => {
const src = await createTestDb();
try {
await seed(src.db);
const ndjson = await streamToString(
src.db.export({ filter: { kinds: ["abode"] } }),
);
const kinds = new Set(parseLines(ndjson).map((l) => l.kind));
assert.ok(kinds.has("abode"));
assert.ok(!kinds.has("user"));
assert.ok(!kinds.has("note"));
} finally {
src.close();
}
});
});
// ---------------------------------------------------------------------------
// 3. forced-filter enforcement
// ---------------------------------------------------------------------------
describe("exportScope: computeForcedExportFilter", () => {
it("returns null for a global admin on a basic/session credential", async () => {
const t = await createTestDb();
try {
const s = await seed(t.db);
const forced = await computeForcedExportFilter(t.db, {
user: s.admin,
session: { source: "basic" },
});
assert.equal(forced, null);
} finally {
t.close();
}
});
it("returns null for a global admin with an unrestricted apikey", async () => {
const t = await createTestDb();
try {
const s = await seed(t.db);
const forced = await computeForcedExportFilter(t.db, {
user: s.admin,
session: {
source: "apikey",
key: {
kid: "k",
uid: s.admin.uid,
name: "k",
permissions: { admin: true, all: true },
created_at: new Date().toISOString(),
expires_at: null,
},
},
});
assert.equal(forced, null);
} finally {
t.close();
}
});
it("forces a non-admin to their abodes + co-residents", async () => {
const t = await createTestDb();
try {
const s = await seed(t.db);
const forced = await computeForcedExportFilter(t.db, {
user: s.normal,
session: { source: "basic" },
});
assert.ok(forced);
assert.deepEqual(forced!.abodes, [s.aid1]);
assert.deepEqual(
new Set(forced!.users),
new Set([s.normal.uid, s.co.uid]),
);
assert.ok(!forced!.users!.includes(s.admin.uid));
// apikeys are self-only, even though co is a co-resident whose user
// record is exported for referential integrity.
assert.deepEqual(forced!.apikeys, [s.normal.uid]);
// A caller requesting a wider abode never gets it: intersection, not union.
const effective = intersectExportFilters(
{ abodes: [s.aid1, s.aid2] },
forced,
);
assert.deepEqual(effective.abodes, [s.aid1]);
assert.ok(!effective.abodes!.includes(s.aid2));
} finally {
t.close();
}
});
it("intersects a non-admin apikey with restrict_abodes", async () => {
const t = await createTestDb();
try {
const s = await seed(t.db);
const forced = await computeForcedExportFilter(t.db, {
user: s.normal,
session: {
source: "apikey",
key: {
kid: "k",
uid: s.normal.uid,
name: "k",
permissions: { restrict_abodes: [s.aid2] },
created_at: new Date().toISOString(),
expires_at: null,
},
},
});
// normal resides only in aid1; restrict to aid2 => empty intersection.
assert.deepEqual(forced!.abodes, []);
} finally {
t.close();
}
});
});
// ---------------------------------------------------------------------------
// 4. cancellation, export side
// ---------------------------------------------------------------------------
describe("export cancellation", () => {
it("stops querying tables after the destination aborts", async () => {
const t = await createTestDb();
try {
const pw = await hashPassword("password");
// Enough users that the first table can't be buffered in one go, so the
// generator backpressures mid-`user` and never reaches later tables.
for (let i = 0; i < 400; i++) {
await t.db.createUser({
email: `bulk-${i}@test.example`,
name: `Bulk ${i}`,
password: pw,
flags: {},
});
}
let allCalls = 0;
const orig = t.wrapped.all.bind(t.wrapped);
(t.wrapped as { all: unknown }).all = (stmt: never) => {
allCalls++;
return orig(stmt);
};
const ac = new AbortController();
const stream = t.db.export({ signal: ac.signal });
const rl = readline.createInterface({ input: stream });
let lines = 0;
let callsAtAbort = -1;
for await (const _ of rl) {
lines++;
if (lines === 3) {
ac.abort();
callsAtAbort = allCalls;
}
}
assert.ok(callsAtAbort >= 1, "at least the first table was queried");
assert.equal(allCalls, callsAtAbort, "no further queries after abort");
assert.ok(allCalls < 5, "did not materialise all five tables");
} finally {
t.close();
}
});
});
// ---------------------------------------------------------------------------
// 5. cancellation, import side
// ---------------------------------------------------------------------------
describe("import cancellation", () => {
it("rolls back and leaves no dangling transaction when the source errors", async () => {
const dst = await createTestDb();
try {
function line(kind: string, data: unknown): string {
return JSON.stringify({ kind, data }) + "\n";
}
const now = new Date().toISOString();
const mkUser = (i: number) => ({
uid: crypto.randomUUID(),
email: `imp-${i}@test.example`,
name: `Imp ${i}`,
flags: {},
created_at: now,
updated_at: now,
});
// Emits a couple of valid user records, then throws mid-stream.
const source = Readable.from(
(async function* () {
yield line("meta", { v: 1 });
yield line("user", mkUser(1));
yield line("user", mkUser(2));
throw new Error("source exploded");
})(),
);
await assert.rejects(() => dst.db.import(source), /source exploded/);
// ROLLBACK happened: nothing persisted.
assert.equal((await dst.db.listUsers()).length, 0);
// No lingering open transaction: a follow-up write succeeds immediately.
const pw = await hashPassword("password");
const created = await dst.db.createUser({
email: "after@test.example",
name: "After",
password: pw,
flags: {},
});
assert.ok(created.uid);
} finally {
dst.close();
}
});
it("aborts cleanly via signal and commits nothing", async () => {
const dst = await createTestDb();
try {
const ac = new AbortController();
const now = new Date().toISOString();
const source = Readable.from(
(async function* () {
yield JSON.stringify({ kind: "meta", data: { v: 1 } }) + "\n";
yield JSON.stringify({
kind: "user",
data: {
uid: crypto.randomUUID(),
email: "abort@test.example",
name: "Abort",
flags: {},
created_at: now,
updated_at: now,
},
}) + "\n";
ac.abort();
// Keep the stream alive so abort — not EOF — ends the import.
await new Promise((r) => setTimeout(r, 1000));
})(),
);
await assert.rejects(() => dst.db.import(source, { signal: ac.signal }));
assert.equal((await dst.db.listUsers()).length, 0);
} finally {
dst.close();
}
});
});
// ---------------------------------------------------------------------------
// 6. api export endpoint
// ---------------------------------------------------------------------------
describe("GET /export endpoint", () => {
let t: TestDb;
let s: Seed;
let url: string;
let close: () => Promise<void>;
const basic = (email: string) =>
"Basic " + Buffer.from(`${email}:password`).toString("base64");
before(async () => {
t = await createTestDb();
s = await seed(t.db);
const app = new Koa();
const router = apirouter(t.db);
app.use(router.routes());
app.use(router.allowedMethods());
const server = createServer(app.callback());
await new Promise<void>((r) => server.listen(0, "127.0.0.1", r));
const { port } = server.address() as AddressInfo;
url = `http://127.0.0.1:${port}`;
close = () =>
new Promise<void>((resolve, reject) =>
server.close((err) => (err ? reject(err) : resolve())),
);
});
after(async () => {
await close();
t.close();
});
it("global admin exports everything", async () => {
const res = await fetch(`${url}/export`, {
headers: { Authorization: basic(s.admin.email) },
});
assert.equal(res.status, 200);
const lines = parseLines(await res.text());
const abodeAids = new Set(
lines.filter((l) => l.kind === "abode").map((l) => l.data.aid),
);
assert.ok(abodeAids.has(s.aid1));
assert.ok(abodeAids.has(s.aid2));
const userUids = new Set(
lines.filter((l) => l.kind === "user").map((l) => l.data.uid),
);
assert.ok(userUids.has(s.admin.uid));
assert.ok(userUids.has(s.normal.uid));
});
it("non-admin is force-scoped even when requesting wider abodes", async () => {
const res = await fetch(`${url}/export?abodes=${s.aid1},${s.aid2}`, {
headers: { Authorization: basic(s.normal.email) },
});
assert.equal(res.status, 200);
const lines = parseLines(await res.text());
const abodeAids = new Set(
lines.filter((l) => l.kind === "abode").map((l) => l.data.aid),
);
assert.ok(abodeAids.has(s.aid1));
assert.ok(!abodeAids.has(s.aid2), "aid2 forced out of scope");
const userUids = new Set(
lines.filter((l) => l.kind === "user").map((l) => l.data.uid),
);
assert.ok(userUids.has(s.normal.uid));
assert.ok(userUids.has(s.co.uid));
assert.ok(!userUids.has(s.admin.uid), "admin not a co-resident of aid1");
// apikeys are self-only: the caller's own key is exported, but a
// co-resident's key metadata is NOT, even though their user record is.
const apikeyUids = lines
.filter((l) => l.kind === "apikey")
.map((l) => l.data.uid);
assert.deepEqual(new Set(apikeyUids), new Set([s.normal.uid]));
assert.ok(
!apikeyUids.includes(s.co.uid),
"co-resident apikey metadata must not leak",
);
// The meta line records the *effective* (narrowed) filter.
const meta = lines.find((l) => l.kind === "meta");
assert.ok(meta);
assert.deepEqual(meta!.data.filter.abodes, [s.aid1]);
});
it("stops server-side querying shortly after the client aborts", async () => {
// Dedicated db/server. Rows are deliberately large (and inserted in bulk so
// the seed stays fast) so the users table alone can't fit in the socket
// buffer — the export generator backpressures mid-`user` and never reaches
// later tables while the client is still holding the connection open.
const big = await createTestDb();
let server: ReturnType<typeof createServer> | undefined;
try {
const pw = await hashPassword("password");
await big.db.createUser({
email: "a@test.example",
name: "A",
password: pw,
flags: { admin: true },
});
const { sql, db: raw } = big.db._;
// ~32MB of user rows — far more than any socket/kernel buffer can hold,
// so the generator is guaranteed to still be suspended mid-`user` (never
// having queried a later table) when the client aborts.
const bigName = "x".repeat(16000);
raw.multi(() => {
for (let i = 0; i < 2000; i++) {
raw.run(sql`
INSERT INTO "users"("uid", "email", "name", "flags")
VALUES(
${{ uuid: crypto.randomUUID() }},
${{ text: `b-${i}@test.example` }},
${{ text: bigName }},
${{ jsonb: {} }}
)
`);
}
});
let allCalls = 0;
const orig = big.wrapped.all.bind(big.wrapped);
(big.wrapped as { all: unknown }).all = (stmt: never) => {
allCalls++;
return orig(stmt);
};
const app = new Koa();
app.on("error", () => {}); // swallow the expected ECONNRESET on abort
const router = apirouter(big.db);
app.use(router.routes());
app.use(router.allowedMethods());
server = createServer(app.callback());
await new Promise<void>((r) => server!.listen(0, "127.0.0.1", r));
const { port } = server.address() as AddressInfo;
const ac = new AbortController();
const res = await fetch(`http://127.0.0.1:${port}/export`, {
headers: { Authorization: basic("a@test.example") },
signal: ac.signal,
});
const reader = res.body!.getReader();
await reader.read(); // first chunk — server has begun streaming users
const callsWhileStreaming = allCalls;
assert.ok(callsWhileStreaming >= 1, "server queried the first table");
ac.abort();
await reader.cancel().catch(() => {});
await new Promise((r) => setTimeout(r, 250));
const settled = allCalls;
await new Promise((r) => setTimeout(r, 250));
assert.equal(allCalls, settled, "no further queries after abort");
assert.ok(allCalls < 5, "did not materialise all five tables");
} finally {
if (server) await new Promise<void>((r) => server!.close(() => r()));
big.close();
}
});
});
// ---------------------------------------------------------------------------
// 7. inspect utility
// ---------------------------------------------------------------------------
describe("inspectExportStream", () => {
it("tallies counts and reads meta without a db", async () => {
const src = await createTestDb();
try {
await seed(src.db);
const ndjson = await streamToString(src.db.export());
const { counts, meta } = await inspectExportStream(
Readable.from([ndjson]),
);
assert.equal(meta?.v, 1);
assert.equal(meta?.source, "sqlite");
assert.ok((counts.user ?? 0) >= 3);
assert.ok((counts.abode ?? 0) >= 2);
assert.ok((counts.note ?? 0) >= 2);
} finally {
src.close();
}
});
it("stopAfterKinds short-circuits once every requested kind is seen", async () => {
const src = await createTestDb();
try {
await seed(src.db);
const ndjson = await streamToString(src.db.export());
const { counts } = await inspectExportStream(Readable.from([ndjson]), {
stopAfterKinds: ["user"],
});
assert.ok((counts.user ?? 0) >= 1);
// stopped as soon as the first user was seen, before later kinds.
assert.equal(counts.note ?? 0, 0);
} finally {
src.close();
}
});
});
// ---------------------------------------------------------------------------
// filter unit checks
// ---------------------------------------------------------------------------
describe("filter helpers", () => {
it("kindAllowed respects kinds/excludeKinds", () => {
assert.equal(kindAllowed({ kinds: ["abode"] }, "abode"), true);
assert.equal(kindAllowed({ kinds: ["abode"] }, "user"), false);
assert.equal(kindAllowed({ excludeKinds: ["note"] }, "note"), false);
assert.equal(kindAllowed(undefined, "note"), true);
});
it("recordAllowed scopes by aid/uid per kind", () => {
assert.equal(
recordAllowed({ abodes: ["a1"] }, "abode", { aid: "a1" }),
true,
);
assert.equal(
recordAllowed({ abodes: ["a1"] }, "abode", { aid: "a2" }),
false,
);
assert.equal(
recordAllowed({ users: ["u1"] }, "apikey", { uid: "u1" }),
true,
);
assert.equal(
recordAllowed({ users: ["u1"] }, "apikey", { uid: "u2" }),
false,
);
// abodes allowlist does not constrain user records
assert.equal(
recordAllowed({ abodes: ["a1"] }, "user", { uid: "u9" }),
true,
);
});
});