feat: add export/import to the postgres backend; import auto-detects backend
CI / format (pull_request) Successful in 23s
CI / lint (pull_request) Successful in 23s
CI / install-and-build (pull_request) Successful in 45s
CI / typecheck-source (pull_request) Successful in 25s
CI / typecheck-tests (pull_request) Successful in 30s
CI / test (pull_request) Successful in 40s
CI / format (pull_request) Successful in 23s
CI / lint (pull_request) Successful in 23s
CI / install-and-build (pull_request) Successful in 45s
CI / typecheck-source (pull_request) Successful in 25s
CI / typecheck-tests (pull_request) Successful in 30s
CI / test (pull_request) Successful in 40s
The export/import plan predated the postgres backend. Bring it up to parity: - PostgresInterface implements Exportable + Importable, mirroring the sqlite backend. Export is a signal-checked async generator (one query per table); import drives a transaction via `WrappedPool.multi`, which rolls back on any error/abort and commits only after the whole stream is consumed cleanly. - The `note` kind is skipped on postgres (its note CRUD is still unimplemented, so a pg database holds none) — a full dump from sqlite imports its user/abode/resident/apikey records and drops notes. - Unlike sqlite (PRAGMA foreign_keys=off), postgres keeps FK enforcement; the FK-safe insertion order keeps a full dump valid, and truly-dangling partial dumps will (correctly) fail. - abode-import now resolves the backend via getDbInterface instead of constructing SqliteInterface directly; isImportable keeps it from ever running over the remote (api) interface, which has no import. - Add isImportable(); make postgres selectClientApikeys' where optional. - Tests: exercise the real PostgresInterface export/import paths against an in-memory fake WrappedPgClient (no pg service in CI) — NDJSON shape, meta.source, filtering, note-skip, insert dispatch, and error/abort rollback. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -43,8 +43,21 @@ import type {
|
||||
UpdateNote,
|
||||
} from "../types/Note.js";
|
||||
import type { WrappedPgClient } from "./pool.js";
|
||||
import { Readable } from "node:stream";
|
||||
import readline from "node:readline";
|
||||
import type {
|
||||
Exportable,
|
||||
ExportKind,
|
||||
ExportOptions,
|
||||
Importable,
|
||||
ImportOptions,
|
||||
ImportResult,
|
||||
} from "../types/ExportImport.js";
|
||||
import { isExportKind, kindAllowed, recordAllowed } from "../export/filter.js";
|
||||
|
||||
export class PostgresInterface implements BackendDbInterface {
|
||||
export class PostgresInterface
|
||||
implements BackendDbInterface, Exportable, Importable
|
||||
{
|
||||
#db: WrappedPgClient;
|
||||
|
||||
constructor(db: WrappedPgClient) {
|
||||
@@ -508,4 +521,197 @@ export class PostgresInterface implements BackendDbInterface {
|
||||
async listNotesByUserId(_uid: string): Promise<PartialNote[]> {
|
||||
throw new Error("Unimplemented");
|
||||
}
|
||||
|
||||
export(options: ExportOptions = {}): NodeJS.ReadableStream {
|
||||
const { filter, signal } = options;
|
||||
const db = this.#db;
|
||||
const source = this.name;
|
||||
|
||||
// `note` is omitted: the postgres backend has no note CRUD yet, so a pg
|
||||
// database can hold none. Each `load()` is a single query, run lazily and
|
||||
// skipped once the destination aborts.
|
||||
const tables: [
|
||||
ExportKind,
|
||||
() => Promise<{ uid?: string; aid?: string }[]>,
|
||||
][] = [
|
||||
["user", () => selectClientUsers(db)],
|
||||
["abode", () => selectAbodes(db)],
|
||||
["resident", () => selectResidents(db)],
|
||||
["apikey", () => selectClientApikeys(db)],
|
||||
];
|
||||
|
||||
async function* generate(): AsyncGenerator<string> {
|
||||
if (signal?.aborted) return;
|
||||
yield JSON.stringify({
|
||||
kind: "meta",
|
||||
data: {
|
||||
v: 1,
|
||||
exportedAt: new Date().toISOString(),
|
||||
source,
|
||||
filter: filter ?? {},
|
||||
},
|
||||
}) + "\n";
|
||||
try {
|
||||
for (const [kind, load] of tables) {
|
||||
if (signal?.aborted) return;
|
||||
if (!kindAllowed(filter, kind)) continue;
|
||||
for (const row of await load()) {
|
||||
if (signal?.aborted) return;
|
||||
if (recordAllowed(filter, kind, row)) {
|
||||
yield JSON.stringify({ kind, data: row }) + "\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
if (signal?.aborted) return;
|
||||
yield JSON.stringify({
|
||||
kind: "error",
|
||||
data: {
|
||||
message: e instanceof Error ? e.message : String(e),
|
||||
code: e instanceof Error ? e.name : undefined,
|
||||
},
|
||||
}) + "\n";
|
||||
}
|
||||
}
|
||||
|
||||
return Readable.from(generate());
|
||||
}
|
||||
|
||||
async import(
|
||||
source: NodeJS.ReadableStream,
|
||||
options: ImportOptions = {},
|
||||
): Promise<ImportResult> {
|
||||
this.#checkReadonly();
|
||||
const { filter, signal } = options;
|
||||
const counts: Partial<Record<ExportKind, number>> = {};
|
||||
|
||||
const rl = readline.createInterface({
|
||||
input: source,
|
||||
crlfDelay: Infinity,
|
||||
signal,
|
||||
});
|
||||
|
||||
// Postgres holds FK enforcement (unlike sqlite, which we toggle off): the
|
||||
// FK-safe insertion order — user, abode, resident, apikey — keeps a full
|
||||
// dump valid. `notes` are skipped entirely (unsupported on this backend).
|
||||
// The transaction commits only if the whole stream is consumed cleanly; an
|
||||
// error/abort rolls it back via `multi`.
|
||||
try {
|
||||
await this.#db.rethrow(() =>
|
||||
this.#db.multi(async (tx) => {
|
||||
for await (const raw of rl) {
|
||||
signal?.throwIfAborted();
|
||||
const line = raw.trim();
|
||||
if (!line) continue;
|
||||
let parsed: { kind?: unknown; data?: unknown };
|
||||
try {
|
||||
parsed = JSON.parse(line);
|
||||
} catch {
|
||||
throw new InvalidAbodeError();
|
||||
}
|
||||
if (parsed.kind === "meta") continue;
|
||||
if (parsed.kind === "error") {
|
||||
throw new Error(
|
||||
`export stream reported an error: ${
|
||||
(parsed.data as { message?: string })?.message ?? "unknown"
|
||||
}`,
|
||||
);
|
||||
}
|
||||
if (!isExportKind(parsed.kind)) continue;
|
||||
if (parsed.kind === "note") continue; // unsupported on postgres
|
||||
if (!kindAllowed(filter, parsed.kind)) continue;
|
||||
const data = parsed.data as { uid?: string; aid?: string };
|
||||
if (!recordAllowed(filter, parsed.kind, data)) continue;
|
||||
await this.#importRecord(tx, parsed.kind, parsed.data);
|
||||
counts[parsed.kind] = (counts[parsed.kind] ?? 0) + 1;
|
||||
}
|
||||
// An abort while blocked on the source closes readline without
|
||||
// throwing, so re-check before the transaction commits.
|
||||
signal?.throwIfAborted();
|
||||
}),
|
||||
);
|
||||
} finally {
|
||||
rl.close();
|
||||
}
|
||||
|
||||
return { counts };
|
||||
}
|
||||
|
||||
async #importRecord(
|
||||
tx: WrappedPgClient,
|
||||
kind: ExportKind,
|
||||
data: unknown,
|
||||
): Promise<void> {
|
||||
switch (kind) {
|
||||
case "user": {
|
||||
const u = data as ClientUser;
|
||||
// `password` is never exported; imported users land on the schema
|
||||
// default ('#unset') and must reset before they can log in.
|
||||
await tx.run(sql`
|
||||
INSERT INTO "users"("uid", "email", "name", "flags", "created_at", "updated_at")
|
||||
VALUES(
|
||||
${{ uuid: u.uid }},
|
||||
${{ text: u.email }},
|
||||
${{ text: u.name }},
|
||||
${{ jsonb: u.flags }},
|
||||
${{ date: u.created_at }},
|
||||
${{ date: u.updated_at }}
|
||||
)
|
||||
`);
|
||||
break;
|
||||
}
|
||||
case "abode": {
|
||||
const a = data as Abode;
|
||||
await tx.run(sql`
|
||||
INSERT INTO "abodes"("aid", "name", "created_at", "created_by", "updated_at", "updated_by")
|
||||
VALUES(
|
||||
${{ uuid: a.aid }},
|
||||
${{ text: a.name }},
|
||||
${{ date: a.created_at }},
|
||||
${a.created_by ? { uuid: a.created_by } : { null: true }},
|
||||
${{ date: a.updated_at }},
|
||||
${a.updated_by ? { uuid: a.updated_by } : { null: true }}
|
||||
)
|
||||
`);
|
||||
break;
|
||||
}
|
||||
case "resident": {
|
||||
const r = data as Resident;
|
||||
await tx.run(sql`
|
||||
INSERT INTO "residents"("uid", "aid", "flags", "created_at", "created_by", "updated_at", "updated_by")
|
||||
VALUES(
|
||||
${{ uuid: r.uid }},
|
||||
${{ uuid: r.aid }},
|
||||
${{ jsonb: r.flags }},
|
||||
${{ date: r.created_at }},
|
||||
${r.created_by ? { uuid: r.created_by } : { null: true }},
|
||||
${{ date: r.updated_at }},
|
||||
${r.updated_by ? { uuid: r.updated_by } : { null: true }}
|
||||
)
|
||||
`);
|
||||
break;
|
||||
}
|
||||
case "apikey": {
|
||||
const k = data as ClientApikey;
|
||||
// `token` is never exported; mint a fresh unique one so the record's
|
||||
// metadata (kid/permissions/expiry) survives even though the original
|
||||
// secret cannot.
|
||||
await tx.run(sql`
|
||||
INSERT INTO "apikeys"("uid", "kid", "token", "name", "permissions", "created_at", "expires_at")
|
||||
VALUES(
|
||||
${{ uuid: k.uid }},
|
||||
${{ uuid: k.kid }},
|
||||
${{ text: createApikeyToken() }},
|
||||
${{ text: k.name }},
|
||||
${{ jsonb: k.permissions }},
|
||||
${{ date: k.created_at }},
|
||||
${k.expires_at ? { date: k.expires_at } : { null: true }}
|
||||
)
|
||||
`);
|
||||
break;
|
||||
}
|
||||
case "note":
|
||||
break; // unsupported on postgres; skipped before reaching here
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -130,10 +130,10 @@ export async function selectClientApikey(
|
||||
}
|
||||
export async function selectClientApikeys(
|
||||
db: WrappedPgClient,
|
||||
where: SqlCode,
|
||||
where?: SqlCode,
|
||||
): Promise<ClientApikey[]> {
|
||||
const rows = await db.all<RawClientApikey>(
|
||||
sql`${sqlClientApikey} WHERE ${where}`,
|
||||
where ? sql`${sqlClientApikey} WHERE ${where}` : sqlClientApikey,
|
||||
);
|
||||
return rows.map(pgToClientApikey);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user