feat: add export/import streaming to the pluggable backends
CI / format (pull_request) Successful in 28s
CI / lint (pull_request) Successful in 28s
CI / install-and-build (pull_request) Successful in 53s
CI / typecheck-tests (pull_request) Successful in 28s
CI / typecheck-source (pull_request) Successful in 32s
CI / test (pull_request) Successful in 43s

Add backend-agnostic data export/import over an NDJSON wire format, plus an
inspect utility, exposed via three new CLIs and an HTTP export endpoint.

- ExportImport types + filter helpers (kind/record scoping, hard-intersection
  of filters) in src/db/{types/ExportImport,export/filter}.ts
- SqliteInterface implements Exportable + Importable: signal-checked async
  generator export (one query per table, per-record yield, trailing error
  sentinel on mid-stream failure) and a manually-driven import transaction
  that rolls back on any error/abort and never commits partial data
- ApiInterface implements Exportable via its own fetch({signal})
- computeForcedExportFilter enforces non-global-admin scope (resided-in abodes
  + co-resident users, intersected with apikey restrict_*); GET /export
  intersects it with the caller's filter and wires an AbortController to the
  response socket
- inspectExportStream reports kinds/counts from any stream without a db
- abode-export / abode-import / abode-inspect CLIs (import is sqlite-only)
- Secrets are not exported: imported users default to '#unset' passwords and
  apikeys are re-minted a token (ClientApikey view round-trips exactly)
- test/tools/export-import.test.ts: round-trip, filter narrowing, forced-scope,
  export/import cancellation, in-process Koa endpoint, inspect

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-07-22 22:29:03 +00:00
co-authored by Claude
parent 31d4636dde
commit ccb970f200
14 changed files with 1586 additions and 5 deletions
+48
View File
@@ -16,6 +16,29 @@ import {
} from "../schema/validators.js";
import { authenticate } from "./middleware/authenticate.js";
import { InvalidAbodeError, NotFoundAbodeError } from "../db/types/errors.js";
import { isExportable } from "../db/types/ExportImport.js";
import type { ExportFilter, ExportKind } from "../db/types/ExportImport.js";
import { intersectExportFilters, isExportKind } from "../db/export/filter.js";
import { computeForcedExportFilter } from "./exportScope.js";
function parseExportFilter(query: Record<string, unknown>): ExportFilter {
const list = (v: unknown): string[] | undefined => {
if (typeof v !== "string" || !v) return undefined;
return v.split(",").filter(Boolean);
};
const kinds = (v: unknown): ExportKind[] | undefined =>
list(v)?.filter(isExportKind);
const filter: ExportFilter = {};
const k = kinds(query.kinds);
if (k) filter.kinds = k;
const ek = kinds(query.excludeKinds);
if (ek) filter.excludeKinds = ek;
const abodes = list(query.abodes);
if (abodes) filter.abodes = abodes;
const users = list(query.users);
if (users) filter.users = users;
return filter;
}
export function apirouter(db: BackendDbInterface): KoaRouter {
const router = new KoaRouter();
@@ -42,6 +65,31 @@ export function apirouter(db: BackendDbInterface): KoaRouter {
ctx.status = 204;
});
router.get("/export", authenticate(db), async (ctx) => {
// Everything that can throw a domain error runs *before* any byte is
// written, so `convertError` still applies. Once `ctx.body` is a stream,
// a mid-stream failure surfaces as a trailing `error` NDJSON line instead.
const forced = await computeForcedExportFilter(db, {
user: ctx.user!,
session: ctx.session!,
});
const effective = intersectExportFilters(
parseExportFilter(ctx.query),
forced,
);
if (!isExportable(db)) {
ctx.status = 501;
ctx.body = { ok: false, error: "export_unsupported" };
return;
}
const ac = new AbortController();
ctx.res.on("close", () => {
if (!ctx.res.writableEnded) ac.abort();
});
ctx.type = "application/x-ndjson";
ctx.body = db.export({ filter: effective, signal: ac.signal });
});
router.use("/users", authenticate(db));
router.get("/users", async (ctx) => {
ctx.body = await db.listUsers();