feat: add export/import streaming to the pluggable backends
CI / format (pull_request) Successful in 28s
CI / lint (pull_request) Successful in 28s
CI / install-and-build (pull_request) Successful in 53s
CI / typecheck-tests (pull_request) Successful in 28s
CI / typecheck-source (pull_request) Successful in 32s
CI / test (pull_request) Successful in 43s
CI / format (pull_request) Successful in 28s
CI / lint (pull_request) Successful in 28s
CI / install-and-build (pull_request) Successful in 53s
CI / typecheck-tests (pull_request) Successful in 28s
CI / typecheck-source (pull_request) Successful in 32s
CI / test (pull_request) Successful in 43s
Add backend-agnostic data export/import over an NDJSON wire format, plus an
inspect utility, exposed via three new CLIs and an HTTP export endpoint.
- ExportImport types + filter helpers (kind/record scoping, hard-intersection
of filters) in src/db/{types/ExportImport,export/filter}.ts
- SqliteInterface implements Exportable + Importable: signal-checked async
generator export (one query per table, per-record yield, trailing error
sentinel on mid-stream failure) and a manually-driven import transaction
that rolls back on any error/abort and never commits partial data
- ApiInterface implements Exportable via its own fetch({signal})
- computeForcedExportFilter enforces non-global-admin scope (resided-in abodes
+ co-resident users, intersected with apikey restrict_*); GET /export
intersects it with the caller's filter and wires an AbortController to the
response socket
- inspectExportStream reports kinds/counts from any stream without a db
- abode-export / abode-import / abode-inspect CLIs (import is sqlite-only)
- Secrets are not exported: imported users default to '#unset' passwords and
apikeys are re-minted a token (ClientApikey view round-trips exactly)
- test/tools/export-import.test.ts: round-trip, filter narrowing, forced-scope,
export/import cancellation, in-process Koa endpoint, inspect
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -16,6 +16,29 @@ import {
|
||||
} from "../schema/validators.js";
|
||||
import { authenticate } from "./middleware/authenticate.js";
|
||||
import { InvalidAbodeError, NotFoundAbodeError } from "../db/types/errors.js";
|
||||
import { isExportable } from "../db/types/ExportImport.js";
|
||||
import type { ExportFilter, ExportKind } from "../db/types/ExportImport.js";
|
||||
import { intersectExportFilters, isExportKind } from "../db/export/filter.js";
|
||||
import { computeForcedExportFilter } from "./exportScope.js";
|
||||
|
||||
function parseExportFilter(query: Record<string, unknown>): ExportFilter {
|
||||
const list = (v: unknown): string[] | undefined => {
|
||||
if (typeof v !== "string" || !v) return undefined;
|
||||
return v.split(",").filter(Boolean);
|
||||
};
|
||||
const kinds = (v: unknown): ExportKind[] | undefined =>
|
||||
list(v)?.filter(isExportKind);
|
||||
const filter: ExportFilter = {};
|
||||
const k = kinds(query.kinds);
|
||||
if (k) filter.kinds = k;
|
||||
const ek = kinds(query.excludeKinds);
|
||||
if (ek) filter.excludeKinds = ek;
|
||||
const abodes = list(query.abodes);
|
||||
if (abodes) filter.abodes = abodes;
|
||||
const users = list(query.users);
|
||||
if (users) filter.users = users;
|
||||
return filter;
|
||||
}
|
||||
|
||||
export function apirouter(db: BackendDbInterface): KoaRouter {
|
||||
const router = new KoaRouter();
|
||||
@@ -42,6 +65,31 @@ export function apirouter(db: BackendDbInterface): KoaRouter {
|
||||
ctx.status = 204;
|
||||
});
|
||||
|
||||
router.get("/export", authenticate(db), async (ctx) => {
|
||||
// Everything that can throw a domain error runs *before* any byte is
|
||||
// written, so `convertError` still applies. Once `ctx.body` is a stream,
|
||||
// a mid-stream failure surfaces as a trailing `error` NDJSON line instead.
|
||||
const forced = await computeForcedExportFilter(db, {
|
||||
user: ctx.user!,
|
||||
session: ctx.session!,
|
||||
});
|
||||
const effective = intersectExportFilters(
|
||||
parseExportFilter(ctx.query),
|
||||
forced,
|
||||
);
|
||||
if (!isExportable(db)) {
|
||||
ctx.status = 501;
|
||||
ctx.body = { ok: false, error: "export_unsupported" };
|
||||
return;
|
||||
}
|
||||
const ac = new AbortController();
|
||||
ctx.res.on("close", () => {
|
||||
if (!ctx.res.writableEnded) ac.abort();
|
||||
});
|
||||
ctx.type = "application/x-ndjson";
|
||||
ctx.body = db.export({ filter: effective, signal: ac.signal });
|
||||
});
|
||||
|
||||
router.use("/users", authenticate(db));
|
||||
router.get("/users", async (ctx) => {
|
||||
ctx.body = await db.listUsers();
|
||||
|
||||
Reference in New Issue
Block a user