feat: add export/import streaming to the pluggable backends
CI / format (pull_request) Successful in 28s
CI / lint (pull_request) Successful in 28s
CI / install-and-build (pull_request) Successful in 53s
CI / typecheck-tests (pull_request) Successful in 28s
CI / typecheck-source (pull_request) Successful in 32s
CI / test (pull_request) Successful in 43s

Add backend-agnostic data export/import over an NDJSON wire format, plus an
inspect utility, exposed via three new CLIs and an HTTP export endpoint.

- ExportImport types + filter helpers (kind/record scoping, hard-intersection
  of filters) in src/db/{types/ExportImport,export/filter}.ts
- SqliteInterface implements Exportable + Importable: signal-checked async
  generator export (one query per table, per-record yield, trailing error
  sentinel on mid-stream failure) and a manually-driven import transaction
  that rolls back on any error/abort and never commits partial data
- ApiInterface implements Exportable via its own fetch({signal})
- computeForcedExportFilter enforces non-global-admin scope (resided-in abodes
  + co-resident users, intersected with apikey restrict_*); GET /export
  intersects it with the caller's filter and wires an AbortController to the
  response socket
- inspectExportStream reports kinds/counts from any stream without a db
- abode-export / abode-import / abode-inspect CLIs (import is sqlite-only)
- Secrets are not exported: imported users default to '#unset' passwords and
  apikeys are re-minted a token (ClientApikey view round-trips exactly)
- test/tools/export-import.test.ts: round-trip, filter narrowing, forced-scope,
  export/import cancellation, in-process Koa endpoint, inspect

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-07-22 22:29:03 +00:00
co-authored by Claude
parent 31d4636dde
commit ccb970f200
14 changed files with 1586 additions and 5 deletions
+106
View File
@@ -0,0 +1,106 @@
import { createReadStream } from "node:fs";
import { getWrappedDb } from "../db/sqlite/impl/index.js";
import { SqliteInterface } from "../db/sqlite/SqliteInterface.js";
import { parseSqliteUrl } from "../db/sqlite/url.js";
import { isExportKind } from "../db/export/filter.js";
import type { ExportFilter } from "../db/types/ExportImport.js";
const args = process.argv.slice(2);
function printUsage(err: boolean | string = false): never {
const log = (err ? console.error : console.log).bind(console);
if (typeof err === "string") {
log(`Error: ${err}`);
log("");
}
log("Usage:");
log("\tabode-import --help");
log("\tabode-import <sqlite-database-url> <input-file|-> [--kinds=...] \\");
log(
"\t [--exclude-kinds=...] [--abodes=aid,...] [--users=uid,...]",
);
log("");
log(
"The target database must already be migrated (run abode-migrate first).",
);
process.exit(err ? 1 : 0);
}
if (["-h", "--help", "help"].some((x) => args.includes(x))) printUsage();
const positional: string[] = [];
const flags = new Map<string, string>();
for (const arg of args) {
if (arg.startsWith("--")) {
const eq = arg.indexOf("=");
if (eq === -1) printUsage(`invalid flag: ${arg}`);
flags.set(arg.slice(2, eq), arg.slice(eq + 1));
} else {
positional.push(arg);
}
}
const url = positional[0];
const input = positional[1];
if (!url) printUsage("missing <sqlite-database-url>");
if (!input) printUsage("missing <input-file|->");
if (positional.length > 2) printUsage("too many arguments");
const knownFlags = ["kinds", "exclude-kinds", "abodes", "users"];
for (const key of flags.keys()) {
if (!knownFlags.includes(key)) printUsage(`unknown flag: --${key}`);
}
function parseList(value: string | undefined): string[] | undefined {
if (value === undefined) return undefined;
return value.split(",").filter(Boolean);
}
function parseKinds(value: string | undefined) {
const list = parseList(value);
if (!list) return undefined;
const bad = list.filter((k) => !isExportKind(k));
if (bad.length) printUsage(`invalid kind(s): ${bad.join(", ")}`);
return list.filter(isExportKind);
}
const filter: ExportFilter = {};
const kinds = parseKinds(flags.get("kinds"));
if (kinds) filter.kinds = kinds;
const excludeKinds = parseKinds(flags.get("exclude-kinds"));
if (excludeKinds) filter.excludeKinds = excludeKinds;
const abodes = parseList(flags.get("abodes"));
if (abodes) filter.abodes = abodes;
const users = parseList(flags.get("users"));
if (users) filter.users = users;
// Import is sqlite-only: construct the backend directly rather than resolving
// generically, so it can never be pointed at a remote (api) target.
const db = new SqliteInterface(getWrappedDb(...parseSqliteUrl(url)));
const source = input === "-" ? process.stdin : createReadStream(input);
const ac = new AbortController();
const onSignal = () => ac.abort();
process.on("SIGINT", onSignal);
process.on("SIGTERM", onSignal);
try {
const result = await db.import(source, { filter, signal: ac.signal });
const total = Object.values(result.counts).reduce((a, b) => a + b, 0);
console.log(`Imported ${total} record(s):`);
for (const [kind, count] of Object.entries(result.counts)) {
console.log(`- ${kind}: ${count}`);
}
} catch (e) {
if (ac.signal.aborted) {
console.error("Import aborted; no changes committed");
process.exit(130);
}
throw e;
} finally {
process.off("SIGINT", onSignal);
process.off("SIGTERM", onSignal);
await db.close().catch(() => {});
}
process.exit(0);