feat: add export/import streaming to the pluggable backends
CI / format (pull_request) Successful in 28s
CI / lint (pull_request) Successful in 28s
CI / install-and-build (pull_request) Successful in 53s
CI / typecheck-tests (pull_request) Successful in 28s
CI / typecheck-source (pull_request) Successful in 32s
CI / test (pull_request) Successful in 43s
CI / format (pull_request) Successful in 28s
CI / lint (pull_request) Successful in 28s
CI / install-and-build (pull_request) Successful in 53s
CI / typecheck-tests (pull_request) Successful in 28s
CI / typecheck-source (pull_request) Successful in 32s
CI / test (pull_request) Successful in 43s
Add backend-agnostic data export/import over an NDJSON wire format, plus an
inspect utility, exposed via three new CLIs and an HTTP export endpoint.
- ExportImport types + filter helpers (kind/record scoping, hard-intersection
of filters) in src/db/{types/ExportImport,export/filter}.ts
- SqliteInterface implements Exportable + Importable: signal-checked async
generator export (one query per table, per-record yield, trailing error
sentinel on mid-stream failure) and a manually-driven import transaction
that rolls back on any error/abort and never commits partial data
- ApiInterface implements Exportable via its own fetch({signal})
- computeForcedExportFilter enforces non-global-admin scope (resided-in abodes
+ co-resident users, intersected with apikey restrict_*); GET /export
intersects it with the caller's filter and wires an AbortController to the
response socket
- inspectExportStream reports kinds/counts from any stream without a db
- abode-export / abode-import / abode-inspect CLIs (import is sqlite-only)
- Secrets are not exported: imported users default to '#unset' passwords and
apikeys are re-minted a token (ClientApikey view round-trips exactly)
- test/tools/export-import.test.ts: round-trip, filter narrowing, forced-scope,
export/import cancellation, in-process Koa endpoint, inspect
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,106 @@
|
||||
import { createReadStream } from "node:fs";
|
||||
import { getWrappedDb } from "../db/sqlite/impl/index.js";
|
||||
import { SqliteInterface } from "../db/sqlite/SqliteInterface.js";
|
||||
import { parseSqliteUrl } from "../db/sqlite/url.js";
|
||||
import { isExportKind } from "../db/export/filter.js";
|
||||
import type { ExportFilter } from "../db/types/ExportImport.js";
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
|
||||
function printUsage(err: boolean | string = false): never {
|
||||
const log = (err ? console.error : console.log).bind(console);
|
||||
if (typeof err === "string") {
|
||||
log(`Error: ${err}`);
|
||||
log("");
|
||||
}
|
||||
log("Usage:");
|
||||
log("\tabode-import --help");
|
||||
log("\tabode-import <sqlite-database-url> <input-file|-> [--kinds=...] \\");
|
||||
log(
|
||||
"\t [--exclude-kinds=...] [--abodes=aid,...] [--users=uid,...]",
|
||||
);
|
||||
log("");
|
||||
log(
|
||||
"The target database must already be migrated (run abode-migrate first).",
|
||||
);
|
||||
process.exit(err ? 1 : 0);
|
||||
}
|
||||
|
||||
if (["-h", "--help", "help"].some((x) => args.includes(x))) printUsage();
|
||||
|
||||
const positional: string[] = [];
|
||||
const flags = new Map<string, string>();
|
||||
for (const arg of args) {
|
||||
if (arg.startsWith("--")) {
|
||||
const eq = arg.indexOf("=");
|
||||
if (eq === -1) printUsage(`invalid flag: ${arg}`);
|
||||
flags.set(arg.slice(2, eq), arg.slice(eq + 1));
|
||||
} else {
|
||||
positional.push(arg);
|
||||
}
|
||||
}
|
||||
|
||||
const url = positional[0];
|
||||
const input = positional[1];
|
||||
if (!url) printUsage("missing <sqlite-database-url>");
|
||||
if (!input) printUsage("missing <input-file|->");
|
||||
if (positional.length > 2) printUsage("too many arguments");
|
||||
|
||||
const knownFlags = ["kinds", "exclude-kinds", "abodes", "users"];
|
||||
for (const key of flags.keys()) {
|
||||
if (!knownFlags.includes(key)) printUsage(`unknown flag: --${key}`);
|
||||
}
|
||||
|
||||
function parseList(value: string | undefined): string[] | undefined {
|
||||
if (value === undefined) return undefined;
|
||||
return value.split(",").filter(Boolean);
|
||||
}
|
||||
function parseKinds(value: string | undefined) {
|
||||
const list = parseList(value);
|
||||
if (!list) return undefined;
|
||||
const bad = list.filter((k) => !isExportKind(k));
|
||||
if (bad.length) printUsage(`invalid kind(s): ${bad.join(", ")}`);
|
||||
return list.filter(isExportKind);
|
||||
}
|
||||
|
||||
const filter: ExportFilter = {};
|
||||
const kinds = parseKinds(flags.get("kinds"));
|
||||
if (kinds) filter.kinds = kinds;
|
||||
const excludeKinds = parseKinds(flags.get("exclude-kinds"));
|
||||
if (excludeKinds) filter.excludeKinds = excludeKinds;
|
||||
const abodes = parseList(flags.get("abodes"));
|
||||
if (abodes) filter.abodes = abodes;
|
||||
const users = parseList(flags.get("users"));
|
||||
if (users) filter.users = users;
|
||||
|
||||
// Import is sqlite-only: construct the backend directly rather than resolving
|
||||
// generically, so it can never be pointed at a remote (api) target.
|
||||
const db = new SqliteInterface(getWrappedDb(...parseSqliteUrl(url)));
|
||||
|
||||
const source = input === "-" ? process.stdin : createReadStream(input);
|
||||
|
||||
const ac = new AbortController();
|
||||
const onSignal = () => ac.abort();
|
||||
process.on("SIGINT", onSignal);
|
||||
process.on("SIGTERM", onSignal);
|
||||
|
||||
try {
|
||||
const result = await db.import(source, { filter, signal: ac.signal });
|
||||
const total = Object.values(result.counts).reduce((a, b) => a + b, 0);
|
||||
console.log(`Imported ${total} record(s):`);
|
||||
for (const [kind, count] of Object.entries(result.counts)) {
|
||||
console.log(`- ${kind}: ${count}`);
|
||||
}
|
||||
} catch (e) {
|
||||
if (ac.signal.aborted) {
|
||||
console.error("Import aborted; no changes committed");
|
||||
process.exit(130);
|
||||
}
|
||||
throw e;
|
||||
} finally {
|
||||
process.off("SIGINT", onSignal);
|
||||
process.off("SIGTERM", onSignal);
|
||||
await db.close().catch(() => {});
|
||||
}
|
||||
|
||||
process.exit(0);
|
||||
Reference in New Issue
Block a user