abode/test/tools/export-import.test.ts

import { describe, it, before, after } from "node:test";
import assert from "node:assert/strict";
import readline from "node:readline";
import { Readable } from "node:stream";
import { createServer } from "node:http";
import type { AddressInfo } from "node:net";
import Koa from "koa";
import { createTestDb, type TestDb } from "../helpers/sqlite.js";
import { apirouter } from "../../src/webapi/apirouter.js";
import { computeForcedExportFilter } from "../../src/webapi/exportScope.js";
import {
  intersectExportFilters,
  recordAllowed,
  kindAllowed,
} from "../../src/db/export/filter.js";
import { inspectExportStream } from "../../src/db/export/inspect.js";
import { hashPassword } from "../../src/util/hash.js";
import type { ClientUser } from "../../src/db/types/User.js";
import {
  EXPORT_KIND_ORDER,
  type ExportKind,
} from "../../src/db/types/ExportImport.js";

/**
 * Assert the record lines of a parsed export are grouped in FK-safe
 * EXPORT_KIND_ORDER: the leading line is `meta`, and every record kind's
 * position in the order is non-decreasing down the stream.
 */
function assertFkSafeOrder(lines: { kind: string }[]): void {
  assert.equal(lines[0]?.kind, "meta", "first line is meta");
  const rank = (k: string) => EXPORT_KIND_ORDER.indexOf(k as ExportKind);
  let last = -1;
  for (const { kind } of lines.slice(1)) {
    if (kind === "error") continue;
    const r = rank(kind);
    assert.notEqual(r, -1, `unexpected kind ${kind}`);
    assert.ok(
      r >= last,
      `kind ${kind} (order ${r}) appears after a later kind (order ${last})`,
    );
    last = r;
  }
}

// ---------------------------------------------------------------------------
// helpers
// ---------------------------------------------------------------------------

async function streamToString(s: NodeJS.ReadableStream): Promise<string> {
  let out = "";
  for await (const chunk of s) out += chunk;
  return out;
}

function parseLines(ndjson: string): { kind: string; data: any }[] {
  return ndjson
    .split("\n")
    .filter(Boolean)
    .map((l) => JSON.parse(l));
}

/** Normalise `*_at` fields to epoch ms so formatting never breaks equality. */
function norm(obj: Record<string, unknown>): Record<string, unknown> {
  const out: Record<string, unknown> = {};
  for (const [k, v] of Object.entries(obj)) {
    out[k] = k.endsWith("_at")
      ? v == null
        ? null
        : new Date(v as string).getTime()
      : v;
  }
  return out;
}
function normSorted(
  arr: Record<string, unknown>[],
  key: (x: any) => string,
): Record<string, unknown>[] {
  return arr.map(norm).sort((a, b) => key(a).localeCompare(key(b)));
}

interface Seed {
  admin: ClientUser;
  normal: ClientUser;
  co: ClientUser;
  aid1: string;
  aid2: string;
  nid1: string;
  nid2: string;
}

async function seed(db: TestDb["db"]): Promise<Seed> {
  const pw = await hashPassword("password");
  const admin = await db.createUser({
    email: "admin@test.example",
    name: "Admin",
    password: pw,
    flags: { admin: true },
  });
  const normal = await db.createUser({
    email: "normal@test.example",
    name: "Normal",
    password: pw,
    flags: {},
  });
  const co = await db.createUser({
    email: "co@test.example",
    name: "Co Resident",
    password: pw,
    flags: {},
  });
  const abode1 = await db.createAbode(
    { name: "Abode One" },
    { uid: admin.uid },
  );
  const abode2 = await db.createAbode(
    { name: "Abode Two" },
    { uid: admin.uid },
  );
  await db.createResident(
    { uid: normal.uid, aid: abode1.aid, flags: {} },
    { uid: admin.uid },
  );
  await db.createResident(
    { uid: co.uid, aid: abode1.aid, flags: {} },
    { uid: admin.uid },
  );
  await db.createResident(
    { uid: admin.uid, aid: abode2.aid, flags: { admin: true } },
    { uid: admin.uid },
  );
  await db.createApikey({
    uid: normal.uid,
    name: "normal key",
    permissions: {},
    expires_at: null,
  });
  await db.createApikey({
    uid: co.uid,
    name: "co key",
    permissions: {},
    expires_at: null,
  });
  const note1 = await db.createNote(
    {
      aid: abode1.aid,
      name: "Note One",
      content: "hello",
      properties: { type: "note" },
    },
    { uid: normal.uid },
  );
  const note2 = await db.createNote(
    {
      aid: abode2.aid,
      name: "Note Two",
      content: "world",
      properties: { type: "note" },
    },
    { uid: admin.uid },
  );
  return {
    admin,
    normal,
    co,
    aid1: abode1.aid,
    aid2: abode2.aid,
    nid1: note1.nid,
    nid2: note2.nid,
  };
}

// ---------------------------------------------------------------------------
// 0. wire-format ordering invariant
// ---------------------------------------------------------------------------

describe("export ordering (FK-safe wire contract)", () => {
  it("EXPORT_KIND_ORDER is the FK-safe dependency order", () => {
    // Change-detector: reordering is a breaking change to the format and must
    // keep every kind after the kinds it references (see ExportImport.ts).
    assert.deepEqual(EXPORT_KIND_ORDER, [
      "user",
      "abode",
      "resident",
      "apikey",
      "note",
    ]);
  });

  it("sqlite export emits record kinds grouped in EXPORT_KIND_ORDER", async () => {
    const src = await createTestDb();
    try {
      await seed(src.db);
      const lines = parseLines(await streamToString(src.db.export()));
      // every kind must be present so the ordering is actually exercised
      const kinds = new Set(lines.map((l) => l.kind));
      for (const k of EXPORT_KIND_ORDER) {
        assert.ok(kinds.has(k), `stream contains ${k}`);
      }
      assertFkSafeOrder(lines);
    } finally {
      src.close();
    }
  });
});

// ---------------------------------------------------------------------------
// 1. round-trip
// ---------------------------------------------------------------------------

describe("export/import: sqlite -> sqlite round-trip", () => {
  it("reproduces users, abodes, residents, apikeys, notes", async () => {
    const src = await createTestDb();
    const dst = await createTestDb();
    try {
      const s = await seed(src.db);

      const result = await dst.db.import(src.db.export());
      assert.ok(result.counts.user && result.counts.user >= 3);
      assert.ok(result.counts.note && result.counts.note >= 2);

      assert.deepEqual(
        normSorted(await dst.db.listUsers(), (u) => u.uid),
        normSorted(await src.db.listUsers(), (u) => u.uid),
      );
      assert.deepEqual(
        normSorted(await dst.db.listAbodes(), (a) => a.aid),
        normSorted(await src.db.listAbodes(), (a) => a.aid),
      );
      assert.deepEqual(
        normSorted(await dst.db.listResidents(), (r) => r.uid + r.aid),
        normSorted(await src.db.listResidents(), (r) => r.uid + r.aid),
      );
      // apikeys: token is regenerated on import, so the ClientApikey view
      // (which omits token) must still match exactly.
      assert.deepEqual(
        normSorted(await dst.db.listApikeysByUser(s.normal.uid), (k) => k.kid),
        normSorted(await src.db.listApikeysByUser(s.normal.uid), (k) => k.kid),
      );
      // notes (full, with content)
      const srcNote = await src.db.getNoteById(s.nid1);
      const dstNote = await dst.db.getNoteById(s.nid1);
      assert.deepEqual(norm(dstNote), norm(srcNote));
    } finally {
      src.close();
      dst.close();
    }
  });
});

// ---------------------------------------------------------------------------
// 2. filter narrowing
// ---------------------------------------------------------------------------

describe("export/import: filter narrowing", () => {
  it("scopes abode/resident/note to the requested abodes", async () => {
    const src = await createTestDb();
    try {
      const s = await seed(src.db);
      const ndjson = await streamToString(
        src.db.export({ filter: { abodes: [s.aid1] } }),
      );
      const lines = parseLines(ndjson);

      const abodeAids = lines
        .filter((l) => l.kind === "abode")
        .map((l) => l.data.aid);
      assert.deepEqual(abodeAids, [s.aid1]);

      const noteAids = new Set(
        lines.filter((l) => l.kind === "note").map((l) => l.data.aid),
      );
      assert.ok(noteAids.has(s.aid1));
      assert.ok(!noteAids.has(s.aid2));

      const residentAids = new Set(
        lines.filter((l) => l.kind === "resident").map((l) => l.data.aid),
      );
      assert.ok(!residentAids.has(s.aid2));
    } finally {
      src.close();
    }
  });

  it("imports the narrowed dump into a fresh db with the same scope", async () => {
    const src = await createTestDb();
    const dst = await createTestDb();
    try {
      const s = await seed(src.db);
      await dst.db.import(src.db.export({ filter: { abodes: [s.aid1] } }));

      const abodes = await dst.db.listAbodes();
      assert.deepEqual(
        abodes.map((a) => a.aid),
        [s.aid1],
      );
      const notes = await dst.db.listNotesByAbodeId(s.aid1);
      assert.equal(notes.length, 1);
      assert.equal((await dst.db.listNotesByAbodeId(s.aid2)).length, 0);
    } finally {
      src.close();
      dst.close();
    }
  });

  it("kinds filter selects only the requested kinds", async () => {
    const src = await createTestDb();
    try {
      await seed(src.db);
      const ndjson = await streamToString(
        src.db.export({ filter: { kinds: ["abode"] } }),
      );
      const kinds = new Set(parseLines(ndjson).map((l) => l.kind));
      assert.ok(kinds.has("abode"));
      assert.ok(!kinds.has("user"));
      assert.ok(!kinds.has("note"));
    } finally {
      src.close();
    }
  });
});

// ---------------------------------------------------------------------------
// 3. forced-filter enforcement
// ---------------------------------------------------------------------------

describe("exportScope: computeForcedExportFilter", () => {
  it("returns null for a global admin on a basic/session credential", async () => {
    const t = await createTestDb();
    try {
      const s = await seed(t.db);
      const forced = await computeForcedExportFilter(t.db, {
        user: s.admin,
        session: { source: "basic" },
      });
      assert.equal(forced, null);
    } finally {
      t.close();
    }
  });

  it("returns null for a global admin with an unrestricted apikey", async () => {
    const t = await createTestDb();
    try {
      const s = await seed(t.db);
      const forced = await computeForcedExportFilter(t.db, {
        user: s.admin,
        session: {
          source: "apikey",
          key: {
            kid: "k",
            uid: s.admin.uid,
            name: "k",
            permissions: { admin: true, all: true },
            created_at: new Date().toISOString(),
            expires_at: null,
          },
        },
      });
      assert.equal(forced, null);
    } finally {
      t.close();
    }
  });

  it("forces a non-admin to their abodes + co-residents", async () => {
    const t = await createTestDb();
    try {
      const s = await seed(t.db);
      const forced = await computeForcedExportFilter(t.db, {
        user: s.normal,
        session: { source: "basic" },
      });
      assert.ok(forced);
      assert.deepEqual(forced!.abodes, [s.aid1]);
      assert.deepEqual(
        new Set(forced!.users),
        new Set([s.normal.uid, s.co.uid]),
      );
      assert.ok(!forced!.users!.includes(s.admin.uid));
      // apikeys are self-only, even though co is a co-resident whose user
      // record is exported for referential integrity.
      assert.deepEqual(forced!.apikeys, [s.normal.uid]);

      // A caller requesting a wider abode never gets it: intersection, not union.
      const effective = intersectExportFilters(
        { abodes: [s.aid1, s.aid2] },
        forced,
      );
      assert.deepEqual(effective.abodes, [s.aid1]);
      assert.ok(!effective.abodes!.includes(s.aid2));
    } finally {
      t.close();
    }
  });

  it("intersects a non-admin apikey with restrict_abodes", async () => {
    const t = await createTestDb();
    try {
      const s = await seed(t.db);
      const forced = await computeForcedExportFilter(t.db, {
        user: s.normal,
        session: {
          source: "apikey",
          key: {
            kid: "k",
            uid: s.normal.uid,
            name: "k",
            permissions: { restrict_abodes: [s.aid2] },
            created_at: new Date().toISOString(),
            expires_at: null,
          },
        },
      });
      // normal resides only in aid1; restrict to aid2 => empty intersection.
      assert.deepEqual(forced!.abodes, []);
    } finally {
      t.close();
    }
  });
});

// ---------------------------------------------------------------------------
// 4. cancellation, export side
// ---------------------------------------------------------------------------

describe("export cancellation", () => {
  it("stops querying tables after the destination aborts", async () => {
    const t = await createTestDb();
    try {
      const pw = await hashPassword("password");
      // Enough users that the first table can't be buffered in one go, so the
      // generator backpressures mid-`user` and never reaches later tables.
      for (let i = 0; i < 400; i++) {
        await t.db.createUser({
          email: `bulk-${i}@test.example`,
          name: `Bulk ${i}`,
          password: pw,
          flags: {},
        });
      }

      let allCalls = 0;
      const orig = t.wrapped.all.bind(t.wrapped);
      (t.wrapped as { all: unknown }).all = (stmt: never) => {
        allCalls++;
        return orig(stmt);
      };

      const ac = new AbortController();
      const stream = t.db.export({ signal: ac.signal });
      const rl = readline.createInterface({ input: stream });

      let lines = 0;
      let callsAtAbort = -1;
      for await (const _ of rl) {
        lines++;
        if (lines === 3) {
          ac.abort();
          callsAtAbort = allCalls;
        }
      }

      assert.ok(callsAtAbort >= 1, "at least the first table was queried");
      assert.equal(allCalls, callsAtAbort, "no further queries after abort");
      assert.ok(allCalls < 5, "did not materialise all five tables");
    } finally {
      t.close();
    }
  });
});

// ---------------------------------------------------------------------------
// 5. cancellation, import side
// ---------------------------------------------------------------------------

describe("import cancellation", () => {
  it("rolls back and leaves no dangling transaction when the source errors", async () => {
    const dst = await createTestDb();
    try {
      function line(kind: string, data: unknown): string {
        return JSON.stringify({ kind, data }) + "\n";
      }
      const now = new Date().toISOString();
      const mkUser = (i: number) => ({
        uid: crypto.randomUUID(),
        email: `imp-${i}@test.example`,
        name: `Imp ${i}`,
        flags: {},
        created_at: now,
        updated_at: now,
      });

      // Emits a couple of valid user records, then throws mid-stream.
      const source = Readable.from(
        (async function* () {
          yield line("meta", { v: 1 });
          yield line("user", mkUser(1));
          yield line("user", mkUser(2));
          throw new Error("source exploded");
        })(),
      );

      await assert.rejects(() => dst.db.import(source), /source exploded/);

      // ROLLBACK happened: nothing persisted.
      assert.equal((await dst.db.listUsers()).length, 0);

      // No lingering open transaction: a follow-up write succeeds immediately.
      const pw = await hashPassword("password");
      const created = await dst.db.createUser({
        email: "after@test.example",
        name: "After",
        password: pw,
        flags: {},
      });
      assert.ok(created.uid);
    } finally {
      dst.close();
    }
  });

  it("aborts cleanly via signal and commits nothing", async () => {
    const dst = await createTestDb();
    try {
      const ac = new AbortController();
      const now = new Date().toISOString();
      const source = Readable.from(
        (async function* () {
          yield JSON.stringify({ kind: "meta", data: { v: 1 } }) + "\n";
          yield JSON.stringify({
            kind: "user",
            data: {
              uid: crypto.randomUUID(),
              email: "abort@test.example",
              name: "Abort",
              flags: {},
              created_at: now,
              updated_at: now,
            },
          }) + "\n";
          ac.abort();
          // Keep the stream alive so abort — not EOF — ends the import.
          await new Promise((r) => setTimeout(r, 1000));
        })(),
      );

      await assert.rejects(() => dst.db.import(source, { signal: ac.signal }));
      assert.equal((await dst.db.listUsers()).length, 0);
    } finally {
      dst.close();
    }
  });
});

// ---------------------------------------------------------------------------
// 6. api export endpoint
// ---------------------------------------------------------------------------

describe("GET /export endpoint", () => {
  let t: TestDb;
  let s: Seed;
  let url: string;
  let close: () => Promise<void>;

  const basic = (email: string) =>
    "Basic " + Buffer.from(`${email}:password`).toString("base64");

  before(async () => {
    t = await createTestDb();
    s = await seed(t.db);
    const app = new Koa();
    const router = apirouter(t.db);
    app.use(router.routes());
    app.use(router.allowedMethods());
    const server = createServer(app.callback());
    await new Promise<void>((r) => server.listen(0, "127.0.0.1", r));
    const { port } = server.address() as AddressInfo;
    url = `http://127.0.0.1:${port}`;
    close = () =>
      new Promise<void>((resolve, reject) =>
        server.close((err) => (err ? reject(err) : resolve())),
      );
  });

  after(async () => {
    await close();
    t.close();
  });

  it("global admin exports everything", async () => {
    const res = await fetch(`${url}/export`, {
      headers: { Authorization: basic(s.admin.email) },
    });
    assert.equal(res.status, 200);
    const lines = parseLines(await res.text());
    const abodeAids = new Set(
      lines.filter((l) => l.kind === "abode").map((l) => l.data.aid),
    );
    assert.ok(abodeAids.has(s.aid1));
    assert.ok(abodeAids.has(s.aid2));
    const userUids = new Set(
      lines.filter((l) => l.kind === "user").map((l) => l.data.uid),
    );
    assert.ok(userUids.has(s.admin.uid));
    assert.ok(userUids.has(s.normal.uid));
  });

  it("non-admin is force-scoped even when requesting wider abodes", async () => {
    const res = await fetch(`${url}/export?abodes=${s.aid1},${s.aid2}`, {
      headers: { Authorization: basic(s.normal.email) },
    });
    assert.equal(res.status, 200);
    const lines = parseLines(await res.text());

    const abodeAids = new Set(
      lines.filter((l) => l.kind === "abode").map((l) => l.data.aid),
    );
    assert.ok(abodeAids.has(s.aid1));
    assert.ok(!abodeAids.has(s.aid2), "aid2 forced out of scope");

    const userUids = new Set(
      lines.filter((l) => l.kind === "user").map((l) => l.data.uid),
    );
    assert.ok(userUids.has(s.normal.uid));
    assert.ok(userUids.has(s.co.uid));
    assert.ok(!userUids.has(s.admin.uid), "admin not a co-resident of aid1");

    // apikeys are self-only: the caller's own key is exported, but a
    // co-resident's key metadata is NOT, even though their user record is.
    const apikeyUids = lines
      .filter((l) => l.kind === "apikey")
      .map((l) => l.data.uid);
    assert.deepEqual(new Set(apikeyUids), new Set([s.normal.uid]));
    assert.ok(
      !apikeyUids.includes(s.co.uid),
      "co-resident apikey metadata must not leak",
    );

    // The meta line records the *effective* (narrowed) filter.
    const meta = lines.find((l) => l.kind === "meta");
    assert.ok(meta);
    assert.deepEqual(meta!.data.filter.abodes, [s.aid1]);
  });

  it("stops server-side querying shortly after the client aborts", async () => {
    // Dedicated db/server. Rows are deliberately large (and inserted in bulk so
    // the seed stays fast) so the users table alone can't fit in the socket
    // buffer — the export generator backpressures mid-`user` and never reaches
    // later tables while the client is still holding the connection open.
    const big = await createTestDb();
    let server: ReturnType<typeof createServer> | undefined;
    try {
      const pw = await hashPassword("password");
      await big.db.createUser({
        email: "a@test.example",
        name: "A",
        password: pw,
        flags: { admin: true },
      });
      const { sql, db: raw } = big.db._;
      // ~32MB of user rows — far more than any socket/kernel buffer can hold,
      // so the generator is guaranteed to still be suspended mid-`user` (never
      // having queried a later table) when the client aborts.
      const bigName = "x".repeat(16000);
      raw.multi(() => {
        for (let i = 0; i < 2000; i++) {
          raw.run(sql`
            INSERT INTO "users"("uid", "email", "name", "flags")
            VALUES(
              ${{ uuid: crypto.randomUUID() }},
              ${{ text: `b-${i}@test.example` }},
              ${{ text: bigName }},
              ${{ jsonb: {} }}
            )
          `);
        }
      });

      let allCalls = 0;
      const orig = big.wrapped.all.bind(big.wrapped);
      (big.wrapped as { all: unknown }).all = (stmt: never) => {
        allCalls++;
        return orig(stmt);
      };

      const app = new Koa();
      app.on("error", () => {}); // swallow the expected ECONNRESET on abort
      const router = apirouter(big.db);
      app.use(router.routes());
      app.use(router.allowedMethods());
      server = createServer(app.callback());
      await new Promise<void>((r) => server!.listen(0, "127.0.0.1", r));
      const { port } = server.address() as AddressInfo;

      const ac = new AbortController();
      const res = await fetch(`http://127.0.0.1:${port}/export`, {
        headers: { Authorization: basic("a@test.example") },
        signal: ac.signal,
      });
      const reader = res.body!.getReader();
      await reader.read(); // first chunk — server has begun streaming users
      const callsWhileStreaming = allCalls;
      assert.ok(callsWhileStreaming >= 1, "server queried the first table");
      ac.abort();
      await reader.cancel().catch(() => {});

      await new Promise((r) => setTimeout(r, 250));
      const settled = allCalls;
      await new Promise((r) => setTimeout(r, 250));
      assert.equal(allCalls, settled, "no further queries after abort");
      assert.ok(allCalls < 5, "did not materialise all five tables");
    } finally {
      if (server) await new Promise<void>((r) => server!.close(() => r()));
      big.close();
    }
  });
});

// ---------------------------------------------------------------------------
// 7. inspect utility
// ---------------------------------------------------------------------------

describe("inspectExportStream", () => {
  it("tallies counts and reads meta without a db", async () => {
    const src = await createTestDb();
    try {
      await seed(src.db);
      const ndjson = await streamToString(src.db.export());
      const { counts, meta } = await inspectExportStream(
        Readable.from([ndjson]),
      );
      assert.equal(meta?.v, 1);
      assert.equal(meta?.source, "sqlite");
      assert.ok((counts.user ?? 0) >= 3);
      assert.ok((counts.abode ?? 0) >= 2);
      assert.ok((counts.note ?? 0) >= 2);
    } finally {
      src.close();
    }
  });

  it("stopAfterKinds short-circuits once every requested kind is seen", async () => {
    const src = await createTestDb();
    try {
      await seed(src.db);
      const ndjson = await streamToString(src.db.export());
      const { counts } = await inspectExportStream(Readable.from([ndjson]), {
        stopAfterKinds: ["user"],
      });
      assert.ok((counts.user ?? 0) >= 1);
      // stopped as soon as the first user was seen, before later kinds.
      assert.equal(counts.note ?? 0, 0);
    } finally {
      src.close();
    }
  });
});

// ---------------------------------------------------------------------------
// filter unit checks
// ---------------------------------------------------------------------------

describe("filter helpers", () => {
  it("kindAllowed respects kinds/excludeKinds", () => {
    assert.equal(kindAllowed({ kinds: ["abode"] }, "abode"), true);
    assert.equal(kindAllowed({ kinds: ["abode"] }, "user"), false);
    assert.equal(kindAllowed({ excludeKinds: ["note"] }, "note"), false);
    assert.equal(kindAllowed(undefined, "note"), true);
  });

  it("recordAllowed scopes by aid/uid per kind", () => {
    assert.equal(
      recordAllowed({ abodes: ["a1"] }, "abode", { aid: "a1" }),
      true,
    );
    assert.equal(
      recordAllowed({ abodes: ["a1"] }, "abode", { aid: "a2" }),
      false,
    );
    assert.equal(
      recordAllowed({ users: ["u1"] }, "apikey", { uid: "u1" }),
      true,
    );
    assert.equal(
      recordAllowed({ users: ["u1"] }, "apikey", { uid: "u2" }),
      false,
    );
    // abodes allowlist does not constrain user records
    assert.equal(
      recordAllowed({ abodes: ["a1"] }, "user", { uid: "u9" }),
      true,
    );
  });
});