Scaffold Pwned Passwords clone: React/wouter frontend, Koa/pg backend, import CLI
- Backend: Koa + @koa/router serving the HIBP-style /range/:prefix k-anonymity API, backed by Postgres via pg, with migrate/seed scripts run through tsx. - Frontend: React + wouter, bundled with webpack (dev server + prod build), thin client that hashes passwords locally and only sends the hash prefix. - Import package: CLI to bulk-load a wordlist or hash list into the database, plus a separate script to pull the full dataset from the HIBP range API. Both support parallelism (worker threads for wordlist hashing, concurrency limiter for DB/HTTP fan-out). - Dev tooling: docker compose for a local Postgres, .env.local/.env.example, DATABASE_URL support as an alternative to discrete PG* vars.
This commit is contained in:
@@ -0,0 +1,12 @@
|
||||
# Either set DATABASE_URL to point at an externally hosted database...
|
||||
# DATABASE_URL=postgres://pwned:pwned@some-host:5432/pwned
|
||||
# ...or leave it unset and use the discrete vars below (used by compose.yaml
|
||||
# for local dev):
|
||||
PGHOST=localhost
|
||||
PGPORT=5432
|
||||
PGUSER=pwned
|
||||
PGPASSWORD=pwned
|
||||
PGDATABASE=pwned
|
||||
|
||||
BACKEND_PORT=3001
|
||||
FRONTEND_PORT=3000
|
||||
@@ -0,0 +1,6 @@
|
||||
node_modules
|
||||
packages/*/dist
|
||||
packages/*/out
|
||||
.env.local
|
||||
*.log
|
||||
/wordlists
|
||||
@@ -0,0 +1,15 @@
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17-alpine
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
POSTGRES_USER: ${PGUSER:-pwned}
|
||||
POSTGRES_PASSWORD: ${PGPASSWORD:-pwned}
|
||||
POSTGRES_DB: ${PGDATABASE:-pwned}
|
||||
ports:
|
||||
- "${PGPORT:-5432}:5432"
|
||||
volumes:
|
||||
- pwned-postgres-data:/var/lib/postgresql/data
|
||||
|
||||
volumes:
|
||||
pwned-postgres-data:
|
||||
Generated
+6420
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"name": "pwned",
|
||||
"version": "0.1.0",
|
||||
"description": "A simple validator that checks if a password is known to be pwned",
|
||||
"private": true,
|
||||
"workspaces": [
|
||||
"packages/*"
|
||||
],
|
||||
"scripts": {
|
||||
"dev": "concurrently -n backend,frontend -c blue,green \"npm:dev -w backend\" \"npm:dev -w frontend\"",
|
||||
"build": "npm run build -w backend && npm run build -w frontend",
|
||||
"db:up": "docker compose --env-file .env.local up -d",
|
||||
"db:down": "docker compose --env-file .env.local down",
|
||||
"db:seed": "npm run seed -w backend",
|
||||
"import": "npm run start -w import --",
|
||||
"import:hibp": "npm run import-hibp -w import --"
|
||||
},
|
||||
"devDependencies": {
|
||||
"concurrently": "^10.0.3",
|
||||
"tsx": "^4.22.4",
|
||||
"typescript": "^6.0.3"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"name": "backend",
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"type": "module",
|
||||
"main": "dist/index.js",
|
||||
"scripts": {
|
||||
"dev": "tsx watch --env-file=../../.env.local src/index.ts",
|
||||
"build": "tsc -p tsconfig.json",
|
||||
"start": "node --env-file=../../.env.local dist/index.js",
|
||||
"migrate": "tsx --env-file=../../.env.local src/migrate.ts",
|
||||
"seed": "tsx --env-file=../../.env.local src/seed.ts",
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
},
|
||||
"keywords": [],
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"@koa/router": "^15.6.0",
|
||||
"koa": "^3.2.1",
|
||||
"pg": "^8.22.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/koa": "^3.0.3",
|
||||
"@types/koa__router": "^12.0.5",
|
||||
"@types/pg": "^8.20.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
import { Pool } from "pg";
|
||||
|
||||
// DATABASE_URL (e.g. postgres://user:pass@host:5432/db) takes precedence over
|
||||
// the discrete PG* vars, for connecting to an externally hosted database.
|
||||
export const pool = new Pool(
|
||||
process.env.DATABASE_URL
|
||||
? { connectionString: process.env.DATABASE_URL }
|
||||
: {
|
||||
host: process.env.PGHOST,
|
||||
port: process.env.PGPORT ? Number(process.env.PGPORT) : undefined,
|
||||
user: process.env.PGUSER,
|
||||
password: process.env.PGPASSWORD,
|
||||
database: process.env.PGDATABASE,
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,23 @@
|
||||
import Koa from "koa";
|
||||
import { rangeRouter } from "./routes/range.ts";
|
||||
|
||||
const app = new Koa();
|
||||
|
||||
app.use(async (ctx, next) => {
|
||||
try {
|
||||
await next();
|
||||
} catch (err) {
|
||||
ctx.status = 500;
|
||||
ctx.body = { error: "internal server error" };
|
||||
console.error(err);
|
||||
}
|
||||
});
|
||||
|
||||
app.use(rangeRouter.routes());
|
||||
app.use(rangeRouter.allowedMethods());
|
||||
|
||||
const port = process.env.BACKEND_PORT ? Number(process.env.BACKEND_PORT) : 3001;
|
||||
|
||||
app.listen(port, () => {
|
||||
console.log(`backend listening on http://localhost:${port}`);
|
||||
});
|
||||
@@ -0,0 +1,10 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { pool } from "./db.ts";
|
||||
|
||||
const schemaPath = fileURLToPath(new URL("./schema.sql", import.meta.url));
|
||||
const schema = readFileSync(schemaPath, "utf8");
|
||||
|
||||
await pool.query(schema);
|
||||
console.log("migration applied");
|
||||
await pool.end();
|
||||
@@ -0,0 +1,26 @@
|
||||
import Router from "@koa/router";
|
||||
import { pool } from "../db.ts";
|
||||
|
||||
const HEX_PREFIX = /^[0-9A-Fa-f]{5}$/;
|
||||
|
||||
export const rangeRouter = new Router();
|
||||
|
||||
// Mirrors the HaveIBeenPwned Pwned Passwords range API:
|
||||
// https://haveibeenpwned.com/API/v3#PwnedPasswords
|
||||
rangeRouter.get("/range/:prefix", async (ctx) => {
|
||||
const { prefix } = ctx.params;
|
||||
|
||||
if (!HEX_PREFIX.test(prefix)) {
|
||||
ctx.status = 400;
|
||||
ctx.body = { error: "prefix must be 5 hex characters" };
|
||||
return;
|
||||
}
|
||||
|
||||
const { rows } = await pool.query<{ suffix: string; count: string }>(
|
||||
"SELECT suffix, count FROM pwned_passwords WHERE prefix = $1",
|
||||
[prefix.toUpperCase()],
|
||||
);
|
||||
|
||||
ctx.type = "text/plain";
|
||||
ctx.body = rows.map((row) => `${row.suffix}:${row.count}`).join("\r\n");
|
||||
});
|
||||
@@ -0,0 +1,8 @@
|
||||
CREATE TABLE IF NOT EXISTS pwned_passwords (
|
||||
prefix CHAR(5) NOT NULL,
|
||||
suffix CHAR(35) NOT NULL,
|
||||
count BIGINT NOT NULL DEFAULT 1,
|
||||
PRIMARY KEY (prefix, suffix)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS pwned_passwords_prefix_idx ON pwned_passwords (prefix);
|
||||
@@ -0,0 +1,34 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { pool } from "./db.ts";
|
||||
|
||||
// A handful of well-known weak passwords, seeded for local testing.
|
||||
// The real HIBP dataset has ~900M entries; this is just enough to exercise the API.
|
||||
const SAMPLE_PASSWORDS: Record<string, number> = {
|
||||
"123456": 30000000,
|
||||
password: 10000000,
|
||||
"123456789": 9000000,
|
||||
qwerty: 4000000,
|
||||
letmein: 500000,
|
||||
admin: 400000,
|
||||
welcome: 300000,
|
||||
"iloveyou": 250000,
|
||||
};
|
||||
|
||||
function sha1(value: string): string {
|
||||
return createHash("sha1").update(value).digest("hex").toUpperCase();
|
||||
}
|
||||
|
||||
for (const [password, count] of Object.entries(SAMPLE_PASSWORDS)) {
|
||||
const hash = sha1(password);
|
||||
const prefix = hash.slice(0, 5);
|
||||
const suffix = hash.slice(5);
|
||||
await pool.query(
|
||||
`INSERT INTO pwned_passwords (prefix, suffix, count)
|
||||
VALUES ($1, $2, $3)
|
||||
ON CONFLICT (prefix, suffix) DO UPDATE SET count = EXCLUDED.count`,
|
||||
[prefix, suffix, count],
|
||||
);
|
||||
console.log(`seeded "${password}" -> ${hash} (${count})`);
|
||||
}
|
||||
|
||||
await pool.end();
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2023",
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"lib": ["ES2023"],
|
||||
"outDir": "dist",
|
||||
"rootDir": "src",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"allowImportingTsExtensions": false,
|
||||
"rewriteRelativeImportExtensions": true,
|
||||
"resolveJsonModule": true,
|
||||
"declaration": false,
|
||||
"sourceMap": true
|
||||
},
|
||||
"include": ["src"]
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"name": "frontend",
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"dev": "webpack serve --config webpack.config.js --mode development",
|
||||
"build": "webpack --config webpack.config.js --mode production",
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
},
|
||||
"keywords": [],
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"react": "^19.2.7",
|
||||
"react-dom": "^19.2.7",
|
||||
"wouter": "^3.10.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/react": "^19.2.17",
|
||||
"@types/react-dom": "^19.2.3",
|
||||
"html-webpack-plugin": "^5.6.7",
|
||||
"ts-loader": "^9.6.2",
|
||||
"typescript": "^6.0.3",
|
||||
"webpack": "^5.108.3",
|
||||
"webpack-cli": "^7.1.0",
|
||||
"webpack-dev-server": "^5.2.5"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>Pwned Passwords</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,17 @@
|
||||
import { Link } from "wouter";
|
||||
|
||||
export function AboutPage() {
|
||||
return (
|
||||
<div>
|
||||
<h1>About</h1>
|
||||
<p>
|
||||
This is a small clone of the{" "}
|
||||
<a href="https://haveibeenpwned.com/API/v3#PwnedPasswords" target="_blank" rel="noreferrer">
|
||||
HaveIBeenPwned Pwned Passwords API
|
||||
</a>
|
||||
, built for local development and testing.
|
||||
</p>
|
||||
<Link href="/">Back to checker</Link>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
import { Link, Route, Switch } from "wouter";
|
||||
import { CheckerPage } from "./CheckerPage";
|
||||
import { AboutPage } from "./AboutPage";
|
||||
|
||||
export function App() {
|
||||
return (
|
||||
<div>
|
||||
<nav>
|
||||
<Link href="/">Checker</Link> | <Link href="/about">About</Link>
|
||||
</nav>
|
||||
<Switch>
|
||||
<Route path="/" component={CheckerPage} />
|
||||
<Route path="/about" component={AboutPage} />
|
||||
<Route>404 - not found</Route>
|
||||
</Switch>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
import { useState } from "react";
|
||||
import { checkPassword, type PwnedResult } from "./api";
|
||||
|
||||
export function CheckerPage() {
|
||||
const [password, setPassword] = useState("");
|
||||
const [result, setResult] = useState<PwnedResult | null>(null);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [loading, setLoading] = useState(false);
|
||||
|
||||
async function handleSubmit(event: React.FormEvent) {
|
||||
event.preventDefault();
|
||||
setLoading(true);
|
||||
setError(null);
|
||||
setResult(null);
|
||||
try {
|
||||
setResult(await checkPassword(password));
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : String(err));
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<div>
|
||||
<h1>Pwned Passwords checker</h1>
|
||||
<p>Your password never leaves the browser: only the first 5 characters of its SHA-1 hash are sent to the server.</p>
|
||||
<form onSubmit={handleSubmit}>
|
||||
<input
|
||||
type="password"
|
||||
value={password}
|
||||
onChange={(event) => setPassword(event.target.value)}
|
||||
placeholder="Enter a password to check"
|
||||
/>
|
||||
<button type="submit" disabled={loading || password.length === 0}>
|
||||
{loading ? "Checking..." : "Check"}
|
||||
</button>
|
||||
</form>
|
||||
{error && <p style={{ color: "red" }}>{error}</p>}
|
||||
{result && result.pwned && (
|
||||
<p style={{ color: "red" }}>
|
||||
This password has been seen {result.count.toLocaleString()} times in data breaches.
|
||||
</p>
|
||||
)}
|
||||
{result && !result.pwned && <p style={{ color: "green" }}>This password was not found in the dataset.</p>}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
export interface PwnedResult {
|
||||
pwned: boolean;
|
||||
count: number;
|
||||
}
|
||||
|
||||
async function sha1(value: string): Promise<string> {
|
||||
const bytes = new TextEncoder().encode(value);
|
||||
const digest = await crypto.subtle.digest("SHA-1", bytes);
|
||||
return Array.from(new Uint8Array(digest))
|
||||
.map((byte) => byte.toString(16).padStart(2, "0"))
|
||||
.join("")
|
||||
.toUpperCase();
|
||||
}
|
||||
|
||||
// Implements the k-anonymity model used by HaveIBeenPwned: only the first 5
|
||||
// characters of the password hash are ever sent to the server.
|
||||
export async function checkPassword(password: string): Promise<PwnedResult> {
|
||||
const hash = await sha1(password);
|
||||
const prefix = hash.slice(0, 5);
|
||||
const suffix = hash.slice(5);
|
||||
|
||||
const response = await fetch(`/range/${prefix}`);
|
||||
if (!response.ok) {
|
||||
throw new Error(`range request failed: ${response.status}`);
|
||||
}
|
||||
|
||||
const body = await response.text();
|
||||
for (const line of body.split(/\r?\n/)) {
|
||||
if (!line) continue;
|
||||
const [lineSuffix, count] = line.split(":");
|
||||
if (lineSuffix === suffix) {
|
||||
return { pwned: true, count: Number(count) };
|
||||
}
|
||||
}
|
||||
|
||||
return { pwned: false, count: 0 };
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
import { StrictMode } from "react";
|
||||
import { createRoot } from "react-dom/client";
|
||||
import { App } from "./App";
|
||||
|
||||
const container = document.getElementById("root");
|
||||
if (!container) {
|
||||
throw new Error("missing #root element");
|
||||
}
|
||||
|
||||
createRoot(container).render(
|
||||
<StrictMode>
|
||||
<App />
|
||||
</StrictMode>,
|
||||
);
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "Bundler",
|
||||
"lib": ["ES2022", "DOM", "DOM.Iterable"],
|
||||
"jsx": "react-jsx",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"resolveJsonModule": true,
|
||||
"isolatedModules": true
|
||||
},
|
||||
"include": ["src"]
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
const path = require("node:path");
|
||||
const HtmlWebpackPlugin = require("html-webpack-plugin");
|
||||
|
||||
const backendPort = process.env.BACKEND_PORT || 3001;
|
||||
const frontendPort = process.env.FRONTEND_PORT || 3000;
|
||||
|
||||
module.exports = (_env, argv) => {
|
||||
const isDev = argv.mode !== "production";
|
||||
|
||||
return {
|
||||
entry: "./src/index.tsx",
|
||||
devtool: isDev ? "eval-source-map" : "source-map",
|
||||
output: {
|
||||
path: path.resolve(__dirname, "dist"),
|
||||
filename: "bundle.[contenthash].js",
|
||||
clean: true,
|
||||
},
|
||||
resolve: {
|
||||
extensions: [".tsx", ".ts", ".js"],
|
||||
},
|
||||
module: {
|
||||
rules: [
|
||||
{
|
||||
test: /\.tsx?$/,
|
||||
use: "ts-loader",
|
||||
exclude: /node_modules/,
|
||||
},
|
||||
],
|
||||
},
|
||||
plugins: [
|
||||
new HtmlWebpackPlugin({
|
||||
template: "./public/index.html",
|
||||
}),
|
||||
],
|
||||
devServer: {
|
||||
port: frontendPort,
|
||||
historyApiFallback: true,
|
||||
hot: true,
|
||||
proxy: [
|
||||
{
|
||||
context: ["/range"],
|
||||
target: `http://localhost:${backendPort}`,
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"name": "import",
|
||||
"version": "1.0.0",
|
||||
"description": "Bulk-loads a wordlist or hash list into the pwned_passwords table",
|
||||
"type": "module",
|
||||
"main": "dist/index.js",
|
||||
"scripts": {
|
||||
"start": "tsx --env-file=../../.env.local src/index.ts",
|
||||
"import-hibp": "tsx --env-file=../../.env.local src/importHibp.ts",
|
||||
"build": "tsc -p tsconfig.json",
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
},
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"pg": "^8.22.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/pg": "^8.20.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
// Bounds how many async tasks run at once. Used both for fanning out
|
||||
// concurrent DB upserts and concurrent HTTP requests.
|
||||
export function createLimiter(concurrency: number) {
|
||||
let active = 0;
|
||||
const queue: Array<() => void> = [];
|
||||
|
||||
function release() {
|
||||
active--;
|
||||
queue.shift()?.();
|
||||
}
|
||||
|
||||
return async function limit<T>(fn: () => Promise<T>): Promise<T> {
|
||||
if (active >= concurrency) {
|
||||
await new Promise<void>((resolve) => queue.push(resolve));
|
||||
}
|
||||
active++;
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
release();
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import { Pool } from "pg";
|
||||
|
||||
// Concurrent imports open several connections at once; bump via PGPOOL_MAX
|
||||
// if running with a high --jobs/--concurrency value.
|
||||
const max = process.env.PGPOOL_MAX ? Number(process.env.PGPOOL_MAX) : 20;
|
||||
|
||||
// DATABASE_URL (e.g. postgres://user:pass@host:5432/db) takes precedence over
|
||||
// the discrete PG* vars, for connecting to an externally hosted database.
|
||||
export const pool = new Pool(
|
||||
process.env.DATABASE_URL
|
||||
? { connectionString: process.env.DATABASE_URL, max }
|
||||
: {
|
||||
host: process.env.PGHOST,
|
||||
port: process.env.PGPORT ? Number(process.env.PGPORT) : undefined,
|
||||
user: process.env.PGUSER,
|
||||
password: process.env.PGPASSWORD,
|
||||
database: process.env.PGDATABASE,
|
||||
max,
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,15 @@
|
||||
import { parentPort } from "node:worker_threads";
|
||||
import { createHash } from "node:crypto";
|
||||
import type { Entry } from "./upsertBatch.ts";
|
||||
|
||||
function sha1(value: string): string {
|
||||
return createHash("sha1").update(value).digest("hex").toUpperCase();
|
||||
}
|
||||
|
||||
parentPort?.on("message", (lines: string[]) => {
|
||||
const entries: Entry[] = lines.map((line) => {
|
||||
const hash = sha1(line);
|
||||
return { prefix: hash.slice(0, 5), suffix: hash.slice(5), count: 1 };
|
||||
});
|
||||
parentPort!.postMessage(entries);
|
||||
});
|
||||
@@ -0,0 +1,112 @@
|
||||
import { parseArgs } from "node:util";
|
||||
import { pool } from "./db.ts";
|
||||
import { upsertBatch, type Entry } from "./upsertBatch.ts";
|
||||
import { createLimiter } from "./concurrencyLimit.ts";
|
||||
|
||||
// Mirrors the HaveIBeenPwned Pwned Passwords range API on the way in:
|
||||
// https://haveibeenpwned.com/API/v3#PwnedPasswords
|
||||
// There's no bulk file for anonymous use, so this walks all 16^5 = 1,048,576
|
||||
// 5-hex-character prefixes and pulls each range individually.
|
||||
const HIBP_RANGE_URL = "https://api.pwnedpasswords.com/range/";
|
||||
const PREFIX_COUNT = 0x100000; // 16^5
|
||||
const MAX_RETRIES = 5;
|
||||
|
||||
function usage(): never {
|
||||
console.error(
|
||||
`Usage: npm run import-hibp -w import -- [options]\n\n` +
|
||||
`Options:\n` +
|
||||
` --concurrency, -c parallel HTTP requests to api.pwnedpasswords.com (default 16)\n` +
|
||||
` --start starting prefix index, for resuming (default 0)\n` +
|
||||
` --end ending prefix index, exclusive (default ${PREFIX_COUNT})\n`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
function prefixFromIndex(index: number): string {
|
||||
return index.toString(16).toUpperCase().padStart(5, "0");
|
||||
}
|
||||
|
||||
function parseRangeBody(prefix: string, body: string): Entry[] {
|
||||
const entries: Entry[] = [];
|
||||
for (const line of body.split(/\r?\n/)) {
|
||||
if (!line) continue;
|
||||
const [suffix, count] = line.split(":");
|
||||
entries.push({ prefix, suffix, count: Number(count) });
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
async function fetchRange(prefix: string): Promise<Entry[]> {
|
||||
for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
|
||||
const response = await fetch(`${HIBP_RANGE_URL}${prefix}`, {
|
||||
headers: { "Add-Padding": "false" },
|
||||
});
|
||||
|
||||
if (response.ok) {
|
||||
return parseRangeBody(prefix, await response.text());
|
||||
}
|
||||
|
||||
if (response.status === 429) {
|
||||
const retryAfterSeconds = Number(response.headers.get("retry-after")) || 2 ** attempt;
|
||||
console.warn(`rate limited on ${prefix}, retrying in ${retryAfterSeconds}s`);
|
||||
await new Promise((resolve) => setTimeout(resolve, retryAfterSeconds * 1000));
|
||||
continue;
|
||||
}
|
||||
|
||||
throw new Error(`unexpected status ${response.status} for prefix ${prefix}`);
|
||||
}
|
||||
|
||||
throw new Error(`giving up on prefix ${prefix} after ${MAX_RETRIES} retries`);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const { values } = parseArgs({
|
||||
options: {
|
||||
concurrency: { type: "string", short: "c", default: "16" },
|
||||
start: { type: "string", default: "0" },
|
||||
end: { type: "string" },
|
||||
},
|
||||
});
|
||||
|
||||
const concurrency = Number(values.concurrency);
|
||||
const start = Number(values.start);
|
||||
const end = values.end ? Number(values.end) : PREFIX_COUNT;
|
||||
if (!Number.isInteger(start) || !Number.isInteger(end) || start < 0 || end > PREFIX_COUNT || start >= end) {
|
||||
usage();
|
||||
}
|
||||
|
||||
const limit = createLimiter(concurrency);
|
||||
let completed = 0;
|
||||
const total = end - start;
|
||||
const startedAt = Date.now();
|
||||
|
||||
const tasks = [];
|
||||
for (let i = start; i < end; i++) {
|
||||
const prefix = prefixFromIndex(i);
|
||||
tasks.push(
|
||||
limit(async () => {
|
||||
const entries = await fetchRange(prefix);
|
||||
await upsertBatch(pool, entries);
|
||||
completed++;
|
||||
if (completed % 1000 === 0) {
|
||||
const elapsedSeconds = (Date.now() - startedAt) / 1000;
|
||||
const rate = completed / elapsedSeconds;
|
||||
const etaMinutes = (total - completed) / rate / 60;
|
||||
console.log(
|
||||
`${completed.toLocaleString()}/${total.toLocaleString()} prefixes ` +
|
||||
`(${rate.toFixed(0)}/s, ~${etaMinutes.toFixed(1)}m remaining)`,
|
||||
);
|
||||
}
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
await Promise.all(tasks);
|
||||
console.log(`done: imported ${total.toLocaleString()} prefixes`);
|
||||
await pool.end();
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
@@ -0,0 +1,155 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { availableParallelism } from "node:os";
|
||||
import { parseArgs } from "node:util";
|
||||
import { pool } from "./db.ts";
|
||||
import { readLines } from "./lines.ts";
|
||||
import { upsertBatch, type Entry } from "./upsertBatch.ts";
|
||||
import { createLimiter } from "./concurrencyLimit.ts";
|
||||
import { WorkerPool } from "./workerPool.ts";
|
||||
|
||||
const HASH_LINE = /^([0-9A-Fa-f]{40})(?::(\d+))?$/;
|
||||
const CHUNK_SIZE = 2000;
|
||||
|
||||
function usage(): never {
|
||||
console.error(
|
||||
`Usage: npm run start -w import -- <file> --format=<wordlist|hashes> [options]\n\n` +
|
||||
` wordlist file contains one plaintext password per line\n` +
|
||||
` hashes file contains one SHA-1 hash per line, optionally "HASH:COUNT"\n` +
|
||||
` file may be gzip-compressed (.gz)\n\n` +
|
||||
`Options:\n` +
|
||||
` --batch-size rows per upsert statement (default 5000)\n` +
|
||||
` --jobs, -j parallelism: worker threads for wordlist hashing, or\n` +
|
||||
` concurrent DB upserts for hashes (default: all CPU cores)\n`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
function sha1(value: string): string {
|
||||
return createHash("sha1").update(value).digest("hex").toUpperCase();
|
||||
}
|
||||
|
||||
function toEntry(hash: string, count: number): Entry {
|
||||
return { prefix: hash.slice(0, 5), suffix: hash.slice(5), count };
|
||||
}
|
||||
|
||||
function parseHashLine(line: string): Entry | null {
|
||||
const match = HASH_LINE.exec(line.trim());
|
||||
if (!match) return null;
|
||||
const [, hash, count] = match;
|
||||
return toEntry(hash.toUpperCase(), count ? Number(count) : 1);
|
||||
}
|
||||
|
||||
// `.ts` when running under tsx, `.js` once built with tsc.
|
||||
const hashWorkerUrl = new URL(`./hashWorker.${import.meta.url.endsWith(".ts") ? "ts" : "js"}`, import.meta.url);
|
||||
|
||||
async function importWordlist(file: string, batchSize: number, jobs: number) {
|
||||
let processed = 0;
|
||||
|
||||
if (jobs <= 1) {
|
||||
let batch: Entry[] = [];
|
||||
for await (const line of readLines(file)) {
|
||||
batch.push(toEntry(sha1(line), 1));
|
||||
processed++;
|
||||
if (batch.length >= batchSize) {
|
||||
await upsertBatch(pool, batch);
|
||||
batch = [];
|
||||
console.log(`imported ${processed.toLocaleString()} lines`);
|
||||
}
|
||||
}
|
||||
await upsertBatch(pool, batch);
|
||||
return processed;
|
||||
}
|
||||
|
||||
const workers = new WorkerPool<string[], Entry[]>(hashWorkerUrl, jobs);
|
||||
const limit = createLimiter(jobs);
|
||||
const inFlight: Promise<void>[] = [];
|
||||
|
||||
let chunk: string[] = [];
|
||||
const submit = (lines: string[]) =>
|
||||
limit(async () => {
|
||||
const entries = await workers.run(lines);
|
||||
await upsertBatch(pool, entries);
|
||||
processed += lines.length;
|
||||
console.log(`imported ${processed.toLocaleString()} lines`);
|
||||
});
|
||||
|
||||
for await (const line of readLines(file)) {
|
||||
chunk.push(line);
|
||||
if (chunk.length >= CHUNK_SIZE) {
|
||||
inFlight.push(submit(chunk));
|
||||
chunk = [];
|
||||
}
|
||||
}
|
||||
if (chunk.length > 0) inFlight.push(submit(chunk));
|
||||
|
||||
await Promise.all(inFlight);
|
||||
await workers.close();
|
||||
return processed;
|
||||
}
|
||||
|
||||
async function importHashes(file: string, batchSize: number, jobs: number) {
|
||||
const limit = createLimiter(Math.max(1, jobs));
|
||||
const inFlight: Promise<void>[] = [];
|
||||
|
||||
let batch: Entry[] = [];
|
||||
let processed = 0;
|
||||
let skipped = 0;
|
||||
|
||||
const flush = (rows: Entry[]) => limit(() => upsertBatch(pool, rows));
|
||||
|
||||
for await (const line of readLines(file)) {
|
||||
const entry = parseHashLine(line);
|
||||
if (!entry) {
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
batch.push(entry);
|
||||
processed++;
|
||||
|
||||
if (batch.length >= batchSize) {
|
||||
inFlight.push(flush(batch));
|
||||
batch = [];
|
||||
console.log(`queued ${processed.toLocaleString()} lines (${skipped} skipped)`);
|
||||
}
|
||||
}
|
||||
if (batch.length > 0) inFlight.push(flush(batch));
|
||||
|
||||
await Promise.all(inFlight);
|
||||
return { processed, skipped };
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const { positionals, values } = parseArgs({
|
||||
allowPositionals: true,
|
||||
options: {
|
||||
format: { type: "string" },
|
||||
"batch-size": { type: "string", default: "5000" },
|
||||
jobs: { type: "string", short: "j" },
|
||||
},
|
||||
});
|
||||
|
||||
const [file] = positionals;
|
||||
const format = values.format;
|
||||
if (!file || (format !== "wordlist" && format !== "hashes")) {
|
||||
usage();
|
||||
}
|
||||
|
||||
const batchSize = Number(values["batch-size"]);
|
||||
const jobs = values.jobs ? Number(values.jobs) : availableParallelism();
|
||||
|
||||
if (format === "wordlist") {
|
||||
const processed = await importWordlist(file, batchSize, jobs);
|
||||
console.log(`done: imported ${processed.toLocaleString()} lines`);
|
||||
} else {
|
||||
const { processed, skipped } = await importHashes(file, batchSize, jobs);
|
||||
console.log(`done: imported ${processed.toLocaleString()} lines, skipped ${skipped}`);
|
||||
}
|
||||
|
||||
await pool.end();
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
@@ -0,0 +1,17 @@
|
||||
import { createReadStream } from "node:fs";
|
||||
import { createGunzip } from "node:zlib";
|
||||
import { createInterface } from "node:readline";
|
||||
|
||||
export async function* readLines(filePath: string): AsyncGenerator<string> {
|
||||
let stream: NodeJS.ReadableStream = createReadStream(filePath);
|
||||
if (filePath.endsWith(".gz")) {
|
||||
stream = stream.pipe(createGunzip());
|
||||
}
|
||||
|
||||
const rl = createInterface({ input: stream, crlfDelay: Infinity });
|
||||
for await (const line of rl) {
|
||||
if (line.length > 0) {
|
||||
yield line;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
import type { Pool } from "pg";
|
||||
|
||||
export interface Entry {
|
||||
prefix: string;
|
||||
suffix: string;
|
||||
count: number;
|
||||
}
|
||||
|
||||
export async function upsertBatch(pool: Pool, entries: Entry[]): Promise<void> {
|
||||
if (entries.length === 0) return;
|
||||
|
||||
// Merge duplicate prefix/suffix pairs within the batch: ON CONFLICT can't
|
||||
// touch the same row twice in one statement.
|
||||
const merged = new Map<string, Entry>();
|
||||
for (const entry of entries) {
|
||||
const key = `${entry.prefix}:${entry.suffix}`;
|
||||
const existing = merged.get(key);
|
||||
if (existing) {
|
||||
existing.count += entry.count;
|
||||
} else {
|
||||
merged.set(key, { ...entry });
|
||||
}
|
||||
}
|
||||
// Concurrent upserts (--jobs > 1) may target overlapping rows across
|
||||
// statements; a consistent lock order avoids deadlocks between them.
|
||||
const rows = [...merged.values()].sort((a, b) => (a.prefix + a.suffix < b.prefix + b.suffix ? -1 : 1));
|
||||
|
||||
await pool.query(
|
||||
`INSERT INTO pwned_passwords (prefix, suffix, count)
|
||||
SELECT * FROM UNNEST($1::char(5)[], $2::char(35)[], $3::bigint[])
|
||||
ON CONFLICT (prefix, suffix) DO UPDATE
|
||||
SET count = pwned_passwords.count + EXCLUDED.count`,
|
||||
[rows.map((r) => r.prefix), rows.map((r) => r.suffix), rows.map((r) => r.count)],
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
import { Worker } from "node:worker_threads";
|
||||
|
||||
interface Task<T, R> {
|
||||
payload: T;
|
||||
resolve: (result: R) => void;
|
||||
reject: (err: unknown) => void;
|
||||
}
|
||||
|
||||
// A fixed-size pool of worker_threads that each run `workerUrl`, used to move
|
||||
// CPU-bound work (hashing) off the main thread. Workers are stateless
|
||||
// request/response: send a payload, get a result back.
|
||||
export class WorkerPool<T, R> {
|
||||
private workers: Worker[] = [];
|
||||
private idle: Worker[] = [];
|
||||
private queue: Task<T, R>[] = [];
|
||||
private pending = new Map<Worker, Task<T, R>>();
|
||||
|
||||
constructor(workerUrl: URL, size: number) {
|
||||
for (let i = 0; i < size; i++) {
|
||||
const worker = new Worker(workerUrl);
|
||||
worker.on("message", (result: R) => {
|
||||
this.pending.get(worker)?.resolve(result);
|
||||
this.pending.delete(worker);
|
||||
this.idle.push(worker);
|
||||
this.dispatch();
|
||||
});
|
||||
worker.on("error", (err) => {
|
||||
this.pending.get(worker)?.reject(err);
|
||||
this.pending.delete(worker);
|
||||
});
|
||||
this.workers.push(worker);
|
||||
this.idle.push(worker);
|
||||
}
|
||||
}
|
||||
|
||||
run(payload: T): Promise<R> {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.queue.push({ payload, resolve, reject });
|
||||
this.dispatch();
|
||||
});
|
||||
}
|
||||
|
||||
private dispatch() {
|
||||
while (this.idle.length > 0 && this.queue.length > 0) {
|
||||
const worker = this.idle.shift()!;
|
||||
const task = this.queue.shift()!;
|
||||
this.pending.set(worker, task);
|
||||
worker.postMessage(task.payload);
|
||||
}
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
await Promise.all(this.workers.map((worker) => worker.terminate()));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2023",
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"lib": ["ES2023"],
|
||||
"outDir": "dist",
|
||||
"rootDir": "src",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"rewriteRelativeImportExtensions": true,
|
||||
"resolveJsonModule": true,
|
||||
"declaration": false,
|
||||
"sourceMap": true
|
||||
},
|
||||
"include": ["src"]
|
||||
}
|
||||
Reference in New Issue
Block a user