- Add TOTP-based MFA with setup, verify, disable, backup codes, and challenge flow - Add PII field encryption middleware with AES-256-GCM and deterministic search hashes - Add agents, inquiries, and leads domain modules with entities, events, value objects - Add web dashboard pages for inquiries and leads with detail dialogs - Add 30+ component tests (valuation, charts, listings, search, providers, UI) - Add Prisma migrations for encryption hash columns and MFA TOTP support - Fix all ESLint errors (unused imports, duplicate imports, lint auto-fixes) - Update dependencies and lock file - Clean up obsolete exploration/QA docs, add audit documentation Co-Authored-By: Paperclip <noreply@paperclip.ing>
232 lines
6.8 KiB
TypeScript
232 lines
6.8 KiB
TypeScript
/**
|
|
* Backfill migration: encrypt all plaintext PII fields and populate hash columns.
|
|
*
|
|
* Usage:
|
|
* FIELD_ENCRYPTION_KEY=<hex-key> npx tsx scripts/encrypt-pii-fields.ts [--dry-run] [--batch-size=500]
|
|
*
|
|
* This script:
|
|
* 1. Iterates over all PII field configurations (User, Agent, Payment, Lead, Inquiry)
|
|
* 2. Reads rows in batches
|
|
* 3. Skips already-encrypted values (idempotent)
|
|
* 4. Encrypts plaintext values + writes deterministic hashes for searchable fields
|
|
* 5. Updates each row in a transaction
|
|
*
|
|
* Safe to re-run — already-encrypted values are detected by the `enc:` prefix.
|
|
*/
|
|
|
|
import crypto from 'node:crypto';
|
|
import { PrismaClient } from '@prisma/client';
|
|
import {
|
|
encryptField,
|
|
isEncrypted,
|
|
type FieldEncryptionConfig,
|
|
} from '../apps/api/src/modules/shared/infrastructure/field-encryption';
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Config
|
|
// ---------------------------------------------------------------------------
|
|
|
|
interface ModelBackfillConfig {
|
|
model: string;
|
|
/** Prisma model accessor name (e.g. 'user', 'agent') */
|
|
accessor: 'user' | 'agent' | 'payment' | 'lead' | 'inquiry';
|
|
/** Fields to encrypt */
|
|
fields: string[];
|
|
/** Fields that also need a hash column populated: { sourceField: hashColumn } */
|
|
hashFields?: Record<string, string>;
|
|
}
|
|
|
|
const MODELS: ModelBackfillConfig[] = [
|
|
{
|
|
model: 'User',
|
|
accessor: 'user',
|
|
fields: ['email', 'phone', 'kycData'],
|
|
hashFields: { email: 'emailHash', phone: 'phoneHash' },
|
|
},
|
|
{
|
|
model: 'Agent',
|
|
accessor: 'agent',
|
|
fields: ['licenseNumber'],
|
|
},
|
|
{
|
|
model: 'Payment',
|
|
accessor: 'payment',
|
|
fields: ['providerTxId', 'callbackData'],
|
|
},
|
|
{
|
|
model: 'Lead',
|
|
accessor: 'lead',
|
|
fields: ['phone', 'email'],
|
|
hashFields: { phone: 'phoneHash', email: 'emailHash' },
|
|
},
|
|
{
|
|
model: 'Inquiry',
|
|
accessor: 'inquiry',
|
|
fields: ['phone'],
|
|
},
|
|
];
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Helpers
|
|
// ---------------------------------------------------------------------------
|
|
|
|
function deriveHmacKey(encryptionKeyHex: string): Buffer {
|
|
return crypto.hkdfSync(
|
|
'sha256',
|
|
Buffer.from(encryptionKeyHex, 'hex'),
|
|
Buffer.alloc(0),
|
|
Buffer.from('goodgo-field-hash', 'utf8'),
|
|
32,
|
|
) as unknown as Buffer;
|
|
}
|
|
|
|
function computeHash(value: string, hmacKey: Buffer): string {
|
|
const normalized = value.toLowerCase().trim();
|
|
return crypto.createHmac('sha256', hmacKey).update(normalized).digest('hex');
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Main
|
|
// ---------------------------------------------------------------------------
|
|
|
|
async function main() {
|
|
const key = process.env['FIELD_ENCRYPTION_KEY'] ?? process.env['KYC_ENCRYPTION_KEY'];
|
|
if (!key) {
|
|
console.error('ERROR: FIELD_ENCRYPTION_KEY env var is required.');
|
|
process.exit(1);
|
|
}
|
|
|
|
const dryRun = process.argv.includes('--dry-run');
|
|
const batchSizeArg = process.argv.find((a) => a.startsWith('--batch-size='));
|
|
const batchSize = batchSizeArg ? parseInt(batchSizeArg.split('=')[1]!, 10) : 500;
|
|
|
|
const config: FieldEncryptionConfig = {
|
|
key,
|
|
keyVersion: parseInt(
|
|
process.env['FIELD_ENCRYPTION_KEY_VERSION'] ??
|
|
process.env['KYC_ENCRYPTION_KEY_VERSION'] ??
|
|
'1',
|
|
10,
|
|
),
|
|
};
|
|
|
|
const hmacKey = deriveHmacKey(key);
|
|
|
|
// Raw PrismaClient — no encryption middleware (we handle it manually)
|
|
const prisma = new PrismaClient();
|
|
await prisma.$connect();
|
|
|
|
const stats = {
|
|
processed: 0,
|
|
encrypted: 0,
|
|
skipped: 0,
|
|
errors: 0,
|
|
};
|
|
|
|
try {
|
|
for (const modelConfig of MODELS) {
|
|
console.warn(`\n--- ${modelConfig.model} ---`);
|
|
await backfillModel(prisma, modelConfig, config, hmacKey, batchSize, dryRun, stats);
|
|
}
|
|
|
|
console.warn('\n=== Summary ===');
|
|
console.warn(` Processed: ${stats.processed}`);
|
|
console.warn(` Encrypted: ${stats.encrypted}`);
|
|
console.warn(` Skipped (already encrypted): ${stats.skipped}`);
|
|
console.warn(` Errors: ${stats.errors}`);
|
|
if (dryRun) console.warn(' [DRY RUN — no changes written]');
|
|
} finally {
|
|
await prisma.$disconnect();
|
|
}
|
|
}
|
|
|
|
async function backfillModel(
|
|
prisma: PrismaClient,
|
|
modelConfig: ModelBackfillConfig,
|
|
encConfig: FieldEncryptionConfig,
|
|
hmacKey: Buffer,
|
|
batchSize: number,
|
|
dryRun: boolean,
|
|
stats: { processed: number; encrypted: number; skipped: number; errors: number },
|
|
): Promise<void> {
|
|
// Build select clause: id + all PII fields
|
|
const select: Record<string, boolean> = { id: true };
|
|
for (const field of modelConfig.fields) {
|
|
select[field] = true;
|
|
}
|
|
|
|
// Paginate through all records
|
|
let cursor: string | undefined;
|
|
let batch: Array<Record<string, unknown>>;
|
|
|
|
do {
|
|
const accessor = (prisma as unknown as Record<string, { findMany: (args: unknown) => Promise<Array<Record<string, unknown>>> }>)[modelConfig.accessor]!;
|
|
batch = await accessor.findMany({
|
|
select,
|
|
take: batchSize,
|
|
...(cursor
|
|
? { skip: 1, cursor: { id: cursor } }
|
|
: {}),
|
|
orderBy: { id: 'asc' },
|
|
});
|
|
|
|
for (const row of batch) {
|
|
stats.processed++;
|
|
const id = row['id'] as string;
|
|
const updateData: Record<string, unknown> = {};
|
|
let needsUpdate = false;
|
|
|
|
for (const field of modelConfig.fields) {
|
|
const value = row[field];
|
|
if (value === null || value === undefined) continue;
|
|
|
|
// Skip already encrypted
|
|
if (isEncrypted(value)) {
|
|
stats.skipped++;
|
|
continue;
|
|
}
|
|
|
|
// Compute hash BEFORE encryption
|
|
if (modelConfig.hashFields?.[field] && typeof value === 'string') {
|
|
const hashCol = modelConfig.hashFields[field]!;
|
|
updateData[hashCol] = computeHash(value, hmacKey);
|
|
}
|
|
|
|
// Encrypt the value
|
|
updateData[field] = encryptField(value, encConfig);
|
|
needsUpdate = true;
|
|
}
|
|
|
|
if (!needsUpdate) continue;
|
|
|
|
if (dryRun) {
|
|
console.warn(`[DRY RUN] Would encrypt ${modelConfig.model} id=${id}: ${Object.keys(updateData).join(', ')}`);
|
|
stats.encrypted++;
|
|
continue;
|
|
}
|
|
|
|
try {
|
|
const accessor = (prisma as unknown as Record<string, { update: (args: unknown) => Promise<unknown> }>)[modelConfig.accessor]!;
|
|
await accessor.update({
|
|
where: { id },
|
|
data: updateData,
|
|
});
|
|
stats.encrypted++;
|
|
} catch (err) {
|
|
console.error(`ERROR encrypting ${modelConfig.model} id=${id}:`, err);
|
|
stats.errors++;
|
|
}
|
|
}
|
|
|
|
if (batch.length > 0) {
|
|
cursor = batch[batch.length - 1]!['id'] as string;
|
|
console.warn(` Processed batch up to id=${cursor} (${batch.length} rows)`);
|
|
}
|
|
} while (batch.length === batchSize);
|
|
}
|
|
|
|
main().catch((err) => {
|
|
console.error('Backfill failed:', err);
|
|
process.exit(1);
|
|
});
|