feat: add MFA/TOTP auth, PII encryption, agents/leads/inquiries modules, and comprehensive tests
- Add TOTP-based MFA with setup, verify, disable, backup codes, and challenge flow - Add PII field encryption middleware with AES-256-GCM and deterministic search hashes - Add agents, inquiries, and leads domain modules with entities, events, value objects - Add web dashboard pages for inquiries and leads with detail dialogs - Add 30+ component tests (valuation, charts, listings, search, providers, UI) - Add Prisma migrations for encryption hash columns and MFA TOTP support - Fix all ESLint errors (unused imports, duplicate imports, lint auto-fixes) - Update dependencies and lock file - Clean up obsolete exploration/QA docs, add audit documentation Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Ho Ngoc Hai
231
scripts/encrypt-pii-fields.ts
Normal file
231
scripts/encrypt-pii-fields.ts
Normal file
@@ -0,0 +1,231 @@
|
||||
/**
|
||||
* Backfill migration: encrypt all plaintext PII fields and populate hash columns.
|
||||
*
|
||||
* Usage:
|
||||
* FIELD_ENCRYPTION_KEY=<hex-key> npx tsx scripts/encrypt-pii-fields.ts [--dry-run] [--batch-size=500]
|
||||
*
|
||||
* This script:
|
||||
* 1. Iterates over all PII field configurations (User, Agent, Payment, Lead, Inquiry)
|
||||
* 2. Reads rows in batches
|
||||
* 3. Skips already-encrypted values (idempotent)
|
||||
* 4. Encrypts plaintext values + writes deterministic hashes for searchable fields
|
||||
* 5. Updates each row in a transaction
|
||||
*
|
||||
* Safe to re-run — already-encrypted values are detected by the `enc:` prefix.
|
||||
*/
|
||||
|
||||
import crypto from 'node:crypto';
|
||||
import { PrismaClient } from '@prisma/client';
|
||||
import {
|
||||
encryptField,
|
||||
isEncrypted,
|
||||
type FieldEncryptionConfig,
|
||||
} from '../apps/api/src/modules/shared/infrastructure/field-encryption';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Config
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface ModelBackfillConfig {
|
||||
model: string;
|
||||
/** Prisma model accessor name (e.g. 'user', 'agent') */
|
||||
accessor: 'user' | 'agent' | 'payment' | 'lead' | 'inquiry';
|
||||
/** Fields to encrypt */
|
||||
fields: string[];
|
||||
/** Fields that also need a hash column populated: { sourceField: hashColumn } */
|
||||
hashFields?: Record<string, string>;
|
||||
}
|
||||
|
||||
const MODELS: ModelBackfillConfig[] = [
|
||||
{
|
||||
model: 'User',
|
||||
accessor: 'user',
|
||||
fields: ['email', 'phone', 'kycData'],
|
||||
hashFields: { email: 'emailHash', phone: 'phoneHash' },
|
||||
},
|
||||
{
|
||||
model: 'Agent',
|
||||
accessor: 'agent',
|
||||
fields: ['licenseNumber'],
|
||||
},
|
||||
{
|
||||
model: 'Payment',
|
||||
accessor: 'payment',
|
||||
fields: ['providerTxId', 'callbackData'],
|
||||
},
|
||||
{
|
||||
model: 'Lead',
|
||||
accessor: 'lead',
|
||||
fields: ['phone', 'email'],
|
||||
hashFields: { phone: 'phoneHash', email: 'emailHash' },
|
||||
},
|
||||
{
|
||||
model: 'Inquiry',
|
||||
accessor: 'inquiry',
|
||||
fields: ['phone'],
|
||||
},
|
||||
];
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function deriveHmacKey(encryptionKeyHex: string): Buffer {
|
||||
return crypto.hkdfSync(
|
||||
'sha256',
|
||||
Buffer.from(encryptionKeyHex, 'hex'),
|
||||
Buffer.alloc(0),
|
||||
Buffer.from('goodgo-field-hash', 'utf8'),
|
||||
32,
|
||||
) as unknown as Buffer;
|
||||
}
|
||||
|
||||
function computeHash(value: string, hmacKey: Buffer): string {
|
||||
const normalized = value.toLowerCase().trim();
|
||||
return crypto.createHmac('sha256', hmacKey).update(normalized).digest('hex');
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Main
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function main() {
|
||||
const key = process.env['FIELD_ENCRYPTION_KEY'] ?? process.env['KYC_ENCRYPTION_KEY'];
|
||||
if (!key) {
|
||||
console.error('ERROR: FIELD_ENCRYPTION_KEY env var is required.');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const dryRun = process.argv.includes('--dry-run');
|
||||
const batchSizeArg = process.argv.find((a) => a.startsWith('--batch-size='));
|
||||
const batchSize = batchSizeArg ? parseInt(batchSizeArg.split('=')[1]!, 10) : 500;
|
||||
|
||||
const config: FieldEncryptionConfig = {
|
||||
key,
|
||||
keyVersion: parseInt(
|
||||
process.env['FIELD_ENCRYPTION_KEY_VERSION'] ??
|
||||
process.env['KYC_ENCRYPTION_KEY_VERSION'] ??
|
||||
'1',
|
||||
10,
|
||||
),
|
||||
};
|
||||
|
||||
const hmacKey = deriveHmacKey(key);
|
||||
|
||||
// Raw PrismaClient — no encryption middleware (we handle it manually)
|
||||
const prisma = new PrismaClient();
|
||||
await prisma.$connect();
|
||||
|
||||
const stats = {
|
||||
processed: 0,
|
||||
encrypted: 0,
|
||||
skipped: 0,
|
||||
errors: 0,
|
||||
};
|
||||
|
||||
try {
|
||||
for (const modelConfig of MODELS) {
|
||||
console.warn(`\n--- ${modelConfig.model} ---`);
|
||||
await backfillModel(prisma, modelConfig, config, hmacKey, batchSize, dryRun, stats);
|
||||
}
|
||||
|
||||
console.warn('\n=== Summary ===');
|
||||
console.warn(` Processed: ${stats.processed}`);
|
||||
console.warn(` Encrypted: ${stats.encrypted}`);
|
||||
console.warn(` Skipped (already encrypted): ${stats.skipped}`);
|
||||
console.warn(` Errors: ${stats.errors}`);
|
||||
if (dryRun) console.warn(' [DRY RUN — no changes written]');
|
||||
} finally {
|
||||
await prisma.$disconnect();
|
||||
}
|
||||
}
|
||||
|
||||
async function backfillModel(
|
||||
prisma: PrismaClient,
|
||||
modelConfig: ModelBackfillConfig,
|
||||
encConfig: FieldEncryptionConfig,
|
||||
hmacKey: Buffer,
|
||||
batchSize: number,
|
||||
dryRun: boolean,
|
||||
stats: { processed: number; encrypted: number; skipped: number; errors: number },
|
||||
): Promise<void> {
|
||||
// Build select clause: id + all PII fields
|
||||
const select: Record<string, boolean> = { id: true };
|
||||
for (const field of modelConfig.fields) {
|
||||
select[field] = true;
|
||||
}
|
||||
|
||||
// Paginate through all records
|
||||
let cursor: string | undefined;
|
||||
let batch: Array<Record<string, unknown>>;
|
||||
|
||||
do {
|
||||
const accessor = (prisma as unknown as Record<string, { findMany: (args: unknown) => Promise<Array<Record<string, unknown>>> }>)[modelConfig.accessor]!;
|
||||
batch = await accessor.findMany({
|
||||
select,
|
||||
take: batchSize,
|
||||
...(cursor
|
||||
? { skip: 1, cursor: { id: cursor } }
|
||||
: {}),
|
||||
orderBy: { id: 'asc' },
|
||||
});
|
||||
|
||||
for (const row of batch) {
|
||||
stats.processed++;
|
||||
const id = row['id'] as string;
|
||||
const updateData: Record<string, unknown> = {};
|
||||
let needsUpdate = false;
|
||||
|
||||
for (const field of modelConfig.fields) {
|
||||
const value = row[field];
|
||||
if (value === null || value === undefined) continue;
|
||||
|
||||
// Skip already encrypted
|
||||
if (isEncrypted(value)) {
|
||||
stats.skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Compute hash BEFORE encryption
|
||||
if (modelConfig.hashFields?.[field] && typeof value === 'string') {
|
||||
const hashCol = modelConfig.hashFields[field]!;
|
||||
updateData[hashCol] = computeHash(value, hmacKey);
|
||||
}
|
||||
|
||||
// Encrypt the value
|
||||
updateData[field] = encryptField(value, encConfig);
|
||||
needsUpdate = true;
|
||||
}
|
||||
|
||||
if (!needsUpdate) continue;
|
||||
|
||||
if (dryRun) {
|
||||
console.warn(`[DRY RUN] Would encrypt ${modelConfig.model} id=${id}: ${Object.keys(updateData).join(', ')}`);
|
||||
stats.encrypted++;
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const accessor = (prisma as unknown as Record<string, { update: (args: unknown) => Promise<unknown> }>)[modelConfig.accessor]!;
|
||||
await accessor.update({
|
||||
where: { id },
|
||||
data: updateData,
|
||||
});
|
||||
stats.encrypted++;
|
||||
} catch (err) {
|
||||
console.error(`ERROR encrypting ${modelConfig.model} id=${id}:`, err);
|
||||
stats.errors++;
|
||||
}
|
||||
}
|
||||
|
||||
if (batch.length > 0) {
|
||||
cursor = batch[batch.length - 1]!['id'] as string;
|
||||
console.warn(` Processed batch up to id=${cursor} (${batch.length} rows)`);
|
||||
}
|
||||
} while (batch.length === batchSize);
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error('Backfill failed:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user