diff --git a/.cursor/skills/api-design/SKILL.md b/.cursor/skills/api-design/SKILL.md index d7eb5933..b61e1214 100644 --- a/.cursor/skills/api-design/SKILL.md +++ b/.cursor/skills/api-design/SKILL.md @@ -1,7 +1,7 @@ --- name: api-design description: RESTful API design standards for GoodGo microservices. Use for new API endpoints, DTOs, controllers, OpenAPI documentation, or standardized responses. -dependencies: "express>=4.18, zod>=3, @types/express" +compatibility: "express>=4.18, zod>=3, @types/express" --- # RESTful API Design Standards @@ -16,7 +16,6 @@ Use this skill when: - Standardizing error responses - Implementing pagination, filtering, and sorting - Setting up API versioning -- Designing resource relationships ## Core Principles @@ -31,14 +30,12 @@ Use this skill when: ``` https://api.goodgo.com/v1/{resource}/{id}/{sub-resource} -Examples: GET /v1/users # List users POST /v1/users # Create user GET /v1/users/123 # Get user by ID PUT /v1/users/123 # Update user DELETE /v1/users/123 # Delete user GET /v1/users/123/orders # Get user's orders -POST /v1/users/123/orders # Create order for user ``` ## HTTP Methods @@ -51,482 +48,94 @@ POST /v1/users/123/orders # Create order for user ## Standard Response Format -### Success Response - ```typescript +// Success interface SuccessResponse { success: true; data: T; - metadata?: { - timestamp: string; - version: string; - requestId: string; - }; - pagination?: { - page: number; - limit: number; - total: number; - totalPages: number; - }; + pagination?: { page: number; limit: number; total: number; totalPages: number }; } -// Example -{ - "success": true, - "data": { - "id": "123", - "email": "user@example.com", - "name": "John Doe" - }, - "metadata": { - "timestamp": "2024-01-01T00:00:00Z", - "version": "1.0.0", - "requestId": "req_abc123" - } -} -``` - -### Error Response - -```typescript +// Error interface ErrorResponse { success: false; - error: { - code: string; - message: string; - details?: any; - field?: string; - stack?: string; // Only in development - }; - metadata?: { - timestamp: string; - requestId: string; - }; -} - -// Example -{ - "success": false, - "error": { - "code": "VALIDATION_ERROR", - "message": "Invalid email format", - "field": "email", - "details": { - "provided": "invalid-email", - "expected": "valid email address" - } - } + error: { code: string; message: string; details?: any; field?: string }; } ``` -## Status Codes +## Key Patterns + +### Request DTO ```typescript -// Success codes -200 OK // GET, PUT, PATCH success -201 Created // POST success with resource creation -204 No Content // DELETE success - -// Client errors -400 Bad Request // Invalid request data -401 Unauthorized // Missing/invalid authentication -403 Forbidden // Valid auth but no permission -404 Not Found // Resource doesn't exist -409 Conflict // Resource conflict (duplicate) -422 Unprocessable // Validation errors - -// Server errors -500 Internal Error // Unexpected server error -502 Bad Gateway // External service error -503 Service Unavailable // Service temporarily down -504 Gateway Timeout // External service timeout -``` - -## DTOs (Data Transfer Objects) - -### Request DTOs - -```typescript -// create.dto.ts -import { IsEmail, IsNotEmpty, IsOptional, MinLength } from 'class-validator'; - export class CreateUserDto { - @IsEmail() - @IsNotEmpty() - email: string; - - @MinLength(6) - @IsNotEmpty() - password: string; - - @IsOptional() - name?: string; + @IsEmail() @IsNotEmpty() email: string; + @MinLength(6) @IsNotEmpty() password: string; + @IsOptional() name?: string; } -// update.dto.ts -export class UpdateUserDto { - @IsEmail() - @IsOptional() - email?: string; - - @IsOptional() - name?: string; - - @IsOptional() - avatar?: string; -} - -// query.dto.ts export class QueryUsersDto { - @IsOptional() - @Type(() => Number) - @Min(1) - page?: number = 1; - - @IsOptional() - @Type(() => Number) - @Min(1) - @Max(100) - limit?: number = 10; - - @IsOptional() - search?: string; - - @IsOptional() - @IsIn(['createdAt', 'name', 'email']) - sortBy?: string = 'createdAt'; - - @IsOptional() - @IsIn(['asc', 'desc']) - order?: 'asc' | 'desc' = 'desc'; + @IsOptional() @Min(1) page?: number = 1; + @IsOptional() @Min(1) @Max(100) limit?: number = 10; + @IsOptional() search?: string; + @IsOptional() @IsIn(['createdAt', 'name']) sortBy?: string = 'createdAt'; + @IsOptional() @IsIn(['asc', 'desc']) order?: 'asc' | 'desc' = 'desc'; } ``` -### Response DTOs +### Controller ```typescript -// user.response.dto.ts -export class UserResponseDto { - id: string; - email: string; - name: string; - avatar?: string; - role: string; - createdAt: Date; - updatedAt: Date; - - // Hide sensitive data - static fromEntity(user: User): UserResponseDto { - const { password, ...data } = user; - return data; - } -} - -// paginated.response.dto.ts -export class PaginatedResponseDto { - data: T[]; - pagination: { - page: number; - limit: number; - total: number; - totalPages: number; +@Get() +async list(@Query() query: QueryUsersDto) { + const { data, total } = await this.userService.findAll(query); + return { + success: true, + data: data.map(UserResponseDto.fromEntity), + pagination: { page: query.page, limit: query.limit, total, totalPages: Math.ceil(total / query.limit) } }; } -``` -## Controller Implementation - -```typescript -// user.controller.ts -@Controller('users') -@ApiTags('Users') -export class UserController { - constructor(private readonly userService: UserService) {} - - @Get() - @ApiOperation({ summary: 'List users' }) - @ApiQuery({ type: QueryUsersDto }) - @ApiResponse({ status: 200, type: PaginatedResponseDto }) - async list(@Query() query: QueryUsersDto): Promise { - const { data, total } = await this.userService.findAll(query); - - return { - success: true, - data: data.map(UserResponseDto.fromEntity), - pagination: { - page: query.page, - limit: query.limit, - total, - totalPages: Math.ceil(total / query.limit) - } - }; - } - - @Get(':id') - @ApiOperation({ summary: 'Get user by ID' }) - @ApiParam({ name: 'id', type: 'string' }) - @ApiResponse({ status: 200, type: UserResponseDto }) - @ApiResponse({ status: 404, description: 'User not found' }) - async getById(@Param('id') id: string): Promise { - const user = await this.userService.findById(id); - - if (!user) { - throw new HttpException( - { - success: false, - error: { - code: 'USER_NOT_FOUND', - message: `User with ID ${id} not found` - } - }, - HttpStatus.NOT_FOUND - ); - } - - return { - success: true, - data: UserResponseDto.fromEntity(user) - }; - } - - @Post() - @ApiOperation({ summary: 'Create user' }) - @ApiBody({ type: CreateUserDto }) - @ApiResponse({ status: 201, type: UserResponseDto }) - async create(@Body() dto: CreateUserDto): Promise { - const user = await this.userService.create(dto); - - return { - success: true, - data: UserResponseDto.fromEntity(user) - }; - } - - @Put(':id') - @ApiOperation({ summary: 'Update user' }) - @UseGuards(AuthGuard) - async update( - @Param('id') id: string, - @Body() dto: UpdateUserDto - ): Promise { - const user = await this.userService.update(id, dto); - - return { - success: true, - data: UserResponseDto.fromEntity(user) - }; - } - - @Delete(':id') - @ApiOperation({ summary: 'Delete user' }) - @UseGuards(AuthGuard, RolesGuard) - @Roles('admin') - async delete(@Param('id') id: string): Promise { - await this.userService.delete(id); - - return { - success: true, - data: { deleted: true } - }; - } -} -``` - -## OpenAPI/Swagger Documentation - -```yaml -# openapi/user-service.yaml -openapi: 3.0.0 -info: - title: User Service API - version: 1.0.0 - description: User management endpoints -servers: - - url: https://api.goodgo.com/v1 -paths: - /users: - get: - summary: List users - parameters: - - name: page - in: query - schema: - type: integer - default: 1 - - name: limit - in: query - schema: - type: integer - default: 10 - responses: - '200': - description: List of users - content: - application/json: - schema: - $ref: '#/components/schemas/UserListResponse' - post: - summary: Create user - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/CreateUserRequest' - responses: - '201': - description: User created - '400': - description: Validation error -``` - -## Pagination Pattern - -```typescript -// pagination.service.ts -export class PaginationService { - paginate( - query: any, - options: { - page: number; - limit: number; - sortBy?: string; - order?: 'asc' | 'desc'; - } - ) { - const skip = (options.page - 1) * options.limit; - - return { - skip, - take: options.limit, - orderBy: options.sortBy ? { - [options.sortBy]: options.order || 'desc' - } : undefined - }; - } -} -``` - -## Error Handling - -```typescript -// error.middleware.ts -export function errorHandler( - err: Error, - req: Request, - res: Response, - next: NextFunction -) { - const isDev = process.env.NODE_ENV === 'development'; - - // Known errors - if (err instanceof ValidationError) { - return res.status(400).json({ - success: false, - error: { - code: 'VALIDATION_ERROR', - message: err.message, - details: err.errors - } - }); - } - - if (err instanceof UnauthorizedError) { - return res.status(401).json({ - success: false, - error: { - code: 'UNAUTHORIZED', - message: 'Authentication required' - } - }); - } - - // Unknown errors - logger.error('Unhandled error:', err); - - res.status(500).json({ - success: false, - error: { - code: 'INTERNAL_ERROR', - message: isDev ? err.message : 'Internal server error', - stack: isDev ? err.stack : undefined - } - }); +@Get(':id') +async getById(@Param('id') id: string) { + const user = await this.userService.findById(id); + if (!user) throw new NotFoundException({ success: false, error: { code: 'NOT_FOUND', message: 'User not found' } }); + return { success: true, data: UserResponseDto.fromEntity(user) }; } ``` ## Best Practices -1. **Resource Naming** - - Use plural nouns (`/users` not `/user`) - - Use kebab-case for multi-word resources - - Keep URLs as short as possible - -2. **Versioning** - - Include version in URL (`/v1/users`) - - Maintain backward compatibility - - Deprecate old versions gracefully - -3. **Security** - - Always use HTTPS - - Implement rate limiting - - Validate all inputs - - Use proper authentication/authorization - -4. **Performance** - - Implement pagination for lists - - Use field filtering when possible - - Cache responses appropriately - - Compress responses (gzip) - -5. **Documentation** - - Keep OpenAPI spec up to date - - Include examples in documentation - - Document error responses - - Version your documentation +- **Resource Naming**: Use plural nouns (`/users`), kebab-case for multi-word +- **Versioning**: Include version in URL (`/v1/users`), maintain backward compatibility +- **Security**: Use HTTPS, implement rate limiting, validate all inputs +- **Performance**: Implement pagination, use field filtering, cache responses +- **Documentation**: Keep OpenAPI spec up to date, include examples ## Common Mistakes 1. **Using Verbs in URLs**: Non-RESTful endpoints ``` - # ❌ BAD: Verb in URL - POST /api/v1/createUser - GET /api/v1/getUserById/123 - - # ✅ GOOD: RESTful resources - POST /api/v1/users - GET /api/v1/users/123 + # BAD: POST /api/v1/createUser + # GOOD: POST /api/v1/users ``` 2. **Inconsistent Response Format**: Different structures for different endpoints ```typescript - // ❌ BAD: Inconsistent - res.json({ user: data }); // One endpoint - res.json({ result: data }); // Another endpoint - - // ✅ GOOD: Consistent structure - res.json({ success: true, data }); // All endpoints + // BAD: res.json({ user: data }) vs res.json({ result: data }) + // GOOD: res.json({ success: true, data }) ``` 3. **Wrong HTTP Status Codes**: Using 200 for errors ```typescript - // ❌ BAD: 200 with error - res.status(200).json({ error: 'Not found' }); - - // ✅ GOOD: Appropriate status - res.status(404).json({ success: false, error: { code: 'NOT_FOUND', message: 'User not found' } }); + // BAD: res.status(200).json({ error: 'Not found' }); + // GOOD: res.status(404).json({ success: false, error: { code: 'NOT_FOUND' } }); ``` 4. **Missing Pagination**: Returning all records ```typescript - // ❌ BAD: Returns everything - const users = await prisma.user.findMany(); - - // ✅ GOOD: Paginated - const users = await prisma.user.findMany({ - skip: (page - 1) * limit, - take: limit, - }); + // BAD: prisma.user.findMany() + // GOOD: prisma.user.findMany({ skip: (page - 1) * limit, take: limit }) ``` ## Quick Reference @@ -541,21 +150,8 @@ export function errorHandler( **Response Format:** ```typescript -// Success -{ success: true, data: T, pagination?: {...} } - -// Error -{ success: false, error: { code: string, message: string, details?: any } } -``` - -**URL Patterns:** -``` -GET /v1/users # List -POST /v1/users # Create -GET /v1/users/:id # Get by ID -PUT /v1/users/:id # Update -DELETE /v1/users/:id # Delete -GET /v1/users/:id/orders # Sub-resource +// Success: { success: true, data: T, pagination?: {...} } +// Error: { success: false, error: { code: string, message: string } } ``` **Common Error Codes:** @@ -571,7 +167,7 @@ GET /v1/users/:id/orders # Sub-resource - [OpenAPI Specification](https://spec.openapis.org/oas/latest.html) - Official OpenAPI docs - [REST API Design](https://restfulapi.net/) - REST best practices +- [Detailed Code Examples](./references/REFERENCE.md) - [API Versioning Strategy](../api-versioning-strategy/SKILL.md) - Versioning patterns - [API Gateway Advanced](../api-gateway-advanced/SKILL.md) - Gateway patterns - [Middleware Patterns](../middleware-patterns/SKILL.md) - Request handling -- [Project Rules](../project-rules/SKILL.md) - GoodGo standards \ No newline at end of file diff --git a/.cursor/skills/api-design/references/REFERENCE.md b/.cursor/skills/api-design/references/REFERENCE.md new file mode 100644 index 00000000..dc2b1e3a --- /dev/null +++ b/.cursor/skills/api-design/references/REFERENCE.md @@ -0,0 +1,418 @@ +# API Design - Detailed Reference + +This reference contains detailed code examples for RESTful API design patterns. + +## Standard Response Format + +### Success Response + +```typescript +interface SuccessResponse { + success: true; + data: T; + metadata?: { + timestamp: string; + version: string; + requestId: string; + }; + pagination?: { + page: number; + limit: number; + total: number; + totalPages: number; + }; +} + +// Example +{ + "success": true, + "data": { + "id": "123", + "email": "user@example.com", + "name": "John Doe" + }, + "metadata": { + "timestamp": "2024-01-01T00:00:00Z", + "version": "1.0.0", + "requestId": "req_abc123" + } +} +``` + +### Error Response + +```typescript +interface ErrorResponse { + success: false; + error: { + code: string; + message: string; + details?: any; + field?: string; + stack?: string; // Only in development + }; + metadata?: { + timestamp: string; + requestId: string; + }; +} + +// Example +{ + "success": false, + "error": { + "code": "VALIDATION_ERROR", + "message": "Invalid email format", + "field": "email", + "details": { + "provided": "invalid-email", + "expected": "valid email address" + } + } +} +``` + +## DTOs (Data Transfer Objects) + +### Request DTOs + +```typescript +// create.dto.ts +import { IsEmail, IsNotEmpty, IsOptional, MinLength } from 'class-validator'; + +export class CreateUserDto { + @IsEmail() + @IsNotEmpty() + email: string; + + @MinLength(6) + @IsNotEmpty() + password: string; + + @IsOptional() + name?: string; +} + +// update.dto.ts +export class UpdateUserDto { + @IsEmail() + @IsOptional() + email?: string; + + @IsOptional() + name?: string; + + @IsOptional() + avatar?: string; +} + +// query.dto.ts +export class QueryUsersDto { + @IsOptional() + @Type(() => Number) + @Min(1) + page?: number = 1; + + @IsOptional() + @Type(() => Number) + @Min(1) + @Max(100) + limit?: number = 10; + + @IsOptional() + search?: string; + + @IsOptional() + @IsIn(['createdAt', 'name', 'email']) + sortBy?: string = 'createdAt'; + + @IsOptional() + @IsIn(['asc', 'desc']) + order?: 'asc' | 'desc' = 'desc'; +} +``` + +### Response DTOs + +```typescript +// user.response.dto.ts +export class UserResponseDto { + id: string; + email: string; + name: string; + avatar?: string; + role: string; + createdAt: Date; + updatedAt: Date; + + // Hide sensitive data + static fromEntity(user: User): UserResponseDto { + const { password, ...data } = user; + return data; + } +} + +// paginated.response.dto.ts +export class PaginatedResponseDto { + data: T[]; + pagination: { + page: number; + limit: number; + total: number; + totalPages: number; + }; +} +``` + +## Controller Implementation + +```typescript +// user.controller.ts +@Controller('users') +@ApiTags('Users') +export class UserController { + constructor(private readonly userService: UserService) {} + + @Get() + @ApiOperation({ summary: 'List users' }) + @ApiQuery({ type: QueryUsersDto }) + @ApiResponse({ status: 200, type: PaginatedResponseDto }) + async list(@Query() query: QueryUsersDto): Promise { + const { data, total } = await this.userService.findAll(query); + + return { + success: true, + data: data.map(UserResponseDto.fromEntity), + pagination: { + page: query.page, + limit: query.limit, + total, + totalPages: Math.ceil(total / query.limit) + } + }; + } + + @Get(':id') + @ApiOperation({ summary: 'Get user by ID' }) + @ApiParam({ name: 'id', type: 'string' }) + @ApiResponse({ status: 200, type: UserResponseDto }) + @ApiResponse({ status: 404, description: 'User not found' }) + async getById(@Param('id') id: string): Promise { + const user = await this.userService.findById(id); + + if (!user) { + throw new HttpException( + { + success: false, + error: { + code: 'USER_NOT_FOUND', + message: `User with ID ${id} not found` + } + }, + HttpStatus.NOT_FOUND + ); + } + + return { + success: true, + data: UserResponseDto.fromEntity(user) + }; + } + + @Post() + @ApiOperation({ summary: 'Create user' }) + @ApiBody({ type: CreateUserDto }) + @ApiResponse({ status: 201, type: UserResponseDto }) + async create(@Body() dto: CreateUserDto): Promise { + const user = await this.userService.create(dto); + + return { + success: true, + data: UserResponseDto.fromEntity(user) + }; + } + + @Put(':id') + @ApiOperation({ summary: 'Update user' }) + @UseGuards(AuthGuard) + async update( + @Param('id') id: string, + @Body() dto: UpdateUserDto + ): Promise { + const user = await this.userService.update(id, dto); + + return { + success: true, + data: UserResponseDto.fromEntity(user) + }; + } + + @Delete(':id') + @ApiOperation({ summary: 'Delete user' }) + @UseGuards(AuthGuard, RolesGuard) + @Roles('admin') + async delete(@Param('id') id: string): Promise { + await this.userService.delete(id); + + return { + success: true, + data: { deleted: true } + }; + } +} +``` + +## OpenAPI/Swagger Documentation + +```yaml +# openapi/user-service.yaml +openapi: 3.0.0 +info: + title: User Service API + version: 1.0.0 + description: User management endpoints +servers: + - url: https://api.goodgo.com/v1 +paths: + /users: + get: + summary: List users + parameters: + - name: page + in: query + schema: + type: integer + default: 1 + - name: limit + in: query + schema: + type: integer + default: 10 + responses: + '200': + description: List of users + content: + application/json: + schema: + $ref: '#/components/schemas/UserListResponse' + post: + summary: Create user + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/CreateUserRequest' + responses: + '201': + description: User created + '400': + description: Validation error +``` + +## Pagination Pattern + +```typescript +// pagination.service.ts +export class PaginationService { + paginate( + query: any, + options: { + page: number; + limit: number; + sortBy?: string; + order?: 'asc' | 'desc'; + } + ) { + const skip = (options.page - 1) * options.limit; + + return { + skip, + take: options.limit, + orderBy: options.sortBy ? { + [options.sortBy]: options.order || 'desc' + } : undefined + }; + } +} +``` + +## Error Handling + +```typescript +// error.middleware.ts +export function errorHandler( + err: Error, + req: Request, + res: Response, + next: NextFunction +) { + const isDev = process.env.NODE_ENV === 'development'; + + // Known errors + if (err instanceof ValidationError) { + return res.status(400).json({ + success: false, + error: { + code: 'VALIDATION_ERROR', + message: err.message, + details: err.errors + } + }); + } + + if (err instanceof UnauthorizedError) { + return res.status(401).json({ + success: false, + error: { + code: 'UNAUTHORIZED', + message: 'Authentication required' + } + }); + } + + // Unknown errors + logger.error('Unhandled error:', err); + + res.status(500).json({ + success: false, + error: { + code: 'INTERNAL_ERROR', + message: isDev ? err.message : 'Internal server error', + stack: isDev ? err.stack : undefined + } + }); +} +``` + +## Zod Validation Alternative + +```typescript +import { z } from 'zod'; + +const CreateUserSchema = z.object({ + email: z.string().email(), + password: z.string().min(6), + name: z.string().optional() +}); + +const QueryUsersSchema = z.object({ + page: z.coerce.number().min(1).default(1), + limit: z.coerce.number().min(1).max(100).default(10), + search: z.string().optional(), + sortBy: z.enum(['createdAt', 'name', 'email']).default('createdAt'), + order: z.enum(['asc', 'desc']).default('desc') +}); + +// Usage in controller +async create(req: Request, res: Response) { + const result = CreateUserSchema.safeParse(req.body); + if (!result.success) { + return res.status(400).json({ + success: false, + error: { code: 'VALIDATION_ERROR', details: result.error.issues } + }); + } + // ... create user with result.data +} +``` diff --git a/.cursor/skills/api-gateway-advanced/SKILL.md b/.cursor/skills/api-gateway-advanced/SKILL.md index 06f052c7..aff4ea6f 100644 --- a/.cursor/skills/api-gateway-advanced/SKILL.md +++ b/.cursor/skills/api-gateway-advanced/SKILL.md @@ -1,7 +1,7 @@ --- name: api-gateway-advanced description: Advanced API Gateway patterns for GoodGo. Use for API composition, request/response transformation, service mesh, or gateway resilience. -dependencies: "traefik>=2.10" +compatibility: "traefik>=2.10" --- # API Gateway Advanced Patterns @@ -15,10 +15,7 @@ Use this skill when: - Implementing advanced routing strategies - Adding gateway-level circuit breakers - Implementing API versioning at gateway -- Building API composition services - Optimizing gateway performance -- Implementing request/response caching at gateway -- Managing complex routing rules ## Core Concepts @@ -34,418 +31,91 @@ Use this skill when: ### API Composition Patterns -**Aggregation**: Combine multiple service responses into single response -**Chaining**: Call services sequentially, use previous response in next call -**Fan-out/Fan-in**: Call multiple services in parallel, aggregate results +- **Aggregation**: Combine multiple service responses into single response +- **Chaining**: Call services sequentially, use previous response in next call +- **Fan-out/Fan-in**: Call multiple services in parallel, aggregate results -## API Composition Service +## Key Patterns + +### API Composition ```typescript -// src/modules/gateway/api-composition.service.ts -// EN: API composition service -// VI: Service API composition -import { ServiceClient } from '../../core/clients/service-client'; -import { logger } from '@goodgo/logger'; +// Fan-out: Call multiple services in parallel +async getUserProfile(userId: string) { + const [user, orders, payments] = await Promise.all([ + this.userClient.get(`/users/${userId}`), + this.orderClient.get(`/orders?userId=${userId}`), + this.paymentClient.get(`/payments?userId=${userId}`), + ]); -// EN: Type definitions for API composition -// VI: Định nghĩa types cho API composition -interface CreateOrderInput { - userId: string; - items: Array<{ productId: string; quantity: number }>; - paymentMethod: string; -} - -interface OrderWithPaymentResult { - success: boolean; - data: { - order: { id: string; total: number }; - payment: { id: string; status: string }; + return { + user: user.data, + orders: orders.data.orders, + paymentHistory: payments.data.payments, }; } -export class ApiCompositionService { - private userClient: ServiceClient; - private orderClient: ServiceClient; - private paymentClient: ServiceClient; - - constructor() { - this.userClient = new ServiceClient({ - baseURL: process.env.USER_SERVICE_URL || 'http://user-service:5002', - serviceName: 'user-service', - }); - - this.orderClient = new ServiceClient({ - baseURL: process.env.ORDER_SERVICE_URL || 'http://order-service:5003', - serviceName: 'order-service', - }); - - this.paymentClient = new ServiceClient({ - baseURL: process.env.PAYMENT_SERVICE_URL || 'http://payment-service:5004', - serviceName: 'payment-service', - }); - } - - /** - * EN: Aggregate user profile with orders and payment history - * VI: Tổng hợp profile user với orders và payment history - */ - async getUserProfile(userId: string): Promise { - try { - // EN: Fan-out: Call multiple services in parallel - // VI: Fan-out: Gọi nhiều services song song - const [user, orders, payments] = await Promise.all([ - this.userClient.get(`/api/v1/users/${userId}`), - this.orderClient.get(`/api/v1/orders?userId=${userId}`), - this.paymentClient.get(`/api/v1/payments?userId=${userId}`), - ]); - - // EN: Transform and aggregate response - // VI: Transform và tổng hợp response - return { - success: true, - data: { - user: user.data, - orders: orders.data.orders, - paymentHistory: payments.data.payments, - summary: { - totalOrders: orders.data.total, - totalSpent: payments.data.totalAmount, - }, - }, - }; - } catch (error) { - logger.error('API composition failed', { userId, error }); - throw error; - } - } - - /** - * EN: Chain services: Create order then process payment - * VI: Chain services: Tạo order rồi xử lý payment - */ - async createOrderWithPayment(orderData: CreateOrderInput): Promise { - // EN: Step 1: Create order - // VI: Bước 1: Tạo order - const order = await this.orderClient.post('/api/v1/orders', orderData); - - try { - // EN: Step 2: Process payment using order data - // VI: Bước 2: Xử lý payment sử dụng order data - const payment = await this.paymentClient.post('/api/v1/payments', { - orderId: order.data.id, - amount: order.data.total, - paymentMethod: orderData.paymentMethod, - }); - - return { - success: true, - data: { - order: order.data, - payment: payment.data, - }, - }; - } catch (error) { - // EN: Compensate: Cancel order if payment fails - // VI: Compensate: Hủy order nếu payment fails - await this.orderClient.delete(`/api/v1/orders/${order.data.id}`); - throw error; - } +// Chaining: Sequential calls with compensation +async createOrderWithPayment(data) { + const order = await this.orderClient.post('/orders', data); + try { + const payment = await this.paymentClient.post('/payments', { orderId: order.data.id }); + return { order: order.data, payment: payment.data }; + } catch (error) { + await this.orderClient.delete(`/orders/${order.data.id}`); // Compensate + throw error; } } ``` -## Request/Response Transformation +### Gateway Caching ```typescript -// src/middlewares/gateway-transform.middleware.ts -// EN: Gateway transformation middleware -// VI: Middleware transformation gateway -import { Request, Response, NextFunction } from 'express'; -import { logger } from '@goodgo/logger'; - -export interface TransformRule { - path: string; - requestTransform?: (req: Request) => Request; - responseTransform?: (res: Response, data: T) => R; -} - -export class GatewayTransformMiddleware { - private rules: TransformRule[] = []; - - addRule(rule: TransformRule): void { - this.rules.push(rule); - } - - middleware() { - return (req: Request, res: Response, next: NextFunction) => { - // EN: Find matching transform rule - // VI: Tìm transform rule khớp - const rule = this.rules.find((r) => req.path.startsWith(r.path)); - - if (rule?.requestTransform) { - // EN: Transform request - // VI: Transform request - Object.assign(req, rule.requestTransform(req)); - } - - // EN: Store original json method - // VI: Lưu method json gốc - const originalJson = res.json.bind(res); - - // EN: Override json to transform response - // VI: Override json để transform response - res.json = (data: unknown) => { - let transformedData = data; - if (rule?.responseTransform) { - transformedData = rule.responseTransform(res, data); - } - return originalJson(transformedData); - }; - - next(); - }; - } -} - -// Usage -const transformer = new GatewayTransformMiddleware(); - -transformer.addRule({ - path: '/api/v1/users', - requestTransform: (req) => { - // EN: Add default pagination - // VI: Thêm pagination mặc định - if (!req.query.page) req.query.page = '1'; - if (!req.query.limit) req.query.limit = '10'; - return req; - }, - responseTransform: (res, data) => { - // EN: Standardize response format - // VI: Chuẩn hóa format response - return { - success: true, - data: data.data || data, - metadata: { - timestamp: new Date().toISOString(), - }, - }; - }, -}); -``` - -## Advanced Traefik Configuration - -```yaml -# infra/traefik/dynamic/routes.yml -http: - routers: - # EN: API composition route - # VI: Route API composition - user-profile: - rule: "Path(`/api/v1/user-profile/{userId}`)" - service: api-composition-service - middlewares: - - cors - - compress - - rate-limit - - # EN: Versioned routes - # VI: Routes có version - user-service-v1: - rule: "PathPrefix(`/api/v1/users`)" - service: user-service-v1 - priority: 10 - - user-service-v2: - rule: "PathPrefix(`/api/v2/users`)" - service: user-service-v2 - priority: 5 # EN: Lower priority / VI: Độ ưu tiên thấp hơn - - services: - api-composition-service: - loadBalancer: - servers: - - url: "http://api-composition-service:5000" - - middlewares: - # EN: Request transformation - # VI: Transform request - add-default-headers: - headers: - customRequestHeaders: - X-Request-ID: "{{.RequestHeader.X-Request-ID}}" - X-Source: "traefik-gateway" - - # EN: Response transformation - # VI: Transform response - add-response-headers: - headers: - customResponseHeaders: - X-Response-Time: "{{.ResponseTime}}" - - # EN: Circuit breaker at gateway - # VI: Circuit breaker ở gateway - circuit-breaker: - circuitBreaker: - expression: "NetworkErrorRatio() > 0.50" -``` - -## Service Mesh Integration - -```yaml -# EN: Traefik with Istio integration -# VI: Traefik với tích hợp Istio -apiVersion: networking.istio.io/v1alpha3 -kind: VirtualService -metadata: - name: traefik-gateway -spec: - hosts: - - api.goodgo.com - gateways: - - traefik-gateway - http: - - match: - - uri: - prefix: "/api/v1" - route: - - destination: - host: user-service - port: - number: 5002 - - destination: - host: order-service - port: - number: 5003 - fault: - delay: - percentage: - value: 0.1 - fixedDelay: 5s - retries: - attempts: 3 - perTryTimeout: 2s -``` - -## Gateway-Level Circuit Breaker - -```typescript -// src/core/gateway/circuit-breaker.middleware.ts -// EN: Gateway-level circuit breaker -// VI: Circuit breaker ở gateway level -import { Request, Response, NextFunction } from 'express'; -import { createCircuitBreaker } from '../resilience/circuit-breaker'; -import { logger } from '@goodgo/logger'; - -const serviceCircuitBreakers = new Map>(); - -export function gatewayCircuitBreaker(serviceName: string) { - return (req: Request, res: Response, next: NextFunction) => { - if (!serviceCircuitBreakers.has(serviceName)) { - serviceCircuitBreakers.set( - serviceName, - createCircuitBreaker( - async () => { - // EN: Continue to next middleware - // VI: Tiếp tục tới middleware tiếp theo - return next(); - }, - `gateway-${serviceName}`, - { - timeout: 5000, - errorThresholdPercentage: 50, - resetTimeout: 30000, - } - ) - ); - } - - const breaker = serviceCircuitBreakers.get(serviceName)!; - - breaker.fire().catch((error) => { - logger.error('Gateway circuit breaker opened', { serviceName, error }); - res.status(503).json({ - success: false, - error: { - code: 'SERVICE_UNAVAILABLE', - message: `Service ${serviceName} is currently unavailable`, - }, - }); - }); - }; -} -``` - -## Caching at Gateway - -```typescript -// src/core/gateway/gateway-cache.middleware.ts -// EN: Gateway-level caching -// VI: Caching ở gateway level -import { Request, Response, NextFunction } from 'express'; -import { cacheService } from '../cache/cache.service'; -import { logger } from '@goodgo/logger'; - export function gatewayCache(ttl: number = 60) { - return async (req: Request, res: Response, next: NextFunction) => { - // EN: Only cache GET requests - // VI: Chỉ cache GET requests - if (req.method !== 'GET') { - return next(); - } + return async (req, res, next) => { + if (req.method !== 'GET') return next(); const cacheKey = `gateway:${req.path}:${JSON.stringify(req.query)}`; + const cached = await cacheService.get(cacheKey); + if (cached) return res.json(cached); - try { - // EN: Check cache - // VI: Kiểm tra cache - const cached = await cacheService.get(cacheKey); - if (cached) { - logger.debug('Gateway cache hit', { path: req.path }); - return res.json(cached); - } - - // EN: Store original json method - // VI: Lưu method json gốc - const originalJson = res.json.bind(res); - - // EN: Override to cache response - // VI: Override để cache response - res.json = (data: unknown) => { - cacheService.set(cacheKey, data, ttl).catch((error) => { - logger.warn('Gateway cache set failed', { error }); - }); - return originalJson(data); - }; - - next(); - } catch (error) { - logger.error('Gateway cache error', { error }); - next(); // EN: Continue on cache error / VI: Tiếp tục khi có lỗi cache - } + const originalJson = res.json.bind(res); + res.json = (data) => { + cacheService.set(cacheKey, data, ttl); + return originalJson(data); + }; + next(); }; } ``` +### Traefik Circuit Breaker + +```yaml +middlewares: + circuit-breaker: + circuitBreaker: + expression: "NetworkErrorRatio() > 0.50" + timeout: + forwardingTimeouts: + dialTimeout: 5s + responseHeaderTimeout: 10s +``` + ## Best Practices -1. **API Composition**: Use for aggregating related data -2. **Caching**: Cache at gateway for frequently accessed data -3. **Circuit Breaker**: Implement at gateway to protect services -4. **Transformation**: Keep transformations simple and testable -5. **Versioning**: Use path-based or header-based versioning -6. **Monitoring**: Monitor gateway metrics (latency, error rate) +- **API Composition**: Use for aggregating related data from multiple services +- **Caching**: Cache at gateway for frequently accessed data +- **Circuit Breaker**: Implement at gateway to protect downstream services +- **Transformation**: Keep transformations simple and testable +- **Versioning**: Use path-based or header-based versioning +- **Monitoring**: Monitor gateway metrics (latency, error rate) ## Common Mistakes 1. **No Timeout at Gateway**: Requests hanging forever ```yaml - # ❌ BAD: No timeout - services: - my-service: - loadBalancer: ... - - # ✅ GOOD: Set timeout + # GOOD: Set timeout middlewares: timeout: forwardingTimeouts: @@ -455,7 +125,7 @@ export function gatewayCache(ttl: number = 60) { 2. **Missing Circuit Breaker**: Cascading failures ```yaml - # ✅ Add circuit breaker for each service + # GOOD: Add circuit breaker for each service middlewares: circuit-breaker: circuitBreaker: @@ -464,22 +134,14 @@ export function gatewayCache(ttl: number = 60) { 3. **No Caching for Static Data**: Unnecessary service load ```typescript - // ❌ BAD: Every request hits service - app.get('/api/config', handler); - - // ✅ GOOD: Cache at gateway + // GOOD: Cache at gateway app.get('/api/config', gatewayCache(3600), handler); ``` 4. **N+1 API Calls from Client**: Multiple round trips ```typescript - // ❌ BAD: Client makes 3 calls - GET /users/123 - GET /orders?userId=123 - GET /payments?userId=123 - - // ✅ GOOD: Use API composition - GET /user-profile/123 // Aggregated response + // BAD: Client makes 3 calls + // GOOD: Use API composition GET /user-profile/123 ``` ## Quick Reference @@ -505,35 +167,18 @@ export function gatewayCache(ttl: number = 60) { **API Composition Patterns:** ```typescript // Fan-out (parallel) -const [users, orders] = await Promise.all([ - userClient.get('/users'), - orderClient.get('/orders') -]); +const [users, orders] = await Promise.all([userClient.get('/users'), orderClient.get('/orders')]); // Chaining (sequential) const order = await orderClient.create(data); const payment = await paymentClient.process(order.id); ``` -**Gateway Config Template:** -```yaml -http: - routers: - service-route: - rule: "PathPrefix(`/api/v1/service`)" - middlewares: [rate-limit, auth, circuit-breaker] - service: backend-service - middlewares: - circuit-breaker: - circuitBreaker: - expression: "NetworkErrorRatio() > 0.5" -``` - ## Resources - [Traefik Documentation](https://doc.traefik.io/traefik/) - Official Traefik docs - [API Gateway Pattern](https://microservices.io/patterns/apigateway.html) - Gateway patterns - [Service Mesh](https://istio.io/) - Istio service mesh +- [Detailed Code Examples](./references/REFERENCE.md) - [Middleware Patterns](../middleware-patterns/SKILL.md) - Middleware patterns - [Resilience Patterns](../resilience-patterns/SKILL.md) - Circuit breaker patterns -- [Project Rules](../project-rules/SKILL.md) - GoodGo coding standards diff --git a/.cursor/skills/api-gateway-advanced/references/REFERENCE.md b/.cursor/skills/api-gateway-advanced/references/REFERENCE.md new file mode 100644 index 00000000..667607dd --- /dev/null +++ b/.cursor/skills/api-gateway-advanced/references/REFERENCE.md @@ -0,0 +1,357 @@ +# API Gateway Advanced - Detailed Reference + +This reference contains detailed code examples for advanced API Gateway patterns. + +## API Composition Service + +```typescript +// src/modules/gateway/api-composition.service.ts +import { ServiceClient } from '../../core/clients/service-client'; +import { logger } from '@goodgo/logger'; + +interface CreateOrderInput { + userId: string; + items: Array<{ productId: string; quantity: number }>; + paymentMethod: string; +} + +interface OrderWithPaymentResult { + success: boolean; + data: { + order: { id: string; total: number }; + payment: { id: string; status: string }; + }; +} + +export class ApiCompositionService { + private userClient: ServiceClient; + private orderClient: ServiceClient; + private paymentClient: ServiceClient; + + constructor() { + this.userClient = new ServiceClient({ + baseURL: process.env.USER_SERVICE_URL || 'http://user-service:5002', + serviceName: 'user-service', + }); + + this.orderClient = new ServiceClient({ + baseURL: process.env.ORDER_SERVICE_URL || 'http://order-service:5003', + serviceName: 'order-service', + }); + + this.paymentClient = new ServiceClient({ + baseURL: process.env.PAYMENT_SERVICE_URL || 'http://payment-service:5004', + serviceName: 'payment-service', + }); + } + + /** + * Aggregate user profile with orders and payment history + */ + async getUserProfile(userId: string): Promise { + try { + // Fan-out: Call multiple services in parallel + const [user, orders, payments] = await Promise.all([ + this.userClient.get(`/api/v1/users/${userId}`), + this.orderClient.get(`/api/v1/orders?userId=${userId}`), + this.paymentClient.get(`/api/v1/payments?userId=${userId}`), + ]); + + // Transform and aggregate response + return { + success: true, + data: { + user: user.data, + orders: orders.data.orders, + paymentHistory: payments.data.payments, + summary: { + totalOrders: orders.data.total, + totalSpent: payments.data.totalAmount, + }, + }, + }; + } catch (error) { + logger.error('API composition failed', { userId, error }); + throw error; + } + } + + /** + * Chain services: Create order then process payment + */ + async createOrderWithPayment(orderData: CreateOrderInput): Promise { + // Step 1: Create order + const order = await this.orderClient.post('/api/v1/orders', orderData); + + try { + // Step 2: Process payment using order data + const payment = await this.paymentClient.post('/api/v1/payments', { + orderId: order.data.id, + amount: order.data.total, + paymentMethod: orderData.paymentMethod, + }); + + return { + success: true, + data: { + order: order.data, + payment: payment.data, + }, + }; + } catch (error) { + // Compensate: Cancel order if payment fails + await this.orderClient.delete(`/api/v1/orders/${order.data.id}`); + throw error; + } + } +} +``` + +## Request/Response Transformation + +```typescript +// src/middlewares/gateway-transform.middleware.ts +import { Request, Response, NextFunction } from 'express'; +import { logger } from '@goodgo/logger'; + +export interface TransformRule { + path: string; + requestTransform?: (req: Request) => Request; + responseTransform?: (res: Response, data: T) => R; +} + +export class GatewayTransformMiddleware { + private rules: TransformRule[] = []; + + addRule(rule: TransformRule): void { + this.rules.push(rule); + } + + middleware() { + return (req: Request, res: Response, next: NextFunction) => { + // Find matching transform rule + const rule = this.rules.find((r) => req.path.startsWith(r.path)); + + if (rule?.requestTransform) { + // Transform request + Object.assign(req, rule.requestTransform(req)); + } + + // Store original json method + const originalJson = res.json.bind(res); + + // Override json to transform response + res.json = (data: unknown) => { + let transformedData = data; + if (rule?.responseTransform) { + transformedData = rule.responseTransform(res, data); + } + return originalJson(transformedData); + }; + + next(); + }; + } +} + +// Usage +const transformer = new GatewayTransformMiddleware(); + +transformer.addRule({ + path: '/api/v1/users', + requestTransform: (req) => { + // Add default pagination + if (!req.query.page) req.query.page = '1'; + if (!req.query.limit) req.query.limit = '10'; + return req; + }, + responseTransform: (res, data) => { + // Standardize response format + return { + success: true, + data: data.data || data, + metadata: { + timestamp: new Date().toISOString(), + }, + }; + }, +}); +``` + +## Advanced Traefik Configuration + +```yaml +# infra/traefik/dynamic/routes.yml +http: + routers: + # API composition route + user-profile: + rule: "Path(`/api/v1/user-profile/{userId}`)" + service: api-composition-service + middlewares: + - cors + - compress + - rate-limit + + # Versioned routes + user-service-v1: + rule: "PathPrefix(`/api/v1/users`)" + service: user-service-v1 + priority: 10 + + user-service-v2: + rule: "PathPrefix(`/api/v2/users`)" + service: user-service-v2 + priority: 5 + + services: + api-composition-service: + loadBalancer: + servers: + - url: "http://api-composition-service:5000" + + middlewares: + # Request transformation + add-default-headers: + headers: + customRequestHeaders: + X-Request-ID: "{{.RequestHeader.X-Request-ID}}" + X-Source: "traefik-gateway" + + # Response transformation + add-response-headers: + headers: + customResponseHeaders: + X-Response-Time: "{{.ResponseTime}}" + + # Circuit breaker at gateway + circuit-breaker: + circuitBreaker: + expression: "NetworkErrorRatio() > 0.50" +``` + +## Service Mesh Integration + +```yaml +# Traefik with Istio integration +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: traefik-gateway +spec: + hosts: + - api.goodgo.com + gateways: + - traefik-gateway + http: + - match: + - uri: + prefix: "/api/v1" + route: + - destination: + host: user-service + port: + number: 5002 + - destination: + host: order-service + port: + number: 5003 + fault: + delay: + percentage: + value: 0.1 + fixedDelay: 5s + retries: + attempts: 3 + perTryTimeout: 2s +``` + +## Gateway-Level Circuit Breaker + +```typescript +// src/core/gateway/circuit-breaker.middleware.ts +import { Request, Response, NextFunction } from 'express'; +import { createCircuitBreaker } from '../resilience/circuit-breaker'; +import { logger } from '@goodgo/logger'; + +const serviceCircuitBreakers = new Map>(); + +export function gatewayCircuitBreaker(serviceName: string) { + return (req: Request, res: Response, next: NextFunction) => { + if (!serviceCircuitBreakers.has(serviceName)) { + serviceCircuitBreakers.set( + serviceName, + createCircuitBreaker( + async () => { + return next(); + }, + `gateway-${serviceName}`, + { + timeout: 5000, + errorThresholdPercentage: 50, + resetTimeout: 30000, + } + ) + ); + } + + const breaker = serviceCircuitBreakers.get(serviceName)!; + + breaker.fire().catch((error) => { + logger.error('Gateway circuit breaker opened', { serviceName, error }); + res.status(503).json({ + success: false, + error: { + code: 'SERVICE_UNAVAILABLE', + message: `Service ${serviceName} is currently unavailable`, + }, + }); + }); + }; +} +``` + +## Caching at Gateway + +```typescript +// src/core/gateway/gateway-cache.middleware.ts +import { Request, Response, NextFunction } from 'express'; +import { cacheService } from '../cache/cache.service'; +import { logger } from '@goodgo/logger'; + +export function gatewayCache(ttl: number = 60) { + return async (req: Request, res: Response, next: NextFunction) => { + // Only cache GET requests + if (req.method !== 'GET') { + return next(); + } + + const cacheKey = `gateway:${req.path}:${JSON.stringify(req.query)}`; + + try { + // Check cache + const cached = await cacheService.get(cacheKey); + if (cached) { + logger.debug('Gateway cache hit', { path: req.path }); + return res.json(cached); + } + + // Store original json method + const originalJson = res.json.bind(res); + + // Override to cache response + res.json = (data: unknown) => { + cacheService.set(cacheKey, data, ttl).catch((error) => { + logger.warn('Gateway cache set failed', { error }); + }); + return originalJson(data); + }; + + next(); + } catch (error) { + logger.error('Gateway cache error', { error }); + next(); + } + }; +} +``` diff --git a/.cursor/skills/data-consistency-patterns/SKILL.md b/.cursor/skills/data-consistency-patterns/SKILL.md index fc40f7fc..3bd2cf5e 100644 --- a/.cursor/skills/data-consistency-patterns/SKILL.md +++ b/.cursor/skills/data-consistency-patterns/SKILL.md @@ -1,7 +1,7 @@ --- name: data-consistency-patterns description: Data consistency patterns for distributed systems. Use for Saga patterns, distributed transactions, eventual consistency, or cross-service data sync. -dependencies: "prisma>=5" +compatibility: "prisma>=5" --- # Data Consistency Patterns @@ -37,9 +37,12 @@ Use this skill when: ### Consistency Models -1. **Strong Consistency**: All nodes see same data at same time -2. **Weak Consistency**: No guarantees about when consistency occurs -3. **Eventual Consistency**: System becomes consistent over time +| Model | Description | Use Case | +|-------|-------------|----------| +| **Strong** | All nodes see same data at same time | Financial transactions | +| **Weak** | No guarantees about when consistency occurs | Caching | +| **Eventual** | System becomes consistent over time | Read models, analytics | +| **Causal** | Related operations maintain order | User sessions | ### Distributed Transaction Challenges @@ -53,686 +56,193 @@ Use this skill when: ### Orchestration vs Choreography -**Orchestration (Centralized):** -- Central orchestrator coordinates steps -- Easier to understand and debug -- Single point of failure -- Use when: Complex workflows, need central control +| Approach | Central Control | Resilience | Complexity | Best For | +|----------|-----------------|------------|------------|----------| +| **Orchestration** | Yes (single coordinator) | Lower (SPOF) | Easier to debug | Complex workflows | +| **Choreography** | No (event-driven) | Higher | Harder to trace | Simple flows, loose coupling | -**Choreography (Decentralized):** -- Each service knows what to do next -- More resilient, no single point of failure -- Harder to understand overall flow -- Use when: Simple workflows, loose coupling preferred +### Saga Execution Flow -### Saga Orchestrator Pattern +``` +Execute: Step1 -> Step2 -> Step3 -> Complete + | | | + v v v +Compensate: <- Step2.undo <- Step1.undo (on failure) +``` + +### Key Saga Interfaces ```typescript -// src/core/saga/saga-orchestrator.ts -// EN: Saga orchestrator for distributed transactions -// VI: Saga orchestrator cho distributed transactions -import { logger } from '@goodgo/logger'; -import { eventPublisher } from '../events/event-publisher'; - -export interface SagaStep { +interface SagaStep { name: string; execute: () => Promise; compensate: (context: any) => Promise; retry?: number; } -export interface SagaContext { +interface SagaContext { sagaId: string; steps: SagaStep[]; currentStep: number; data: Record; status: 'pending' | 'running' | 'completed' | 'compensating' | 'failed'; } - -export class SagaOrchestrator { - /** - * EN: Execute saga with all steps - * VI: Thực thi saga với tất cả các bước - */ - async execute(context: SagaContext): Promise { - context.status = 'running'; - - try { - for (let i = 0; i < context.steps.length; i++) { - context.currentStep = i; - const step = context.steps[i]; - - logger.info('Executing saga step', { - sagaId: context.sagaId, - step: step.name, - stepIndex: i, - }); - - try { - const result = await this.executeWithRetry(step, step.retry || 3); - context.data[step.name] = result; - - // EN: Publish step completed event - // VI: Publish event step đã hoàn thành - await eventPublisher.publish('saga.step.completed', { - eventType: 'saga.step.completed', - eventVersion: '1.0.0', - data: { - sagaId: context.sagaId, - step: step.name, - stepIndex: i, - }, - }); - } catch (error) { - logger.error('Saga step failed', { - sagaId: context.sagaId, - step: step.name, - error: error.message, - }); - - // EN: Compensate all completed steps - // VI: Compensate tất cả các bước đã hoàn thành - await this.compensate(context, i - 1); - throw error; - } - } - - context.status = 'completed'; - logger.info('Saga completed successfully', { sagaId: context.sagaId }); - } catch (error) { - context.status = 'failed'; - logger.error('Saga failed', { - sagaId: context.sagaId, - error: error.message, - }); - throw error; - } - } - - private async executeWithRetry(step: SagaStep, maxRetries: number): Promise { - let lastError: Error; - - for (let attempt = 0; attempt <= maxRetries; attempt++) { - try { - return await step.execute(); - } catch (error) { - lastError = error; - if (attempt < maxRetries) { - const delay = Math.pow(2, attempt) * 1000; // Exponential backoff - await new Promise((resolve) => setTimeout(resolve, delay)); - } - } - } - - throw lastError!; - } - - /** - * EN: Compensate completed steps in reverse order - * VI: Compensate các bước đã hoàn thành theo thứ tự ngược - */ - private async compensate(context: SagaContext, lastCompletedStep: number): Promise { - context.status = 'compensating'; - - for (let i = lastCompletedStep; i >= 0; i--) { - const step = context.steps[i]; - - try { - logger.info('Compensating saga step', { - sagaId: context.sagaId, - step: step.name, - stepIndex: i, - }); - - await step.compensate(context.data); - - await eventPublisher.publish('saga.step.compensated', { - eventType: 'saga.step.compensated', - eventVersion: '1.0.0', - data: { - sagaId: context.sagaId, - step: step.name, - stepIndex: i, - }, - }); - } catch (error) { - logger.error('Compensation failed', { - sagaId: context.sagaId, - step: step.name, - error: error.message, - }); - // EN: Log but continue compensating other steps - // VI: Log nhưng tiếp tục compensate các bước khác - } - } - } -} ``` -### Saga Example: Order Processing +See [./references/REFERENCE.md](./references/REFERENCE.md) for full Saga orchestrator implementation. + +## Idempotency Pattern + +Ensures operations can be safely retried without side effects. + +### Key Concepts + +- **Idempotency Key**: Unique identifier for each operation +- **Result Storage**: Cache results to return on duplicates +- **TTL**: Time-to-live for idempotency records + +### Pattern ```typescript -// src/modules/order/order.saga.ts -// EN: Order processing saga -// VI: Saga xử lý order -import { SagaOrchestrator, SagaContext, SagaStep } from '../../core/saga/saga-orchestrator'; -import { orderService } from './order.service'; -import { paymentService } from '../payment/payment.service'; -import { inventoryService } from '../inventory/inventory.service'; -import { notificationService } from '../notification/notification.service'; - -export class OrderProcessingSaga { - private orchestrator: SagaOrchestrator; - - constructor() { - this.orchestrator = new SagaOrchestrator(); - } - - async processOrder(orderData: any): Promise { - const sagaId = `saga_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`; - - const steps: SagaStep[] = [ - { - name: 'create-order', - execute: async () => { - return await orderService.create(orderData); - }, - compensate: async (context) => { - await orderService.cancel(context.data['create-order'].id); - }, - }, - { - name: 'reserve-inventory', - execute: async () => { - const order = context.data['create-order']; - return await inventoryService.reserve(order.items); - }, - compensate: async (context) => { - const reservation = context.data['reserve-inventory']; - await inventoryService.release(reservation.id); - }, - }, - { - name: 'process-payment', - execute: async () => { - const order = context.data['create-order']; - return await paymentService.charge(order.total, order.paymentMethod); - }, - compensate: async (context) => { - const payment = context.data['process-payment']; - await paymentService.refund(payment.id); - }, - }, - { - name: 'send-confirmation', - execute: async () => { - const order = context.data['create-order']; - await notificationService.sendOrderConfirmation(order.userId, order.id); - // EN: Non-critical step, no compensation needed - // VI: Bước không quan trọng, không cần compensation - }, - compensate: async () => { - // EN: No compensation needed for notification - // VI: Không cần compensation cho notification - }, - }, - ]; - - const context: SagaContext = { - sagaId, - steps, - currentStep: 0, - data: {}, - status: 'pending', - }; - - await this.orchestrator.execute(context); - } -} +const key = `${operation}:${userId}:${hash(requestData)}`; +await idempotencyHandler.execute(key, () => operation()); ``` -### Saga Choreography Pattern - -```typescript -// EN: Choreography pattern - services react to events -// VI: Pattern choreography - services phản ứng với events -// src/modules/order/order.service.ts -export class OrderService { - async createOrder(data: CreateOrderDto): Promise { - const order = await this.orderRepository.create(data); - - // EN: Publish event for next step - // VI: Publish event cho bước tiếp theo - await eventPublisher.publish('order.created', { - eventType: 'order.created', - eventVersion: '1.0.0', - data: { - orderId: order.id, - items: order.items, - total: order.total, - }, - }); - - return order; - } - - async cancelOrder(orderId: string): Promise { - await this.orderRepository.update(orderId, { status: 'cancelled' }); - } -} - -// src/modules/inventory/inventory.consumer.ts -// EN: Inventory service reacts to order.created event -// VI: Inventory service phản ứng với order.created event -eventConsumer.on('order.created', { - handle: async (event) => { - try { - const reservation = await inventoryService.reserve(event.data.items); - - // EN: Publish next step event - // VI: Publish event bước tiếp theo - await eventPublisher.publish('inventory.reserved', { - eventType: 'inventory.reserved', - eventVersion: '1.0.0', - data: { - orderId: event.data.orderId, - reservationId: reservation.id, - }, - }); - } catch (error) { - // EN: Publish compensation event - // VI: Publish event compensation - await eventPublisher.publish('order.cancelled', { - eventType: 'order.cancelled', - eventVersion: '1.0.0', - data: { - orderId: event.data.orderId, - reason: 'inventory_reservation_failed', - }, - }); - } - }, -}); - -// src/modules/payment/payment.consumer.ts -eventConsumer.on('inventory.reserved', { - handle: async (event) => { - try { - const order = await orderService.findById(event.data.orderId); - const payment = await paymentService.charge(order.total, order.paymentMethod); - - await eventPublisher.publish('payment.processed', { - eventType: 'payment.processed', - eventVersion: '1.0.0', - data: { - orderId: event.data.orderId, - paymentId: payment.id, - }, - }); - } catch (error) { - // EN: Trigger compensation - // VI: Kích hoạt compensation - await eventPublisher.publish('order.cancelled', { - eventType: 'order.cancelled', - eventVersion: '1.0.0', - data: { - orderId: event.data.orderId, - reason: 'payment_failed', - }, - }); - } - }, -}); -``` - -## Idempotency Patterns - -### Idempotent Operations - -```typescript -// src/core/idempotency/idempotency.ts -// EN: Idempotency handler -// VI: Handler idempotency -import { PrismaClient } from '@prisma/client'; -import { logger } from '@goodgo/logger'; -import { v4 as uuidv4 } from 'uuid'; - -export class IdempotencyHandler { - constructor(private prisma: PrismaClient) {} - - /** - * EN: Execute operation with idempotency check - * VI: Thực thi operation với idempotency check - */ - async execute( - idempotencyKey: string, - operation: () => Promise, - ttl: number = 3600 // EN: 1 hour default / VI: Mặc định 1 giờ - ): Promise { - // EN: Check if already processed - // VI: Kiểm tra xem đã được xử lý chưa - const existing = await this.prisma.idempotencyRecord.findUnique({ - where: { idempotencyKey }, - }); - - if (existing) { - logger.info('Idempotent operation skipped', { idempotencyKey }); - return existing.result as T; - } - - // EN: Execute operation - // VI: Thực thi operation - try { - const result = await operation(); - - // EN: Store result - // VI: Lưu kết quả - await this.prisma.idempotencyRecord.create({ - data: { - idempotencyKey, - result: result as any, - expiresAt: new Date(Date.now() + ttl * 1000), - }, - }); - - return result; - } catch (error) { - // EN: Store error for idempotency (optional) - // VI: Lưu lỗi cho idempotency (tùy chọn) - logger.error('Idempotent operation failed', { idempotencyKey, error }); - throw error; - } - } -} - -// Usage -const idempotencyHandler = new IdempotencyHandler(prisma); - -await idempotencyHandler.execute( - `create-user-${userId}`, - async () => { - return await userService.create(userData); - } -); -``` - -### Idempotency Key Generation - -```typescript -// EN: Generate idempotency key from request -// VI: Tạo idempotency key từ request -export function generateIdempotencyKey( - userId: string, - operation: string, - requestData: any -): string { - const hash = createHash('sha256') - .update(JSON.stringify({ userId, operation, requestData })) - .digest('hex'); - - return `${operation}:${userId}:${hash.substr(0, 16)}`; -} -``` +See [./references/REFERENCE.md](./references/REFERENCE.md) for full implementation. ## Optimistic Locking -### Version-Based Locking +Prevents lost updates in concurrent scenarios using version fields. + +### Pattern ```typescript -// src/core/locking/optimistic-lock.ts -// EN: Optimistic locking with version field -// VI: Optimistic locking với trường version -export class OptimisticLockService { - /** - * EN: Update with optimistic lock - * VI: Update với optimistic lock - */ - async updateWithLock( - repository: any, - id: string, - updateFn: (current: T) => Partial, - maxRetries: number = 3 - ): Promise { - for (let attempt = 0; attempt < maxRetries; attempt++) { - // EN: Read current version - // VI: Đọc version hiện tại - const current = await repository.findById(id); - - if (!current) { - throw new Error('Resource not found'); - } - - // EN: Apply update - // VI: Áp dụng update - const updates = updateFn(current); - - try { - // EN: Update with version check - // VI: Update với kiểm tra version - const updated = await repository.update(id, { - ...updates, - version: current.version + 1, - }, { - where: { - id, - version: current.version, // EN: Only update if version matches / VI: Chỉ update nếu version khớp - }, - }); - - return updated; - } catch (error) { - if (attempt === maxRetries - 1) { - throw new Error('Optimistic lock conflict - max retries exceeded'); - } - - // EN: Retry with exponential backoff - // VI: Retry với exponential backoff - await new Promise((resolve) => - setTimeout(resolve, Math.pow(2, attempt) * 100) - ); - } - } - - throw new Error('Optimistic lock failed'); - } -} -``` - -## Eventual Consistency Strategies - -### Read Models and CQRS - -```typescript -// EN: Separate read and write models -// VI: Tách biệt read và write models -// src/modules/user/user.write.service.ts -export class UserWriteService { - async createUser(data: CreateUserDto): Promise { - const user = await this.userRepository.create(data); - - // EN: Publish event for read model update - // VI: Publish event để cập nhật read model - await eventPublisher.publish('user.created', { - eventType: 'user.created', - eventVersion: '1.0.0', - data: { - userId: user.id, - email: user.email, - name: user.name, - }, - }); - - return user; - } -} - -// src/modules/user/user.read.service.ts -// EN: Read model updated via events -// VI: Read model được cập nhật qua events -export class UserReadService { - async getUser(userId: string): Promise { - return await this.userReadRepository.findById(userId); - } -} - -// src/modules/user/user.read.consumer.ts -eventConsumer.on('user.created', { - handle: async (event) => { - // EN: Update read model - // VI: Cập nhật read model - await userReadRepository.create({ - userId: event.data.userId, - email: event.data.email, - name: event.data.name, - // EN: Additional denormalized data for queries - // VI: Dữ liệu denormalized thêm cho queries - }); - }, +await prisma.entity.update({ + where: { id, version: currentVersion }, + data: { ...updates, version: { increment: 1 } } }); ``` -### Conflict Resolution +### Prisma Schema -```typescript -// src/core/consistency/conflict-resolver.ts -// EN: Conflict resolution strategies -// VI: Các chiến lược giải quyết conflict -export enum ConflictResolutionStrategy { - LAST_WRITE_WINS = 'last_write_wins', - FIRST_WRITE_WINS = 'first_write_wins', - MERGE = 'merge', - MANUAL = 'manual', -} - -export class ConflictResolver { - resolve( - strategy: ConflictResolutionStrategy, - current: any, - incoming: any - ): any { - switch (strategy) { - case ConflictResolutionStrategy.LAST_WRITE_WINS: - return incoming.timestamp > current.timestamp ? incoming : current; - - case ConflictResolutionStrategy.FIRST_WRITE_WINS: - return incoming.timestamp < current.timestamp ? incoming : current; - - case ConflictResolutionStrategy.MERGE: - return { ...current, ...incoming, timestamp: Date.now() }; - - case ConflictResolutionStrategy.MANUAL: - // EN: Store conflict for manual resolution - // VI: Lưu conflict để giải quyết thủ công - this.storeConflict(current, incoming); - return current; - - default: - return current; - } - } +```prisma +model Entity { + id String @id @default(cuid()) + version Int @default(1) + // ... other fields } ``` +See [./references/REFERENCE.md](./references/REFERENCE.md) for full optimistic locking service. + +## CQRS Pattern + +Command Query Responsibility Segregation separates read and write operations. + +### Architecture + +``` +Write Path: Command -> Write Model -> Event -> Event Store +Read Path: Query -> Read Model (denormalized, optimized for reads) +``` + +### Key Benefits + +- Optimized read models for query performance +- Independent scaling of read/write operations +- Eventual consistency between models + +See [./references/REFERENCE.md](./references/REFERENCE.md) for implementation details. + +## Conflict Resolution Strategies + +| Strategy | Description | Use Case | +|----------|-------------|----------| +| **Last Write Wins** | Latest timestamp wins | Simple scenarios | +| **First Write Wins** | Earliest timestamp wins | Preserving original data | +| **Merge** | Combine both versions | Non-conflicting fields | +| **Manual** | Store for human review | Critical data conflicts | + +## Outbox Pattern + +Ensures reliable event publishing by storing events in the same transaction as business data. + +### Flow + +1. Execute business operation in transaction +2. Store event in outbox table (same transaction) +3. Background processor publishes and marks as sent + +See [./references/REFERENCE.md](./references/REFERENCE.md) for implementation. + ## Best Practices ### Saga Pattern - -1. **Design Compensations**: Every step needs compensation -2. **Idempotent Steps**: Make steps idempotent for retries -3. **Timeout Handling**: Set timeouts for saga execution -4. **Monitoring**: Track saga execution and compensation -5. **Choreography vs Orchestration**: Choose based on complexity +- Design compensations for every step +- Make steps idempotent for retries +- Set timeouts for saga execution +- Monitor saga execution and compensation +- Choose orchestration for complex workflows, choreography for simple ones ### Idempotency - -1. **Idempotency Keys**: Generate from request data -2. **Key Storage**: Store keys with results -3. **TTL**: Set appropriate TTL for idempotency records -4. **Cleanup**: Regularly clean expired records +- Generate keys from request data +- Store keys with results +- Set appropriate TTL for records +- Regularly clean expired records ### Eventual Consistency - -1. **Accept Delays**: Accept that consistency is eventual -2. **Read Models**: Use separate read models for queries -3. **Conflict Resolution**: Define resolution strategies -4. **Monitoring**: Monitor consistency lag +- Accept that consistency is eventual +- Use separate read models for queries +- Define resolution strategies upfront +- Monitor consistency lag ### Optimistic Locking - -1. **Version Fields**: Add version fields to entities -2. **Retry Logic**: Implement retry with backoff -3. **Conflict Handling**: Handle conflicts gracefully -4. **Performance**: Better than pessimistic locking for read-heavy workloads - -## Testing Patterns - -### Testing Sagas - -```typescript -// src/__tests__/saga/order-processing.saga.test.ts -describe('OrderProcessingSaga', () => { - it('should complete saga successfully', async () => { - const saga = new OrderProcessingSaga(); - await saga.processOrder(orderData); - - expect(orderService.create).toHaveBeenCalled(); - expect(inventoryService.reserve).toHaveBeenCalled(); - expect(paymentService.charge).toHaveBeenCalled(); - }); - - it('should compensate on payment failure', async () => { - paymentService.charge.mockRejectedValue(new Error('Payment failed')); - - await expect(saga.processOrder(orderData)).rejects.toThrow(); - - expect(orderService.cancel).toHaveBeenCalled(); - expect(inventoryService.release).toHaveBeenCalled(); - expect(paymentService.refund).not.toHaveBeenCalled(); - }); -}); -``` +- Add version fields to entities +- Implement retry with exponential backoff +- Handle conflicts gracefully +- Use for read-heavy workloads ## Common Mistakes -1. **No Compensation Logic**: Saga steps without rollback - ```typescript - // ❌ BAD: No compensation - steps: [{ execute: () => createOrder() }] +### No Compensation Logic +```typescript +// BAD: No compensation +steps: [{ execute: () => createOrder() }] - // ✅ GOOD: Always define compensation - steps: [{ - execute: () => createOrder(), - compensate: (ctx) => cancelOrder(ctx.orderId) - }] - ``` +// GOOD: Always define compensation +steps: [{ + execute: () => createOrder(), + compensate: (ctx) => cancelOrder(ctx.orderId) +}] +``` -2. **Missing Idempotency**: Duplicate processing on retry - ```typescript - // ❌ BAD: Creates duplicate - await createPayment(orderId); +### Missing Idempotency +```typescript +// BAD: Creates duplicate on retry +await createPayment(orderId); - // ✅ GOOD: Idempotent check - const existing = await findPayment(idempotencyKey); - if (existing) return existing; - await createPayment(orderId); - ``` +// GOOD: Idempotent check +const existing = await findPayment(idempotencyKey); +if (existing) return existing; +await createPayment(orderId); +``` -3. **Ignoring Partial Failures**: Not handling step failures - ```typescript - // ❌ BAD: No error handling - await step1(); await step2(); await step3(); +### Ignoring Partial Failures +```typescript +// BAD: No error handling +await step1(); await step2(); await step3(); - // ✅ GOOD: Saga orchestration - await sagaOrchestrator.execute(context); - ``` +// GOOD: Saga orchestration +await sagaOrchestrator.execute(context); +``` -4. **No Version Field**: Concurrent update conflicts - ```prisma - // ✅ Add version for optimistic locking - model Entity { - version Int @default(1) - } - ``` +### No Version Field +```prisma +// GOOD: Add version for optimistic locking +model Entity { + version Int @default(1) +} +``` ## Quick Reference @@ -743,40 +253,19 @@ describe('OrderProcessingSaga', () => { | **Outbox Pattern** | Guaranteed event publishing | Medium | | **Idempotency** | Retry-safe operations | Low | | **Optimistic Lock** | Concurrent updates | Low | - -**Saga Steps:** -``` -Execute: Step1 → Step2 → Step3 → Complete -Compensate: ← Step2.undo ← Step1.undo (on failure) -``` - -**Idempotency Key Pattern:** -```typescript -const key = `${operation}:${userId}:${hash(requestData)}`; -await idempotencyHandler.execute(key, () => operation()); -``` - -**Optimistic Lock Query:** -```typescript -await prisma.entity.update({ - where: { id, version: currentVersion }, - data: { ...updates, version: { increment: 1 } } -}); -``` - -**Consistency Models:** -| Model | Latency | Use Case | -|-------|---------|----------| -| **Strong** | High | Financial transactions | -| **Eventual** | Low | Read models, analytics | -| **Causal** | Medium | User sessions | +| **CQRS** | Read/write optimization | High | +| **Dead Letter Queue** | Failed message handling | Medium | ## Resources -- [Saga Pattern](https://microservices.io/patterns/data/saga.html) - Saga pattern overview -- [Event Sourcing](https://martinfowler.com/eaaDev/EventSourcing.html) - Event sourcing pattern -- [CQRS Pattern](https://martinfowler.com/bliki/CQRS.html) - CQRS pattern +### Internal References +- [Detailed Code Examples](./references/REFERENCE.md) - Full implementations for all patterns - [Event-Driven Architecture](../event-driven-architecture/SKILL.md) - Event patterns - [Error Handling Patterns](../error-handling-patterns/SKILL.md) - Error handling - [Database & Prisma](../database-prisma/SKILL.md) - Database patterns - [Project Rules](../project-rules/SKILL.md) - GoodGo coding standards + +### External Resources +- [Saga Pattern](https://microservices.io/patterns/data/saga.html) - Saga pattern overview +- [Event Sourcing](https://martinfowler.com/eaaDev/EventSourcing.html) - Event sourcing pattern +- [CQRS Pattern](https://martinfowler.com/bliki/CQRS.html) - CQRS pattern diff --git a/.cursor/skills/data-consistency-patterns/references/REFERENCE.md b/.cursor/skills/data-consistency-patterns/references/REFERENCE.md new file mode 100644 index 00000000..95c3a58b --- /dev/null +++ b/.cursor/skills/data-consistency-patterns/references/REFERENCE.md @@ -0,0 +1,926 @@ +# Data Consistency Patterns - Reference + +This reference contains detailed implementations and code examples for data consistency patterns in distributed systems. + +## Table of Contents + +- [Saga Orchestrator Implementation](#saga-orchestrator-implementation) +- [Saga Example: Order Processing](#saga-example-order-processing) +- [Saga Choreography Pattern](#saga-choreography-pattern) +- [Idempotency Handler](#idempotency-handler) +- [Idempotency Key Generation](#idempotency-key-generation) +- [Optimistic Locking](#optimistic-locking) +- [CQRS Pattern](#cqrs-pattern) +- [Conflict Resolution](#conflict-resolution) +- [Testing Patterns](#testing-patterns) + +--- + +## Saga Orchestrator Implementation + +```typescript +// src/core/saga/saga-orchestrator.ts +// EN: Saga orchestrator for distributed transactions +// VI: Saga orchestrator cho distributed transactions +import { logger } from '@goodgo/logger'; +import { eventPublisher } from '../events/event-publisher'; + +export interface SagaStep { + name: string; + execute: () => Promise; + compensate: (context: any) => Promise; + retry?: number; +} + +export interface SagaContext { + sagaId: string; + steps: SagaStep[]; + currentStep: number; + data: Record; + status: 'pending' | 'running' | 'completed' | 'compensating' | 'failed'; +} + +export class SagaOrchestrator { + /** + * EN: Execute saga with all steps + * VI: Thuc thi saga voi tat ca cac buoc + */ + async execute(context: SagaContext): Promise { + context.status = 'running'; + + try { + for (let i = 0; i < context.steps.length; i++) { + context.currentStep = i; + const step = context.steps[i]; + + logger.info('Executing saga step', { + sagaId: context.sagaId, + step: step.name, + stepIndex: i, + }); + + try { + const result = await this.executeWithRetry(step, step.retry || 3); + context.data[step.name] = result; + + // EN: Publish step completed event + // VI: Publish event step da hoan thanh + await eventPublisher.publish('saga.step.completed', { + eventType: 'saga.step.completed', + eventVersion: '1.0.0', + data: { + sagaId: context.sagaId, + step: step.name, + stepIndex: i, + }, + }); + } catch (error) { + logger.error('Saga step failed', { + sagaId: context.sagaId, + step: step.name, + error: error.message, + }); + + // EN: Compensate all completed steps + // VI: Compensate tat ca cac buoc da hoan thanh + await this.compensate(context, i - 1); + throw error; + } + } + + context.status = 'completed'; + logger.info('Saga completed successfully', { sagaId: context.sagaId }); + } catch (error) { + context.status = 'failed'; + logger.error('Saga failed', { + sagaId: context.sagaId, + error: error.message, + }); + throw error; + } + } + + private async executeWithRetry(step: SagaStep, maxRetries: number): Promise { + let lastError: Error; + + for (let attempt = 0; attempt <= maxRetries; attempt++) { + try { + return await step.execute(); + } catch (error) { + lastError = error; + if (attempt < maxRetries) { + const delay = Math.pow(2, attempt) * 1000; // Exponential backoff + await new Promise((resolve) => setTimeout(resolve, delay)); + } + } + } + + throw lastError!; + } + + /** + * EN: Compensate completed steps in reverse order + * VI: Compensate cac buoc da hoan thanh theo thu tu nguoc + */ + private async compensate(context: SagaContext, lastCompletedStep: number): Promise { + context.status = 'compensating'; + + for (let i = lastCompletedStep; i >= 0; i--) { + const step = context.steps[i]; + + try { + logger.info('Compensating saga step', { + sagaId: context.sagaId, + step: step.name, + stepIndex: i, + }); + + await step.compensate(context.data); + + await eventPublisher.publish('saga.step.compensated', { + eventType: 'saga.step.compensated', + eventVersion: '1.0.0', + data: { + sagaId: context.sagaId, + step: step.name, + stepIndex: i, + }, + }); + } catch (error) { + logger.error('Compensation failed', { + sagaId: context.sagaId, + step: step.name, + error: error.message, + }); + // EN: Log but continue compensating other steps + // VI: Log nhung tiep tuc compensate cac buoc khac + } + } + } +} +``` + +--- + +## Saga Example: Order Processing + +```typescript +// src/modules/order/order.saga.ts +// EN: Order processing saga +// VI: Saga xu ly order +import { SagaOrchestrator, SagaContext, SagaStep } from '../../core/saga/saga-orchestrator'; +import { orderService } from './order.service'; +import { paymentService } from '../payment/payment.service'; +import { inventoryService } from '../inventory/inventory.service'; +import { notificationService } from '../notification/notification.service'; + +export class OrderProcessingSaga { + private orchestrator: SagaOrchestrator; + + constructor() { + this.orchestrator = new SagaOrchestrator(); + } + + async processOrder(orderData: any): Promise { + const sagaId = `saga_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`; + + const steps: SagaStep[] = [ + { + name: 'create-order', + execute: async () => { + return await orderService.create(orderData); + }, + compensate: async (context) => { + await orderService.cancel(context.data['create-order'].id); + }, + }, + { + name: 'reserve-inventory', + execute: async () => { + const order = context.data['create-order']; + return await inventoryService.reserve(order.items); + }, + compensate: async (context) => { + const reservation = context.data['reserve-inventory']; + await inventoryService.release(reservation.id); + }, + }, + { + name: 'process-payment', + execute: async () => { + const order = context.data['create-order']; + return await paymentService.charge(order.total, order.paymentMethod); + }, + compensate: async (context) => { + const payment = context.data['process-payment']; + await paymentService.refund(payment.id); + }, + }, + { + name: 'send-confirmation', + execute: async () => { + const order = context.data['create-order']; + await notificationService.sendOrderConfirmation(order.userId, order.id); + // EN: Non-critical step, no compensation needed + // VI: Buoc khong quan trong, khong can compensation + }, + compensate: async () => { + // EN: No compensation needed for notification + // VI: Khong can compensation cho notification + }, + }, + ]; + + const context: SagaContext = { + sagaId, + steps, + currentStep: 0, + data: {}, + status: 'pending', + }; + + await this.orchestrator.execute(context); + } +} +``` + +--- + +## Saga Choreography Pattern + +```typescript +// EN: Choreography pattern - services react to events +// VI: Pattern choreography - services phan ung voi events +// src/modules/order/order.service.ts +export class OrderService { + async createOrder(data: CreateOrderDto): Promise { + const order = await this.orderRepository.create(data); + + // EN: Publish event for next step + // VI: Publish event cho buoc tiep theo + await eventPublisher.publish('order.created', { + eventType: 'order.created', + eventVersion: '1.0.0', + data: { + orderId: order.id, + items: order.items, + total: order.total, + }, + }); + + return order; + } + + async cancelOrder(orderId: string): Promise { + await this.orderRepository.update(orderId, { status: 'cancelled' }); + } +} + +// src/modules/inventory/inventory.consumer.ts +// EN: Inventory service reacts to order.created event +// VI: Inventory service phan ung voi order.created event +eventConsumer.on('order.created', { + handle: async (event) => { + try { + const reservation = await inventoryService.reserve(event.data.items); + + // EN: Publish next step event + // VI: Publish event buoc tiep theo + await eventPublisher.publish('inventory.reserved', { + eventType: 'inventory.reserved', + eventVersion: '1.0.0', + data: { + orderId: event.data.orderId, + reservationId: reservation.id, + }, + }); + } catch (error) { + // EN: Publish compensation event + // VI: Publish event compensation + await eventPublisher.publish('order.cancelled', { + eventType: 'order.cancelled', + eventVersion: '1.0.0', + data: { + orderId: event.data.orderId, + reason: 'inventory_reservation_failed', + }, + }); + } + }, +}); + +// src/modules/payment/payment.consumer.ts +eventConsumer.on('inventory.reserved', { + handle: async (event) => { + try { + const order = await orderService.findById(event.data.orderId); + const payment = await paymentService.charge(order.total, order.paymentMethod); + + await eventPublisher.publish('payment.processed', { + eventType: 'payment.processed', + eventVersion: '1.0.0', + data: { + orderId: event.data.orderId, + paymentId: payment.id, + }, + }); + } catch (error) { + // EN: Trigger compensation + // VI: Kich hoat compensation + await eventPublisher.publish('order.cancelled', { + eventType: 'order.cancelled', + eventVersion: '1.0.0', + data: { + orderId: event.data.orderId, + reason: 'payment_failed', + }, + }); + } + }, +}); +``` + +--- + +## Idempotency Handler + +```typescript +// src/core/idempotency/idempotency.ts +// EN: Idempotency handler +// VI: Handler idempotency +import { PrismaClient } from '@prisma/client'; +import { logger } from '@goodgo/logger'; +import { v4 as uuidv4 } from 'uuid'; + +export class IdempotencyHandler { + constructor(private prisma: PrismaClient) {} + + /** + * EN: Execute operation with idempotency check + * VI: Thuc thi operation voi idempotency check + */ + async execute( + idempotencyKey: string, + operation: () => Promise, + ttl: number = 3600 // EN: 1 hour default / VI: Mac dinh 1 gio + ): Promise { + // EN: Check if already processed + // VI: Kiem tra xem da duoc xu ly chua + const existing = await this.prisma.idempotencyRecord.findUnique({ + where: { idempotencyKey }, + }); + + if (existing) { + logger.info('Idempotent operation skipped', { idempotencyKey }); + return existing.result as T; + } + + // EN: Execute operation + // VI: Thuc thi operation + try { + const result = await operation(); + + // EN: Store result + // VI: Luu ket qua + await this.prisma.idempotencyRecord.create({ + data: { + idempotencyKey, + result: result as any, + expiresAt: new Date(Date.now() + ttl * 1000), + }, + }); + + return result; + } catch (error) { + // EN: Store error for idempotency (optional) + // VI: Luu loi cho idempotency (tuy chon) + logger.error('Idempotent operation failed', { idempotencyKey, error }); + throw error; + } + } +} + +// Usage +const idempotencyHandler = new IdempotencyHandler(prisma); + +await idempotencyHandler.execute( + `create-user-${userId}`, + async () => { + return await userService.create(userData); + } +); +``` + +--- + +## Idempotency Key Generation + +```typescript +// EN: Generate idempotency key from request +// VI: Tao idempotency key tu request +export function generateIdempotencyKey( + userId: string, + operation: string, + requestData: any +): string { + const hash = createHash('sha256') + .update(JSON.stringify({ userId, operation, requestData })) + .digest('hex'); + + return `${operation}:${userId}:${hash.substr(0, 16)}`; +} +``` + +--- + +## Optimistic Locking + +### Version-Based Locking + +```typescript +// src/core/locking/optimistic-lock.ts +// EN: Optimistic locking with version field +// VI: Optimistic locking voi truong version +export class OptimisticLockService { + /** + * EN: Update with optimistic lock + * VI: Update voi optimistic lock + */ + async updateWithLock( + repository: any, + id: string, + updateFn: (current: T) => Partial, + maxRetries: number = 3 + ): Promise { + for (let attempt = 0; attempt < maxRetries; attempt++) { + // EN: Read current version + // VI: Doc version hien tai + const current = await repository.findById(id); + + if (!current) { + throw new Error('Resource not found'); + } + + // EN: Apply update + // VI: Ap dung update + const updates = updateFn(current); + + try { + // EN: Update with version check + // VI: Update voi kiem tra version + const updated = await repository.update(id, { + ...updates, + version: current.version + 1, + }, { + where: { + id, + version: current.version, // EN: Only update if version matches / VI: Chi update neu version khop + }, + }); + + return updated; + } catch (error) { + if (attempt === maxRetries - 1) { + throw new Error('Optimistic lock conflict - max retries exceeded'); + } + + // EN: Retry with exponential backoff + // VI: Retry voi exponential backoff + await new Promise((resolve) => + setTimeout(resolve, Math.pow(2, attempt) * 100) + ); + } + } + + throw new Error('Optimistic lock failed'); + } +} +``` + +### Prisma Schema for Optimistic Locking + +```prisma +// Add version field to entities requiring optimistic locking +model Entity { + id String @id @default(cuid()) + version Int @default(1) + data String + updatedAt DateTime @updatedAt +} +``` + +--- + +## CQRS Pattern + +### Write Service + +```typescript +// EN: Separate read and write models +// VI: Tach biet read va write models +// src/modules/user/user.write.service.ts +export class UserWriteService { + async createUser(data: CreateUserDto): Promise { + const user = await this.userRepository.create(data); + + // EN: Publish event for read model update + // VI: Publish event de cap nhat read model + await eventPublisher.publish('user.created', { + eventType: 'user.created', + eventVersion: '1.0.0', + data: { + userId: user.id, + email: user.email, + name: user.name, + }, + }); + + return user; + } +} +``` + +### Read Service + +```typescript +// src/modules/user/user.read.service.ts +// EN: Read model updated via events +// VI: Read model duoc cap nhat qua events +export class UserReadService { + async getUser(userId: string): Promise { + return await this.userReadRepository.findById(userId); + } +} +``` + +### Read Model Consumer + +```typescript +// src/modules/user/user.read.consumer.ts +eventConsumer.on('user.created', { + handle: async (event) => { + // EN: Update read model + // VI: Cap nhat read model + await userReadRepository.create({ + userId: event.data.userId, + email: event.data.email, + name: event.data.name, + // EN: Additional denormalized data for queries + // VI: Du lieu denormalized them cho queries + }); + }, +}); +``` + +--- + +## Conflict Resolution + +```typescript +// src/core/consistency/conflict-resolver.ts +// EN: Conflict resolution strategies +// VI: Cac chien luoc giai quyet conflict +export enum ConflictResolutionStrategy { + LAST_WRITE_WINS = 'last_write_wins', + FIRST_WRITE_WINS = 'first_write_wins', + MERGE = 'merge', + MANUAL = 'manual', +} + +export class ConflictResolver { + resolve( + strategy: ConflictResolutionStrategy, + current: any, + incoming: any + ): any { + switch (strategy) { + case ConflictResolutionStrategy.LAST_WRITE_WINS: + return incoming.timestamp > current.timestamp ? incoming : current; + + case ConflictResolutionStrategy.FIRST_WRITE_WINS: + return incoming.timestamp < current.timestamp ? incoming : current; + + case ConflictResolutionStrategy.MERGE: + return { ...current, ...incoming, timestamp: Date.now() }; + + case ConflictResolutionStrategy.MANUAL: + // EN: Store conflict for manual resolution + // VI: Luu conflict de giai quyet thu cong + this.storeConflict(current, incoming); + return current; + + default: + return current; + } + } + + private storeConflict(current: any, incoming: any): void { + // Store conflict in database for manual resolution + logger.warn('Conflict stored for manual resolution', { current, incoming }); + } +} +``` + +### Conflict Resolution Usage Example + +```typescript +// Example: Resolving conflicts in a sync operation +const resolver = new ConflictResolver(); + +async function syncData(localData: any, remoteData: any): Promise { + if (localData.version !== remoteData.version) { + return resolver.resolve( + ConflictResolutionStrategy.LAST_WRITE_WINS, + localData, + remoteData + ); + } + return localData; +} +``` + +--- + +## Testing Patterns + +### Testing Sagas + +```typescript +// src/__tests__/saga/order-processing.saga.test.ts +describe('OrderProcessingSaga', () => { + let saga: OrderProcessingSaga; + let orderService: jest.Mocked; + let inventoryService: jest.Mocked; + let paymentService: jest.Mocked; + + beforeEach(() => { + orderService = { + create: jest.fn(), + cancel: jest.fn(), + } as any; + inventoryService = { + reserve: jest.fn(), + release: jest.fn(), + } as any; + paymentService = { + charge: jest.fn(), + refund: jest.fn(), + } as any; + + saga = new OrderProcessingSaga(); + }); + + it('should complete saga successfully', async () => { + orderService.create.mockResolvedValue({ id: 'order-1', items: [], total: 100 }); + inventoryService.reserve.mockResolvedValue({ id: 'reservation-1' }); + paymentService.charge.mockResolvedValue({ id: 'payment-1' }); + + await saga.processOrder(orderData); + + expect(orderService.create).toHaveBeenCalled(); + expect(inventoryService.reserve).toHaveBeenCalled(); + expect(paymentService.charge).toHaveBeenCalled(); + }); + + it('should compensate on payment failure', async () => { + orderService.create.mockResolvedValue({ id: 'order-1', items: [], total: 100 }); + inventoryService.reserve.mockResolvedValue({ id: 'reservation-1' }); + paymentService.charge.mockRejectedValue(new Error('Payment failed')); + + await expect(saga.processOrder(orderData)).rejects.toThrow('Payment failed'); + + expect(orderService.cancel).toHaveBeenCalled(); + expect(inventoryService.release).toHaveBeenCalled(); + expect(paymentService.refund).not.toHaveBeenCalled(); + }); + + it('should compensate on inventory failure', async () => { + orderService.create.mockResolvedValue({ id: 'order-1', items: [], total: 100 }); + inventoryService.reserve.mockRejectedValue(new Error('Out of stock')); + + await expect(saga.processOrder(orderData)).rejects.toThrow('Out of stock'); + + expect(orderService.cancel).toHaveBeenCalled(); + expect(inventoryService.release).not.toHaveBeenCalled(); + }); +}); +``` + +### Testing Idempotency + +```typescript +// src/__tests__/idempotency/idempotency-handler.test.ts +describe('IdempotencyHandler', () => { + let handler: IdempotencyHandler; + let prisma: PrismaClient; + + beforeEach(() => { + prisma = new PrismaClient(); + handler = new IdempotencyHandler(prisma); + }); + + it('should execute operation once', async () => { + const operation = jest.fn().mockResolvedValue({ id: 'result-1' }); + + const result1 = await handler.execute('key-1', operation); + const result2 = await handler.execute('key-1', operation); + + expect(operation).toHaveBeenCalledTimes(1); + expect(result1).toEqual(result2); + }); + + it('should execute different keys independently', async () => { + const operation = jest.fn().mockResolvedValue({ id: 'result' }); + + await handler.execute('key-1', operation); + await handler.execute('key-2', operation); + + expect(operation).toHaveBeenCalledTimes(2); + }); +}); +``` + +### Testing Optimistic Locking + +```typescript +// src/__tests__/locking/optimistic-lock.test.ts +describe('OptimisticLockService', () => { + let service: OptimisticLockService; + let repository: jest.Mocked; + + beforeEach(() => { + repository = { + findById: jest.fn(), + update: jest.fn(), + } as any; + service = new OptimisticLockService(); + }); + + it('should update successfully on first attempt', async () => { + repository.findById.mockResolvedValue({ id: '1', version: 1, data: 'old' }); + repository.update.mockResolvedValue({ id: '1', version: 2, data: 'new' }); + + const result = await service.updateWithLock( + repository, + '1', + (current) => ({ data: 'new' }) + ); + + expect(result.version).toBe(2); + expect(result.data).toBe('new'); + }); + + it('should retry on version conflict', async () => { + repository.findById + .mockResolvedValueOnce({ id: '1', version: 1, data: 'old' }) + .mockResolvedValueOnce({ id: '1', version: 2, data: 'concurrent' }); + repository.update + .mockRejectedValueOnce(new Error('Version conflict')) + .mockResolvedValueOnce({ id: '1', version: 3, data: 'new' }); + + const result = await service.updateWithLock( + repository, + '1', + (current) => ({ data: 'new' }) + ); + + expect(repository.findById).toHaveBeenCalledTimes(2); + expect(result.version).toBe(3); + }); + + it('should throw after max retries', async () => { + repository.findById.mockResolvedValue({ id: '1', version: 1, data: 'old' }); + repository.update.mockRejectedValue(new Error('Version conflict')); + + await expect( + service.updateWithLock(repository, '1', (current) => ({ data: 'new' }), 3) + ).rejects.toThrow('Optimistic lock conflict - max retries exceeded'); + + expect(repository.findById).toHaveBeenCalledTimes(3); + }); +}); +``` + +--- + +## Additional Patterns + +### Outbox Pattern for Reliable Event Publishing + +```typescript +// src/core/outbox/outbox.service.ts +// EN: Outbox pattern for reliable event publishing +// VI: Outbox pattern cho viec publish event dang tin cay +export class OutboxService { + constructor(private prisma: PrismaClient) {} + + async publishWithOutbox( + operation: () => Promise, + event: DomainEvent + ): Promise { + return await this.prisma.$transaction(async (tx) => { + // Execute business operation + const result = await operation(); + + // Store event in outbox (same transaction) + await tx.outboxEvent.create({ + data: { + eventType: event.eventType, + payload: event.data as any, + status: 'pending', + createdAt: new Date(), + }, + }); + + return result; + }); + } +} + +// Background processor to publish outbox events +export class OutboxProcessor { + async processOutbox(): Promise { + const events = await this.prisma.outboxEvent.findMany({ + where: { status: 'pending' }, + orderBy: { createdAt: 'asc' }, + take: 100, + }); + + for (const event of events) { + try { + await eventPublisher.publish(event.eventType, event.payload); + await this.prisma.outboxEvent.update({ + where: { id: event.id }, + data: { status: 'published', publishedAt: new Date() }, + }); + } catch (error) { + await this.prisma.outboxEvent.update({ + where: { id: event.id }, + data: { + status: 'failed', + retryCount: { increment: 1 }, + lastError: error.message, + }, + }); + } + } + } +} +``` + +### Dead Letter Queue Pattern + +```typescript +// src/core/dlq/dead-letter-queue.ts +// EN: Dead letter queue for failed messages +// VI: Dead letter queue cho cac message that bai +export class DeadLetterQueue { + constructor(private prisma: PrismaClient) {} + + async moveToDeadLetter( + originalQueue: string, + message: any, + error: Error, + retryCount: number + ): Promise { + await this.prisma.deadLetterMessage.create({ + data: { + originalQueue, + payload: message as any, + errorMessage: error.message, + errorStack: error.stack, + retryCount, + createdAt: new Date(), + }, + }); + + logger.warn('Message moved to dead letter queue', { + originalQueue, + retryCount, + error: error.message, + }); + } + + async reprocessDeadLetters(queue: string): Promise { + const messages = await this.prisma.deadLetterMessage.findMany({ + where: { originalQueue: queue, reprocessed: false }, + }); + + let processed = 0; + for (const msg of messages) { + try { + await eventPublisher.publish(queue, msg.payload); + await this.prisma.deadLetterMessage.update({ + where: { id: msg.id }, + data: { reprocessed: true, reprocessedAt: new Date() }, + }); + processed++; + } catch (error) { + logger.error('Failed to reprocess dead letter', { id: msg.id, error }); + } + } + + return processed; + } +} +``` diff --git a/.cursor/skills/database-prisma/SKILL.md b/.cursor/skills/database-prisma/SKILL.md index b4ccf3e4..4ceec3b0 100644 --- a/.cursor/skills/database-prisma/SKILL.md +++ b/.cursor/skills/database-prisma/SKILL.md @@ -1,7 +1,7 @@ --- name: database-prisma description: Prisma ORM and database patterns for GoodGo services. Use for schemas, migrations, repositories, or query optimization. -dependencies: "prisma>=5, @prisma/client>=5" +compatibility: "prisma>=5, @prisma/client>=5" --- # Prisma Database Patterns @@ -27,547 +27,142 @@ Use this skill when: - Connection pooling for performance - Transaction support for data consistency -## Prisma Setup +## Key Patterns -### Installation +### Prisma Setup ```bash npm install @prisma/client prisma -npm install --save-dev @types/node +npx prisma init ``` -### Configuration +### Schema Definition -```typescript -// prisma/schema.prisma -generator client { - provider = "prisma-client-js" -} - -datasource db { - provider = "postgresql" - url = env("DATABASE_URL") -} - -// Base model with common fields +```prisma model User { id String @id @default(cuid()) email String @unique name String? - password String role Role @default(USER) createdAt DateTime @default(now()) updatedAt DateTime @updatedAt - - // Relations posts Post[] - profile Profile? - - // Indexes for performance + @@index([email]) @@index([createdAt]) @@map("users") } - -model Post { - id String @id @default(cuid()) - title String - content String? - published Boolean @default(false) - authorId String - author User @relation(fields: [authorId], references: [id]) - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt - - @@index([authorId]) - @@index([published, createdAt]) - @@map("posts") -} - -model Profile { - id String @id @default(cuid()) - bio String? - avatar String? - userId String @unique - user User @relation(fields: [userId], references: [id]) - - @@map("profiles") -} - -enum Role { - USER - ADMIN - MODERATOR -} ``` -## Database Connection +### Database Connection ```typescript -// src/lib/prisma.ts import { PrismaClient } from '@prisma/client'; -const globalForPrisma = global as unknown as { - prisma: PrismaClient | undefined; -}; +const globalForPrisma = global as unknown as { prisma: PrismaClient | undefined }; -export const prisma = globalForPrisma.prisma ?? - new PrismaClient({ - log: process.env.NODE_ENV === 'development' - ? ['query', 'error', 'warn'] - : ['error'], - }); - -if (process.env.NODE_ENV !== 'production') { - globalForPrisma.prisma = prisma; -} - -// Middleware for soft delete -prisma.$use(async (params, next) => { - if (params.model && params.action === 'delete') { - return next({ - ...params, - action: 'update', - args: { - ...params.args, - data: { deletedAt: new Date() } - } - }); - } - return next(params); +export const prisma = globalForPrisma.prisma ?? new PrismaClient({ + log: process.env.NODE_ENV === 'development' ? ['query', 'error'] : ['error'], }); + +if (process.env.NODE_ENV !== 'production') globalForPrisma.prisma = prisma; ``` -## Repository Pattern +### Repository Pattern ```typescript -// src/repositories/base.repository.ts -export abstract class BaseRepository { - constructor(protected prisma: PrismaClient) {} - - abstract findById(id: string): Promise; - abstract findAll(options?: any): Promise; - abstract create(data: any): Promise; - abstract update(id: string, data: any): Promise; - abstract delete(id: string): Promise; -} - -// src/repositories/user.repository.ts -export class UserRepository extends BaseRepository { - async findById(id: string): Promise { - return this.prisma.user.findUnique({ - where: { id }, - include: { profile: true } - }); +export class UserRepository { + async findById(id: string) { + return prisma.user.findUnique({ where: { id }, include: { profile: true } }); } - async findByEmail(email: string): Promise { - return this.prisma.user.findUnique({ - where: { email } - }); - } - - async findAll(options: { - page?: number; - limit?: number; - search?: string; - sortBy?: string; - order?: 'asc' | 'desc'; - } = {}): Promise<{ data: User[]; total: number }> { - const { - page = 1, - limit = 10, - search, - sortBy = 'createdAt', - order = 'desc' - } = options; - - const where = search ? { - OR: [ - { email: { contains: search, mode: 'insensitive' } }, - { name: { contains: search, mode: 'insensitive' } } - ] - } : {}; + async findAll({ page = 1, limit = 10, search }: QueryOptions) { + const where = search ? { OR: [ + { email: { contains: search, mode: 'insensitive' } }, + { name: { contains: search, mode: 'insensitive' } } + ]} : {}; const [data, total] = await Promise.all([ - this.prisma.user.findMany({ - where, - skip: (page - 1) * limit, - take: limit, - orderBy: { [sortBy]: order }, - include: { profile: true } - }), - this.prisma.user.count({ where }) + prisma.user.findMany({ where, skip: (page - 1) * limit, take: limit }), + prisma.user.count({ where }) ]); - return { data, total }; } - async create(data: CreateUserDto): Promise { - return this.prisma.user.create({ - data: { - email: data.email, - password: data.password, - name: data.name, - profile: data.bio ? { - create: { bio: data.bio } - } : undefined - }, - include: { profile: true } - }); - } - - async update(id: string, data: UpdateUserDto): Promise { - return this.prisma.user.update({ - where: { id }, - data, - include: { profile: true } - }); - } - - async delete(id: string): Promise { - await this.prisma.user.delete({ - where: { id } - }); + async create(data: CreateUserDto) { + return prisma.user.create({ data }); } } ``` -## Transactions +### Transactions ```typescript -// Transaction example -export class TransferService { - async transferFunds( - fromAccountId: string, - toAccountId: string, - amount: number - ) { - return await this.prisma.$transaction(async (tx) => { - // Check balance - const fromAccount = await tx.account.findUnique({ - where: { id: fromAccountId } - }); - - if (!fromAccount || fromAccount.balance < amount) { - throw new Error('Insufficient funds'); - } - - // Deduct from sender - const updatedFrom = await tx.account.update({ - where: { id: fromAccountId }, - data: { balance: { decrement: amount } } - }); - - // Add to receiver - const updatedTo = await tx.account.update({ - where: { id: toAccountId }, - data: { balance: { increment: amount } } - }); - - // Create transaction record - const transaction = await tx.transaction.create({ - data: { - fromAccountId, - toAccountId, - amount, - type: 'TRANSFER', - status: 'COMPLETED' - } - }); - - return transaction; - }, { - maxWait: 5000, - timeout: 10000, - }); - } -} -``` - -## Migrations - -```bash -# Create migration -npx prisma migrate dev --name add_user_table - -# Apply migrations -npx prisma migrate deploy - -# Reset database -npx prisma migrate reset - -# Generate Prisma Client -npx prisma generate -``` - -### Migration Files - -```sql --- migrations/20240101000000_add_user_table/migration.sql -CREATE TABLE "users" ( - "id" TEXT NOT NULL, - "email" TEXT NOT NULL, - "name" TEXT, - "password" TEXT NOT NULL, - "role" TEXT NOT NULL DEFAULT 'USER', - "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, - "updatedAt" TIMESTAMP(3) NOT NULL, - - CONSTRAINT "users_pkey" PRIMARY KEY ("id") -); - -CREATE UNIQUE INDEX "users_email_key" ON "users"("email"); -CREATE INDEX "users_createdAt_idx" ON "users"("createdAt"); -``` - -## Query Optimization - -```typescript -// Optimized queries -export class OptimizedUserRepository { - // Select only needed fields - async findUsersLight() { - return this.prisma.user.findMany({ - select: { - id: true, - email: true, - name: true - } - }); - } - - // Use pagination - async findPaginated(cursor?: string) { - return this.prisma.user.findMany({ - take: 10, - skip: cursor ? 1 : 0, - cursor: cursor ? { id: cursor } : undefined, - orderBy: { createdAt: 'desc' } - }); - } - - // Batch operations - async createMany(users: CreateUserDto[]) { - return this.prisma.user.createMany({ - data: users, - skipDuplicates: true - }); - } - - // Use raw SQL for complex queries - async getStatistics() { - return this.prisma.$queryRaw` - SELECT - COUNT(*) as total, - COUNT(CASE WHEN role = 'ADMIN' THEN 1 END) as admins, - COUNT(CASE WHEN created_at > NOW() - INTERVAL '30 days' THEN 1 END) as new_users - FROM users - `; - } -} -``` - -## Seeding - -```typescript -// prisma/seed.ts -import { PrismaClient } from '@prisma/client'; -import bcrypt from 'bcrypt'; - -const prisma = new PrismaClient(); - -async function main() { - // Create admin user - const adminPassword = await bcrypt.hash('admin123', 10); - const admin = await prisma.user.upsert({ - where: { email: 'admin@goodgo.com' }, - update: {}, - create: { - email: 'admin@goodgo.com', - name: 'Admin User', - password: adminPassword, - role: 'ADMIN' - } - }); - - // Create test users - const testUsers = Array.from({ length: 10 }, (_, i) => ({ - email: `user${i}@example.com`, - name: `Test User ${i}`, - password: bcrypt.hashSync('password123', 10) - })); - - await prisma.user.createMany({ - data: testUsers, - skipDuplicates: true - }); - - console.log('Database seeded successfully'); -} - -main() - .catch(console.error) - .finally(() => prisma.$disconnect()); -``` - -## Neon PostgreSQL Configuration - -```typescript -// .env -DATABASE_URL="postgresql://user:password@ep-xxx.us-east-1.aws.neon.tech/dbname?sslmode=require" - -// Connection pooling for serverless -DIRECT_URL="postgresql://user:password@ep-xxx.us-east-1.aws.neon.tech/dbname?sslmode=require" -``` - -## Testing with Prisma - -```typescript -// __tests__/user.repository.test.ts -import { mockDeep, mockReset } from 'jest-mock-extended'; -import { PrismaClient } from '@prisma/client'; - -jest.mock('../src/lib/prisma', () => ({ - __esModule: true, - prisma: mockDeep() -})); - -describe('UserRepository', () => { - beforeEach(() => { - mockReset(prismaMock); - }); - - it('should create user', async () => { - const user = { id: '1', email: 'test@example.com' }; - prismaMock.user.create.mockResolvedValue(user); - - const result = await repository.create({ - email: 'test@example.com', - password: 'password' - }); - - expect(result).toEqual(user); - }); -}); +await prisma.$transaction(async (tx) => { + await tx.account.update({ where: { id: from }, data: { balance: { decrement: amount } } }); + await tx.account.update({ where: { id: to }, data: { balance: { increment: amount } } }); +}, { maxWait: 5000, timeout: 10000 }); ``` ## Best Practices -1. **Schema Design** - - Use appropriate field types - - Add indexes for frequently queried fields - - Use relations instead of storing JSON - - Implement soft deletes when needed - -2. **Performance** - - Use select to fetch only needed fields - - Implement pagination for large datasets - - Use connection pooling - - Cache frequently accessed data - -3. **Security** - - Never expose sensitive fields - - Use parameterized queries - - Validate input before database operations - - Implement row-level security - -4. **Maintenance** - - Keep migrations small and focused - - Test migrations before production - - Backup before major changes - - Monitor query performance +- **Schema Design**: Use appropriate field types, add indexes for frequently queried fields +- **Performance**: Use select to fetch only needed fields, implement pagination +- **Security**: Never expose sensitive fields, use parameterized queries +- **Maintenance**: Keep migrations small and focused, test before production ## Common Mistakes 1. **N+1 Query Problem**: Fetching related data in a loop ```typescript - // ❌ BAD: N+1 queries - const users = await prisma.user.findMany(); - for (const user of users) { - const posts = await prisma.post.findMany({ where: { authorId: user.id } }); - } - - // ✅ GOOD: Include relations - const users = await prisma.user.findMany({ - include: { posts: true } - }); + // BAD: N+1 queries + for (const user of users) { await prisma.post.findMany({ where: { authorId: user.id } }); } + // GOOD: Include relations + await prisma.user.findMany({ include: { posts: true } }); ``` 2. **No Indexes**: Missing indexes on frequently queried columns ```prisma - // ❌ BAD: No index - model User { - email String @unique - createdAt DateTime @default(now()) - } - - // ✅ GOOD: Add index for queries - model User { - email String @unique - createdAt DateTime @default(now()) - @@index([createdAt]) - } + // GOOD: Add index + @@index([createdAt]) ``` 3. **Raw Queries Without Parameters**: SQL injection risk ```typescript - // ❌ BAD: SQL injection risk - prisma.$queryRawUnsafe(`SELECT * FROM users WHERE email = '${email}'`); - - // ✅ GOOD: Parameterized query - prisma.$queryRaw`SELECT * FROM users WHERE email = ${email}`; + // BAD: prisma.$queryRawUnsafe(`SELECT * FROM users WHERE email = '${email}'`); + // GOOD: prisma.$queryRaw`SELECT * FROM users WHERE email = ${email}`; ``` 4. **Not Using Transactions**: Data inconsistency risk ```typescript - // ❌ BAD: No transaction - await prisma.account.update({ where: { id: from }, data: { balance: { decrement: amount } } }); - await prisma.account.update({ where: { id: to }, data: { balance: { increment: amount } } }); - - // ✅ GOOD: Use transaction - await prisma.$transaction([ - prisma.account.update({ where: { id: from }, data: { balance: { decrement: amount } } }), - prisma.account.update({ where: { id: to }, data: { balance: { increment: amount } } }) - ]); + // GOOD: Use transaction for related operations + await prisma.$transaction([update1, update2]); ``` 5. **Exposing Internal IDs**: Leaking database structure - ```typescript - // ❌ BAD: Exposing auto-increment ID - model User { id Int @id @default(autoincrement()) } - - // ✅ GOOD: Use CUID or UUID + ```prisma + // GOOD: Use CUID or UUID model User { id String @id @default(cuid()) } ``` ## Quick Reference -| Operation | Prisma Command | -|-----------|----------------| +| Operation | Command | +|-----------|---------| | **Create migration** | `npx prisma migrate dev --name ` | | **Apply migrations** | `npx prisma migrate deploy` | | **Reset database** | `npx prisma migrate reset` | | **Generate client** | `npx prisma generate` | -| **Open Prisma Studio** | `npx prisma studio` | +| **Open Studio** | `npx prisma studio` | | **Seed database** | `npx prisma db seed` | **Common Query Patterns:** ```typescript -// Find unique -await prisma.user.findUnique({ where: { id } }); - -// Find with relations -await prisma.user.findMany({ include: { posts: true } }); - -// Select specific fields -await prisma.user.findMany({ select: { id: true, email: true } }); - -// Pagination -await prisma.user.findMany({ skip: 10, take: 10 }); - -// Transaction -await prisma.$transaction(async (tx) => { /* operations */ }); +await prisma.user.findUnique({ where: { id } }); // Find unique +await prisma.user.findMany({ include: { posts: true } }); // With relations +await prisma.user.findMany({ select: { id: true } }); // Select fields +await prisma.user.findMany({ skip: 10, take: 10 }); // Pagination +await prisma.$transaction(async (tx) => { ... }); // Transaction ``` **Schema Field Types:** @@ -584,8 +179,8 @@ await prisma.$transaction(async (tx) => { /* operations */ }); ## Resources - [Prisma Documentation](https://www.prisma.io/docs) - Official Prisma docs -- [Prisma Schema Reference](https://www.prisma.io/docs/reference/api-reference/prisma-schema-reference) - Schema syntax +- [Prisma Schema Reference](https://www.prisma.io/docs/reference/api-reference/prisma-schema-reference) +- [Detailed Code Examples](./references/REFERENCE.md) - [Repository Pattern](../repository-pattern/SKILL.md) - Data access patterns - [Caching Patterns](../caching-patterns/SKILL.md) - Query caching - [Testing Patterns](../testing-patterns/SKILL.md) - Database mocking -- [Project Rules](../project-rules/SKILL.md) - GoodGo standards \ No newline at end of file diff --git a/.cursor/skills/database-prisma/references/REFERENCE.md b/.cursor/skills/database-prisma/references/REFERENCE.md new file mode 100644 index 00000000..c85327fe --- /dev/null +++ b/.cursor/skills/database-prisma/references/REFERENCE.md @@ -0,0 +1,403 @@ +# Prisma Database Patterns - Detailed Reference + +This reference contains detailed code examples for Prisma ORM and database patterns. + +## Prisma Schema + +```prisma +// prisma/schema.prisma +generator client { + provider = "prisma-client-js" +} + +datasource db { + provider = "postgresql" + url = env("DATABASE_URL") +} + +// Base model with common fields +model User { + id String @id @default(cuid()) + email String @unique + name String? + password String + role Role @default(USER) + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + // Relations + posts Post[] + profile Profile? + + // Indexes for performance + @@index([email]) + @@index([createdAt]) + @@map("users") +} + +model Post { + id String @id @default(cuid()) + title String + content String? + published Boolean @default(false) + authorId String + author User @relation(fields: [authorId], references: [id]) + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + @@index([authorId]) + @@index([published, createdAt]) + @@map("posts") +} + +model Profile { + id String @id @default(cuid()) + bio String? + avatar String? + userId String @unique + user User @relation(fields: [userId], references: [id]) + + @@map("profiles") +} + +enum Role { + USER + ADMIN + MODERATOR +} +``` + +## Database Connection + +```typescript +// src/lib/prisma.ts +import { PrismaClient } from '@prisma/client'; + +const globalForPrisma = global as unknown as { + prisma: PrismaClient | undefined; +}; + +export const prisma = globalForPrisma.prisma ?? + new PrismaClient({ + log: process.env.NODE_ENV === 'development' + ? ['query', 'error', 'warn'] + : ['error'], + }); + +if (process.env.NODE_ENV !== 'production') { + globalForPrisma.prisma = prisma; +} + +// Middleware for soft delete +prisma.$use(async (params, next) => { + if (params.model && params.action === 'delete') { + return next({ + ...params, + action: 'update', + args: { + ...params.args, + data: { deletedAt: new Date() } + } + }); + } + return next(params); +}); +``` + +## Repository Pattern + +```typescript +// src/repositories/base.repository.ts +export abstract class BaseRepository { + constructor(protected prisma: PrismaClient) {} + + abstract findById(id: string): Promise; + abstract findAll(options?: any): Promise; + abstract create(data: any): Promise; + abstract update(id: string, data: any): Promise; + abstract delete(id: string): Promise; +} + +// src/repositories/user.repository.ts +export class UserRepository extends BaseRepository { + async findById(id: string): Promise { + return this.prisma.user.findUnique({ + where: { id }, + include: { profile: true } + }); + } + + async findByEmail(email: string): Promise { + return this.prisma.user.findUnique({ + where: { email } + }); + } + + async findAll(options: { + page?: number; + limit?: number; + search?: string; + sortBy?: string; + order?: 'asc' | 'desc'; + } = {}): Promise<{ data: User[]; total: number }> { + const { + page = 1, + limit = 10, + search, + sortBy = 'createdAt', + order = 'desc' + } = options; + + const where = search ? { + OR: [ + { email: { contains: search, mode: 'insensitive' } }, + { name: { contains: search, mode: 'insensitive' } } + ] + } : {}; + + const [data, total] = await Promise.all([ + this.prisma.user.findMany({ + where, + skip: (page - 1) * limit, + take: limit, + orderBy: { [sortBy]: order }, + include: { profile: true } + }), + this.prisma.user.count({ where }) + ]); + + return { data, total }; + } + + async create(data: CreateUserDto): Promise { + return this.prisma.user.create({ + data: { + email: data.email, + password: data.password, + name: data.name, + profile: data.bio ? { + create: { bio: data.bio } + } : undefined + }, + include: { profile: true } + }); + } + + async update(id: string, data: UpdateUserDto): Promise { + return this.prisma.user.update({ + where: { id }, + data, + include: { profile: true } + }); + } + + async delete(id: string): Promise { + await this.prisma.user.delete({ + where: { id } + }); + } +} +``` + +## Transactions + +```typescript +// Transaction example +export class TransferService { + async transferFunds( + fromAccountId: string, + toAccountId: string, + amount: number + ) { + return await this.prisma.$transaction(async (tx) => { + // Check balance + const fromAccount = await tx.account.findUnique({ + where: { id: fromAccountId } + }); + + if (!fromAccount || fromAccount.balance < amount) { + throw new Error('Insufficient funds'); + } + + // Deduct from sender + const updatedFrom = await tx.account.update({ + where: { id: fromAccountId }, + data: { balance: { decrement: amount } } + }); + + // Add to receiver + const updatedTo = await tx.account.update({ + where: { id: toAccountId }, + data: { balance: { increment: amount } } + }); + + // Create transaction record + const transaction = await tx.transaction.create({ + data: { + fromAccountId, + toAccountId, + amount, + type: 'TRANSFER', + status: 'COMPLETED' + } + }); + + return transaction; + }, { + maxWait: 5000, + timeout: 10000, + }); + } +} +``` + +## Query Optimization + +```typescript +// Optimized queries +export class OptimizedUserRepository { + // Select only needed fields + async findUsersLight() { + return this.prisma.user.findMany({ + select: { + id: true, + email: true, + name: true + } + }); + } + + // Use pagination + async findPaginated(cursor?: string) { + return this.prisma.user.findMany({ + take: 10, + skip: cursor ? 1 : 0, + cursor: cursor ? { id: cursor } : undefined, + orderBy: { createdAt: 'desc' } + }); + } + + // Batch operations + async createMany(users: CreateUserDto[]) { + return this.prisma.user.createMany({ + data: users, + skipDuplicates: true + }); + } + + // Use raw SQL for complex queries + async getStatistics() { + return this.prisma.$queryRaw` + SELECT + COUNT(*) as total, + COUNT(CASE WHEN role = 'ADMIN' THEN 1 END) as admins, + COUNT(CASE WHEN created_at > NOW() - INTERVAL '30 days' THEN 1 END) as new_users + FROM users + `; + } +} +``` + +## Seeding + +```typescript +// prisma/seed.ts +import { PrismaClient } from '@prisma/client'; +import bcrypt from 'bcrypt'; + +const prisma = new PrismaClient(); + +async function main() { + // Create admin user + const adminPassword = await bcrypt.hash('admin123', 10); + const admin = await prisma.user.upsert({ + where: { email: 'admin@goodgo.com' }, + update: {}, + create: { + email: 'admin@goodgo.com', + name: 'Admin User', + password: adminPassword, + role: 'ADMIN' + } + }); + + // Create test users + const testUsers = Array.from({ length: 10 }, (_, i) => ({ + email: `user${i}@example.com`, + name: `Test User ${i}`, + password: bcrypt.hashSync('password123', 10) + })); + + await prisma.user.createMany({ + data: testUsers, + skipDuplicates: true + }); + + console.log('Database seeded successfully'); +} + +main() + .catch(console.error) + .finally(() => prisma.$disconnect()); +``` + +## Testing with Prisma + +```typescript +// __tests__/user.repository.test.ts +import { mockDeep, mockReset } from 'jest-mock-extended'; +import { PrismaClient } from '@prisma/client'; + +jest.mock('../src/lib/prisma', () => ({ + __esModule: true, + prisma: mockDeep() +})); + +describe('UserRepository', () => { + beforeEach(() => { + mockReset(prismaMock); + }); + + it('should create user', async () => { + const user = { id: '1', email: 'test@example.com' }; + prismaMock.user.create.mockResolvedValue(user); + + const result = await repository.create({ + email: 'test@example.com', + password: 'password' + }); + + expect(result).toEqual(user); + }); +}); +``` + +## Migration Files + +```sql +-- migrations/20240101000000_add_user_table/migration.sql +CREATE TABLE "users" ( + "id" TEXT NOT NULL, + "email" TEXT NOT NULL, + "name" TEXT, + "password" TEXT NOT NULL, + "role" TEXT NOT NULL DEFAULT 'USER', + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "users_pkey" PRIMARY KEY ("id") +); + +CREATE UNIQUE INDEX "users_email_key" ON "users"("email"); +CREATE INDEX "users_createdAt_idx" ON "users"("createdAt"); +``` + +## Neon PostgreSQL Configuration + +```typescript +// .env +DATABASE_URL="postgresql://user:password@ep-xxx.us-east-1.aws.neon.tech/dbname?sslmode=require" + +// Connection pooling for serverless +DIRECT_URL="postgresql://user:password@ep-xxx.us-east-1.aws.neon.tech/dbname?sslmode=require" +``` diff --git a/.cursor/skills/deployment-kubernetes/SKILL.md b/.cursor/skills/deployment-kubernetes/SKILL.md index 2c22858d..0c4845ed 100644 --- a/.cursor/skills/deployment-kubernetes/SKILL.md +++ b/.cursor/skills/deployment-kubernetes/SKILL.md @@ -1,7 +1,7 @@ --- name: deployment-kubernetes description: Kubernetes deployment for GoodGo services. Use for K8s manifests, HPA, ingress, staging/production deployments, or troubleshooting. -dependencies: "kubernetes>=1.28, helm>=3" +compatibility: "kubernetes>=1.28, helm>=3" --- # Kubernetes Deployment Patterns @@ -25,103 +25,48 @@ Use this skill when: - Resource limits and requests for stability - Health checks (liveness/readiness probes) - Horizontal Pod Autoscaler (HPA) for auto-scaling -- ConfigMaps for configuration -- Secrets for sensitive data +- ConfigMaps for configuration, Secrets for sensitive data -## Service Deployment Manifest +## Key Patterns + +### Deployment Manifest ```yaml -# kubernetes/auth-service.yaml apiVersion: apps/v1 kind: Deployment metadata: name: auth-service namespace: goodgo - labels: - app: auth-service - version: v1 spec: replicas: 3 selector: matchLabels: app: auth-service template: - metadata: - labels: - app: auth-service - version: v1 spec: containers: - name: auth-service - image: goodgo/auth-service:latest - imagePullPolicy: IfNotPresent - ports: - - containerPort: 3000 - name: http + image: goodgo/auth-service:v1.0.0 + resources: + requests: { memory: "256Mi", cpu: "250m" } + limits: { memory: "512Mi", cpu: "500m" } + livenessProbe: + httpGet: { path: /health, port: 3000 } + initialDelaySeconds: 30 + readinessProbe: + httpGet: { path: /ready, port: 3000 } + initialDelaySeconds: 5 env: - - name: NODE_ENV - value: "production" - - name: PORT - value: "3000" - name: DATABASE_URL valueFrom: - secretKeyRef: - name: database-secrets - key: url - - name: JWT_SECRET - valueFrom: - secretKeyRef: - name: auth-secrets - key: jwt-secret - - name: REDIS_URL - valueFrom: - configMapKeyRef: - name: redis-config - key: url - resources: - requests: - memory: "256Mi" - cpu: "250m" - limits: - memory: "512Mi" - cpu: "500m" - livenessProbe: - httpGet: - path: /health - port: 3000 - initialDelaySeconds: 30 - periodSeconds: 10 - readinessProbe: - httpGet: - path: /ready - port: 3000 - initialDelaySeconds: 5 - periodSeconds: 5 ---- -apiVersion: v1 -kind: Service -metadata: - name: auth-service - namespace: goodgo -spec: - type: ClusterIP - selector: - app: auth-service - ports: - - port: 80 - targetPort: 3000 - protocol: TCP + secretKeyRef: { name: db-secrets, key: url } ``` -## Horizontal Pod Autoscaler +### HPA Configuration ```yaml -# kubernetes/hpa.yaml apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler -metadata: - name: auth-service-hpa - namespace: goodgo spec: scaleTargetRef: apiVersion: apps/v1 @@ -133,86 +78,20 @@ spec: - type: Resource resource: name: cpu - target: - type: Utilization - averageUtilization: 70 - - type: Resource - resource: - name: memory - target: - type: Utilization - averageUtilization: 80 - behavior: - scaleDown: - stabilizationWindowSeconds: 300 - policies: - - type: Percent - value: 50 - periodSeconds: 60 - scaleUp: - stabilizationWindowSeconds: 0 - policies: - - type: Percent - value: 100 - periodSeconds: 15 + target: { type: Utilization, averageUtilization: 70 } ``` -## ConfigMap & Secrets +### Ingress ```yaml -# kubernetes/configmap.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: app-config - namespace: goodgo -data: - NODE_ENV: "production" - LOG_LEVEL: "info" - REDIS_URL: "redis://redis-service:6379" - METRICS_ENABLED: "true" - ---- -# kubernetes/secrets.yaml (example - use sealed-secrets in production) -apiVersion: v1 -kind: Secret -metadata: - name: database-secrets - namespace: goodgo -type: Opaque -stringData: - url: "postgresql://user:pass@postgres:5432/db" - ---- -apiVersion: v1 -kind: Secret -metadata: - name: auth-secrets - namespace: goodgo -type: Opaque -stringData: - jwt-secret: "your-secret-key" - refresh-secret: "your-refresh-secret" -``` - -## Ingress Configuration - -```yaml -# kubernetes/ingress.yaml apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: api-ingress - namespace: goodgo annotations: - kubernetes.io/ingress.class: nginx cert-manager.io/cluster-issuer: letsencrypt-prod - nginx.ingress.kubernetes.io/rate-limit: "100" - nginx.ingress.kubernetes.io/ssl-redirect: "true" spec: tls: - - hosts: - - api.goodgo.com + - hosts: [api.goodgo.com] secretName: api-tls-secret rules: - host: api.goodgo.com @@ -221,321 +100,47 @@ spec: - path: /auth pathType: Prefix backend: - service: - name: auth-service - port: - number: 80 - - path: /users - pathType: Prefix - backend: - service: - name: user-service - port: - number: 80 -``` - -## Database Deployment (Development Only) - -```yaml -# kubernetes/postgres.yaml -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: postgres - namespace: goodgo -spec: - serviceName: postgres - replicas: 1 - selector: - matchLabels: - app: postgres - template: - metadata: - labels: - app: postgres - spec: - containers: - - name: postgres - image: postgres:14-alpine - ports: - - containerPort: 5432 - env: - - name: POSTGRES_DB - value: goodgo - - name: POSTGRES_USER - value: postgres - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: postgres-secret - key: password - volumeMounts: - - name: postgres-storage - mountPath: /var/lib/postgresql/data - volumeClaimTemplates: - - metadata: - name: postgres-storage - spec: - accessModes: ["ReadWriteOnce"] - resources: - requests: - storage: 10Gi -``` - -## Deployment Scripts - -```bash -#!/bin/bash -# scripts/deploy-k8s.sh - -# Set namespace -NAMESPACE="goodgo" -ENVIRONMENT="${1:-staging}" - -# Create namespace if not exists -kubectl create namespace $NAMESPACE --dry-run=client -o yaml | kubectl apply -f - - -# Apply configurations -echo "Applying ConfigMaps..." -kubectl apply -f kubernetes/configmap-$ENVIRONMENT.yaml - -echo "Applying Secrets..." -kubectl apply -f kubernetes/secrets-$ENVIRONMENT.yaml - -echo "Deploying services..." -kubectl apply -f kubernetes/auth-service.yaml -kubectl apply -f kubernetes/user-service.yaml - -echo "Configuring autoscaling..." -kubectl apply -f kubernetes/hpa.yaml - -echo "Setting up ingress..." -kubectl apply -f kubernetes/ingress.yaml - -# Wait for rollout -kubectl rollout status deployment/auth-service -n $NAMESPACE -kubectl rollout status deployment/user-service -n $NAMESPACE - -echo "Deployment complete!" -``` - -## Health Check Implementation - -```typescript -// src/modules/health/health.controller.ts -export class HealthController { - constructor( - private prisma: PrismaClient, - private redis: Redis - ) {} - - // Liveness probe - is the service alive? - async liveness(req: Request, res: Response) { - res.status(200).json({ status: 'ok' }); - } - - // Readiness probe - is the service ready to accept traffic? - async readiness(req: Request, res: Response) { - try { - // Check database connection - await this.prisma.$queryRaw`SELECT 1`; - - // Check Redis connection - await this.redis.ping(); - - res.status(200).json({ - status: 'ready', - checks: { - database: 'ok', - redis: 'ok' - } - }); - } catch (error) { - res.status(503).json({ - status: 'not ready', - error: error.message - }); - } - } -} -``` - -## Monitoring with Prometheus - -```yaml -# kubernetes/servicemonitor.yaml -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: auth-service-monitor - namespace: goodgo -spec: - selector: - matchLabels: - app: auth-service - endpoints: - - port: http - path: /metrics - interval: 30s -``` - -## Common Commands - -```bash -# Deploy to staging -kubectl apply -f kubernetes/ -n goodgo-staging - -# Check deployment status -kubectl get deployments -n goodgo -kubectl get pods -n goodgo -kubectl get svc -n goodgo - -# View logs -kubectl logs -f deployment/auth-service -n goodgo -kubectl logs -f pod-name -n goodgo --tail=100 - -# Scale manually -kubectl scale deployment auth-service --replicas=5 -n goodgo - -# Update image -kubectl set image deployment/auth-service auth-service=goodgo/auth-service:v1.2.3 -n goodgo - -# Rollback -kubectl rollout undo deployment/auth-service -n goodgo - -# Port forward for debugging -kubectl port-forward service/auth-service 3000:80 -n goodgo - -# Execute command in pod -kubectl exec -it pod-name -n goodgo -- /bin/sh - -# View HPA status -kubectl get hpa -n goodgo -kubectl describe hpa auth-service-hpa -n goodgo - -# View resource usage -kubectl top nodes -kubectl top pods -n goodgo -``` - -## Troubleshooting - -### Pod Not Starting - -```bash -# Check pod status -kubectl describe pod pod-name -n goodgo - -# Check events -kubectl get events -n goodgo --sort-by='.lastTimestamp' - -# Check logs -kubectl logs pod-name -n goodgo --previous -``` - -### ImagePullBackOff - -```bash -# Check image name and tag -kubectl describe pod pod-name -n goodgo | grep -i image - -# Check image pull secrets -kubectl get secrets -n goodgo -``` - -### CrashLoopBackOff - -```bash -# Check logs of crashed container -kubectl logs pod-name -n goodgo --previous - -# Check resource limits -kubectl describe pod pod-name -n goodgo | grep -A 5 Limits + service: { name: auth-service, port: { number: 80 } } ``` ## Best Practices -1. **Resource Management** - - Always set resource requests and limits - - Monitor actual usage and adjust accordingly - - Use HPA for automatic scaling - -2. **Configuration** - - Use ConfigMaps for non-sensitive config - - Use Secrets for sensitive data - - Never hardcode configuration in images - -3. **Health Checks** - - Implement both liveness and readiness probes - - Set appropriate timeouts and thresholds - - Include dependency checks in readiness probe - -4. **Deployment** - - Use rolling updates for zero-downtime - - Set maxSurge and maxUnavailable appropriately - - Test deployments in staging first - -5. **Security** - - Run containers as non-root user - - Use network policies to restrict traffic - - Regularly update base images - - Use sealed-secrets or external secret manager - -6. **Monitoring** - - Expose metrics endpoint - - Set up alerts for critical issues - - Monitor resource usage and performance +- **Resource Management**: Always set resource requests and limits, use HPA for scaling +- **Configuration**: Use ConfigMaps for config, Secrets for sensitive data +- **Health Checks**: Implement both liveness and readiness probes +- **Deployment**: Use rolling updates, set maxSurge/maxUnavailable appropriately +- **Security**: Run as non-root, use network policies, update base images regularly +- **Monitoring**: Expose metrics endpoint, set up alerts ## Common Mistakes 1. **No Resource Limits**: Pods consuming all node resources ```yaml - # ❌ BAD: No limits - containers: - - name: app - image: app:latest - - # ✅ GOOD: Set limits - containers: - - name: app - image: app:latest - resources: - requests: { memory: "256Mi", cpu: "250m" } - limits: { memory: "512Mi", cpu: "500m" } + # GOOD: Set limits + resources: + requests: { memory: "256Mi", cpu: "250m" } + limits: { memory: "512Mi", cpu: "500m" } ``` 2. **Missing Health Checks**: K8s can't detect unhealthy pods ```yaml - # ✅ Always add liveness and readiness probes + # GOOD: Add probes livenessProbe: httpGet: { path: /health, port: 3000 } - initialDelaySeconds: 30 readinessProbe: httpGet: { path: /ready, port: 3000 } - initialDelaySeconds: 5 ``` 3. **Hardcoded Secrets**: Exposing sensitive data ```yaml - # ❌ BAD: Hardcoded - env: - - name: DB_PASSWORD - value: "secret123" - - # ✅ GOOD: Use secrets - env: - - name: DB_PASSWORD - valueFrom: - secretKeyRef: { name: db-secrets, key: password } + # BAD: value: "secret123" + # GOOD: valueFrom: secretKeyRef: { name: secrets, key: password } ``` 4. **Using `latest` Tag**: Unpredictable deployments ```yaml - # ❌ BAD - image: app:latest - - # ✅ GOOD - image: app:v1.2.3 + # BAD: image: app:latest + # GOOD: image: app:v1.2.3 ``` ## Quick Reference @@ -574,7 +179,7 @@ readinessProbe: - [Kubernetes Documentation](https://kubernetes.io/docs/) - Official K8s docs - [Helm](https://helm.sh/docs/) - K8s package manager +- [Detailed Manifests](./references/REFERENCE.md) - [Infrastructure as Code](../infrastructure-as-code/SKILL.md) - Terraform patterns - [Observability & Monitoring](../observability-monitoring/SKILL.md) - Health checks - [Service Discovery](../service-discovery-registry/SKILL.md) - K8s DNS patterns -- [Project Rules](../project-rules/SKILL.md) - GoodGo standards \ No newline at end of file diff --git a/.cursor/skills/deployment-kubernetes/references/REFERENCE.md b/.cursor/skills/deployment-kubernetes/references/REFERENCE.md new file mode 100644 index 00000000..b6c38875 --- /dev/null +++ b/.cursor/skills/deployment-kubernetes/references/REFERENCE.md @@ -0,0 +1,428 @@ +# Kubernetes Deployment - Detailed Reference + +This reference contains detailed Kubernetes manifests and deployment patterns. + +## Service Deployment Manifest + +```yaml +# kubernetes/auth-service.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: auth-service + namespace: goodgo + labels: + app: auth-service + version: v1 +spec: + replicas: 3 + selector: + matchLabels: + app: auth-service + template: + metadata: + labels: + app: auth-service + version: v1 + spec: + containers: + - name: auth-service + image: goodgo/auth-service:latest + imagePullPolicy: IfNotPresent + ports: + - containerPort: 3000 + name: http + env: + - name: NODE_ENV + value: "production" + - name: PORT + value: "3000" + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: database-secrets + key: url + - name: JWT_SECRET + valueFrom: + secretKeyRef: + name: auth-secrets + key: jwt-secret + - name: REDIS_URL + valueFrom: + configMapKeyRef: + name: redis-config + key: url + resources: + requests: + memory: "256Mi" + cpu: "250m" + limits: + memory: "512Mi" + cpu: "500m" + livenessProbe: + httpGet: + path: /health + port: 3000 + initialDelaySeconds: 30 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /ready + port: 3000 + initialDelaySeconds: 5 + periodSeconds: 5 +--- +apiVersion: v1 +kind: Service +metadata: + name: auth-service + namespace: goodgo +spec: + type: ClusterIP + selector: + app: auth-service + ports: + - port: 80 + targetPort: 3000 + protocol: TCP +``` + +## Horizontal Pod Autoscaler + +```yaml +# kubernetes/hpa.yaml +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: auth-service-hpa + namespace: goodgo +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: auth-service + minReplicas: 2 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 70 + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: 80 + behavior: + scaleDown: + stabilizationWindowSeconds: 300 + policies: + - type: Percent + value: 50 + periodSeconds: 60 + scaleUp: + stabilizationWindowSeconds: 0 + policies: + - type: Percent + value: 100 + periodSeconds: 15 +``` + +## ConfigMap & Secrets + +```yaml +# kubernetes/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: app-config + namespace: goodgo +data: + NODE_ENV: "production" + LOG_LEVEL: "info" + REDIS_URL: "redis://redis-service:6379" + METRICS_ENABLED: "true" + +--- +# kubernetes/secrets.yaml (example - use sealed-secrets in production) +apiVersion: v1 +kind: Secret +metadata: + name: database-secrets + namespace: goodgo +type: Opaque +stringData: + url: "postgresql://user:pass@postgres:5432/db" + +--- +apiVersion: v1 +kind: Secret +metadata: + name: auth-secrets + namespace: goodgo +type: Opaque +stringData: + jwt-secret: "your-secret-key" + refresh-secret: "your-refresh-secret" +``` + +## Ingress Configuration + +```yaml +# kubernetes/ingress.yaml +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: api-ingress + namespace: goodgo + annotations: + kubernetes.io/ingress.class: nginx + cert-manager.io/cluster-issuer: letsencrypt-prod + nginx.ingress.kubernetes.io/rate-limit: "100" + nginx.ingress.kubernetes.io/ssl-redirect: "true" +spec: + tls: + - hosts: + - api.goodgo.com + secretName: api-tls-secret + rules: + - host: api.goodgo.com + http: + paths: + - path: /auth + pathType: Prefix + backend: + service: + name: auth-service + port: + number: 80 + - path: /users + pathType: Prefix + backend: + service: + name: user-service + port: + number: 80 +``` + +## Database Deployment (Development Only) + +```yaml +# kubernetes/postgres.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: postgres + namespace: goodgo +spec: + serviceName: postgres + replicas: 1 + selector: + matchLabels: + app: postgres + template: + metadata: + labels: + app: postgres + spec: + containers: + - name: postgres + image: postgres:14-alpine + ports: + - containerPort: 5432 + env: + - name: POSTGRES_DB + value: goodgo + - name: POSTGRES_USER + value: postgres + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: postgres-secret + key: password + volumeMounts: + - name: postgres-storage + mountPath: /var/lib/postgresql/data + volumeClaimTemplates: + - metadata: + name: postgres-storage + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 10Gi +``` + +## Deployment Scripts + +```bash +#!/bin/bash +# scripts/deploy-k8s.sh + +# Set namespace +NAMESPACE="goodgo" +ENVIRONMENT="${1:-staging}" + +# Create namespace if not exists +kubectl create namespace $NAMESPACE --dry-run=client -o yaml | kubectl apply -f - + +# Apply configurations +echo "Applying ConfigMaps..." +kubectl apply -f kubernetes/configmap-$ENVIRONMENT.yaml + +echo "Applying Secrets..." +kubectl apply -f kubernetes/secrets-$ENVIRONMENT.yaml + +echo "Deploying services..." +kubectl apply -f kubernetes/auth-service.yaml +kubectl apply -f kubernetes/user-service.yaml + +echo "Configuring autoscaling..." +kubectl apply -f kubernetes/hpa.yaml + +echo "Setting up ingress..." +kubectl apply -f kubernetes/ingress.yaml + +# Wait for rollout +kubectl rollout status deployment/auth-service -n $NAMESPACE +kubectl rollout status deployment/user-service -n $NAMESPACE + +echo "Deployment complete!" +``` + +## Health Check Implementation + +```typescript +// src/modules/health/health.controller.ts +export class HealthController { + constructor( + private prisma: PrismaClient, + private redis: Redis + ) {} + + // Liveness probe - is the service alive? + async liveness(req: Request, res: Response) { + res.status(200).json({ status: 'ok' }); + } + + // Readiness probe - is the service ready to accept traffic? + async readiness(req: Request, res: Response) { + try { + // Check database connection + await this.prisma.$queryRaw`SELECT 1`; + + // Check Redis connection + await this.redis.ping(); + + res.status(200).json({ + status: 'ready', + checks: { + database: 'ok', + redis: 'ok' + } + }); + } catch (error) { + res.status(503).json({ + status: 'not ready', + error: error.message + }); + } + } +} +``` + +## Monitoring with Prometheus + +```yaml +# kubernetes/servicemonitor.yaml +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: auth-service-monitor + namespace: goodgo +spec: + selector: + matchLabels: + app: auth-service + endpoints: + - port: http + path: /metrics + interval: 30s +``` + +## Common Commands + +```bash +# Deploy to staging +kubectl apply -f kubernetes/ -n goodgo-staging + +# Check deployment status +kubectl get deployments -n goodgo +kubectl get pods -n goodgo +kubectl get svc -n goodgo + +# View logs +kubectl logs -f deployment/auth-service -n goodgo +kubectl logs -f pod-name -n goodgo --tail=100 + +# Scale manually +kubectl scale deployment auth-service --replicas=5 -n goodgo + +# Update image +kubectl set image deployment/auth-service auth-service=goodgo/auth-service:v1.2.3 -n goodgo + +# Rollback +kubectl rollout undo deployment/auth-service -n goodgo + +# Port forward for debugging +kubectl port-forward service/auth-service 3000:80 -n goodgo + +# Execute command in pod +kubectl exec -it pod-name -n goodgo -- /bin/sh + +# View HPA status +kubectl get hpa -n goodgo +kubectl describe hpa auth-service-hpa -n goodgo + +# View resource usage +kubectl top nodes +kubectl top pods -n goodgo +``` + +## Troubleshooting + +### Pod Not Starting + +```bash +# Check pod status +kubectl describe pod pod-name -n goodgo + +# Check events +kubectl get events -n goodgo --sort-by='.lastTimestamp' + +# Check logs +kubectl logs pod-name -n goodgo --previous +``` + +### ImagePullBackOff + +```bash +# Check image name and tag +kubectl describe pod pod-name -n goodgo | grep -i image + +# Check image pull secrets +kubectl get secrets -n goodgo +``` + +### CrashLoopBackOff + +```bash +# Check logs of crashed container +kubectl logs pod-name -n goodgo --previous + +# Check resource limits +kubectl describe pod pod-name -n goodgo | grep -A 5 Limits +``` diff --git a/.cursor/skills/event-driven-architecture/SKILL.md b/.cursor/skills/event-driven-architecture/SKILL.md index 128de897..d0095992 100644 --- a/.cursor/skills/event-driven-architecture/SKILL.md +++ b/.cursor/skills/event-driven-architecture/SKILL.md @@ -1,7 +1,7 @@ --- name: event-driven-architecture description: Event-driven patterns with Kafka for GoodGo services. Use for async communication, event sourcing, CQRS, or event stream integration. -dependencies: "kafkajs>=2" +compatibility: "kafkajs>=2" --- # Event-Driven Architecture Patterns @@ -24,19 +24,13 @@ Use this skill when: ### Event-Driven vs Request-Response -**Request-Response (Synchronous):** -- Client waits for response -- Tight coupling between services -- Blocking operations -- Immediate consistency -- Use Traefik API Gateway for HTTP/REST - -**Event-Driven (Asynchronous):** -- Fire-and-forget publishing -- Loose coupling between services -- Non-blocking operations -- Eventual consistency -- Use Kafka for message broker +| Aspect | Request-Response | Event-Driven | +|--------|------------------|--------------| +| Communication | Synchronous | Asynchronous | +| Coupling | Tight | Loose | +| Blocking | Yes | No | +| Consistency | Immediate | Eventual | +| Infrastructure | Traefik API Gateway | Kafka | ### Kafka Fundamentals @@ -58,1538 +52,116 @@ Use this skill when: **Consumers**: Services that subscribe to topics and process events -### Event Schema Versioning - -Events evolve over time. Use Schema Registry (Avro) for: -- Schema validation -- Backward/forward compatibility -- Version management -- Type safety - -### Traefik Integration for Event Streaming - -Traefik serves dual purpose: -- **API Gateway**: Routes synchronous HTTP/REST requests -- **Event Streaming Gateway**: Routes SSE/WebSocket connections to event streaming endpoints - -Services publish events to Kafka, then expose SSE/WebSocket endpoints that consume from Kafka for HTTP clients. - -## Kafka Setup & Configuration - -### Infrastructure Setup - -**Docker Compose (Local Development):** - -```yaml -# deployments/local/docker-compose.yml -services: - zookeeper: - image: confluentinc/cp-zookeeper:7.4.0 - environment: - ZOOKEEPER_CLIENT_PORT: 2181 - ZOOKEEPER_TICK_TIME: 2000 - networks: - - microservices-network - - kafka: - image: confluentinc/cp-kafka:7.4.0 - depends_on: - - zookeeper - ports: - - "9092:9092" - environment: - KAFKA_BROKER_ID: 1 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092 - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true' - networks: - - microservices-network - - schema-registry: - image: confluentinc/cp-schema-registry:7.4.0 - depends_on: - - kafka - ports: - - "8081:8081" - environment: - SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092 - networks: - - microservices-network -``` - -**Kubernetes (Production):** - -Use Confluent Operator or Strimzi operator for Kafka cluster management. - -### KafkaJS Client Configuration - -```typescript -// src/config/kafka.config.ts -// EN: Kafka client configuration -// VI: Cấu hình Kafka client -import { Kafka, logLevel } from 'kafkajs'; -import { logger } from '@goodgo/logger'; - -const kafkaConfig = { - brokers: (process.env.KAFKA_BROKERS || 'localhost:9092').split(','), - clientId: process.env.KAFKA_CLIENT_ID || process.env.SERVICE_NAME || 'unknown-service', - retry: { - initialRetryTime: 100, - retries: 8, - multiplier: 2, - maxRetryTime: 30000, - }, - logLevel: process.env.KAFKA_LOG_LEVEL === 'debug' ? logLevel.DEBUG : logLevel.INFO, - logCreator: () => ({ level, log }) => { - const { message, ...extra } = log; - if (level >= logLevel.ERROR) { - logger.error('Kafka Error', { message, ...extra }); - } else if (level >= logLevel.WARN) { - logger.warn('Kafka Warning', { message, ...extra }); - } else { - logger.debug('Kafka Log', { message, ...extra }); - } - }, -}; - -export const kafka = new Kafka(kafkaConfig); - -// EN: Producer instance (reuse across service) -// VI: Instance producer (tái sử dụng trong service) -export const producer = kafka.producer({ - idempotent: true, // EN: Ensure exactly-once semantics / VI: Đảm bảo exactly-once semantics - maxInFlightRequests: 1, // EN: Required for idempotency / VI: Cần thiết cho idempotency - transactionTimeout: 30000, -}); - -// EN: Admin client for topic management -// VI: Admin client để quản lý topics -export const admin = kafka.admin(); -``` - -### Environment Variables - -```bash -# Kafka Configuration -KAFKA_BROKERS=localhost:9092 -KAFKA_CLIENT_ID=my-service -KAFKA_LOG_LEVEL=info - -# Schema Registry -SCHEMA_REGISTRY_URL=http://localhost:8081 - -# Consumer Configuration -KAFKA_CONSUMER_GROUP_ID=my-service-consumers -KAFKA_CONSUMER_SESSION_TIMEOUT=30000 -KAFKA_CONSUMER_HEARTBEAT_INTERVAL=3000 -``` - -### Connection Management - -```typescript -// src/core/kafka/client.ts -// EN: Initialize Kafka connections -// VI: Khởi tạo kết nối Kafka -import { producer, admin } from '../config/kafka.config'; -import { logger } from '@goodgo/logger'; - -let isConnected = false; - -export async function connectKafka(): Promise { - if (isConnected) { - logger.warn('Kafka already connected'); - return; - } - - try { - await producer.connect(); - await admin.connect(); - isConnected = true; - logger.info('Kafka connected successfully'); - } catch (error) { - logger.error('Failed to connect to Kafka', { error }); - throw error; - } -} - -export async function disconnectKafka(): Promise { - if (!isConnected) { - return; - } - - try { - await producer.disconnect(); - await admin.disconnect(); - isConnected = false; - logger.info('Kafka disconnected'); - } catch (error) { - logger.error('Error disconnecting from Kafka', { error }); - throw error; - } -} - -// EN: Graceful shutdown handler -// VI: Xử lý graceful shutdown -process.on('SIGTERM', async () => { - await disconnectKafka(); - process.exit(0); -}); -``` - -## Event Publishing Patterns - ### Event Structure ```typescript -// src/types/event.types.ts -// EN: Base event interface -// VI: Interface event cơ bản -export interface BaseEvent { - eventId: string; // EN: Unique event identifier / VI: Định danh event duy nhất - eventType: string; // EN: Event type (e.g., "user.created") / VI: Loại event - eventVersion: string; // EN: Schema version (e.g., "1.0.0") / VI: Phiên bản schema - timestamp: string; // EN: ISO 8601 timestamp / VI: Timestamp ISO 8601 - source: string; // EN: Service that published the event / VI: Service phát hành event - correlationId?: string; // EN: Request correlation ID / VI: Correlation ID của request - traceId?: string; // EN: Distributed tracing ID / VI: ID phân tán tracing - data: unknown; // EN: Event payload / VI: Payload của event -} - -// EN: Example domain event -// VI: Ví dụ domain event -export interface UserCreatedEvent extends BaseEvent { - eventType: 'user.created'; - eventVersion: '1.0.0'; - data: { - userId: string; - email: string; - name: string; - createdAt: string; - }; -} -``` - -### Event Publisher Service - -```typescript -// src/core/events/event-publisher.ts -// EN: Event publisher service -// VI: Service phát hành events -import { producer } from '../config/kafka.config'; -import { logger } from '@goodgo/logger'; -import { v4 as uuidv4 } from 'uuid'; -import type { BaseEvent } from '../../types/event.types'; - -export class EventPublisher { - private serviceName: string; - - constructor(serviceName: string = process.env.SERVICE_NAME || 'unknown') { - this.serviceName = serviceName; - } - - /** - * EN: Publish single event to Kafka topic - * VI: Phát hành một event tới Kafka topic - */ - async publish( - topic: string, - event: Omit, - options?: { - partitionKey?: string; // EN: Partition key for ordering / VI: Partition key để đảm bảo thứ tự - headers?: Record; // EN: Additional headers / VI: Headers bổ sung - } - ): Promise { - const fullEvent: T = { - ...event, - eventId: uuidv4(), - timestamp: new Date().toISOString(), - source: this.serviceName, - } as T; - - try { - await producer.send({ - topic, - messages: [ - { - key: options?.partitionKey || fullEvent.eventId, - value: JSON.stringify(fullEvent), - headers: { - 'event-type': event.eventType, - 'event-version': event.eventVersion, - 'correlation-id': event.correlationId || '', - ...options?.headers, - }, - }, - ], - }); - - logger.info('Event published', { - topic, - eventType: event.eventType, - eventId: fullEvent.eventId, - correlationId: event.correlationId, - }); - } catch (error) { - logger.error('Failed to publish event', { - topic, - eventType: event.eventType, - error, - }); - throw error; - } - } - - /** - * EN: Publish multiple events in batch - * VI: Phát hành nhiều events cùng lúc - */ - async publishBatch( - events: Array<{ - topic: string; - event: Omit; - partitionKey?: string; - }> - ): Promise { - const messages = events.map(({ topic, event, partitionKey }) => { - const fullEvent: T = { - ...event, - eventId: uuidv4(), - timestamp: new Date().toISOString(), - source: this.serviceName, - } as T; - - return { - topic, - messages: [ - { - key: partitionKey || fullEvent.eventId, - value: JSON.stringify(fullEvent), - headers: { - 'event-type': event.eventType, - 'event-version': event.eventVersion, - 'correlation-id': event.correlationId || '', - }, - }, - ], - }; - }); - - try { - await producer.sendBatch({ - topicMessages: messages, - }); - - logger.info('Events published in batch', { - count: events.length, - }); - } catch (error) { - logger.error('Failed to publish events batch', { error }); - throw error; - } - } -} - -export const eventPublisher = new EventPublisher(); -``` - -### Publishing Events from Services - -```typescript -// src/modules/user/user.service.ts -// EN: Example: Publishing event after user creation -// VI: Ví dụ: Phát hành event sau khi tạo user -import { eventPublisher } from '../../core/events/event-publisher'; -import { logger } from '@goodgo/logger'; -import type { UserCreatedEvent } from '../../types/event.types'; - -export class UserService { - async createUser(data: CreateUserDto, correlationId?: string): Promise { - // EN: Create user in database - // VI: Tạo user trong database - const user = await this.userRepository.create(data); - - // EN: Publish event after successful creation - // VI: Phát hành event sau khi tạo thành công - try { - await eventPublisher.publish( - 'user.created', - { - eventType: 'user.created', - eventVersion: '1.0.0', - correlationId, - data: { - userId: user.id, - email: user.email, - name: user.name, - createdAt: user.createdAt.toISOString(), - }, - }, - { - partitionKey: user.id, // EN: Ensure events for same user are ordered / VI: Đảm bảo events của cùng user có thứ tự - } - ); - } catch (error) { - // EN: Log error but don't fail user creation - // VI: Ghi log lỗi nhưng không fail việc tạo user - logger.error('Failed to publish user.created event', { error, userId: user.id }); - } - - return user; - } -} -``` - -### Transactional Publishing (Outbox Pattern) - -```typescript -// src/core/events/outbox-pattern.ts -// EN: Outbox pattern for transactional event publishing -// VI: Outbox pattern để phát hành events trong transaction -import { PrismaClient } from '@prisma/client'; -import { EventPublisher } from './event-publisher'; -import { logger } from '@goodgo/logger'; - -export class OutboxService { - constructor(private prisma: PrismaClient) {} - - /** - * EN: Store event in outbox table within transaction - * VI: Lưu event vào outbox table trong transaction - */ - async addToOutbox( - eventType: string, - eventData: unknown, - topic: string, - correlationId?: string - ): Promise { - await this.prisma.outboxEvent.create({ - data: { - eventType, - eventData: eventData as any, - topic, - correlationId, - status: 'PENDING', - }, - }); - } - - /** - * EN: Process outbox events and publish to Kafka - * VI: Xử lý outbox events và phát hành tới Kafka - */ - async processOutbox(eventPublisher: EventPublisher): Promise { - const events = await this.prisma.outboxEvent.findMany({ - where: { status: 'PENDING' }, - take: 100, - orderBy: { createdAt: 'asc' }, - }); - - for (const event of events) { - try { - await eventPublisher.publish( - event.topic, - { - eventType: event.eventType, - eventVersion: '1.0.0', - correlationId: event.correlationId || undefined, - data: event.eventData, - }, - { - partitionKey: event.id, - } - ); - - await this.prisma.outboxEvent.update({ - where: { id: event.id }, - data: { status: 'PUBLISHED', publishedAt: new Date() }, - }); - } catch (error) { - logger.error('Failed to process outbox event', { eventId: event.id, error }); - // EN: Mark as failed, will retry later - // VI: Đánh dấu failed, sẽ retry sau - await this.prisma.outboxEvent.update({ - where: { id: event.id }, - data: { status: 'FAILED', retryCount: { increment: 1 } }, - }); - } - } - } -} - -// EN: Example: Using outbox in transaction -// VI: Ví dụ: Sử dụng outbox trong transaction -export class UserService { - constructor( - private prisma: PrismaClient, - private outboxService: OutboxService - ) {} - - async createUserWithOutbox(data: CreateUserDto): Promise { - return await this.prisma.$transaction(async (tx) => { - // EN: Create user - // VI: Tạo user - const user = await tx.user.create({ data }); - - // EN: Add event to outbox (within same transaction) - // VI: Thêm event vào outbox (trong cùng transaction) - await this.outboxService.addToOutbox( - 'user.created', - { - userId: user.id, - email: user.email, - name: user.name, - }, - 'user.created' - ); - - return user; - }); - } +interface BaseEvent { + eventId: string; // Unique event identifier + eventType: string; // Event type (e.g., "user.created") + eventVersion: string; // Schema version (e.g., "1.0.0") + timestamp: string; // ISO 8601 timestamp + source: string; // Service that published the event + correlationId?: string; // Request correlation ID + traceId?: string; // Distributed tracing ID + data: unknown; // Event payload } ``` ### Event Naming Conventions -**Event Type Format**: `{domain}.{action}.{version}` - -Examples: +**Event Type Format**: `{domain}.{action}.v{version}` - `user.created.v1` -- `user.updated.v1` - `order.placed.v1` - `payment.processed.v2` **Topic Naming**: `{domain}.{entity}.{action}` - -Examples: - `user.created` -- `user.updated` - `order.placed` - `payment.processed` -## Event Consuming Patterns +## Key Patterns -### Event Consumer Service +### 1. Event Publishing ```typescript -// src/core/events/event-consumer.ts -// EN: Event consumer service -// VI: Service tiêu thụ events -import { kafka } from '../config/kafka.config'; -import { logger } from '@goodgo/logger'; -import type { BaseEvent } from '../../types/event.types'; - -export interface EventHandler { - handle(event: T): Promise; -} - -export class EventConsumer { - private consumer: ReturnType; - private handlers: Map = new Map(); - - constructor( - groupId: string = process.env.KAFKA_CONSUMER_GROUP_ID || 'default-group' - ) { - this.consumer = kafka.consumer({ - groupId, - sessionTimeout: 30000, - heartbeatInterval: 3000, - }); - } - - /** - * EN: Register event handler for specific event type - * VI: Đăng ký event handler cho loại event cụ thể - */ - on(eventType: string, handler: EventHandler): void { - if (!this.handlers.has(eventType)) { - this.handlers.set(eventType, []); - } - this.handlers.get(eventType)!.push(handler as EventHandler); - } - - /** - * EN: Start consuming from topics - * VI: Bắt đầu tiêu thụ từ topics - */ - async start(topics: string[]): Promise { - await this.consumer.connect(); - await this.consumer.subscribe({ topics, fromBeginning: false }); - - await this.consumer.run({ - eachMessage: async ({ topic, partition, message }) => { - try { - const event: BaseEvent = JSON.parse(message.value?.toString() || '{}'); - - // EN: Get handlers for this event type - // VI: Lấy handlers cho loại event này - const handlers = this.handlers.get(event.eventType) || []; - - if (handlers.length === 0) { - logger.warn('No handler registered for event type', { - eventType: event.eventType, - topic, - }); - return; - } - - // EN: Execute all handlers - // VI: Thực thi tất cả handlers - await Promise.all(handlers.map((handler) => handler.handle(event))); - - logger.debug('Event processed successfully', { - eventType: event.eventType, - eventId: event.eventId, - topic, - partition, - }); - } catch (error) { - logger.error('Error processing event', { - topic, - partition, - offset: message.offset, - error, - }); - // EN: Event will be retried by consumer - // VI: Event sẽ được retry bởi consumer - throw error; - } - }, - }); - - logger.info('Event consumer started', { topics, groupId: this.consumer.groupId }); - } - - /** - * EN: Stop consuming - * VI: Dừng tiêu thụ - */ - async stop(): Promise { - await this.consumer.disconnect(); - logger.info('Event consumer stopped'); - } -} +// Fire-and-forget with error logging +eventPublisher.publish('user.created', event, { partitionKey: user.id }) + .catch(err => logger.error('Failed to publish', { err })); ``` -### Consumer Implementation +### 2. Event Consuming ```typescript -// src/modules/notification/notification.consumer.ts -// EN: Example: Consuming user.created events -// VI: Ví dụ: Tiêu thụ user.created events -import { EventConsumer, EventHandler } from '../../core/events/event-consumer'; -import { logger } from '@goodgo/logger'; -import type { UserCreatedEvent } from '../../types/event.types'; - -class UserCreatedHandler implements EventHandler { - async handle(event: UserCreatedEvent): Promise { - logger.info('Processing user.created event', { eventId: event.eventId, userId: event.data.userId }); - - // EN: Send welcome email - // VI: Gửi email chào mừng - await this.sendWelcomeEmail(event.data.email, event.data.name); - - // EN: Create notification preference - // VI: Tạo notification preference - await this.createNotificationPreferences(event.data.userId); - } - - private async sendWelcomeEmail(email: string, name: string): Promise { - // EN: Email sending logic - // VI: Logic gửi email - } - - private async createNotificationPreferences(userId: string): Promise { - // EN: Create default preferences - // VI: Tạo preferences mặc định - } -} - -// EN: Initialize consumer -// VI: Khởi tạo consumer -export const notificationConsumer = new EventConsumer('notification-service'); - -// EN: Register handlers -// VI: Đăng ký handlers -notificationConsumer.on('user.created', new UserCreatedHandler()); - -// EN: Start consuming -// VI: Bắt đầu tiêu thụ -await notificationConsumer.start(['user.created']); -``` - -### Dead Letter Queue (DLQ) Pattern - -```typescript -// src/core/events/dlq-handler.ts -// EN: Dead Letter Queue handler -// VI: Handler cho Dead Letter Queue -import { eventPublisher } from './event-publisher'; -import { logger } from '@goodgo/logger'; - -export class DLQHandler { - /** - * EN: Send failed event to DLQ topic - * VI: Gửi event failed tới DLQ topic - */ - async sendToDLQ( - originalEvent: BaseEvent, - originalTopic: string, - error: Error, - retryCount: number - ): Promise { - const dlqEvent = { - ...originalEvent, - eventType: `${originalEvent.eventType}.dlq`, - data: { - originalEvent, - originalTopic, - error: { - message: error.message, - stack: error.stack, - }, - retryCount, - dlqTimestamp: new Date().toISOString(), - }, - }; - - await eventPublisher.publish(`${originalTopic}.dlq`, dlqEvent); - - logger.error('Event sent to DLQ', { - originalEventType: originalEvent.eventType, - originalTopic, - retryCount, - error: error.message, - }); - } -} - -// EN: Consumer with DLQ support -// VI: Consumer với hỗ trợ DLQ -export class EventConsumerWithDLQ extends EventConsumer { - private dlqHandler = new DLQHandler(); - private maxRetries = 3; - - async start(topics: string[]): Promise { - await this.consumer.connect(); - await this.consumer.subscribe({ topics, fromBeginning: false }); - - await this.consumer.run({ - eachMessage: async ({ topic, partition, message }) => { - let retryCount = 0; - const maxAttempts = this.maxRetries + 1; - - while (retryCount < maxAttempts) { - try { - const event: BaseEvent = JSON.parse(message.value?.toString() || '{}'); - const handlers = this.handlers.get(event.eventType) || []; - - if (handlers.length === 0) { - logger.warn('No handler registered for event type', { - eventType: event.eventType, - topic, - }); - return; - } - - await Promise.all(handlers.map((handler) => handler.handle(event))); - return; // EN: Success, exit retry loop / VI: Thành công, thoát khỏi vòng lặp retry - } catch (error) { - retryCount++; - if (retryCount >= maxAttempts) { - // EN: Max retries exceeded, send to DLQ - // VI: Vượt quá số lần retry, gửi tới DLQ - const event: BaseEvent = JSON.parse(message.value?.toString() || '{}'); - await this.dlqHandler.sendToDLQ(event, topic, error as Error, retryCount); - return; - } - - // EN: Wait before retry with exponential backoff - // VI: Đợi trước khi retry với exponential backoff - const delay = Math.min(1000 * Math.pow(2, retryCount), 30000); - await new Promise((resolve) => setTimeout(resolve, delay)); - - logger.warn('Retrying event processing', { - topic, - partition, - offset: message.offset, - retryCount, - delay, - }); - } - } - }, - }); - } -} -``` - -### Processing Strategies - -**At-Least-Once Delivery:** -- Default Kafka behavior -- Messages may be delivered multiple times -- Consumers must be idempotent -- Use idempotency keys in event data - -**Exactly-Once Semantics:** -- Use transactional producer -- Consumer processes with idempotency checks -- More complex, higher latency - -## Schema Versioning - -### Schema Registry Setup - -```typescript -// src/core/schema/schema-registry.ts -// EN: Schema Registry client -// VI: Client Schema Registry -import { SchemaRegistry } from '@kafkajs/confluent-schema-registry'; -import { logger } from '@goodgo/logger'; - -const registry = new SchemaRegistry({ - host: process.env.SCHEMA_REGISTRY_URL || 'http://localhost:8081', -}); - -export async function registerSchema( - subject: string, - schema: object -): Promise { - try { - const { id } = await registry.register({ type: 'AVRO', schema: JSON.stringify(schema) }); - logger.info('Schema registered', { subject, id }); - return id; - } catch (error) { - logger.error('Failed to register schema', { subject, error }); - throw error; - } -} - -export async function getSchema(subject: string, version?: number) { - try { - const schema = await registry.getSchema(subject, version); - return schema; - } catch (error) { - logger.error('Failed to get schema', { subject, version, error }); - throw error; - } -} -``` - -### Avro Schema Definition - -```json -// schemas/user.created.v1.avsc -{ - "type": "record", - "name": "UserCreated", - "namespace": "com.goodgo.events.user", - "fields": [ - { - "name": "userId", - "type": "string" - }, - { - "name": "email", - "type": "string" - }, - { - "name": "name", - "type": "string" - }, - { - "name": "createdAt", - "type": "string" - } - ] -} -``` - -### Schema Evolution - -**Backward Compatible Changes (Safe):** -- Adding optional fields -- Removing fields (old consumers ignore) - -**Forward Compatible Changes:** -- Removing optional fields -- Adding required fields (with defaults) - -**Breaking Changes:** -- Change field type -- Remove required field -- Rename field - -**Versioning Strategy:** -- Increment version for breaking changes: `user.created.v2` -- Use same topic with versioned event type -- Consumers handle multiple versions - -## Traefik Integration (SSE/WebSocket) - -### SSE Endpoint Implementation - -```typescript -// src/modules/events/events.controller.ts -// EN: SSE endpoint for event streaming -// VI: SSE endpoint để stream events -import { Request, Response } from 'express'; -import { kafka } from '../../config/kafka.config'; -import { logger } from '@goodgo/logger'; - -export class EventsController { - /** - * EN: Server-Sent Events endpoint - * VI: Endpoint Server-Sent Events - */ - async streamEvents(req: Request, res: Response): Promise { - // EN: Set SSE headers - // VI: Thiết lập SSE headers - res.setHeader('Content-Type', 'text/event-stream'); - res.setHeader('Cache-Control', 'no-cache'); - res.setHeader('Connection', 'keep-alive'); - res.setHeader('X-Accel-Buffering', 'no'); // EN: Disable nginx buffering / VI: Tắt nginx buffering - - const topic = req.query.topic as string; - const eventType = req.query.eventType as string | undefined; - - if (!topic) { - res.status(400).json({ success: false, error: { code: 'TOPIC_REQUIRED', message: 'Topic parameter required' } }); - return; - } - - // EN: Create consumer for this connection - // VI: Tạo consumer cho connection này - const consumer = kafka.consumer({ - groupId: `sse-${Date.now()}`, // EN: Unique group per connection / VI: Group duy nhất cho mỗi connection - }); - - try { - await consumer.connect(); - await consumer.subscribe({ topic, fromBeginning: false }); - - // EN: Send initial connection message - // VI: Gửi message kết nối ban đầu - res.write(`data: ${JSON.stringify({ type: 'connected', topic })}\n\n`); - - await consumer.run({ - eachMessage: async ({ message }) => { - try { - const event = JSON.parse(message.value?.toString() || '{}'); - - // EN: Filter by event type if specified - // VI: Lọc theo event type nếu được chỉ định - if (eventType && event.eventType !== eventType) { - return; - } - - // EN: Send event to client - // VI: Gửi event tới client - res.write(`data: ${JSON.stringify(event)}\n\n`); - } catch (error) { - logger.error('Error processing SSE message', { error }); - } - }, - }); - - // EN: Handle client disconnect - // VI: Xử lý khi client ngắt kết nối - req.on('close', async () => { - await consumer.disconnect(); - logger.info('SSE connection closed', { topic }); - }); - } catch (error) { - logger.error('SSE connection error', { error }); - res.write(`data: ${JSON.stringify({ type: 'error', message: 'Connection failed' })}\n\n`); - await consumer.disconnect(); - } - } -} -``` - -### Traefik Routing Configuration - -```yaml -# infra/traefik/dynamic/routes.yml -http: - routers: - events-sse: - rule: "PathPrefix(`/api/v1/events/stream`)" - entrypoints: - - web - service: events-service - middlewares: - - cors - - compress - - events-ratelimit - - services: - events-service: - loadBalancer: - servers: - - url: "http://events-service:5000" - - middlewares: - events-ratelimit: - rateLimit: - average: 100 - period: 1m - burst: 50 -``` - -### Route Registration - -```typescript -// src/routes/events.routes.ts -// EN: Event streaming routes -// VI: Routes cho event streaming -import { Router } from 'express'; -import { EventsController } from '../modules/events/events.controller'; -import { authenticate } from '../middlewares/auth.middleware'; - -const router = Router(); -const eventsController = new EventsController(); - -// EN: SSE endpoint (requires authentication) -// VI: SSE endpoint (yêu cầu authentication) -router.get('/stream', authenticate(), (req, res) => { - eventsController.streamEvents(req, res); -}); - -export default router; -``` - -### Client Subscription Example - -```typescript -// EN: Client-side SSE subscription -// VI: Client-side subscription SSE -const eventSource = new EventSource('/api/v1/events/stream?topic=user.created&eventType=user.created'); - -eventSource.onmessage = (event) => { - const data = JSON.parse(event.data); - console.log('Received event:', data); -}; - -eventSource.onerror = (error) => { - console.error('SSE error:', error); - eventSource.close(); -}; -``` - -## Error Handling & Resilience - -### Event Publishing Errors - -```typescript -// EN: Retry logic for publishing failures -// VI: Logic retry cho lỗi phát hành -import { retryWithBackoff } from '../resilience/retry'; - -async function publishWithRetry( - topic: string, - event: T, - maxRetries: number = 3 -): Promise { - try { - await retryWithBackoff( - async () => { - await eventPublisher.publish(topic, event); - }, - maxRetries, - 1000 // EN: Base delay 1s / VI: Delay cơ bản 1s - ); - } catch (error) { - // EN: After max retries, store in DLQ or persistent storage - // VI: Sau khi retry tối đa, lưu vào DLQ hoặc persistent storage - logger.error('Failed to publish event after retries', { - topic, - eventType: event.eventType, - eventId: event.eventId, - error, - }); - throw error; - } -} -``` - -### Circuit Breaker for Kafka Operations - -```typescript -// EN: Circuit breaker for Kafka producer -// VI: Circuit breaker cho Kafka producer -import { createCircuitBreaker } from '../resilience/circuit-breaker'; - -const kafkaCircuitBreaker = createCircuitBreaker( - async (topic: string, event: BaseEvent) => { - await eventPublisher.publish(topic, event); - }, - 'kafka-producer', - { - timeout: 5000, - errorThresholdPercentage: 50, - resetTimeout: 30000, - } -); - -// EN: Usage with circuit breaker -// VI: Sử dụng với circuit breaker -try { - await kafkaCircuitBreaker.fire('user.created', event); -} catch (error) { - if (kafkaCircuitBreaker.opened) { - // EN: Circuit is open, use fallback (e.g., store in DB for later) - // VI: Circuit đang mở, sử dụng fallback (ví dụ: lưu vào DB để xử lý sau) - await storeEventForLaterProcessing(event); - } - throw error; -} -``` - -### Event Replay - -```typescript -// src/core/events/event-replay.ts -// EN: Event replay service -// VI: Service replay events -export class EventReplayService { - /** - * EN: Replay events from specific offset - * VI: Replay events từ offset cụ thể - */ - async replayEvents( - topic: string, - fromOffset: string, - toOffset?: string, - eventType?: string - ): Promise { - const consumer = kafka.consumer({ - groupId: `replay-${Date.now()}`, - }); - - await consumer.connect(); - await consumer.subscribe({ topic, fromBeginning: false }); - - // EN: Seek to specific offset - // VI: Tìm tới offset cụ thể - // Implementation depends on partition selection - - logger.info('Event replay started', { topic, fromOffset, toOffset }); - } -} -``` - -## Observability - -### Event Logging - -```typescript -// EN: Structured logging for events -// VI: Structured logging cho events -import { logger } from '@goodgo/logger'; - -// EN: Log when publishing -// VI: Log khi phát hành -logger.info('Event published', { - eventType: event.eventType, - eventId: event.eventId, - topic, - correlationId: event.correlationId, - traceId: event.traceId, - source: event.source, -}); - -// EN: Log when consuming -// VI: Log khi tiêu thụ -logger.info('Event consumed', { - eventType: event.eventType, - eventId: event.eventId, - topic, - partition, - offset, - processingTime: Date.now() - eventTimestamp, -}); -``` - -### Metrics Collection - -```typescript -// src/core/events/event-metrics.ts -// EN: Event metrics -// VI: Metrics cho events -import { Counter, Histogram, Gauge } from 'prom-client'; - -const eventsPublished = new Counter({ - name: 'events_published_total', - help: 'Total number of events published', - labelNames: ['topic', 'event_type'], -}); - -const eventsConsumed = new Counter({ - name: 'events_consumed_total', - help: 'Total number of events consumed', - labelNames: ['topic', 'event_type'], -}); - -const eventProcessingDuration = new Histogram({ - name: 'event_processing_duration_seconds', - help: 'Event processing duration in seconds', - labelNames: ['topic', 'event_type'], -}); - -const consumerLag = new Gauge({ - name: 'kafka_consumer_lag', - help: 'Consumer lag per topic and partition', - labelNames: ['topic', 'partition'], -}); - -// EN: Usage in publisher -// VI: Sử dụng trong publisher -eventsPublished.inc({ topic, event_type: event.eventType }); - -// EN: Usage in consumer -// VI: Sử dụng trong consumer -const timer = eventProcessingDuration.startTimer({ topic, event_type: event.eventType }); -try { - await handler.handle(event); - eventsConsumed.inc({ topic, event_type: event.eventType }); -} finally { - timer(); -} -``` - -### Distributed Tracing - -```typescript -// EN: Add tracing to events -// VI: Thêm tracing vào events -import { trace, context, SpanStatusCode } from '@opentelemetry/api'; - -const tracer = trace.getTracer('event-service'); - -// EN: Create span for event publishing -// VI: Tạo span cho việc phát hành event -const span = tracer.startSpan('event.publish', { - attributes: { - 'event.type': event.eventType, - 'event.topic': topic, - 'event.id': event.eventId, +consumer.on('user.created', { + handle: async (event) => { + await processEvent(event); }, }); - -try { - await eventPublisher.publish(topic, event); - span.setStatus({ code: SpanStatusCode.OK }); -} catch (error) { - span.setStatus({ code: SpanStatusCode.ERROR, message: error.message }); - throw error; -} finally { - span.end(); -} +await consumer.start(['user.created']); ``` -### Health Checks +### 3. Outbox Pattern (Transactional) +Store events in database within same transaction, then publish asynchronously: ```typescript -// src/routes/health.routes.ts -// EN: Health check for Kafka connectivity -// VI: Health check cho kết nối Kafka -import { admin } from '../config/kafka.config'; - -router.get('/health/kafka', async (req, res) => { - try { - await admin.listTopics(); - res.json({ status: 'healthy', kafka: 'connected' }); - } catch (error) { - res.status(503).json({ status: 'unhealthy', kafka: 'disconnected', error: error.message }); - } +await prisma.$transaction(async (tx) => { + const user = await tx.user.create({ data }); + await outboxService.addToOutbox('user.created', userData, 'user.created'); + return user; }); ``` -## Testing Patterns - -### Unit Testing Publishers +### 4. Dead Letter Queue (DLQ) +After max retries, send failed events to DLQ topic for manual inspection: ```typescript -// src/core/events/__tests__/event-publisher.test.ts -// EN: Unit tests for event publisher -// VI: Unit tests cho event publisher -import { EventPublisher } from '../event-publisher'; -import { producer } from '../../config/kafka.config'; - -jest.mock('../../config/kafka.config'); - -describe('EventPublisher', () => { - let publisher: EventPublisher; - const mockSend = jest.fn(); - - beforeEach(() => { - publisher = new EventPublisher('test-service'); - (producer.send as jest.Mock) = mockSend; - mockSend.mockResolvedValue({}); - }); - - it('should publish event successfully', async () => { - const event = { - eventType: 'user.created', - eventVersion: '1.0.0', - data: { userId: '123' }, - }; - - await publisher.publish('user.created', event); - - expect(mockSend).toHaveBeenCalledWith({ - topic: 'user.created', - messages: expect.arrayContaining([ - expect.objectContaining({ - key: expect.any(String), - value: expect.stringContaining('"eventType":"user.created"'), - }), - ]), - }); - }); -}); +after maxRetries → send to topic.dlq ``` -### Integration Testing with Test Containers +### 5. Idempotency +Consumers must handle duplicate events: ```typescript -// src/__tests__/integration/event-flow.e2e.ts -// EN: E2E test with Kafka test containers -// VI: E2E test với Kafka test containers -import { KafkaContainer, StartedKafkaContainer } from '@testcontainers/kafka'; -import { EventPublisher } from '../../core/events/event-publisher'; -import { EventConsumer } from '../../core/events/event-consumer'; -import type { BaseEvent } from '../../types/event.types'; - -describe('Event Flow E2E', () => { - let kafkaContainer: StartedKafkaContainer; - let publisher: EventPublisher; - let consumer: EventConsumer; - - beforeAll(async () => { - kafkaContainer = await new KafkaContainer().start(); - process.env.KAFKA_BROKERS = kafkaContainer.getBootstrapServer(); - - publisher = new EventPublisher('test-service'); - consumer = new EventConsumer('test-group'); - }); - - afterAll(async () => { - await kafkaContainer.stop(); - }); - - it('should publish and consume event', async () => { - const receivedEvents: BaseEvent[] = []; - - consumer.on('user.created', { - handle: async (event) => { - receivedEvents.push(event); - }, - }); - - await consumer.start(['user.created']); - - const event = { - eventType: 'user.created', - eventVersion: '1.0.0', - data: { userId: '123', email: 'test@example.com' }, - }; - - await publisher.publish('user.created', event); - - // EN: Wait for event processing - // VI: Đợi xử lý event - await new Promise((resolve) => setTimeout(resolve, 1000)); - - expect(receivedEvents).toHaveLength(1); - expect(receivedEvents[0].data.userId).toBe('123'); - }); -}); -``` - -### Mocking Kafka - -```typescript -// src/__tests__/mocks/kafka.mock.ts -// EN: Kafka mocks for testing -// VI: Kafka mocks cho testing -export const createMockKafka = () => { - const messages: any[] = []; - - return { - producer: () => ({ - connect: jest.fn(), - send: jest.fn((params) => { - messages.push(...params.messages); - return Promise.resolve({}); - }), - disconnect: jest.fn(), - }), - consumer: () => ({ - connect: jest.fn(), - subscribe: jest.fn(), - run: jest.fn(), - disconnect: jest.fn(), - }), - admin: () => ({ - connect: jest.fn(), - listTopics: jest.fn().mockResolvedValue([]), - disconnect: jest.fn(), - }), - messages, // EN: Access sent messages / VI: Truy cập messages đã gửi - }; -}; +if (await this.isProcessed(event.eventId)) return; +await processEvent(event); +await this.markProcessed(event.eventId); ``` ## Best Practices -### Event Naming Conventions - -- **Event Type**: `{domain}.{action}.v{version}` (e.g., `user.created.v1`) -- **Topic**: `{domain}.{entity}.{action}` (e.g., `user.created`) -- Use lowercase with dots as separators -- Keep names descriptive and consistent - ### Partition Key Selection - - Use entity ID for ordering guarantees (same entity → same partition) - Use correlation ID for request tracing - Use user ID for user-scoped events - Avoid high-cardinality keys (distributes evenly) ### Event Ordering Guarantees - - Kafka guarantees ordering **per partition** - Use partition key to ensure related events go to same partition - Events in different partitions have no ordering guarantee - Don't rely on global ordering across all events ### Event Size Limits - - Recommended: < 1MB per event - Kafka default: 1MB (configurable) - For large payloads: Store data elsewhere, send reference in event -- Consider compression for large events ### Performance Optimization - - **Batch Publishing**: Group multiple events for better throughput - **Async Publishing**: Don't block request handlers - **Consumer Parallelism**: Use multiple partitions and consumers - **Connection Pooling**: Reuse Kafka client instances - **Compression**: Enable compression for better network usage -### Common Anti-Patterns to Avoid - -1. **Publishing in Request Handler**: Use async/outbox pattern -2. **Blocking on Event Publishing**: Fire-and-forget with error handling -3. **No Idempotency**: Consumers must handle duplicate events -4. **Ignoring Consumer Lag**: Monitor and alert on lag -5. **Breaking Schema Changes**: Use versioning strategy -6. **Global Ordering Expectations**: Understand partition ordering only - -## Complete Examples - -### Example 1: User Created Event Flow - -```typescript -// EN: Auth Service publishes user.created event -// VI: Auth Service phát hành user.created event -// services/auth-service/src/modules/user/user.service.ts -export class UserService { - async createUser(data: CreateUserDto): Promise { - const user = await this.userRepository.create(data); - - await eventPublisher.publish( - 'user.created', - { - eventType: 'user.created', - eventVersion: '1.0.0', - data: { - userId: user.id, - email: user.email, - name: user.name, - }, - }, - { partitionKey: user.id } - ); - - return user; - } -} - -// EN: Notification Service consumes and sends welcome email -// VI: Notification Service tiêu thụ và gửi email chào mừng -// services/notification-service/src/modules/notification/notification.consumer.ts -notificationConsumer.on('user.created', { - handle: async (event: UserCreatedEvent) => { - await emailService.sendWelcomeEmail(event.data.email, event.data.name); - await notificationPreferenceService.createDefaults(event.data.userId); - }, -}); -``` - -### Example 2: Order Placed with Multiple Consumers - -```typescript -// EN: Order Service publishes order.placed -// VI: Order Service phát hành order.placed -await eventPublisher.publish('order.placed', { - eventType: 'order.placed', - eventVersion: '1.0.0', - data: { - orderId: order.id, - userId: order.userId, - total: order.total, - items: order.items, - }, -}); - -// EN: Multiple consumers handle the event -// VI: Nhiều consumers xử lý event - -// Payment Service: Process payment -paymentConsumer.on('order.placed', { handle: processPayment }); - -// Inventory Service: Reserve items -inventoryConsumer.on('order.placed', { handle: reserveItems }); - -// Notification Service: Send confirmation -notificationConsumer.on('order.placed', { handle: sendConfirmation }); -``` - -### Example 3: SSE Endpoint with Filtering - -```typescript -// EN: Client subscribes to user events -// VI: Client subscribe tới user events -const eventSource = new EventSource( - '/api/v1/events/stream?topic=user.created&eventType=user.created', - { - headers: { - Authorization: `Bearer ${token}`, - }, - } -); - -eventSource.addEventListener('user.created', (event) => { - const data = JSON.parse(event.data); - updateUI(data); -}); -``` - ## Common Mistakes 1. **Blocking on Publish**: Slowing down request handlers ```typescript - // ❌ BAD: Await in request handler + // BAD: Await in request handler await eventPublisher.publish('user.created', event); res.json({ success: true }); - // ✅ GOOD: Fire and forget with error logging + // GOOD: Fire and forget with error logging eventPublisher.publish('user.created', event) .catch(err => logger.error('Failed to publish', { err })); res.json({ success: true }); @@ -1597,12 +169,12 @@ eventSource.addEventListener('user.created', (event) => { 2. **No Idempotency**: Duplicate event processing issues ```typescript - // ❌ BAD: No duplicate check + // BAD: No duplicate check async handle(event) { await createUser(event.data); } - // ✅ GOOD: Check for duplicates + // GOOD: Check for duplicates async handle(event) { if (await this.isProcessed(event.eventId)) return; await createUser(event.data); @@ -1612,19 +184,23 @@ eventSource.addEventListener('user.created', (event) => { 3. **Missing Partition Key**: Events for same entity out of order ```typescript - // ❌ BAD: No partition key + // BAD: No partition key await publish('user.updated', event); - // ✅ GOOD: Use entity ID as partition key + // GOOD: Use entity ID as partition key await publish('user.updated', event, { partitionKey: userId }); ``` 4. **No Dead Letter Queue**: Lost events on failure ```typescript - // ✅ Always implement DLQ for failed events + // GOOD: Always implement DLQ for failed events after maxRetries → send to topic.dlq ``` +5. **Breaking Schema Changes**: Use versioning strategy instead + +6. **Global Ordering Expectations**: Understand partition ordering only + ## Quick Reference | Concept | Description | @@ -1676,13 +252,21 @@ const producer = kafka.producer(); const consumer = kafka.consumer({ groupId: 'my-group' }); ``` +**Environment Variables:** +```bash +KAFKA_BROKERS=localhost:9092 +KAFKA_CLIENT_ID=my-service +KAFKA_CONSUMER_GROUP_ID=my-service-consumers +SCHEMA_REGISTRY_URL=http://localhost:8081 +``` + ## Resources +- [Detailed Reference](./references/REFERENCE.md) - Full code examples and implementation details - [KafkaJS Documentation](https://kafka.js.org/) - Node.js Kafka client - [Confluent Schema Registry](https://docs.confluent.io/platform/current/schema-registry/index.html) - Schema versioning - [Kafka Best Practices](https://kafka.apache.org/documentation/#best_practices) - Official Kafka documentation - [Resilience Patterns](../resilience-patterns/SKILL.md) - Circuit breaker, retry patterns - [Error Handling Patterns](../error-handling-patterns/SKILL.md) - Error handling best practices - [Observability & Monitoring](../observability-monitoring/SKILL.md) - Logging, metrics, tracing -- [Middleware Patterns](../middleware-patterns/SKILL.md) - Express middleware patterns - [Project Rules](../project-rules/SKILL.md) - GoodGo coding standards diff --git a/.cursor/skills/event-driven-architecture/references/REFERENCE.md b/.cursor/skills/event-driven-architecture/references/REFERENCE.md new file mode 100644 index 00000000..d75672db --- /dev/null +++ b/.cursor/skills/event-driven-architecture/references/REFERENCE.md @@ -0,0 +1,1207 @@ +# Event-Driven Architecture - Detailed Reference + +This document contains detailed code examples and implementation patterns for event-driven architecture. + +## Kafka Setup & Configuration + +### Infrastructure Setup + +**Docker Compose (Local Development):** + +```yaml +# deployments/local/docker-compose.yml +services: + zookeeper: + image: confluentinc/cp-zookeeper:7.4.0 + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + ZOOKEEPER_TICK_TIME: 2000 + networks: + - microservices-network + + kafka: + image: confluentinc/cp-kafka:7.4.0 + depends_on: + - zookeeper + ports: + - "9092:9092" + environment: + KAFKA_BROKER_ID: 1 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true' + networks: + - microservices-network + + schema-registry: + image: confluentinc/cp-schema-registry:7.4.0 + depends_on: + - kafka + ports: + - "8081:8081" + environment: + SCHEMA_REGISTRY_HOST_NAME: schema-registry + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092 + networks: + - microservices-network +``` + +**Kubernetes (Production):** + +Use Confluent Operator or Strimzi operator for Kafka cluster management. + +### KafkaJS Client Configuration + +```typescript +// src/config/kafka.config.ts +import { Kafka, logLevel } from 'kafkajs'; +import { logger } from '@goodgo/logger'; + +const kafkaConfig = { + brokers: (process.env.KAFKA_BROKERS || 'localhost:9092').split(','), + clientId: process.env.KAFKA_CLIENT_ID || process.env.SERVICE_NAME || 'unknown-service', + retry: { + initialRetryTime: 100, + retries: 8, + multiplier: 2, + maxRetryTime: 30000, + }, + logLevel: process.env.KAFKA_LOG_LEVEL === 'debug' ? logLevel.DEBUG : logLevel.INFO, + logCreator: () => ({ level, log }) => { + const { message, ...extra } = log; + if (level >= logLevel.ERROR) { + logger.error('Kafka Error', { message, ...extra }); + } else if (level >= logLevel.WARN) { + logger.warn('Kafka Warning', { message, ...extra }); + } else { + logger.debug('Kafka Log', { message, ...extra }); + } + }, +}; + +export const kafka = new Kafka(kafkaConfig); + +// Producer instance (reuse across service) +export const producer = kafka.producer({ + idempotent: true, + maxInFlightRequests: 1, + transactionTimeout: 30000, +}); + +// Admin client for topic management +export const admin = kafka.admin(); +``` + +### Environment Variables + +```bash +# Kafka Configuration +KAFKA_BROKERS=localhost:9092 +KAFKA_CLIENT_ID=my-service +KAFKA_LOG_LEVEL=info + +# Schema Registry +SCHEMA_REGISTRY_URL=http://localhost:8081 + +# Consumer Configuration +KAFKA_CONSUMER_GROUP_ID=my-service-consumers +KAFKA_CONSUMER_SESSION_TIMEOUT=30000 +KAFKA_CONSUMER_HEARTBEAT_INTERVAL=3000 +``` + +### Connection Management + +```typescript +// src/core/kafka/client.ts +import { producer, admin } from '../config/kafka.config'; +import { logger } from '@goodgo/logger'; + +let isConnected = false; + +export async function connectKafka(): Promise { + if (isConnected) { + logger.warn('Kafka already connected'); + return; + } + + try { + await producer.connect(); + await admin.connect(); + isConnected = true; + logger.info('Kafka connected successfully'); + } catch (error) { + logger.error('Failed to connect to Kafka', { error }); + throw error; + } +} + +export async function disconnectKafka(): Promise { + if (!isConnected) { + return; + } + + try { + await producer.disconnect(); + await admin.disconnect(); + isConnected = false; + logger.info('Kafka disconnected'); + } catch (error) { + logger.error('Error disconnecting from Kafka', { error }); + throw error; + } +} + +// Graceful shutdown handler +process.on('SIGTERM', async () => { + await disconnectKafka(); + process.exit(0); +}); +``` + +## Event Publishing Patterns + +### Event Structure + +```typescript +// src/types/event.types.ts +export interface BaseEvent { + eventId: string; + eventType: string; + eventVersion: string; + timestamp: string; + source: string; + correlationId?: string; + traceId?: string; + data: unknown; +} + +export interface UserCreatedEvent extends BaseEvent { + eventType: 'user.created'; + eventVersion: '1.0.0'; + data: { + userId: string; + email: string; + name: string; + createdAt: string; + }; +} +``` + +### Event Publisher Service + +```typescript +// src/core/events/event-publisher.ts +import { producer } from '../config/kafka.config'; +import { logger } from '@goodgo/logger'; +import { v4 as uuidv4 } from 'uuid'; +import type { BaseEvent } from '../../types/event.types'; + +export class EventPublisher { + private serviceName: string; + + constructor(serviceName: string = process.env.SERVICE_NAME || 'unknown') { + this.serviceName = serviceName; + } + + async publish( + topic: string, + event: Omit, + options?: { + partitionKey?: string; + headers?: Record; + } + ): Promise { + const fullEvent: T = { + ...event, + eventId: uuidv4(), + timestamp: new Date().toISOString(), + source: this.serviceName, + } as T; + + try { + await producer.send({ + topic, + messages: [ + { + key: options?.partitionKey || fullEvent.eventId, + value: JSON.stringify(fullEvent), + headers: { + 'event-type': event.eventType, + 'event-version': event.eventVersion, + 'correlation-id': event.correlationId || '', + ...options?.headers, + }, + }, + ], + }); + + logger.info('Event published', { + topic, + eventType: event.eventType, + eventId: fullEvent.eventId, + correlationId: event.correlationId, + }); + } catch (error) { + logger.error('Failed to publish event', { + topic, + eventType: event.eventType, + error, + }); + throw error; + } + } + + async publishBatch( + events: Array<{ + topic: string; + event: Omit; + partitionKey?: string; + }> + ): Promise { + const messages = events.map(({ topic, event, partitionKey }) => { + const fullEvent: T = { + ...event, + eventId: uuidv4(), + timestamp: new Date().toISOString(), + source: this.serviceName, + } as T; + + return { + topic, + messages: [ + { + key: partitionKey || fullEvent.eventId, + value: JSON.stringify(fullEvent), + headers: { + 'event-type': event.eventType, + 'event-version': event.eventVersion, + 'correlation-id': event.correlationId || '', + }, + }, + ], + }; + }); + + try { + await producer.sendBatch({ + topicMessages: messages, + }); + + logger.info('Events published in batch', { + count: events.length, + }); + } catch (error) { + logger.error('Failed to publish events batch', { error }); + throw error; + } + } +} + +export const eventPublisher = new EventPublisher(); +``` + +### Publishing Events from Services + +```typescript +// src/modules/user/user.service.ts +import { eventPublisher } from '../../core/events/event-publisher'; +import { logger } from '@goodgo/logger'; +import type { UserCreatedEvent } from '../../types/event.types'; + +export class UserService { + async createUser(data: CreateUserDto, correlationId?: string): Promise { + const user = await this.userRepository.create(data); + + try { + await eventPublisher.publish( + 'user.created', + { + eventType: 'user.created', + eventVersion: '1.0.0', + correlationId, + data: { + userId: user.id, + email: user.email, + name: user.name, + createdAt: user.createdAt.toISOString(), + }, + }, + { + partitionKey: user.id, + } + ); + } catch (error) { + logger.error('Failed to publish user.created event', { error, userId: user.id }); + } + + return user; + } +} +``` + +### Transactional Publishing (Outbox Pattern) + +```typescript +// src/core/events/outbox-pattern.ts +import { PrismaClient } from '@prisma/client'; +import { EventPublisher } from './event-publisher'; +import { logger } from '@goodgo/logger'; + +export class OutboxService { + constructor(private prisma: PrismaClient) {} + + async addToOutbox( + eventType: string, + eventData: unknown, + topic: string, + correlationId?: string + ): Promise { + await this.prisma.outboxEvent.create({ + data: { + eventType, + eventData: eventData as any, + topic, + correlationId, + status: 'PENDING', + }, + }); + } + + async processOutbox(eventPublisher: EventPublisher): Promise { + const events = await this.prisma.outboxEvent.findMany({ + where: { status: 'PENDING' }, + take: 100, + orderBy: { createdAt: 'asc' }, + }); + + for (const event of events) { + try { + await eventPublisher.publish( + event.topic, + { + eventType: event.eventType, + eventVersion: '1.0.0', + correlationId: event.correlationId || undefined, + data: event.eventData, + }, + { + partitionKey: event.id, + } + ); + + await this.prisma.outboxEvent.update({ + where: { id: event.id }, + data: { status: 'PUBLISHED', publishedAt: new Date() }, + }); + } catch (error) { + logger.error('Failed to process outbox event', { eventId: event.id, error }); + await this.prisma.outboxEvent.update({ + where: { id: event.id }, + data: { status: 'FAILED', retryCount: { increment: 1 } }, + }); + } + } + } +} + +// Using outbox in transaction +export class UserService { + constructor( + private prisma: PrismaClient, + private outboxService: OutboxService + ) {} + + async createUserWithOutbox(data: CreateUserDto): Promise { + return await this.prisma.$transaction(async (tx) => { + const user = await tx.user.create({ data }); + + await this.outboxService.addToOutbox( + 'user.created', + { + userId: user.id, + email: user.email, + name: user.name, + }, + 'user.created' + ); + + return user; + }); + } +} +``` + +## Event Consuming Patterns + +### Event Consumer Service + +```typescript +// src/core/events/event-consumer.ts +import { kafka } from '../config/kafka.config'; +import { logger } from '@goodgo/logger'; +import type { BaseEvent } from '../../types/event.types'; + +export interface EventHandler { + handle(event: T): Promise; +} + +export class EventConsumer { + private consumer: ReturnType; + private handlers: Map = new Map(); + + constructor( + groupId: string = process.env.KAFKA_CONSUMER_GROUP_ID || 'default-group' + ) { + this.consumer = kafka.consumer({ + groupId, + sessionTimeout: 30000, + heartbeatInterval: 3000, + }); + } + + on(eventType: string, handler: EventHandler): void { + if (!this.handlers.has(eventType)) { + this.handlers.set(eventType, []); + } + this.handlers.get(eventType)!.push(handler as EventHandler); + } + + async start(topics: string[]): Promise { + await this.consumer.connect(); + await this.consumer.subscribe({ topics, fromBeginning: false }); + + await this.consumer.run({ + eachMessage: async ({ topic, partition, message }) => { + try { + const event: BaseEvent = JSON.parse(message.value?.toString() || '{}'); + const handlers = this.handlers.get(event.eventType) || []; + + if (handlers.length === 0) { + logger.warn('No handler registered for event type', { + eventType: event.eventType, + topic, + }); + return; + } + + await Promise.all(handlers.map((handler) => handler.handle(event))); + + logger.debug('Event processed successfully', { + eventType: event.eventType, + eventId: event.eventId, + topic, + partition, + }); + } catch (error) { + logger.error('Error processing event', { + topic, + partition, + offset: message.offset, + error, + }); + throw error; + } + }, + }); + + logger.info('Event consumer started', { topics, groupId: this.consumer.groupId }); + } + + async stop(): Promise { + await this.consumer.disconnect(); + logger.info('Event consumer stopped'); + } +} +``` + +### Consumer Implementation + +```typescript +// src/modules/notification/notification.consumer.ts +import { EventConsumer, EventHandler } from '../../core/events/event-consumer'; +import { logger } from '@goodgo/logger'; +import type { UserCreatedEvent } from '../../types/event.types'; + +class UserCreatedHandler implements EventHandler { + async handle(event: UserCreatedEvent): Promise { + logger.info('Processing user.created event', { eventId: event.eventId, userId: event.data.userId }); + + await this.sendWelcomeEmail(event.data.email, event.data.name); + await this.createNotificationPreferences(event.data.userId); + } + + private async sendWelcomeEmail(email: string, name: string): Promise { + // Email sending logic + } + + private async createNotificationPreferences(userId: string): Promise { + // Create default preferences + } +} + +export const notificationConsumer = new EventConsumer('notification-service'); +notificationConsumer.on('user.created', new UserCreatedHandler()); +await notificationConsumer.start(['user.created']); +``` + +### Dead Letter Queue (DLQ) Pattern + +```typescript +// src/core/events/dlq-handler.ts +import { eventPublisher } from './event-publisher'; +import { logger } from '@goodgo/logger'; + +export class DLQHandler { + async sendToDLQ( + originalEvent: BaseEvent, + originalTopic: string, + error: Error, + retryCount: number + ): Promise { + const dlqEvent = { + ...originalEvent, + eventType: `${originalEvent.eventType}.dlq`, + data: { + originalEvent, + originalTopic, + error: { + message: error.message, + stack: error.stack, + }, + retryCount, + dlqTimestamp: new Date().toISOString(), + }, + }; + + await eventPublisher.publish(`${originalTopic}.dlq`, dlqEvent); + + logger.error('Event sent to DLQ', { + originalEventType: originalEvent.eventType, + originalTopic, + retryCount, + error: error.message, + }); + } +} + +// Consumer with DLQ support +export class EventConsumerWithDLQ extends EventConsumer { + private dlqHandler = new DLQHandler(); + private maxRetries = 3; + + async start(topics: string[]): Promise { + await this.consumer.connect(); + await this.consumer.subscribe({ topics, fromBeginning: false }); + + await this.consumer.run({ + eachMessage: async ({ topic, partition, message }) => { + let retryCount = 0; + const maxAttempts = this.maxRetries + 1; + + while (retryCount < maxAttempts) { + try { + const event: BaseEvent = JSON.parse(message.value?.toString() || '{}'); + const handlers = this.handlers.get(event.eventType) || []; + + if (handlers.length === 0) { + logger.warn('No handler registered for event type', { + eventType: event.eventType, + topic, + }); + return; + } + + await Promise.all(handlers.map((handler) => handler.handle(event))); + return; + } catch (error) { + retryCount++; + if (retryCount >= maxAttempts) { + const event: BaseEvent = JSON.parse(message.value?.toString() || '{}'); + await this.dlqHandler.sendToDLQ(event, topic, error as Error, retryCount); + return; + } + + const delay = Math.min(1000 * Math.pow(2, retryCount), 30000); + await new Promise((resolve) => setTimeout(resolve, delay)); + + logger.warn('Retrying event processing', { + topic, + partition, + offset: message.offset, + retryCount, + delay, + }); + } + } + }, + }); + } +} +``` + +## Schema Versioning + +### Schema Registry Setup + +```typescript +// src/core/schema/schema-registry.ts +import { SchemaRegistry } from '@kafkajs/confluent-schema-registry'; +import { logger } from '@goodgo/logger'; + +const registry = new SchemaRegistry({ + host: process.env.SCHEMA_REGISTRY_URL || 'http://localhost:8081', +}); + +export async function registerSchema( + subject: string, + schema: object +): Promise { + try { + const { id } = await registry.register({ type: 'AVRO', schema: JSON.stringify(schema) }); + logger.info('Schema registered', { subject, id }); + return id; + } catch (error) { + logger.error('Failed to register schema', { subject, error }); + throw error; + } +} + +export async function getSchema(subject: string, version?: number) { + try { + const schema = await registry.getSchema(subject, version); + return schema; + } catch (error) { + logger.error('Failed to get schema', { subject, version, error }); + throw error; + } +} +``` + +### Avro Schema Definition + +```json +// schemas/user.created.v1.avsc +{ + "type": "record", + "name": "UserCreated", + "namespace": "com.goodgo.events.user", + "fields": [ + { "name": "userId", "type": "string" }, + { "name": "email", "type": "string" }, + { "name": "name", "type": "string" }, + { "name": "createdAt", "type": "string" } + ] +} +``` + +### Schema Evolution + +**Backward Compatible Changes (Safe):** +- Adding optional fields +- Removing fields (old consumers ignore) + +**Forward Compatible Changes:** +- Removing optional fields +- Adding required fields (with defaults) + +**Breaking Changes:** +- Change field type +- Remove required field +- Rename field + +## Traefik Integration (SSE/WebSocket) + +### SSE Endpoint Implementation + +```typescript +// src/modules/events/events.controller.ts +import { Request, Response } from 'express'; +import { kafka } from '../../config/kafka.config'; +import { logger } from '@goodgo/logger'; + +export class EventsController { + async streamEvents(req: Request, res: Response): Promise { + res.setHeader('Content-Type', 'text/event-stream'); + res.setHeader('Cache-Control', 'no-cache'); + res.setHeader('Connection', 'keep-alive'); + res.setHeader('X-Accel-Buffering', 'no'); + + const topic = req.query.topic as string; + const eventType = req.query.eventType as string | undefined; + + if (!topic) { + res.status(400).json({ success: false, error: { code: 'TOPIC_REQUIRED', message: 'Topic parameter required' } }); + return; + } + + const consumer = kafka.consumer({ + groupId: `sse-${Date.now()}`, + }); + + try { + await consumer.connect(); + await consumer.subscribe({ topic, fromBeginning: false }); + + res.write(`data: ${JSON.stringify({ type: 'connected', topic })}\n\n`); + + await consumer.run({ + eachMessage: async ({ message }) => { + try { + const event = JSON.parse(message.value?.toString() || '{}'); + + if (eventType && event.eventType !== eventType) { + return; + } + + res.write(`data: ${JSON.stringify(event)}\n\n`); + } catch (error) { + logger.error('Error processing SSE message', { error }); + } + }, + }); + + req.on('close', async () => { + await consumer.disconnect(); + logger.info('SSE connection closed', { topic }); + }); + } catch (error) { + logger.error('SSE connection error', { error }); + res.write(`data: ${JSON.stringify({ type: 'error', message: 'Connection failed' })}\n\n`); + await consumer.disconnect(); + } + } +} +``` + +### Traefik Routing Configuration + +```yaml +# infra/traefik/dynamic/routes.yml +http: + routers: + events-sse: + rule: "PathPrefix(`/api/v1/events/stream`)" + entrypoints: + - web + service: events-service + middlewares: + - cors + - compress + - events-ratelimit + + services: + events-service: + loadBalancer: + servers: + - url: "http://events-service:5000" + + middlewares: + events-ratelimit: + rateLimit: + average: 100 + period: 1m + burst: 50 +``` + +### Client Subscription Example + +```typescript +// Client-side SSE subscription +const eventSource = new EventSource('/api/v1/events/stream?topic=user.created&eventType=user.created'); + +eventSource.onmessage = (event) => { + const data = JSON.parse(event.data); + console.log('Received event:', data); +}; + +eventSource.onerror = (error) => { + console.error('SSE error:', error); + eventSource.close(); +}; +``` + +## Error Handling & Resilience + +### Event Publishing Errors + +```typescript +import { retryWithBackoff } from '../resilience/retry'; + +async function publishWithRetry( + topic: string, + event: T, + maxRetries: number = 3 +): Promise { + try { + await retryWithBackoff( + async () => { + await eventPublisher.publish(topic, event); + }, + maxRetries, + 1000 + ); + } catch (error) { + logger.error('Failed to publish event after retries', { + topic, + eventType: event.eventType, + eventId: event.eventId, + error, + }); + throw error; + } +} +``` + +### Circuit Breaker for Kafka Operations + +```typescript +import { createCircuitBreaker } from '../resilience/circuit-breaker'; + +const kafkaCircuitBreaker = createCircuitBreaker( + async (topic: string, event: BaseEvent) => { + await eventPublisher.publish(topic, event); + }, + 'kafka-producer', + { + timeout: 5000, + errorThresholdPercentage: 50, + resetTimeout: 30000, + } +); + +try { + await kafkaCircuitBreaker.fire('user.created', event); +} catch (error) { + if (kafkaCircuitBreaker.opened) { + await storeEventForLaterProcessing(event); + } + throw error; +} +``` + +## Observability + +### Event Logging + +```typescript +import { logger } from '@goodgo/logger'; + +// Log when publishing +logger.info('Event published', { + eventType: event.eventType, + eventId: event.eventId, + topic, + correlationId: event.correlationId, + traceId: event.traceId, + source: event.source, +}); + +// Log when consuming +logger.info('Event consumed', { + eventType: event.eventType, + eventId: event.eventId, + topic, + partition, + offset, + processingTime: Date.now() - eventTimestamp, +}); +``` + +### Metrics Collection + +```typescript +// src/core/events/event-metrics.ts +import { Counter, Histogram, Gauge } from 'prom-client'; + +const eventsPublished = new Counter({ + name: 'events_published_total', + help: 'Total number of events published', + labelNames: ['topic', 'event_type'], +}); + +const eventsConsumed = new Counter({ + name: 'events_consumed_total', + help: 'Total number of events consumed', + labelNames: ['topic', 'event_type'], +}); + +const eventProcessingDuration = new Histogram({ + name: 'event_processing_duration_seconds', + help: 'Event processing duration in seconds', + labelNames: ['topic', 'event_type'], +}); + +const consumerLag = new Gauge({ + name: 'kafka_consumer_lag', + help: 'Consumer lag per topic and partition', + labelNames: ['topic', 'partition'], +}); + +// Usage in publisher +eventsPublished.inc({ topic, event_type: event.eventType }); + +// Usage in consumer +const timer = eventProcessingDuration.startTimer({ topic, event_type: event.eventType }); +try { + await handler.handle(event); + eventsConsumed.inc({ topic, event_type: event.eventType }); +} finally { + timer(); +} +``` + +### Distributed Tracing + +```typescript +import { trace, context, SpanStatusCode } from '@opentelemetry/api'; + +const tracer = trace.getTracer('event-service'); + +const span = tracer.startSpan('event.publish', { + attributes: { + 'event.type': event.eventType, + 'event.topic': topic, + 'event.id': event.eventId, + }, +}); + +try { + await eventPublisher.publish(topic, event); + span.setStatus({ code: SpanStatusCode.OK }); +} catch (error) { + span.setStatus({ code: SpanStatusCode.ERROR, message: error.message }); + throw error; +} finally { + span.end(); +} +``` + +### Health Checks + +```typescript +// src/routes/health.routes.ts +import { admin } from '../config/kafka.config'; + +router.get('/health/kafka', async (req, res) => { + try { + await admin.listTopics(); + res.json({ status: 'healthy', kafka: 'connected' }); + } catch (error) { + res.status(503).json({ status: 'unhealthy', kafka: 'disconnected', error: error.message }); + } +}); +``` + +## Testing Patterns + +### Unit Testing Publishers + +```typescript +// src/core/events/__tests__/event-publisher.test.ts +import { EventPublisher } from '../event-publisher'; +import { producer } from '../../config/kafka.config'; + +jest.mock('../../config/kafka.config'); + +describe('EventPublisher', () => { + let publisher: EventPublisher; + const mockSend = jest.fn(); + + beforeEach(() => { + publisher = new EventPublisher('test-service'); + (producer.send as jest.Mock) = mockSend; + mockSend.mockResolvedValue({}); + }); + + it('should publish event successfully', async () => { + const event = { + eventType: 'user.created', + eventVersion: '1.0.0', + data: { userId: '123' }, + }; + + await publisher.publish('user.created', event); + + expect(mockSend).toHaveBeenCalledWith({ + topic: 'user.created', + messages: expect.arrayContaining([ + expect.objectContaining({ + key: expect.any(String), + value: expect.stringContaining('"eventType":"user.created"'), + }), + ]), + }); + }); +}); +``` + +### Integration Testing with Test Containers + +```typescript +// src/__tests__/integration/event-flow.e2e.ts +import { KafkaContainer, StartedKafkaContainer } from '@testcontainers/kafka'; +import { EventPublisher } from '../../core/events/event-publisher'; +import { EventConsumer } from '../../core/events/event-consumer'; +import type { BaseEvent } from '../../types/event.types'; + +describe('Event Flow E2E', () => { + let kafkaContainer: StartedKafkaContainer; + let publisher: EventPublisher; + let consumer: EventConsumer; + + beforeAll(async () => { + kafkaContainer = await new KafkaContainer().start(); + process.env.KAFKA_BROKERS = kafkaContainer.getBootstrapServer(); + + publisher = new EventPublisher('test-service'); + consumer = new EventConsumer('test-group'); + }); + + afterAll(async () => { + await kafkaContainer.stop(); + }); + + it('should publish and consume event', async () => { + const receivedEvents: BaseEvent[] = []; + + consumer.on('user.created', { + handle: async (event) => { + receivedEvents.push(event); + }, + }); + + await consumer.start(['user.created']); + + const event = { + eventType: 'user.created', + eventVersion: '1.0.0', + data: { userId: '123', email: 'test@example.com' }, + }; + + await publisher.publish('user.created', event); + + await new Promise((resolve) => setTimeout(resolve, 1000)); + + expect(receivedEvents).toHaveLength(1); + expect(receivedEvents[0].data.userId).toBe('123'); + }); +}); +``` + +### Mocking Kafka + +```typescript +// src/__tests__/mocks/kafka.mock.ts +export const createMockKafka = () => { + const messages: any[] = []; + + return { + producer: () => ({ + connect: jest.fn(), + send: jest.fn((params) => { + messages.push(...params.messages); + return Promise.resolve({}); + }), + disconnect: jest.fn(), + }), + consumer: () => ({ + connect: jest.fn(), + subscribe: jest.fn(), + run: jest.fn(), + disconnect: jest.fn(), + }), + admin: () => ({ + connect: jest.fn(), + listTopics: jest.fn().mockResolvedValue([]), + disconnect: jest.fn(), + }), + messages, + }; +}; +``` + +## Complete Examples + +### Example 1: User Created Event Flow + +```typescript +// Auth Service publishes user.created event +// services/auth-service/src/modules/user/user.service.ts +export class UserService { + async createUser(data: CreateUserDto): Promise { + const user = await this.userRepository.create(data); + + await eventPublisher.publish( + 'user.created', + { + eventType: 'user.created', + eventVersion: '1.0.0', + data: { + userId: user.id, + email: user.email, + name: user.name, + }, + }, + { partitionKey: user.id } + ); + + return user; + } +} + +// Notification Service consumes and sends welcome email +// services/notification-service/src/modules/notification/notification.consumer.ts +notificationConsumer.on('user.created', { + handle: async (event: UserCreatedEvent) => { + await emailService.sendWelcomeEmail(event.data.email, event.data.name); + await notificationPreferenceService.createDefaults(event.data.userId); + }, +}); +``` + +### Example 2: Order Placed with Multiple Consumers + +```typescript +// Order Service publishes order.placed +await eventPublisher.publish('order.placed', { + eventType: 'order.placed', + eventVersion: '1.0.0', + data: { + orderId: order.id, + userId: order.userId, + total: order.total, + items: order.items, + }, +}); + +// Multiple consumers handle the event + +// Payment Service: Process payment +paymentConsumer.on('order.placed', { handle: processPayment }); + +// Inventory Service: Reserve items +inventoryConsumer.on('order.placed', { handle: reserveItems }); + +// Notification Service: Send confirmation +notificationConsumer.on('order.placed', { handle: sendConfirmation }); +``` + +### Example 3: SSE Endpoint with Filtering + +```typescript +// Client subscribes to user events +const eventSource = new EventSource( + '/api/v1/events/stream?topic=user.created&eventType=user.created', + { + headers: { + Authorization: `Bearer ${token}`, + }, + } +); + +eventSource.addEventListener('user.created', (event) => { + const data = JSON.parse(event.data); + updateUI(data); +}); +``` diff --git a/.cursor/skills/inter-service-communication/SKILL.md b/.cursor/skills/inter-service-communication/SKILL.md index ccaad734..991e93eb 100644 --- a/.cursor/skills/inter-service-communication/SKILL.md +++ b/.cursor/skills/inter-service-communication/SKILL.md @@ -1,7 +1,7 @@ --- name: inter-service-communication description: Inter-service communication for GoodGo microservices. Use for gRPC, GraphQL, service auth, protocol selection, or building service clients. -dependencies: "@grpc/grpc-js, graphql" +compatibility: "@grpc/grpc-js, graphql" --- # Inter-Service Communication Patterns @@ -18,1124 +18,173 @@ Use this skill when: - Managing connection pooling for service clients - Implementing request/response interceptors - Handling service discovery for internal calls -- Optimizing inter-service communication performance -## Core Concepts +## Protocol Comparison -### Communication Protocol Options - -**HTTP/REST:** -- Human-readable, easy to debug -- Browser-compatible -- Standard HTTP semantics -- JSON payloads -- Good for external APIs -- Use for: Client-facing APIs, simple CRUD operations - -**gRPC:** -- Binary protocol (Protocol Buffers) -- High performance, low latency -- Streaming support (unary, server streaming, client streaming, bidirectional) -- Strong typing with .proto files -- HTTP/2 based -- Use for: Internal service-to-service calls, high-throughput scenarios, streaming data - -**GraphQL:** -- Flexible query language -- Single endpoint -- Client-controlled data fetching -- Strong typing with schema -- Use for: Complex data requirements, mobile apps, reducing over-fetching +| Protocol | Use Case | Latency | Complexity | Best For | +|----------|----------|---------|------------|----------| +| **REST** | External APIs, CRUD | Medium | Low | Browser clients, simple APIs | +| **gRPC** | Internal high-perf | Low | High | Service-to-service, streaming | +| **GraphQL** | Flexible queries | Medium | Medium | Mobile apps, complex data | ### Protocol Selection Guidelines -Choose protocol based on: - -1. **Use Case**: - - External APIs → REST - - Internal microservices → gRPC (performance) or REST (simplicity) - - Complex queries → GraphQL - -2. **Performance Requirements**: - - High throughput/low latency → gRPC - - Standard web APIs → REST - - Flexible data needs → GraphQL - -3. **Team Expertise**: - - REST: Widely known, easy onboarding - - gRPC: Requires Protocol Buffers knowledge - - GraphQL: Requires schema design skills - -4. **Ecosystem**: - - Browser clients → REST or GraphQL - - Mobile apps → REST or GraphQL - - Service-to-service → gRPC or REST - -## HTTP/REST Service Client - -### Service Client Setup - -```typescript -// src/core/clients/service-client.ts -// EN: Base service client for HTTP/REST communication -// VI: Service client cơ bản cho giao tiếp HTTP/REST -import axios, { AxiosInstance, AxiosRequestConfig } from 'axios'; -import { logger } from '@goodgo/logger'; -import { createCircuitBreaker } from '../resilience/circuit-breaker'; - -export interface ServiceClientConfig { - baseURL: string; - serviceName: string; - timeout?: number; - retries?: number; - enableCircuitBreaker?: boolean; -} - -export class ServiceClient { - private client: AxiosInstance; - private serviceName: string; - private circuitBreaker?: ReturnType; - - constructor(config: ServiceClientConfig) { - this.serviceName = config.serviceName; - - this.client = axios.create({ - baseURL: config.baseURL, - timeout: config.timeout || 5000, - headers: { - 'Content-Type': 'application/json', - 'User-Agent': `${process.env.SERVICE_NAME || 'unknown'}/1.0`, - }, - }); - - this.setupInterceptors(); - - // EN: Setup circuit breaker if enabled - // VI: Thiết lập circuit breaker nếu được bật - if (config.enableCircuitBreaker !== false) { - this.circuitBreaker = createCircuitBreaker( - async (requestConfig: AxiosRequestConfig) => { - return await this.client.request(requestConfig); - }, - `service-client-${config.serviceName}`, - { - timeout: config.timeout || 5000, - errorThresholdPercentage: 50, - resetTimeout: 30000, - } - ); - } - } - - private setupInterceptors(): void { - // EN: Request interceptor - add authentication and correlation ID - // VI: Request interceptor - thêm authentication và correlation ID - this.client.interceptors.request.use( - (config) => { - // EN: Add service-to-service authentication token - // VI: Thêm token xác thực service-to-service - if (process.env.INTERNAL_API_KEY) { - config.headers['X-Service-Auth'] = process.env.INTERNAL_API_KEY; - } - - // EN: Add correlation ID for tracing - // VI: Thêm correlation ID để tracing - const correlationId = config.headers['x-correlation-id'] || this.generateCorrelationId(); - config.headers['x-correlation-id'] = correlationId; - - logger.debug('Service request', { - service: this.serviceName, - method: config.method, - url: config.url, - correlationId, - }); - - return config; - }, - (error) => { - logger.error('Service request error', { service: this.serviceName, error }); - return Promise.reject(error); - } - ); - - // EN: Response interceptor - handle errors - // VI: Response interceptor - xử lý lỗi - this.client.interceptors.response.use( - (response) => { - logger.debug('Service response', { - service: this.serviceName, - status: response.status, - url: response.config.url, - }); - return response; - }, - async (error) => { - if (error.response) { - logger.warn('Service error response', { - service: this.serviceName, - status: error.response.status, - url: error.config?.url, - data: error.response.data, - }); - } else if (error.request) { - logger.error('Service request timeout', { - service: this.serviceName, - url: error.config?.url, - }); - } - - return Promise.reject(error); - } - ); - } - - private generateCorrelationId(): string { - return `corr_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`; - } - - /** - * EN: Make request with circuit breaker and retry - * VI: Thực hiện request với circuit breaker và retry - */ - async request(config: AxiosRequestConfig): Promise { - try { - const response = this.circuitBreaker - ? await this.circuitBreaker.fire(config) - : await this.client.request(config); - - return response.data; - } catch (error) { - logger.error('Service request failed', { - service: this.serviceName, - url: config.url, - error: error.message, - }); - throw error; - } - } - - async get(url: string, config?: AxiosRequestConfig): Promise { - return this.request({ ...config, method: 'GET', url }); - } - - async post(url: string, data?: any, config?: AxiosRequestConfig): Promise { - return this.request({ ...config, method: 'POST', url, data }); - } - - async put(url: string, data?: any, config?: AxiosRequestConfig): Promise { - return this.request({ ...config, method: 'PUT', url, data }); - } - - async delete(url: string, config?: AxiosRequestConfig): Promise { - return this.request({ ...config, method: 'DELETE', url }); - } -} +``` +External/Public API → REST +Internal service-to-service → gRPC (performance) or REST (simplicity) +Complex data fetching → GraphQL +Real-time streaming → gRPC or WebSocket ``` -### Usage Example +## HTTP/REST Client Pattern ```typescript -// src/modules/notification/notification-client.service.ts -// EN: Example: Notification service client -// VI: Ví dụ: Notification service client -import { ServiceClient } from '../../core/clients/service-client'; +import axios from 'axios'; -const notificationClient = new ServiceClient({ - baseURL: process.env.NOTIFICATION_SERVICE_URL || 'http://notification-service:5003', - serviceName: 'notification-service', +const client = axios.create({ + baseURL: process.env.USER_SERVICE_URL, timeout: 5000, - enableCircuitBreaker: true, + headers: { + 'Content-Type': 'application/json', + 'x-service-auth': process.env.INTERNAL_API_KEY, + }, }); -export class NotificationClientService { - async sendNotification(userId: string, message: string): Promise { - try { - await notificationClient.post('/api/v1/notifications', { - userId, - message, - }); - } catch (error) { - logger.error('Failed to send notification', { userId, error }); - throw error; - } - } -} +// Add correlation ID for tracing +client.interceptors.request.use((config) => { + config.headers['x-correlation-id'] = generateCorrelationId(); + return config; +}); ``` -## gRPC Communication - -### Protocol Buffer Definitions +## gRPC Pattern +### Proto Definition ```protobuf -// proto/user_service.proto syntax = "proto3"; - package goodgo.user.v1; -option go_package = "github.com/goodgo/user-service/proto"; - -// EN: User service definition -// VI: Định nghĩa user service service UserService { - // EN: Get user by ID - // VI: Lấy user theo ID rpc GetUser(GetUserRequest) returns (GetUserResponse); - - // EN: Create user - // VI: Tạo user - rpc CreateUser(CreateUserRequest) returns (CreateUserResponse); - - // EN: Stream user updates - // VI: Stream cập nhật user - rpc StreamUserUpdates(StreamUserUpdatesRequest) returns (stream UserUpdate); + rpc StreamUserUpdates(StreamRequest) returns (stream UserUpdate); } -message GetUserRequest { - string user_id = 1; -} - -message GetUserResponse { - User user = 1; -} - -message CreateUserRequest { - string email = 1; - string name = 2; -} - -message CreateUserResponse { - User user = 1; -} - -message User { - string id = 1; - string email = 2; - string name = 3; - int64 created_at = 4; -} - -message StreamUserUpdatesRequest { - string user_id = 1; -} - -message UserUpdate { - string user_id = 1; - string action = 2; - User user = 3; - int64 timestamp = 4; -} +message GetUserRequest { string user_id = 1; } +message GetUserResponse { User user = 1; } ``` -### gRPC Server Implementation - +### Client Usage ```typescript -// src/modules/user/user.grpc.service.ts -// EN: gRPC server implementation -// VI: Implementation gRPC server -import * as grpc from '@grpc/grpc-js'; -import * as protoLoader from '@grpc/proto-loader'; -import { UserService } from './user.service'; -import { logger } from '@goodgo/logger'; - -const PROTO_PATH = './proto/user_service.proto'; - -const packageDefinition = protoLoader.loadSync(PROTO_PATH, { - keepCase: true, - longs: String, - enums: String, - defaults: true, - oneofs: true, -}); - -const userProto = grpc.loadPackageDefinition(packageDefinition).goodgo?.user?.v1; - -export class UserGrpcServer { - private server: grpc.Server; - private userService: UserService; - - constructor(userService: UserService) { - this.userService = userService; - this.server = new grpc.Server(); - this.setupServices(); - } - - private setupServices(): void { - if (!userProto?.UserService) { - throw new Error('UserService proto not loaded'); - } - - this.server.addService(userProto.UserService.service, { - getUser: this.getUser.bind(this), - createUser: this.createUser.bind(this), - streamUserUpdates: this.streamUserUpdates.bind(this), - }); - } - - private async getUser( - call: grpc.ServerUnaryCall, - callback: grpc.sendUnaryData - ): Promise { - try { - const { user_id } = call.request; - const user = await this.userService.findById(user_id); - - if (!user) { - return callback({ - code: grpc.status.NOT_FOUND, - message: 'User not found', - }); - } - - callback(null, { - user: { - id: user.id, - email: user.email, - name: user.name, - created_at: user.createdAt.getTime(), - }, - }); - } catch (error) { - logger.error('gRPC getUser error', { error }); - callback({ - code: grpc.status.INTERNAL, - message: error.message, - }); - } - } - - private async createUser( - call: grpc.ServerUnaryCall, - callback: grpc.sendUnaryData - ): Promise { - try { - const { email, name } = call.request; - const user = await this.userService.create({ email, name }); - - callback(null, { - user: { - id: user.id, - email: user.email, - name: user.name, - created_at: user.createdAt.getTime(), - }, - }); - } catch (error) { - logger.error('gRPC createUser error', { error }); - callback({ - code: grpc.status.INTERNAL, - message: error.message, - }); - } - } - - private streamUserUpdates(call: grpc.ServerWritableStream): void { - const { user_id } = call.request; - - // EN: Stream user updates - // VI: Stream cập nhật user - const subscription = this.userService.subscribeToUpdates(user_id, (update) => { - call.write({ - user_id: update.userId, - action: update.action, - user: update.user, - timestamp: Date.now(), - }); - }); - - call.on('cancelled', () => { - subscription.unsubscribe(); - }); - } - - start(port: number): void { - this.server.bindAsync( - `0.0.0.0:${port}`, - grpc.ServerCredentials.createInsecure(), - (error, port) => { - if (error) { - logger.error('Failed to start gRPC server', { error }); - return; - } - - this.server.start(); - logger.info('gRPC server started', { port }); - } - ); - } - - stop(): Promise { - return new Promise((resolve) => { - this.server.tryShutdown(() => { - logger.info('gRPC server stopped'); - resolve(); - }); - }); - } -} -``` - -### gRPC Client Implementation - -```typescript -// src/core/clients/grpc-client.ts -// EN: gRPC client wrapper -// VI: Wrapper gRPC client -import * as grpc from '@grpc/grpc-js'; -import * as protoLoader from '@grpc/proto-loader'; -import { logger } from '@goodgo/logger'; - -export interface GrpcClientConfig { - protoPath: string; - packageName: string; - serviceName: string; - serverUrl: string; - options?: protoLoader.Options; -} - -export class GrpcClient { - private client: any; - private serviceName: string; - - constructor(config: GrpcClientConfig) { - this.serviceName = config.serviceName; - - const packageDefinition = protoLoader.loadSync(config.protoPath, { - keepCase: true, - longs: String, - enums: String, - defaults: true, - oneofs: true, - ...config.options, - }); - - const proto = grpc.loadPackageDefinition(packageDefinition); - const service = this.getNestedProperty(proto, config.packageName); - - if (!service?.[config.serviceName]) { - throw new Error(`Service ${config.serviceName} not found in proto`); - } - - this.client = new service[config.serviceName]( - config.serverUrl, - grpc.credentials.createInsecure() - ); - } - - private getNestedProperty(obj: any, path: string): any { - return path.split('.').reduce((current, prop) => current?.[prop], obj); - } - - /** - * EN: Make unary RPC call - * VI: Thực hiện unary RPC call - */ - async call( - methodName: string, - request: TRequest, - metadata?: grpc.Metadata - ): Promise { - return new Promise((resolve, reject) => { - // EN: Add service authentication metadata - // VI: Thêm metadata xác thực service - const callMetadata = metadata || new grpc.Metadata(); - if (process.env.INTERNAL_API_KEY) { - callMetadata.add('x-service-auth', process.env.INTERNAL_API_KEY); - } - - this.client[methodName](request, callMetadata, (error: any, response: TResponse) => { - if (error) { - logger.error('gRPC call error', { - service: this.serviceName, - method: methodName, - error: error.message, - }); - reject(error); - return; - } - - resolve(response); - }); - }); - } - - /** - * EN: Create streaming client - * VI: Tạo streaming client - */ - createStream( - methodName: string, - request: TRequest, - metadata?: grpc.Metadata - ): grpc.ClientReadableStream { - const callMetadata = metadata || new grpc.Metadata(); - if (process.env.INTERNAL_API_KEY) { - callMetadata.add('x-service-auth', process.env.INTERNAL_API_KEY); - } - - return this.client[methodName](request, callMetadata); - } -} -``` - -### gRPC Client Usage - -```typescript -// src/modules/user/user.grpc.client.ts -// EN: User service gRPC client -// VI: User service gRPC client -import { GrpcClient } from '../../core/clients/grpc-client'; - -const userGrpcClient = new GrpcClient({ +const client = new GrpcClient({ protoPath: './proto/user_service.proto', packageName: 'goodgo.user.v1', serviceName: 'UserService', - serverUrl: process.env.USER_SERVICE_GRPC_URL || 'localhost:50051', + serverUrl: 'localhost:50051', }); -export class UserGrpcClientService { - async getUser(userId: string) { - return userGrpcClient.call('getUser', { user_id: userId }); - } - - async createUser(email: string, name: string) { - return userGrpcClient.call('createUser', { email, name }); - } - - streamUserUpdates(userId: string, callback: (update: any) => void) { - const stream = userGrpcClient.createStream('streamUserUpdates', { - user_id: userId, - }); - - stream.on('data', callback); - stream.on('error', (error) => { - logger.error('gRPC stream error', { error }); - }); - } -} +const user = await client.call('getUser', { user_id: '123' }); ``` -## GraphQL Communication - -### GraphQL Schema Definition - -```graphql -# schema/user.graphql -# EN: User service GraphQL schema -# VI: GraphQL schema cho user service -type Query { - user(id: ID!): User - users(filter: UserFilter, pagination: Pagination): UserConnection -} - -type Mutation { - createUser(input: CreateUserInput!): User! - updateUser(id: ID!, input: UpdateUserInput!): User! - deleteUser(id: ID!): Boolean! -} - -type User { - id: ID! - email: String! - name: String - createdAt: DateTime! - updatedAt: DateTime! -} - -input CreateUserInput { - email: String! - name: String -} - -input UpdateUserInput { - name: String -} - -input UserFilter { - email: String - name: String -} - -input Pagination { - skip: Int - take: Int -} - -type UserConnection { - nodes: [User!]! - total: Int! - hasNextPage: Boolean! -} - -scalar DateTime -``` - -### GraphQL Server (Apollo) +## GraphQL Pattern +### Client Usage ```typescript -// src/modules/user/user.graphql.service.ts -// EN: GraphQL server implementation -// VI: Implementation GraphQL server -import { ApolloServer } from '@apollo/server'; -import { expressMiddleware } from '@apollo/server/express4'; -import { UserService } from './user.service'; -import { logger } from '@goodgo/logger'; -import { readFileSync } from 'fs'; -import { resolvers } from './user.resolvers'; - -const typeDefs = readFileSync('./schema/user.graphql', 'utf-8'); - -export function createGraphQLServer(userService: UserService): ApolloServer { - return new ApolloServer({ - typeDefs, - resolvers: resolvers(userService), - plugins: [ - { - requestDidStart() { - return { - didResolveOperation(requestContext) { - logger.debug('GraphQL operation', { - operation: requestContext.operationName, - query: requestContext.request.query, - }); - }, - didEncounterErrors(requestContext) { - logger.error('GraphQL errors', { - errors: requestContext.errors, - }); - }, - }; - }, - }, - ], - }); -} - -// src/modules/user/user.resolvers.ts -export function resolvers(userService: UserService) { - return { - Query: { - user: async (_: any, { id }: { id: string }) => { - return await userService.findById(id); - }, - users: async ( - _: any, - { filter, pagination }: { filter?: any; pagination?: any } - ) => { - const result = await userService.findAll({ filter, pagination }); - return { - nodes: result.users, - total: result.total, - hasNextPage: result.users.length === (pagination?.take || 10), - }; - }, - }, - Mutation: { - createUser: async (_: any, { input }: { input: any }) => { - return await userService.create(input); - }, - updateUser: async (_: any, { id, input }: { id: string; input: any }) => { - return await userService.update(id, input); - }, - deleteUser: async (_: any, { id }: { id: string }) => { - await userService.delete(id); - return true; - }, - }, - }; -} -``` - -### GraphQL Client - -```typescript -// src/core/clients/graphql-client.ts -// EN: GraphQL client wrapper -// VI: Wrapper GraphQL client -import { GraphQLClient } from 'graphql-request'; -import { logger } from '@goodgo/logger'; - -export interface GraphQLClientConfig { - endpoint: string; - headers?: Record; - timeout?: number; -} - -export class GraphQLServiceClient { - private client: GraphQLClient; - private endpoint: string; - - constructor(config: GraphQLClientConfig) { - this.endpoint = config.endpoint; - this.client = new GraphQLClient(config.endpoint, { - headers: { - 'Content-Type': 'application/json', - ...(process.env.INTERNAL_API_KEY && { - 'x-service-auth': process.env.INTERNAL_API_KEY, - }), - ...config.headers, - }, - timeout: config.timeout || 5000, - }); - } - - /** - * EN: Execute GraphQL query - * VI: Thực hiện GraphQL query - */ - async query(query: string, variables?: any): Promise { - try { - const data = await this.client.request(query, variables); - return data; - } catch (error) { - logger.error('GraphQL query error', { - endpoint: this.endpoint, - error: error.message, - }); - throw error; - } - } - - /** - * EN: Execute GraphQL mutation - * VI: Thực hiện GraphQL mutation - */ - async mutate(mutation: string, variables?: any): Promise { - try { - const data = await this.client.request(mutation, variables); - return data; - } catch (error) { - logger.error('GraphQL mutation error', { - endpoint: this.endpoint, - error: error.message, - }); - throw error; - } - } -} - -// Usage example -const userGraphQLClient = new GraphQLServiceClient({ - endpoint: process.env.USER_SERVICE_GRAPHQL_URL || 'http://user-service:5002/graphql', +const client = new GraphQLClient(process.env.USER_SERVICE_GRAPHQL_URL, { + headers: { 'x-service-auth': process.env.INTERNAL_API_KEY }, }); -const GET_USER_QUERY = ` - query GetUser($id: ID!) { - user(id: $id) { - id - email - name - createdAt - } - } -`; - -export async function getUser(userId: string) { - return userGraphQLClient.query(GET_USER_QUERY, { id: userId }); -} +const GET_USER = `query GetUser($id: ID!) { user(id: $id) { id email name } }`; +const data = await client.request(GET_USER, { id: '123' }); ``` -## Service-to-Service Authentication - -### Internal Authentication Middleware +## Service Authentication +### Internal Auth Middleware ```typescript -// src/middlewares/internal-auth.middleware.ts -// EN: Middleware for authenticating internal service requests -// VI: Middleware để xác thực requests từ services nội bộ -import { Request, Response, NextFunction } from 'express'; -import { logger } from '@goodgo/logger'; +export const internalAuthMiddleware = (req, res, next) => { + const token = req.headers['x-service-auth']; -export const internalAuthMiddleware = ( - req: Request, - res: Response, - next: NextFunction -): void => { - const serviceAuthToken = req.headers['x-service-auth']; - - if (!serviceAuthToken) { - logger.warn('Missing service auth token', { - ip: req.ip, - path: req.path, - }); - res.status(401).json({ + if (token !== process.env.INTERNAL_API_KEY) { + return res.status(403).json({ success: false, - error: { - code: 'INTERNAL_AUTH_REQUIRED', - message: 'Service authentication required', - }, + error: { code: 'INVALID_SERVICE_AUTH', message: 'Invalid authentication' } }); - return; } - // EN: Validate service token - // VI: Xác thực service token - if (serviceAuthToken !== process.env.INTERNAL_API_KEY) { - logger.warn('Invalid service auth token', { - ip: req.ip, - path: req.path, - }); - res.status(403).json({ - success: false, - error: { - code: 'INVALID_SERVICE_AUTH', - message: 'Invalid service authentication', - }, - }); - return; - } - - // EN: Add service context to request - // VI: Thêm context service vào request - req.serviceContext = { - authenticated: true, - timestamp: new Date(), - }; - next(); }; ``` -### Mutual TLS (mTLS) for gRPC +### Essential Headers -```typescript -// src/config/grpc-tls.config.ts -// EN: mTLS configuration for gRPC -// VI: Cấu hình mTLS cho gRPC -import * as grpc from '@grpc/grpc-js'; -import { readFileSync } from 'fs'; - -export function createServerCredentials(): grpc.ServerCredentials { - return grpc.ServerCredentials.createSsl( - null, - [ - { - cert_chain: readFileSync(process.env.GRPC_SERVER_CERT_PATH!), - private_key: readFileSync(process.env.GRPC_SERVER_KEY_PATH!), - }, - ], - true // EN: Request client certificate / VI: Yêu cầu client certificate - ); -} - -export function createClientCredentials(): grpc.ChannelCredentials { - return grpc.credentials.createSsl( - readFileSync(process.env.GRPC_CA_CERT_PATH!), - readFileSync(process.env.GRPC_CLIENT_KEY_PATH!), - readFileSync(process.env.GRPC_CLIENT_CERT_PATH!) - ); -} -``` - -## Connection Pooling - -### HTTP Connection Pool - -```typescript -// src/core/clients/connection-pool.ts -// EN: Connection pool for service clients -// VI: Connection pool cho service clients -import { ServiceClient } from './service-client'; - -export class ServiceClientPool { - private pools: Map = new Map(); - - getClient(serviceName: string, config: ServiceClientConfig): ServiceClient { - if (!this.pools.has(serviceName)) { - this.pools.set(serviceName, new ServiceClient(config)); - } - return this.pools.get(serviceName)!; - } - - clearPool(serviceName: string): void { - this.pools.delete(serviceName); - } - - clearAll(): void { - this.pools.clear(); - } -} - -export const serviceClientPool = new ServiceClientPool(); -``` - -## Error Handling - -### Service Error Classes - -```typescript -// src/core/errors/service-errors.ts -// EN: Custom errors for service communication -// VI: Custom errors cho giao tiếp service -export class ServiceUnavailableError extends Error { - constructor(public serviceName: string, message?: string) { - super(message || `Service ${serviceName} is unavailable`); - this.name = 'ServiceUnavailableError'; - } -} - -export class ServiceTimeoutError extends Error { - constructor(public serviceName: string, public timeout: number) { - super(`Service ${serviceName} request timed out after ${timeout}ms`); - this.name = 'ServiceTimeoutError'; - } -} - -export class ServiceError extends Error { - constructor( - public serviceName: string, - public statusCode: number, - message: string, - public details?: any - ) { - super(message); - this.name = 'ServiceError'; - } -} -``` +| Header | Purpose | +|--------|---------| +| `x-service-auth` | Internal authentication token | +| `x-correlation-id` | Request tracing across services | +| `x-request-id` | Unique request identification | ## Best Practices -### Protocol Selection - -1. **Use REST for**: - - External/public APIs - - Browser clients - - Simple CRUD operations - - When team is familiar with REST - -2. **Use gRPC for**: - - Internal service-to-service calls - - High-performance requirements - - Streaming data - - Strong typing needs - -3. **Use GraphQL for**: - - Complex data requirements - - Mobile applications - - Reducing over-fetching - - Flexible querying - ### Performance Optimization - -- **Connection Pooling**: Reuse connections for HTTP clients -- **Keep-Alive**: Enable HTTP keep-alive for persistent connections -- **Compression**: Use gzip compression for HTTP responses -- **Timeouts**: Set appropriate timeouts for all service calls -- **Circuit Breaker**: Implement circuit breakers to prevent cascade failures +- **Connection Pooling**: Reuse HTTP connections +- **Keep-Alive**: Enable persistent connections +- **Compression**: Use gzip for HTTP responses +- **Timeouts**: Always set appropriate timeouts +- **Circuit Breaker**: Prevent cascade failures ### Security - -- **Service Authentication**: Always authenticate internal service calls -- **TLS/mTLS**: Use TLS for all service communication, mTLS for gRPC -- **Secrets Management**: Store API keys in environment variables or secrets manager -- **Rate Limiting**: Implement rate limiting for service clients +- **Service Authentication**: Always authenticate internal calls +- **TLS/mTLS**: Use TLS for all communication +- **Secrets Management**: Use environment variables +- **Rate Limiting**: Implement for service clients ### Observability - -- **Logging**: Log all service calls with correlation IDs -- **Metrics**: Track service call duration, success rate, error rate -- **Tracing**: Add distributed tracing to service calls -- **Health Checks**: Monitor service health before making calls - -## Testing Patterns - -### Mocking Service Clients - -```typescript -// src/__tests__/mocks/service-client.mock.ts -// EN: Mock service client for testing -// VI: Mock service client cho testing -export const createMockServiceClient = () => { - return { - get: jest.fn(), - post: jest.fn(), - put: jest.fn(), - delete: jest.fn(), - request: jest.fn(), - }; -}; - -// Usage in tests -const mockUserClient = createMockServiceClient(); -mockUserClient.get.mockResolvedValue({ id: '123', email: 'test@example.com' }); -``` - -### Integration Testing - -```typescript -// src/__tests__/integration/service-communication.e2e.ts -// EN: Integration tests for service communication -// VI: Integration tests cho giao tiếp service -describe('Service Communication E2E', () => { - it('should communicate with user service via HTTP', async () => { - const client = new ServiceClient({ - baseURL: process.env.USER_SERVICE_URL || 'http://localhost:5002', - serviceName: 'user-service', - }); - - const user = await client.get('/api/v1/users/123'); - expect(user).toHaveProperty('id'); - }); -}); -``` +- **Logging**: Log all calls with correlation IDs +- **Metrics**: Track duration, success rate, errors +- **Tracing**: Add distributed tracing +- **Health Checks**: Monitor service health ## Common Mistakes -1. **No Service Authentication**: Unsecured internal calls +1. **No Service Authentication** ```typescript - // ❌ BAD: No auth header + // BAD: No auth header await client.get('/api/users'); - // ✅ GOOD: Include service auth + // GOOD: Include service auth await client.get('/api/users', { headers: { 'x-service-auth': process.env.INTERNAL_API_KEY } }); ``` -2. **Missing Timeouts**: Hanging requests +2. **Missing Timeouts** ```typescript - // ❌ BAD: No timeout + // BAD: No timeout await axios.get(url); - // ✅ GOOD: Set timeout + // GOOD: Set timeout await axios.get(url, { timeout: 5000 }); ``` -3. **No Circuit Breaker**: Cascade failures +3. **No Circuit Breaker** ```typescript - // ✅ Use circuit breaker for external calls + // GOOD: Use circuit breaker for external calls await circuitBreaker.fire(() => serviceClient.get('/api/users')); ``` -4. **Hardcoded Service URLs**: Breaks in different environments +4. **Hardcoded Service URLs** ```typescript - // ❌ BAD + // BAD const url = 'http://user-service:5000'; - // ✅ GOOD + // GOOD const url = process.env.USER_SERVICE_URL; ``` ## Quick Reference -| Protocol | Use Case | Latency | Complexity | -|----------|----------|---------|------------| -| **REST** | External APIs, CRUD | Medium | Low | -| **gRPC** | Internal high-perf | Low | High | -| **GraphQL** | Flexible queries | Medium | Medium | - -**Protocol Selection:** -``` -External/Public API → REST -Internal service-to-service → gRPC -Complex data fetching → GraphQL -Real-time streaming → gRPC/WebSocket -``` - **HTTP Client Setup:** ```typescript -import axios from 'axios'; - const client = axios.create({ baseURL: process.env.SERVICE_URL, timeout: 5000, @@ -1152,19 +201,26 @@ const client = new GrpcClient({ }); ``` -**Essential Headers:** -| Header | Purpose | -|--------|---------| -| `x-service-auth` | Internal authentication | -| `x-correlation-id` | Request tracing | -| `x-request-id` | Request identification | +**GraphQL Client Setup:** +```typescript +const client = new GraphQLClient(endpoint, { + headers: { 'x-service-auth': process.env.INTERNAL_API_KEY } +}); +``` + +**Error Classes:** +```typescript +ServiceUnavailableError // Service is down +ServiceTimeoutError // Request timeout +ServiceError // Generic service error +``` ## Resources +- [Detailed Reference](./references/REFERENCE.md) - Full code examples and implementation details - [gRPC Documentation](https://grpc.io/docs/) - Official gRPC documentation - [GraphQL Documentation](https://graphql.org/learn/) - GraphQL learning resources - [Protocol Buffers](https://developers.google.com/protocol-buffers) - Protocol Buffer guide -- [Apollo Server](https://www.apollographql.com/docs/apollo-server/) - GraphQL server framework - [Resilience Patterns](../resilience-patterns/SKILL.md) - Circuit breaker, retry patterns - [Security](../security/SKILL.md) - Authentication, authorization patterns - [API Design](../api-design/SKILL.md) - RESTful API patterns diff --git a/.cursor/skills/inter-service-communication/references/REFERENCE.md b/.cursor/skills/inter-service-communication/references/REFERENCE.md new file mode 100644 index 00000000..6fd58936 --- /dev/null +++ b/.cursor/skills/inter-service-communication/references/REFERENCE.md @@ -0,0 +1,891 @@ +# Inter-Service Communication - Detailed Reference + +This document contains detailed code examples and implementation patterns for inter-service communication. + +## HTTP/REST Service Client + +### Service Client Setup + +```typescript +// src/core/clients/service-client.ts +import axios, { AxiosInstance, AxiosRequestConfig } from 'axios'; +import { logger } from '@goodgo/logger'; +import { createCircuitBreaker } from '../resilience/circuit-breaker'; + +export interface ServiceClientConfig { + baseURL: string; + serviceName: string; + timeout?: number; + retries?: number; + enableCircuitBreaker?: boolean; +} + +export class ServiceClient { + private client: AxiosInstance; + private serviceName: string; + private circuitBreaker?: ReturnType; + + constructor(config: ServiceClientConfig) { + this.serviceName = config.serviceName; + + this.client = axios.create({ + baseURL: config.baseURL, + timeout: config.timeout || 5000, + headers: { + 'Content-Type': 'application/json', + 'User-Agent': `${process.env.SERVICE_NAME || 'unknown'}/1.0`, + }, + }); + + this.setupInterceptors(); + + if (config.enableCircuitBreaker !== false) { + this.circuitBreaker = createCircuitBreaker( + async (requestConfig: AxiosRequestConfig) => { + return await this.client.request(requestConfig); + }, + `service-client-${config.serviceName}`, + { + timeout: config.timeout || 5000, + errorThresholdPercentage: 50, + resetTimeout: 30000, + } + ); + } + } + + private setupInterceptors(): void { + this.client.interceptors.request.use( + (config) => { + if (process.env.INTERNAL_API_KEY) { + config.headers['X-Service-Auth'] = process.env.INTERNAL_API_KEY; + } + + const correlationId = config.headers['x-correlation-id'] || this.generateCorrelationId(); + config.headers['x-correlation-id'] = correlationId; + + logger.debug('Service request', { + service: this.serviceName, + method: config.method, + url: config.url, + correlationId, + }); + + return config; + }, + (error) => { + logger.error('Service request error', { service: this.serviceName, error }); + return Promise.reject(error); + } + ); + + this.client.interceptors.response.use( + (response) => { + logger.debug('Service response', { + service: this.serviceName, + status: response.status, + url: response.config.url, + }); + return response; + }, + async (error) => { + if (error.response) { + logger.warn('Service error response', { + service: this.serviceName, + status: error.response.status, + url: error.config?.url, + data: error.response.data, + }); + } else if (error.request) { + logger.error('Service request timeout', { + service: this.serviceName, + url: error.config?.url, + }); + } + + return Promise.reject(error); + } + ); + } + + private generateCorrelationId(): string { + return `corr_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`; + } + + async request(config: AxiosRequestConfig): Promise { + try { + const response = this.circuitBreaker + ? await this.circuitBreaker.fire(config) + : await this.client.request(config); + + return response.data; + } catch (error) { + logger.error('Service request failed', { + service: this.serviceName, + url: config.url, + error: error.message, + }); + throw error; + } + } + + async get(url: string, config?: AxiosRequestConfig): Promise { + return this.request({ ...config, method: 'GET', url }); + } + + async post(url: string, data?: any, config?: AxiosRequestConfig): Promise { + return this.request({ ...config, method: 'POST', url, data }); + } + + async put(url: string, data?: any, config?: AxiosRequestConfig): Promise { + return this.request({ ...config, method: 'PUT', url, data }); + } + + async delete(url: string, config?: AxiosRequestConfig): Promise { + return this.request({ ...config, method: 'DELETE', url }); + } +} +``` + +### Usage Example + +```typescript +// src/modules/notification/notification-client.service.ts +import { ServiceClient } from '../../core/clients/service-client'; + +const notificationClient = new ServiceClient({ + baseURL: process.env.NOTIFICATION_SERVICE_URL || 'http://notification-service:5003', + serviceName: 'notification-service', + timeout: 5000, + enableCircuitBreaker: true, +}); + +export class NotificationClientService { + async sendNotification(userId: string, message: string): Promise { + try { + await notificationClient.post('/api/v1/notifications', { + userId, + message, + }); + } catch (error) { + logger.error('Failed to send notification', { userId, error }); + throw error; + } + } +} +``` + +## gRPC Communication + +### Protocol Buffer Definitions + +```protobuf +// proto/user_service.proto +syntax = "proto3"; + +package goodgo.user.v1; + +option go_package = "github.com/goodgo/user-service/proto"; + +service UserService { + rpc GetUser(GetUserRequest) returns (GetUserResponse); + rpc CreateUser(CreateUserRequest) returns (CreateUserResponse); + rpc StreamUserUpdates(StreamUserUpdatesRequest) returns (stream UserUpdate); +} + +message GetUserRequest { + string user_id = 1; +} + +message GetUserResponse { + User user = 1; +} + +message CreateUserRequest { + string email = 1; + string name = 2; +} + +message CreateUserResponse { + User user = 1; +} + +message User { + string id = 1; + string email = 2; + string name = 3; + int64 created_at = 4; +} + +message StreamUserUpdatesRequest { + string user_id = 1; +} + +message UserUpdate { + string user_id = 1; + string action = 2; + User user = 3; + int64 timestamp = 4; +} +``` + +### gRPC Server Implementation + +```typescript +// src/modules/user/user.grpc.service.ts +import * as grpc from '@grpc/grpc-js'; +import * as protoLoader from '@grpc/proto-loader'; +import { UserService } from './user.service'; +import { logger } from '@goodgo/logger'; + +const PROTO_PATH = './proto/user_service.proto'; + +const packageDefinition = protoLoader.loadSync(PROTO_PATH, { + keepCase: true, + longs: String, + enums: String, + defaults: true, + oneofs: true, +}); + +const userProto = grpc.loadPackageDefinition(packageDefinition).goodgo?.user?.v1; + +export class UserGrpcServer { + private server: grpc.Server; + private userService: UserService; + + constructor(userService: UserService) { + this.userService = userService; + this.server = new grpc.Server(); + this.setupServices(); + } + + private setupServices(): void { + if (!userProto?.UserService) { + throw new Error('UserService proto not loaded'); + } + + this.server.addService(userProto.UserService.service, { + getUser: this.getUser.bind(this), + createUser: this.createUser.bind(this), + streamUserUpdates: this.streamUserUpdates.bind(this), + }); + } + + private async getUser( + call: grpc.ServerUnaryCall, + callback: grpc.sendUnaryData + ): Promise { + try { + const { user_id } = call.request; + const user = await this.userService.findById(user_id); + + if (!user) { + return callback({ + code: grpc.status.NOT_FOUND, + message: 'User not found', + }); + } + + callback(null, { + user: { + id: user.id, + email: user.email, + name: user.name, + created_at: user.createdAt.getTime(), + }, + }); + } catch (error) { + logger.error('gRPC getUser error', { error }); + callback({ + code: grpc.status.INTERNAL, + message: error.message, + }); + } + } + + private async createUser( + call: grpc.ServerUnaryCall, + callback: grpc.sendUnaryData + ): Promise { + try { + const { email, name } = call.request; + const user = await this.userService.create({ email, name }); + + callback(null, { + user: { + id: user.id, + email: user.email, + name: user.name, + created_at: user.createdAt.getTime(), + }, + }); + } catch (error) { + logger.error('gRPC createUser error', { error }); + callback({ + code: grpc.status.INTERNAL, + message: error.message, + }); + } + } + + private streamUserUpdates(call: grpc.ServerWritableStream): void { + const { user_id } = call.request; + + const subscription = this.userService.subscribeToUpdates(user_id, (update) => { + call.write({ + user_id: update.userId, + action: update.action, + user: update.user, + timestamp: Date.now(), + }); + }); + + call.on('cancelled', () => { + subscription.unsubscribe(); + }); + } + + start(port: number): void { + this.server.bindAsync( + `0.0.0.0:${port}`, + grpc.ServerCredentials.createInsecure(), + (error, port) => { + if (error) { + logger.error('Failed to start gRPC server', { error }); + return; + } + + this.server.start(); + logger.info('gRPC server started', { port }); + } + ); + } + + stop(): Promise { + return new Promise((resolve) => { + this.server.tryShutdown(() => { + logger.info('gRPC server stopped'); + resolve(); + }); + }); + } +} +``` + +### gRPC Client Implementation + +```typescript +// src/core/clients/grpc-client.ts +import * as grpc from '@grpc/grpc-js'; +import * as protoLoader from '@grpc/proto-loader'; +import { logger } from '@goodgo/logger'; + +export interface GrpcClientConfig { + protoPath: string; + packageName: string; + serviceName: string; + serverUrl: string; + options?: protoLoader.Options; +} + +export class GrpcClient { + private client: any; + private serviceName: string; + + constructor(config: GrpcClientConfig) { + this.serviceName = config.serviceName; + + const packageDefinition = protoLoader.loadSync(config.protoPath, { + keepCase: true, + longs: String, + enums: String, + defaults: true, + oneofs: true, + ...config.options, + }); + + const proto = grpc.loadPackageDefinition(packageDefinition); + const service = this.getNestedProperty(proto, config.packageName); + + if (!service?.[config.serviceName]) { + throw new Error(`Service ${config.serviceName} not found in proto`); + } + + this.client = new service[config.serviceName]( + config.serverUrl, + grpc.credentials.createInsecure() + ); + } + + private getNestedProperty(obj: any, path: string): any { + return path.split('.').reduce((current, prop) => current?.[prop], obj); + } + + async call( + methodName: string, + request: TRequest, + metadata?: grpc.Metadata + ): Promise { + return new Promise((resolve, reject) => { + const callMetadata = metadata || new grpc.Metadata(); + if (process.env.INTERNAL_API_KEY) { + callMetadata.add('x-service-auth', process.env.INTERNAL_API_KEY); + } + + this.client[methodName](request, callMetadata, (error: any, response: TResponse) => { + if (error) { + logger.error('gRPC call error', { + service: this.serviceName, + method: methodName, + error: error.message, + }); + reject(error); + return; + } + + resolve(response); + }); + }); + } + + createStream( + methodName: string, + request: TRequest, + metadata?: grpc.Metadata + ): grpc.ClientReadableStream { + const callMetadata = metadata || new grpc.Metadata(); + if (process.env.INTERNAL_API_KEY) { + callMetadata.add('x-service-auth', process.env.INTERNAL_API_KEY); + } + + return this.client[methodName](request, callMetadata); + } +} +``` + +### gRPC Client Usage + +```typescript +// src/modules/user/user.grpc.client.ts +import { GrpcClient } from '../../core/clients/grpc-client'; + +const userGrpcClient = new GrpcClient({ + protoPath: './proto/user_service.proto', + packageName: 'goodgo.user.v1', + serviceName: 'UserService', + serverUrl: process.env.USER_SERVICE_GRPC_URL || 'localhost:50051', +}); + +export class UserGrpcClientService { + async getUser(userId: string) { + return userGrpcClient.call('getUser', { user_id: userId }); + } + + async createUser(email: string, name: string) { + return userGrpcClient.call('createUser', { email, name }); + } + + streamUserUpdates(userId: string, callback: (update: any) => void) { + const stream = userGrpcClient.createStream('streamUserUpdates', { + user_id: userId, + }); + + stream.on('data', callback); + stream.on('error', (error) => { + logger.error('gRPC stream error', { error }); + }); + } +} +``` + +## GraphQL Communication + +### GraphQL Schema Definition + +```graphql +# schema/user.graphql +type Query { + user(id: ID!): User + users(filter: UserFilter, pagination: Pagination): UserConnection +} + +type Mutation { + createUser(input: CreateUserInput!): User! + updateUser(id: ID!, input: UpdateUserInput!): User! + deleteUser(id: ID!): Boolean! +} + +type User { + id: ID! + email: String! + name: String + createdAt: DateTime! + updatedAt: DateTime! +} + +input CreateUserInput { + email: String! + name: String +} + +input UpdateUserInput { + name: String +} + +input UserFilter { + email: String + name: String +} + +input Pagination { + skip: Int + take: Int +} + +type UserConnection { + nodes: [User!]! + total: Int! + hasNextPage: Boolean! +} + +scalar DateTime +``` + +### GraphQL Server (Apollo) + +```typescript +// src/modules/user/user.graphql.service.ts +import { ApolloServer } from '@apollo/server'; +import { expressMiddleware } from '@apollo/server/express4'; +import { UserService } from './user.service'; +import { logger } from '@goodgo/logger'; +import { readFileSync } from 'fs'; +import { resolvers } from './user.resolvers'; + +const typeDefs = readFileSync('./schema/user.graphql', 'utf-8'); + +export function createGraphQLServer(userService: UserService): ApolloServer { + return new ApolloServer({ + typeDefs, + resolvers: resolvers(userService), + plugins: [ + { + requestDidStart() { + return { + didResolveOperation(requestContext) { + logger.debug('GraphQL operation', { + operation: requestContext.operationName, + query: requestContext.request.query, + }); + }, + didEncounterErrors(requestContext) { + logger.error('GraphQL errors', { + errors: requestContext.errors, + }); + }, + }; + }, + }, + ], + }); +} + +// src/modules/user/user.resolvers.ts +export function resolvers(userService: UserService) { + return { + Query: { + user: async (_: any, { id }: { id: string }) => { + return await userService.findById(id); + }, + users: async ( + _: any, + { filter, pagination }: { filter?: any; pagination?: any } + ) => { + const result = await userService.findAll({ filter, pagination }); + return { + nodes: result.users, + total: result.total, + hasNextPage: result.users.length === (pagination?.take || 10), + }; + }, + }, + Mutation: { + createUser: async (_: any, { input }: { input: any }) => { + return await userService.create(input); + }, + updateUser: async (_: any, { id, input }: { id: string; input: any }) => { + return await userService.update(id, input); + }, + deleteUser: async (_: any, { id }: { id: string }) => { + await userService.delete(id); + return true; + }, + }, + }; +} +``` + +### GraphQL Client + +```typescript +// src/core/clients/graphql-client.ts +import { GraphQLClient } from 'graphql-request'; +import { logger } from '@goodgo/logger'; + +export interface GraphQLClientConfig { + endpoint: string; + headers?: Record; + timeout?: number; +} + +export class GraphQLServiceClient { + private client: GraphQLClient; + private endpoint: string; + + constructor(config: GraphQLClientConfig) { + this.endpoint = config.endpoint; + this.client = new GraphQLClient(config.endpoint, { + headers: { + 'Content-Type': 'application/json', + ...(process.env.INTERNAL_API_KEY && { + 'x-service-auth': process.env.INTERNAL_API_KEY, + }), + ...config.headers, + }, + timeout: config.timeout || 5000, + }); + } + + async query(query: string, variables?: any): Promise { + try { + const data = await this.client.request(query, variables); + return data; + } catch (error) { + logger.error('GraphQL query error', { + endpoint: this.endpoint, + error: error.message, + }); + throw error; + } + } + + async mutate(mutation: string, variables?: any): Promise { + try { + const data = await this.client.request(mutation, variables); + return data; + } catch (error) { + logger.error('GraphQL mutation error', { + endpoint: this.endpoint, + error: error.message, + }); + throw error; + } + } +} + +// Usage example +const userGraphQLClient = new GraphQLServiceClient({ + endpoint: process.env.USER_SERVICE_GRAPHQL_URL || 'http://user-service:5002/graphql', +}); + +const GET_USER_QUERY = ` + query GetUser($id: ID!) { + user(id: $id) { + id + email + name + createdAt + } + } +`; + +export async function getUser(userId: string) { + return userGraphQLClient.query(GET_USER_QUERY, { id: userId }); +} +``` + +## Service-to-Service Authentication + +### Internal Authentication Middleware + +```typescript +// src/middlewares/internal-auth.middleware.ts +import { Request, Response, NextFunction } from 'express'; +import { logger } from '@goodgo/logger'; + +export const internalAuthMiddleware = ( + req: Request, + res: Response, + next: NextFunction +): void => { + const serviceAuthToken = req.headers['x-service-auth']; + + if (!serviceAuthToken) { + logger.warn('Missing service auth token', { + ip: req.ip, + path: req.path, + }); + res.status(401).json({ + success: false, + error: { + code: 'INTERNAL_AUTH_REQUIRED', + message: 'Service authentication required', + }, + }); + return; + } + + if (serviceAuthToken !== process.env.INTERNAL_API_KEY) { + logger.warn('Invalid service auth token', { + ip: req.ip, + path: req.path, + }); + res.status(403).json({ + success: false, + error: { + code: 'INVALID_SERVICE_AUTH', + message: 'Invalid service authentication', + }, + }); + return; + } + + req.serviceContext = { + authenticated: true, + timestamp: new Date(), + }; + + next(); +}; +``` + +### Mutual TLS (mTLS) for gRPC + +```typescript +// src/config/grpc-tls.config.ts +import * as grpc from '@grpc/grpc-js'; +import { readFileSync } from 'fs'; + +export function createServerCredentials(): grpc.ServerCredentials { + return grpc.ServerCredentials.createSsl( + null, + [ + { + cert_chain: readFileSync(process.env.GRPC_SERVER_CERT_PATH!), + private_key: readFileSync(process.env.GRPC_SERVER_KEY_PATH!), + }, + ], + true + ); +} + +export function createClientCredentials(): grpc.ChannelCredentials { + return grpc.credentials.createSsl( + readFileSync(process.env.GRPC_CA_CERT_PATH!), + readFileSync(process.env.GRPC_CLIENT_KEY_PATH!), + readFileSync(process.env.GRPC_CLIENT_CERT_PATH!) + ); +} +``` + +## Connection Pooling + +### HTTP Connection Pool + +```typescript +// src/core/clients/connection-pool.ts +import { ServiceClient } from './service-client'; + +export class ServiceClientPool { + private pools: Map = new Map(); + + getClient(serviceName: string, config: ServiceClientConfig): ServiceClient { + if (!this.pools.has(serviceName)) { + this.pools.set(serviceName, new ServiceClient(config)); + } + return this.pools.get(serviceName)!; + } + + clearPool(serviceName: string): void { + this.pools.delete(serviceName); + } + + clearAll(): void { + this.pools.clear(); + } +} + +export const serviceClientPool = new ServiceClientPool(); +``` + +## Error Handling + +### Service Error Classes + +```typescript +// src/core/errors/service-errors.ts +export class ServiceUnavailableError extends Error { + constructor(public serviceName: string, message?: string) { + super(message || `Service ${serviceName} is unavailable`); + this.name = 'ServiceUnavailableError'; + } +} + +export class ServiceTimeoutError extends Error { + constructor(public serviceName: string, public timeout: number) { + super(`Service ${serviceName} request timed out after ${timeout}ms`); + this.name = 'ServiceTimeoutError'; + } +} + +export class ServiceError extends Error { + constructor( + public serviceName: string, + public statusCode: number, + message: string, + public details?: any + ) { + super(message); + this.name = 'ServiceError'; + } +} +``` + +## Testing Patterns + +### Mocking Service Clients + +```typescript +// src/__tests__/mocks/service-client.mock.ts +export const createMockServiceClient = () => { + return { + get: jest.fn(), + post: jest.fn(), + put: jest.fn(), + delete: jest.fn(), + request: jest.fn(), + }; +}; + +// Usage in tests +const mockUserClient = createMockServiceClient(); +mockUserClient.get.mockResolvedValue({ id: '123', email: 'test@example.com' }); +``` + +### Integration Testing + +```typescript +// src/__tests__/integration/service-communication.e2e.ts +describe('Service Communication E2E', () => { + it('should communicate with user service via HTTP', async () => { + const client = new ServiceClient({ + baseURL: process.env.USER_SERVICE_URL || 'http://localhost:5002', + serviceName: 'user-service', + }); + + const user = await client.get('/api/v1/users/123'); + expect(user).toHaveProperty('id'); + }); +}); +``` diff --git a/.cursor/skills/microservices-development-process/references/REFERENCE.md b/.cursor/skills/microservices-development-process/references/REFERENCE.md new file mode 100644 index 00000000..53b8edd4 --- /dev/null +++ b/.cursor/skills/microservices-development-process/references/REFERENCE.md @@ -0,0 +1,1212 @@ +# Microservices Development Process - Detailed Reference + +This document contains detailed phase descriptions, diagrams, verification steps, and rollback strategies for microservices development. + +## Table of Contents + +- [Process Flow Diagrams](#process-flow-diagrams) +- [Phase 1: Planning and Impact Analysis](#phase-1-planning-and-impact-analysis) +- [Phase 2: Foundation Setup](#phase-2-foundation-setup) +- [Phase 3: Core Implementation](#phase-3-core-implementation) +- [Phase 4: Integration](#phase-4-integration) +- [Phase 5: Testing](#phase-5-testing) +- [Phase 6: Documentation](#phase-6-documentation) +- [Phase 7: Cleanup and Verification](#phase-7-cleanup-and-verification) +- [Phase 8: Deployment](#phase-8-deployment) +- [Rollback Strategies](#rollback-strategies) +- [Common Pitfalls](#common-pitfalls) + +--- + +## Process Flow Diagrams + +### Main Process Flow + +```mermaid +graph TD + Start([Start: New Service Requirements]) --> Phase1[Phase 1: Planning & Impact Analysis] + Phase1 --> ImpactCheck{Impact Analysis
Complete?} + ImpactCheck -->|No| Phase1 + ImpactCheck -->|Yes| Phase2[Phase 2: Foundation Setup] + + Phase2 --> FoundationCheck{Service Starts
& Health Check Passes?} + FoundationCheck -->|No| Phase2 + FoundationCheck -->|Yes| Phase3[Phase 3: Core Implementation] + + Phase3 --> ImplementationCheck{Business Logic
Implemented?} + ImplementationCheck -->|No| Phase3 + ImplementationCheck -->|Yes| Phase4[Phase 4: Integration] + + Phase4 --> IntegrationCheck{Routes & Middleware
Working?} + IntegrationCheck -->|No| Phase4 + IntegrationCheck -->|Yes| Phase5[Phase 5: Testing] + + Phase5 --> TestCheck{Tests Pass
& Coverage Met?} + TestCheck -->|No| Phase5 + TestCheck -->|Yes| Phase6[Phase 6: Documentation] + + Phase6 --> DocCheck{Docs
Complete?} + DocCheck -->|No| Phase6 + DocCheck -->|Yes| Phase7[Phase 7: Cleanup & Verification] + + Phase7 --> VerificationCheck{All Checks
Pass?} + VerificationCheck -->|No| Phase7 + VerificationCheck -->|Yes| Phase8[Phase 8: Deployment] + + Phase8 --> DeployCheck{Staging
Deployed?} + DeployCheck -->|No| Phase8 + DeployCheck -->|Yes| Production{Deploy to
Production?} + Production -->|Yes| ProdDeploy[Production Deployment] + Production -->|No| Complete([Complete]) + ProdDeploy --> Complete + + style Phase1 fill:#e1f5ff + style Phase2 fill:#fff4e1 + style Phase3 fill:#f0e1ff + style Phase4 fill:#e1ffe1 + style Phase5 fill:#ffe1e1 + style Phase6 fill:#e1ffff + style Phase7 fill:#fff0e1 + style Phase8 fill:#ffe1f5 + style Complete fill:#d4edda +``` + +### Detailed Phase Flow + +```mermaid +graph LR + subgraph Planning["Phase 1: Planning"] + P1A[Define Scope] --> P1B[Impact Analysis] + P1B --> P1C[Dependencies Map] + P1C --> P1D[Acceptance Criteria] + end + + subgraph Foundation["Phase 2: Foundation"] + F2A[Copy Template] --> F2B[Configure Package] + F2B --> F2C[Setup Database] + F2C --> F2D[Configure Docker] + F2D --> F2E[Setup Traefik] + end + + subgraph Implementation["Phase 3: Implementation"] + I3A[DTOs] --> I3B[Repository] + I3B --> I3C[Service] + I3C --> I3D[Controller] + I3D --> I3E[Module] + end + + subgraph Integration["Phase 4: Integration"] + IN4A[Register Routes] --> IN4B[Setup Middleware] + IN4B --> IN4C[External Services] + IN4C --> IN4D[Health Checks] + end + + subgraph Testing["Phase 5: Testing"] + T5A[Unit Tests] --> T5B[Integration Tests] + T5B --> T5C[E2E Tests] + T5C --> T5D[Coverage Check] + end + + subgraph Documentation["Phase 6: Documentation"] + D6A[README] --> D6B[API Docs] + D6B --> D6C[Architecture Docs] + end + + subgraph Cleanup["Phase 7: Cleanup"] + C7A[Remove Temp Files] --> C7B[Update References] + C7B --> C7C[Verify Everything] + end + + subgraph Deployment["Phase 8: Deployment"] + DEP8A[Staging] --> DEP8B[Verification] + DEP8B --> DEP8C[Production] + end + + Planning --> Foundation + Foundation --> Implementation + Implementation --> Integration + Integration --> Testing + Testing --> Documentation + Documentation --> Cleanup + Cleanup --> Deployment + + style Planning fill:#e1f5ff + style Foundation fill:#fff4e1 + style Implementation fill:#f0e1ff + style Integration fill:#e1ffe1 + style Testing fill:#ffe1e1 + style Documentation fill:#e1ffff + style Cleanup fill:#fff0e1 + style Deployment fill:#ffe1f5 +``` + +--- + +## Phase 1: Planning and Impact Analysis + +### Scope Definition + +Define clearly before starting any implementation: + +- **Service Purpose**: What business capability does it provide? +- **API Surface**: What endpoints are needed? +- **Data Models**: What data structures are required? +- **Dependencies**: What services/packages does it depend on? +- **Breaking Changes**: Any backward compatibility concerns? + +### Impact Analysis Checklist + +Before starting implementation, identify all affected areas: + +#### Files to Create + +- [ ] Service directory: `services/service-name/` +- [ ] Prisma schema: `services/service-name/prisma/schema.prisma` +- [ ] Dockerfile: `services/service-name/Dockerfile` +- [ ] Service README: `services/service-name/README.md` +- [ ] Source files: `services/service-name/src/` +- [ ] Test files: `services/service-name/src/__tests__/` +- [ ] Configuration: `services/service-name/src/config/` + +#### Files to Update + +- [ ] Root `package.json` workspace config +- [ ] `deployments/local/docker-compose.yml` - Add service +- [ ] `infra/traefik/dynamic/routes.yml` - Add routes +- [ ] `.github/workflows/ci-*.yml` - Add CI workflow (if needed) +- [ ] Documentation: `docs/en/guides/`, `docs/vi/guides/` +- [ ] Scripts: `scripts/db/*.sh`, `scripts/dev/*.sh` (if service-specific) + +#### Infrastructure Changes + +- [ ] Database: New schema/tables +- [ ] Redis: New cache keys/patterns (if needed) +- [ ] Traefik: New routes and services +- [ ] Observability: New service metrics/traces +- [ ] Kubernetes: Deployment manifests (if deploying to K8s) + +#### Dependencies Mapping + +- [ ] External: Database, Redis, third-party APIs +- [ ] Internal: Shared packages (@goodgo/logger, @goodgo/types, etc.) +- [ ] Other Services: List dependent services + +### Acceptance Criteria for Phase 1 + +- [ ] Service purpose clearly defined +- [ ] All endpoints identified +- [ ] Data models designed +- [ ] Dependencies mapped +- [ ] Files to create/update listed +- [ ] Infrastructure changes identified + +--- + +## Phase 2: Foundation Setup + +### Service Structure Creation + +#### Template Usage + +```bash +# Copy from template +cp -r services/_template services/new-service-name +cd services/new-service-name + +# Update package.json name to @goodgo/new-service-name +# Update other package.json fields as needed +``` + +#### Required Files + +| File | Purpose | +|------|---------| +| `package.json` | Package configuration with correct name and dependencies | +| `src/config/app.config.ts` | Configuration with Zod validation | +| `.env.example` | Environment variables template | +| `prisma/schema.prisma` | Database schema | +| `Dockerfile` | Container configuration | +| `jest.config.ts` | Test configuration | +| `tsconfig.json` | TypeScript configuration | + +### Database Setup + +```bash +# Navigate to service directory +cd services/service-name + +# Create initial migration +pnpm prisma migrate dev --name init + +# Generate Prisma client +pnpm prisma generate + +# Verify database connection +pnpm prisma db push --dry-run +``` + +### Docker and Infrastructure Setup + +#### Docker Compose Integration + +Add service to `deployments/local/docker-compose.yml`: + +```yaml +services: + new-service-name: + build: + context: ../../services/new-service-name + dockerfile: Dockerfile + environment: + - DATABASE_URL=postgresql://user:pass@postgres:5432/db + - NODE_ENV=development + labels: + - "traefik.enable=true" + - "traefik.http.routers.new-service-name.rule=PathPrefix(`/api/v1/new-service-name`)" + - "traefik.http.services.new-service-name.loadbalancer.server.port=5000" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:5000/health"] + interval: 30s + timeout: 10s + retries: 3 + depends_on: + postgres: + condition: service_healthy +``` + +#### Traefik Routes + +Update `infra/traefik/dynamic/routes.yml`: + +```yaml +http: + routers: + new-service-name: + rule: "PathPrefix(`/api/v1/new-service-name`)" + service: new-service-name + middlewares: + - cors + - rate-limit + - auth + + services: + new-service-name: + loadBalancer: + servers: + - url: "http://new-service-name:5000" +``` + +### Verification Steps for Phase 2 + +```bash +# 1. Verify service starts +cd services/service-name +pnpm dev + +# 2. Check health endpoint +curl http://localhost:5000/health + +# 3. TypeScript check +pnpm typecheck + +# 4. Docker build +docker build -t service-name . + +# 5. Full stack with Docker Compose +cd deployments/local +docker-compose up new-service-name + +# 6. Verify Traefik routing +curl http://localhost/api/v1/new-service-name/health +``` + +### Acceptance Criteria for Phase 2 + +- [ ] Service directory created from template +- [ ] `package.json` configured correctly +- [ ] Environment variables defined in `.env.example` +- [ ] Prisma schema created and migration run +- [ ] Service starts: `pnpm dev` (health check passes) +- [ ] Docker build succeeds +- [ ] Service accessible via Traefik +- [ ] No TypeScript errors: `pnpm typecheck` + +--- + +## Phase 3: Core Implementation + +### Module Structure + +Each feature module follows this pattern: + +``` +modules/feature-name/ +├── feature.controller.ts # HTTP handlers +├── feature.service.ts # Business logic +├── feature.repository.ts # Data access +├── feature.dto.ts # Validation schemas (Zod) +├── feature.module.ts # Module registration +├── feature.test.ts # Unit tests +└── index.ts # Public exports +``` + +### Implementation Flow + +```mermaid +graph TD + Start[Start Implementation] --> DTOs[1. Create DTOs
Zod Validation Schemas] + DTOs --> Repo[2. Create Repository
Prisma Data Access] + Repo --> Service[3. Create Service
Business Logic] + Service --> Controller[4. Create Controller
HTTP Handlers] + Controller --> Module[5. Create Module
Wire Up Components] + Module --> Test[Manual Testing] + Test --> Pass{Tests Pass?} + Pass -->|No| Repo + Pass -->|Yes| Next[Next Feature Module] + + style DTOs fill:#e1f5ff + style Repo fill:#fff4e1 + style Service fill:#f0e1ff + style Controller fill:#e1ffe1 + style Module fill:#ffe1e1 +``` + +### Implementation Details + +#### 1. DTOs (Data Transfer Objects) + +Create Zod schemas for request/response validation: + +```typescript +// feature.dto.ts +import { z } from 'zod'; + +export const CreateFeatureSchema = z.object({ + name: z.string().min(1).max(100), + description: z.string().optional(), + isActive: z.boolean().default(true), +}); + +export const UpdateFeatureSchema = CreateFeatureSchema.partial(); + +export const FeatureResponseSchema = z.object({ + id: z.string().uuid(), + name: z.string(), + description: z.string().nullable(), + isActive: z.boolean(), + createdAt: z.date(), + updatedAt: z.date(), +}); + +export type CreateFeatureDto = z.infer; +export type UpdateFeatureDto = z.infer; +export type FeatureResponse = z.infer; +``` + +#### 2. Repository + +Prisma-based data access layer: + +```typescript +// feature.repository.ts +import { PrismaClient, Feature } from '@prisma/client'; +import { CreateFeatureDto, UpdateFeatureDto } from './feature.dto'; + +export class FeatureRepository { + constructor(private prisma: PrismaClient) {} + + async findAll(): Promise { + return this.prisma.feature.findMany(); + } + + async findById(id: string): Promise { + return this.prisma.feature.findUnique({ where: { id } }); + } + + async create(data: CreateFeatureDto): Promise { + return this.prisma.feature.create({ data }); + } + + async update(id: string, data: UpdateFeatureDto): Promise { + return this.prisma.feature.update({ where: { id }, data }); + } + + async delete(id: string): Promise { + await this.prisma.feature.delete({ where: { id } }); + } +} +``` + +#### 3. Service + +Business logic layer: + +```typescript +// feature.service.ts +import { Logger } from '@goodgo/logger'; +import { FeatureRepository } from './feature.repository'; +import { CreateFeatureDto, UpdateFeatureDto } from './feature.dto'; +import { NotFoundError } from '@goodgo/errors'; + +export class FeatureService { + private logger = new Logger('FeatureService'); + + constructor(private repository: FeatureRepository) {} + + async getAll() { + this.logger.info('Fetching all features'); + return this.repository.findAll(); + } + + async getById(id: string) { + const feature = await this.repository.findById(id); + if (!feature) { + throw new NotFoundError(`Feature with id ${id} not found`); + } + return feature; + } + + async create(data: CreateFeatureDto) { + this.logger.info('Creating feature', { name: data.name }); + return this.repository.create(data); + } + + async update(id: string, data: UpdateFeatureDto) { + await this.getById(id); // Ensure exists + return this.repository.update(id, data); + } + + async delete(id: string) { + await this.getById(id); // Ensure exists + await this.repository.delete(id); + } +} +``` + +#### 4. Controller + +HTTP request handlers: + +```typescript +// feature.controller.ts +import { Router, Request, Response } from 'express'; +import { FeatureService } from './feature.service'; +import { CreateFeatureSchema, UpdateFeatureSchema } from './feature.dto'; +import { validateBody } from '@goodgo/middleware'; + +export class FeatureController { + public router = Router(); + + constructor(private service: FeatureService) { + this.initializeRoutes(); + } + + private initializeRoutes() { + this.router.get('/', this.getAll.bind(this)); + this.router.get('/:id', this.getById.bind(this)); + this.router.post('/', validateBody(CreateFeatureSchema), this.create.bind(this)); + this.router.patch('/:id', validateBody(UpdateFeatureSchema), this.update.bind(this)); + this.router.delete('/:id', this.delete.bind(this)); + } + + async getAll(req: Request, res: Response) { + const features = await this.service.getAll(); + res.json({ success: true, data: features }); + } + + async getById(req: Request, res: Response) { + const feature = await this.service.getById(req.params.id); + res.json({ success: true, data: feature }); + } + + async create(req: Request, res: Response) { + const feature = await this.service.create(req.body); + res.status(201).json({ success: true, data: feature }); + } + + async update(req: Request, res: Response) { + const feature = await this.service.update(req.params.id, req.body); + res.json({ success: true, data: feature }); + } + + async delete(req: Request, res: Response) { + await this.service.delete(req.params.id); + res.status(204).send(); + } +} +``` + +#### 5. Module + +Wire up components: + +```typescript +// feature.module.ts +import { PrismaClient } from '@prisma/client'; +import { FeatureRepository } from './feature.repository'; +import { FeatureService } from './feature.service'; +import { FeatureController } from './feature.controller'; + +export function createFeatureModule(prisma: PrismaClient) { + const repository = new FeatureRepository(prisma); + const service = new FeatureService(repository); + const controller = new FeatureController(service); + + return { + repository, + service, + controller, + router: controller.router, + }; +} +``` + +### Acceptance Criteria for Phase 3 + +- [ ] All DTOs defined with Zod validation +- [ ] Repository methods implemented (CRUD operations) +- [ ] Service business logic implemented +- [ ] Controllers handle requests correctly +- [ ] Modules configured properly +- [ ] No TypeScript errors +- [ ] Manual API testing successful + +--- + +## Phase 4: Integration + +### Route Registration + +Update `src/routes/index.ts`: + +```typescript +import { Router } from 'express'; +import { createFeatureModule } from '../modules/feature'; +import { prisma } from '../lib/prisma'; + +export function createRoutes(): Router { + const router = Router(); + + // Create modules + const featureModule = createFeatureModule(prisma); + + // Register routes + router.use('/features', featureModule.router); + + return router; +} + +// In app.ts +app.use('/api/v1/service-name', createRoutes()); +``` + +### Middleware Setup + +Required middlewares in order: + +```typescript +// app.ts +import express from 'express'; +import { + correlationMiddleware, + loggingMiddleware, + metricsMiddleware, + corsMiddleware, + rateLimitMiddleware, + authMiddleware, + errorMiddleware, +} from '@goodgo/middleware'; + +const app = express(); + +// 1. Correlation ID (first - sets up request context) +app.use(correlationMiddleware()); + +// 2. Logging (early - logs all requests) +app.use(loggingMiddleware()); + +// 3. Metrics (early - tracks all requests) +app.use(metricsMiddleware()); + +// 4. CORS (before routes) +app.use(corsMiddleware()); + +// 5. Rate limiting (before routes) +app.use(rateLimitMiddleware()); + +// 6. Body parsing +app.use(express.json()); + +// 7. Authentication (for protected routes) +app.use('/api/v1', authMiddleware()); + +// 8. Routes +app.use('/api/v1/service-name', createRoutes()); + +// 9. Error handling (always last) +app.use(errorMiddleware()); +``` + +### External Service Integration + +#### HTTP Client Setup + +```typescript +import { HttpClient } from '@goodgo/http-client'; + +const externalService = new HttpClient({ + baseUrl: process.env.EXTERNAL_SERVICE_URL, + timeout: 5000, + retries: 3, +}); +``` + +#### Redis Caching Pattern + +```typescript +import { Redis } from 'ioredis'; + +const redis = new Redis(process.env.REDIS_URL); + +async function getCachedData(key: string, fetcher: () => Promise, ttl = 300): Promise { + const cached = await redis.get(key); + if (cached) { + return JSON.parse(cached); + } + + const data = await fetcher(); + await redis.setex(key, ttl, JSON.stringify(data)); + return data; +} +``` + +### Acceptance Criteria for Phase 4 + +- [ ] All routes registered and accessible +- [ ] Middlewares applied in correct order +- [ ] Error handling works for all scenarios +- [ ] External services integrated (if any) +- [ ] Caching implemented (if needed) +- [ ] Health check endpoint works: `/health` +- [ ] Metrics endpoint works: `/metrics` + +--- + +## Phase 5: Testing + +### Test Structure + +| Type | Location | Purpose | +|------|----------|---------| +| Unit Tests | Next to source (`*.test.ts`) | Test isolated components with mocks | +| Integration Tests | `src/__tests__/*.integration.ts` | Test component interactions | +| E2E Tests | `src/__tests__/*.e2e.ts` | Test full API workflows | + +### Test Coverage Targets + +| Component | Minimum | Recommended | +|-----------|---------|-------------| +| Overall | 70% | 80% | +| Critical Paths | 90% | 95% | +| Repositories | 80% | 90% | +| Services | 80% | 90% | +| Controllers | 70% | 80% | + +### Testing Checklist + +#### Unit Tests + +- [ ] Repository tests: All CRUD operations +- [ ] Service tests: Business logic, error handling, edge cases +- [ ] Controller tests: Request/response handling, validation +- [ ] DTO tests: Validation rules, edge cases +- [ ] Utility tests: Helper functions + +#### Integration Tests + +- [ ] Module integration: Controller -> Service -> Repository +- [ ] Database operations: Real Prisma client with test DB +- [ ] Middleware chain: Request flow through middlewares +- [ ] External service mocks: HTTP client integrations + +#### E2E Tests + +- [ ] API endpoints: Full request/response cycle +- [ ] Authentication: Protected routes, token validation +- [ ] Error scenarios: 400, 401, 403, 404, 500 responses +- [ ] Health checks: /health endpoint +- [ ] Concurrent requests: Race conditions + +### Running Tests + +```bash +# Run all tests +pnpm test + +# Run with coverage +pnpm test:coverage + +# Run specific test file +pnpm test -- feature.test.ts + +# Run E2E tests +pnpm test:e2e + +# Watch mode +pnpm test:watch +``` + +### Acceptance Criteria for Phase 5 + +- [ ] All unit tests pass: `pnpm test` +- [ ] Integration tests pass +- [ ] E2E tests pass +- [ ] Coverage meets thresholds: `pnpm test:coverage` +- [ ] No test warnings or errors +- [ ] Tests run in CI pipeline successfully + +--- + +## Phase 6: Documentation + +### Required Documentation + +#### Service README + +Required sections: +- Service overview (bilingual EN/VI) +- Features list +- Prerequisites +- Quick start guide +- Configuration reference (environment variables table) +- API endpoints overview +- Development guide +- Testing instructions + +#### API Documentation (Swagger/OpenAPI) + +```typescript +// src/docs/swagger.ts +import swaggerJSDoc from 'swagger-jsdoc'; + +const options = { + definition: { + openapi: '3.0.0', + info: { + title: 'Service Name API', + version: '1.0.0', + description: 'API documentation for Service Name', + }, + servers: [ + { url: '/api/v1/service-name' }, + ], + }, + apis: ['./src/modules/**/*.controller.ts'], +}; + +export const swaggerSpec = swaggerJSDoc(options); +``` + +#### Architecture Documentation (if complex) + +- `ARCHITECTURE.en.md` / `ARCHITECTURE.vi.md` +- System design diagrams +- Data flow descriptions +- Component interactions + +### Documentation Checklist + +- [ ] README is comprehensive and bilingual +- [ ] Swagger docs accessible: `/api-docs` +- [ ] All endpoints appear in Swagger +- [ ] Request/response examples provided +- [ ] Environment variables documented +- [ ] Error responses documented +- [ ] Architecture docs created (if needed) + +### Acceptance Criteria for Phase 6 + +- [ ] README is comprehensive and bilingual +- [ ] Swagger docs accessible: `/api-docs` +- [ ] All endpoints documented with examples +- [ ] Documentation reviewed and accurate + +--- + +## Phase 7: Cleanup and Verification + +### Verification Process Flow + +```mermaid +graph TD + Start[Start Cleanup] --> Remove[Remove Temporary Files] + Remove --> Update{Is Migration?} + Update -->|Yes| RefUpdate[Update References
grep & replace] + Update -->|No| Verify[Run Verification] + RefUpdate --> Verify + + Verify --> TypeCheck[TypeScript Check] + TypeCheck --> LintCheck[Lint Check] + LintCheck --> TestCheck[Test Check] + TestCheck --> BuildCheck[Build Check] + BuildCheck --> DockerCheck[Docker Build] + DockerCheck --> HealthCheck[Health Check] + HealthCheck --> TraefikCheck[Traefik Check] + TraefikCheck --> AllPass{All Pass?} + + AllPass -->|No| Fix[Fix Issues] + Fix --> Verify + AllPass -->|Yes| Complete[Phase Complete] + + style Remove fill:#ffe1e1 + style RefUpdate fill:#fff4e1 + style Verify fill:#e1ffe1 + style Complete fill:#d4edda +``` + +### Cleanup Checklist + +#### Remove Temporary Files + +- [ ] Remove backup directories (e.g., `service-name.backup/`) +- [ ] Remove temporary status files (e.g., `*_STATUS.md`, `*_CHECKLIST.md`) +- [ ] Remove debug/scratch files +- [ ] Clean up unused imports +- [ ] Remove commented-out code +- [ ] Remove console.log statements (use logger instead) + +#### Reference Updates (for migrations/renames) + +```bash +# Find all references +grep -r "old-service-name" . --exclude-dir=node_modules --exclude-dir=.git + +# Update checklist: +- [ ] Package names: `@goodgo/old-name` -> `@goodgo/new-name` +- [ ] Service paths: `services/old-name` -> `services/new-name` +- [ ] Docker images: `goodgo/old-name` -> `goodgo/new-name` +- [ ] Deployment names: `old-name` -> `new-name` +- [ ] Environment variables updated +- [ ] CI/CD workflows updated +- [ ] Scripts updated (if needed) +- [ ] Documentation updated (except historical context) +``` + +### Comprehensive Verification Steps + +```bash +# 1. Service starts successfully +cd services/service-name +pnpm dev & +sleep 5 +curl http://localhost:5000/health + +# 2. Type checking passes +pnpm typecheck + +# 3. Linting passes +pnpm lint + +# 4. Tests pass with coverage +pnpm test +pnpm test:coverage + +# 5. Build succeeds +pnpm build + +# 6. Docker build succeeds +docker build -t service-name . + +# 7. Docker Compose works +cd deployments/local +docker-compose up -d service-name +docker-compose ps + +# 8. Service accessible via Traefik +curl http://localhost/api/v1/service-name/health + +# 9. No broken references (if migration) +grep -r "old-reference" . --exclude-dir=node_modules --exclude-dir=.git +``` + +### Final Verification Checklist + +#### Code Quality + +- [ ] No TypeScript errors +- [ ] No linting errors +- [ ] No unused imports/variables +- [ ] Code follows project conventions +- [ ] Comments are clear (bilingual if needed) + +#### Functionality + +- [ ] Service starts without errors +- [ ] Health check works +- [ ] All API endpoints functional +- [ ] Database operations work +- [ ] External integrations work (if any) + +#### Testing + +- [ ] All tests pass +- [ ] Coverage meets requirements +- [ ] E2E tests verify full workflows + +#### Documentation + +- [ ] README is complete and accurate +- [ ] API documentation is up-to-date +- [ ] Code comments are helpful + +#### Infrastructure + +- [ ] Docker image builds +- [ ] Service works in Docker Compose +- [ ] Traefik routes configured correctly +- [ ] Environment variables documented + +#### Cleanup + +- [ ] Temporary files removed +- [ ] All references updated (if migration) +- [ ] No orphaned files + +### Acceptance Criteria for Phase 7 + +- [ ] All cleanup tasks completed +- [ ] All verification steps pass +- [ ] No broken references or links +- [ ] Code is production-ready +- [ ] Documentation is complete + +--- + +## Phase 8: Deployment + +### Staging Deployment + +#### Pre-deployment Checklist + +- [ ] All Phase 7 verification passed +- [ ] Database migrations tested: `pnpm prisma migrate deploy` +- [ ] Environment variables configured in staging +- [ ] Kubernetes manifests reviewed +- [ ] Secrets configured in Kubernetes +- [ ] Health checks configured +- [ ] Resource limits set appropriately + +#### Deployment Steps + +```bash +# 1. Build and push Docker image +docker build -t goodgo/service-name:latest . +docker push goodgo/service-name:latest + +# 2. Apply Kubernetes configs +kubectl apply -f deployments/staging/kubernetes/service-name.yaml +kubectl apply -f deployments/staging/kubernetes/service-name-configmap.yaml + +# 3. Wait for rollout +kubectl rollout status deployment/service-name -n staging + +# 4. Verify deployment +kubectl get pods -n staging -l app=service-name +kubectl logs -f deployment/service-name -n staging + +# 5. Health check +curl https://staging-api.example.com/api/v1/service-name/health + +# 6. Run smoke tests +pnpm test:smoke -- --env=staging +``` + +### Production Deployment + +#### Pre-production Checklist + +- [ ] Staging tests passed for at least 24 hours +- [ ] Database backup created +- [ ] Rollback plan documented and tested +- [ ] Monitoring dashboards ready +- [ ] Alerting configured +- [ ] On-call team notified +- [ ] Deployment window approved + +#### Production Deployment Steps + +```bash +# 1. Create database backup +kubectl exec -n production deployment/postgres -- pg_dump -U postgres db > backup.sql + +# 2. Tag release +git tag v1.0.0 +docker tag goodgo/service-name:latest goodgo/service-name:v1.0.0 +docker push goodgo/service-name:v1.0.0 + +# 3. Update Kubernetes manifest with new image tag +# Edit deployments/production/kubernetes/service-name.yaml + +# 4. Apply to production +kubectl apply -f deployments/production/kubernetes/service-name.yaml + +# 5. Monitor rollout +kubectl rollout status deployment/service-name -n production + +# 6. Verify +curl https://api.example.com/api/v1/service-name/health +``` + +### Acceptance Criteria for Phase 8 + +- [ ] Service deployed to staging successfully +- [ ] All staging tests pass +- [ ] Monitoring shows healthy metrics +- [ ] Production deployment completed (if applicable) +- [ ] Post-deployment verification successful + +--- + +## Rollback Strategies + +### When to Rollback + +Trigger a rollback when: +- Critical errors in staging/production +- Performance degradation (response time > 2x normal) +- Data integrity issues detected +- Security vulnerabilities discovered +- Error rate exceeds threshold (> 1%) + +### Quick Rollback Steps + +```bash +# 1. Identify previous working version +kubectl rollout history deployment/service-name -n staging + +# 2. Rollback to previous version +kubectl rollout undo deployment/service-name -n staging + +# 3. Verify rollback +kubectl rollout status deployment/service-name -n staging + +# 4. Check health +curl https://staging-api.example.com/api/v1/service-name/health +``` + +### Database Rollback + +If schema changes were made: + +```bash +# 1. Identify the migration to revert to +pnpm prisma migrate status + +# 2. Restore from backup (if data changes) +kubectl exec -n staging deployment/postgres -- psql -U postgres db < backup.sql + +# 3. Reset migrations (development only) +pnpm prisma migrate reset + +# 4. Or revert specific migration +pnpm prisma migrate resolve --rolled-back +``` + +### Rollback Verification + +After rollback, verify: +- [ ] Service health check passes +- [ ] All endpoints responding +- [ ] No error spikes in logs +- [ ] Database connectivity restored +- [ ] External service integrations working + +--- + +## Common Pitfalls + +### 1. Skipping Impact Analysis + +**Problem**: Missing updates in scripts, configs, or documentation. + +**Solution**: Always complete the impact analysis checklist before coding: +```bash +# Find all files that might reference the service +grep -r "service-name" . --exclude-dir=node_modules --exclude-dir=.git +``` + +### 2. No Phase Verification + +**Problem**: Accumulated issues that are hard to debug later. + +**Solution**: Complete phase checklist before moving on: +```bash +pnpm typecheck && pnpm lint && pnpm test +``` + +### 3. Deferring Cleanup + +**Problem**: Technical debt accumulates, temporary files forgotten. + +**Solution**: Clean up as you go, not at the end: +```bash +# Regular cleanup +rm -rf *.backup/ *_STATUS.md *_CHECKLIST.md +``` + +### 4. Incomplete Testing + +**Problem**: Missing edge cases and error scenarios in production. + +**Solution**: Write tests alongside implementation: +- Test happy path AND error cases +- Test boundary conditions +- Test concurrent access + +### 5. Poor Documentation + +**Problem**: Difficult maintenance, onboarding issues. + +**Solution**: Document as you implement: +- Add JSDoc comments to functions +- Update README with new features +- Add Swagger annotations to endpoints + +### 6. No Rollback Plan + +**Problem**: Difficult recovery from deployment failures. + +**Solution**: Always prepare before deployment: +- Database backup +- Previous image tag recorded +- Rollback commands documented and tested + +### 7. Hardcoded Configuration + +**Problem**: Different behavior in different environments. + +**Solution**: Use environment variables with Zod validation: +```typescript +const config = z.object({ + DATABASE_URL: z.string().url(), + PORT: z.coerce.number().default(5000), +}).parse(process.env); +``` + +### 8. Missing Health Checks + +**Problem**: Unhealthy services not detected by load balancer. + +**Solution**: Implement comprehensive health checks: +```typescript +app.get('/health', async (req, res) => { + const dbHealthy = await checkDatabase(); + const redisHealthy = await checkRedis(); + + if (dbHealthy && redisHealthy) { + res.json({ status: 'healthy' }); + } else { + res.status(503).json({ status: 'unhealthy' }); + } +}); +``` diff --git a/.cursor/skills/observability-monitoring/SKILL.md b/.cursor/skills/observability-monitoring/SKILL.md index 8ddca66b..47a72333 100644 --- a/.cursor/skills/observability-monitoring/SKILL.md +++ b/.cursor/skills/observability-monitoring/SKILL.md @@ -1,7 +1,7 @@ --- name: observability-monitoring description: Observability and monitoring for GoodGo services. Use for metrics, logging, tracing, health checks, or production debugging. -dependencies: "prom-client>=15, winston>=3, @opentelemetry/sdk-node" +compatibility: "prom-client>=15, winston>=3, @opentelemetry/sdk-node" --- # Observability & Monitoring Patterns @@ -31,76 +31,41 @@ Use this skill when: - **Tracing**: OpenTelemetry + Jaeger - **APM**: DataDog or New Relic (optional) -## Structured Logging +## Key Patterns + +### Structured Logging ```typescript -// src/lib/logger.ts import winston from 'winston'; -const logFormat = winston.format.combine( - winston.format.timestamp(), - winston.format.errors({ stack: true }), - winston.format.json() -); - export const logger = winston.createLogger({ level: process.env.LOG_LEVEL || 'info', - format: logFormat, - defaultMeta: { - service: process.env.SERVICE_NAME || 'unknown', - environment: process.env.NODE_ENV || 'development' - }, - transports: [ - new winston.transports.Console({ - format: process.env.NODE_ENV === 'development' - ? winston.format.combine( - winston.format.colorize(), - winston.format.simple() - ) - : logFormat - }), - // Production: Send to log aggregation service - ...(process.env.NODE_ENV === 'production' - ? [new winston.transports.Http({ - host: 'logs.example.com', - path: '/collect', - ssl: true - })] - : []) - ] + format: winston.format.combine( + winston.format.timestamp(), + winston.format.json() + ), + defaultMeta: { service: process.env.SERVICE_NAME } }); -// Request logger middleware -export const requestLogger = (req: Request, res: Response, next: NextFunction) => { +// Request logging middleware +export const requestLogger = (req, res, next) => { const start = Date.now(); - res.on('finish', () => { - const duration = Date.now() - start; - logger.info('HTTP Request', { - method: req.method, - url: req.url, - status: res.statusCode, - duration, - ip: req.ip, - userAgent: req.get('user-agent'), + method: req.method, url: req.url, + status: res.statusCode, duration: Date.now() - start, correlationId: req.headers['x-correlation-id'] }); }); - next(); }; ``` -## Metrics Collection +### Metrics Collection ```typescript -// src/lib/metrics.ts -import { Registry, Counter, Histogram, Gauge } from 'prom-client'; +import { Counter, Histogram, Gauge } from 'prom-client'; -export const register = new Registry(); - -// HTTP metrics export const httpRequestDuration = new Histogram({ name: 'http_request_duration_seconds', help: 'Duration of HTTP requests in seconds', @@ -113,104 +78,22 @@ export const httpRequestTotal = new Counter({ help: 'Total number of HTTP requests', labelNames: ['method', 'route', 'status'] }); - -// Business metrics -export const userRegistrations = new Counter({ - name: 'user_registrations_total', - help: 'Total number of user registrations', - labelNames: ['type'] -}); - -export const activeUsers = new Gauge({ - name: 'active_users', - help: 'Number of active users', - labelNames: ['status'] -}); - -// Register metrics -register.registerMetric(httpRequestDuration); -register.registerMetric(httpRequestTotal); -register.registerMetric(userRegistrations); -register.registerMetric(activeUsers); - -// Metrics middleware -export const metricsMiddleware = (req: Request, res: Response, next: NextFunction) => { - const start = Date.now(); - - res.on('finish', () => { - const duration = (Date.now() - start) / 1000; - const route = req.route?.path || req.path; - - httpRequestDuration - .labels(req.method, route, res.statusCode.toString()) - .observe(duration); - - httpRequestTotal - .labels(req.method, route, res.statusCode.toString()) - .inc(); - }); - - next(); -}; - -// Metrics endpoint -export const metricsHandler = async (req: Request, res: Response) => { - res.set('Content-Type', register.contentType); - res.end(await register.metrics()); -}; ``` -## Distributed Tracing +### Distributed Tracing ```typescript -// src/lib/tracing.ts import { NodeSDK } from '@opentelemetry/sdk-node'; -import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node'; -import { Resource } from '@opentelemetry/resources'; -import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions'; -import { JaegerExporter } from '@opentelemetry/exporter-jaeger'; - -export const initTracing = () => { - const jaegerExporter = new JaegerExporter({ - endpoint: process.env.JAEGER_ENDPOINT || 'http://localhost:14268/api/traces', - }); - - const sdk = new NodeSDK({ - resource: new Resource({ - [SemanticResourceAttributes.SERVICE_NAME]: process.env.SERVICE_NAME || 'unknown', - [SemanticResourceAttributes.SERVICE_VERSION]: process.env.SERVICE_VERSION || '1.0.0', - }), - traceExporter: jaegerExporter, - instrumentations: [getNodeAutoInstrumentations()] - }); - - sdk.start(); - - process.on('SIGTERM', () => { - sdk.shutdown() - .then(() => console.log('Tracing terminated')) - .catch((error) => console.log('Error terminating tracing', error)) - .finally(() => process.exit(0)); - }); -}; - -// Custom span creation import { trace, SpanStatusCode } from '@opentelemetry/api'; export const tracedOperation = async (name: string, fn: Function) => { - const tracer = trace.getTracer('application'); - const span = tracer.startSpan(name); - + const span = trace.getTracer('app').startSpan(name); try { const result = await fn(); span.setStatus({ code: SpanStatusCode.OK }); return result; } catch (error) { - span.setStatus({ - code: SpanStatusCode.ERROR, - message: error.message - }); - span.recordException(error); + span.setStatus({ code: SpanStatusCode.ERROR, message: error.message }); throw error; } finally { span.end(); @@ -218,323 +101,58 @@ export const tracedOperation = async (name: string, fn: Function) => { }; ``` -## Health Checks +### Health Checks ```typescript -// src/modules/health/health.controller.ts -export class HealthController { - constructor( - private prisma: PrismaClient, - private redis: Redis - ) {} +// Liveness - is the service running? +app.get('/health/live', (req, res) => res.json({ status: 'ok' })); - // Liveness probe - is the service running? - async liveness(req: Request, res: Response) { - res.json({ - status: 'ok', - timestamp: new Date().toISOString() - }); - } - - // Readiness probe - is the service ready for traffic? - async readiness(req: Request, res: Response) { - const checks = await this.runHealthChecks(); - const isHealthy = Object.values(checks).every(check => check.status === 'healthy'); - - res.status(isHealthy ? 200 : 503).json({ - status: isHealthy ? 'ready' : 'not ready', - checks, - timestamp: new Date().toISOString() - }); - } - - // Detailed health check - async health(req: Request, res: Response) { - const checks = await this.runHealthChecks(); - const isHealthy = Object.values(checks).every(check => check.status === 'healthy'); - - res.status(isHealthy ? 200 : 503).json({ - status: isHealthy ? 'healthy' : 'unhealthy', - version: process.env.SERVICE_VERSION || '1.0.0', - uptime: process.uptime(), - checks, - timestamp: new Date().toISOString() - }); - } - - private async runHealthChecks() { - const checks: Record = {}; - - // Database check - try { - const start = Date.now(); - await this.prisma.$queryRaw`SELECT 1`; - checks.database = { - status: 'healthy', - responseTime: Date.now() - start - }; - } catch (error) { - checks.database = { - status: 'unhealthy', - error: error.message - }; - } - - // Redis check - try { - const start = Date.now(); - await this.redis.ping(); - checks.redis = { - status: 'healthy', - responseTime: Date.now() - start - }; - } catch (error) { - checks.redis = { - status: 'unhealthy', - error: error.message - }; - } - - // Memory check - const memUsage = process.memoryUsage(); - checks.memory = { - status: memUsage.heapUsed < 500 * 1024 * 1024 ? 'healthy' : 'warning', - heapUsed: Math.round(memUsage.heapUsed / 1024 / 1024), - heapTotal: Math.round(memUsage.heapTotal / 1024 / 1024), - rss: Math.round(memUsage.rss / 1024 / 1024) - }; - - return checks; - } -} -``` - -## Error Tracking - -```typescript -// src/lib/error-tracking.ts -import * as Sentry from '@sentry/node'; - -export const initErrorTracking = () => { - if (process.env.SENTRY_DSN) { - Sentry.init({ - dsn: process.env.SENTRY_DSN, - environment: process.env.NODE_ENV, - tracesSampleRate: 0.1, - beforeSend(event, hint) { - // Filter sensitive data - if (event.request?.cookies) { - delete event.request.cookies; - } - return event; - } - }); - } -}; - -// Error handler middleware -export const errorHandler = ( - err: Error, - req: Request, - res: Response, - next: NextFunction -) => { - // Log error - logger.error('Unhandled error', { - error: err.message, - stack: err.stack, - url: req.url, - method: req.method, - correlationId: req.headers['x-correlation-id'] - }); - - // Report to Sentry - Sentry.captureException(err, { - tags: { - service: process.env.SERVICE_NAME - }, - user: { - id: req.user?.id - } - }); - - // Send response - res.status(500).json({ - success: false, - error: { - code: 'INTERNAL_ERROR', - message: process.env.NODE_ENV === 'production' - ? 'Internal server error' - : err.message - } - }); -}; -``` - -## Performance Monitoring - -```typescript -// src/middlewares/performance.middleware.ts -export const performanceMiddleware = (req: Request, res: Response, next: NextFunction) => { - const start = process.hrtime.bigint(); - - res.on('finish', () => { - const end = process.hrtime.bigint(); - const duration = Number(end - start) / 1000000; // Convert to milliseconds - - // Log slow requests - if (duration > 1000) { - logger.warn('Slow request detected', { - method: req.method, - url: req.url, - duration, - threshold: 1000 - }); - } - - // Add to response header - res.set('X-Response-Time', `${duration}ms`); - }); - - next(); -}; -``` - -## Grafana Dashboard Config - -```json -{ - "dashboard": { - "title": "Service Metrics", - "panels": [ - { - "title": "Request Rate", - "targets": [{ - "expr": "rate(http_requests_total[5m])" - }] - }, - { - "title": "Request Duration", - "targets": [{ - "expr": "histogram_quantile(0.95, http_request_duration_seconds)" - }] - }, - { - "title": "Error Rate", - "targets": [{ - "expr": "rate(http_requests_total{status=~\"5..\"}[5m])" - }] - }, - { - "title": "Active Users", - "targets": [{ - "expr": "active_users" - }] - } - ] - } -} -``` - -## Alerting Rules - -```yaml -# prometheus/alerts.yml -groups: - - name: service_alerts - rules: - - alert: HighErrorRate - expr: rate(http_requests_total{status=~"5.."}[5m]) > 0.05 - for: 5m - annotations: - summary: "High error rate detected" - description: "Error rate is above 5% for 5 minutes" - - - alert: HighLatency - expr: histogram_quantile(0.95, http_request_duration_seconds) > 1 - for: 5m - annotations: - summary: "High latency detected" - description: "95th percentile latency is above 1s" - - - alert: ServiceDown - expr: up{job="service"} == 0 - for: 1m - annotations: - summary: "Service is down" - description: "Service has been down for 1 minute" +// Readiness - is the service ready for traffic? +app.get('/health/ready', async (req, res) => { + const dbOk = await prisma.$queryRaw`SELECT 1`.then(() => true).catch(() => false); + const redisOk = await redis.ping().then(() => true).catch(() => false); + const ready = dbOk && redisOk; + res.status(ready ? 200 : 503).json({ ready, db: dbOk, redis: redisOk }); +}); ``` ## Best Practices -1. **Logging** - - Use structured logging (JSON format) - - Include correlation IDs for request tracing - - Log at appropriate levels (ERROR, WARN, INFO, DEBUG) - - Avoid logging sensitive data - -2. **Metrics** - - Use standard metric types (Counter, Gauge, Histogram) - - Keep cardinality low (avoid high-cardinality labels) - - Define SLIs and SLOs for critical paths - - Monitor business metrics, not just technical ones - -3. **Tracing** - - Add traces for critical operations - - Include relevant context in spans - - Sample appropriately to control costs - - Use distributed tracing for microservices - -4. **Alerting** - - Alert on symptoms, not causes - - Include runbook links in alerts - - Avoid alert fatigue with proper thresholds - - Test alerting rules regularly +- **Logging**: Use structured JSON format with correlation IDs +- **Metrics**: Use standard types (Counter, Gauge, Histogram) with low-cardinality labels +- **Tracing**: Add traces for critical operations, sample appropriately +- **Alerting**: Alert on symptoms, include runbook links, avoid alert fatigue +- **Security**: Never log sensitive data (passwords, tokens, PII) ## Common Mistakes 1. **Logging Sensitive Data**: Exposing PII in logs ```typescript - // ❌ BAD: Logging sensitive data - logger.info('User login', { email, password, token }); - - // ✅ GOOD: Sanitize sensitive fields - logger.info('User login', { email, userId }); + // BAD: logger.info('User login', { email, password, token }); + // GOOD: logger.info('User login', { email, userId }); ``` 2. **High Cardinality Labels**: Too many metric label values ```typescript - // ❌ BAD: userId as label (millions of values) - httpRequests.labels(method, route, userId).inc(); - - // ✅ GOOD: Low cardinality labels only - httpRequests.labels(method, route, statusCode).inc(); + // BAD: httpRequests.labels(method, route, userId).inc(); + // GOOD: httpRequests.labels(method, route, statusCode).inc(); ``` 3. **No Correlation IDs**: Can't trace requests across services ```typescript - // ❌ BAD: No correlation - logger.info('Processing request'); - - // ✅ GOOD: Include correlation ID - logger.info('Processing request', { correlationId: req.headers['x-correlation-id'] }); + // GOOD: Include correlationId in all logs + logger.info('Processing', { correlationId: req.headers['x-correlation-id'] }); ``` 4. **Wrong Log Levels**: Using ERROR for non-errors ```typescript - // ❌ BAD: Wrong level - logger.error('User not found'); // Not an error, expected case - - // ✅ GOOD: Appropriate level - logger.info('User not found', { userId }); - logger.error('Database connection failed', { error }); + // BAD: logger.error('User not found'); + // GOOD: logger.info('User not found', { userId }); ``` 5. **No Health Checks**: Service status unknown ```typescript - // ❌ BAD: No health endpoint - - // ✅ GOOD: Comprehensive health checks + // GOOD: Implement both endpoints app.get('/health/live', livenessCheck); app.get('/health/ready', readinessCheck); ``` @@ -543,7 +161,7 @@ groups: | Pillar | Tool | Endpoint | |--------|------|----------| -| **Logs** | Winston/Loki | stdout → Loki | +| **Logs** | Winston/Loki | stdout -> Loki | | **Metrics** | Prometheus | `/metrics` | | **Traces** | Jaeger | `http://jaeger:14268` | @@ -557,17 +175,9 @@ groups: **Essential Metrics:** ```typescript -// Request rate -rate(http_requests_total[5m]) - -// Error rate -rate(http_requests_total{status=~"5.."}[5m]) - -// Latency (p95) -histogram_quantile(0.95, http_request_duration_seconds_bucket) - -// Active connections -active_connections +rate(http_requests_total[5m]) // Request rate +rate(http_requests_total{status=~"5.."}[5m]) // Error rate +histogram_quantile(0.95, http_request_duration_bucket) // Latency p95 ``` **Health Check Endpoints:** @@ -577,18 +187,11 @@ active_connections | `/health/ready` | Ready for traffic? | K8s readiness probe | | `/health` | Full status | Monitoring | -**Essential Imports:** -```typescript -import { logger } from '@goodgo/logger'; -import { Counter, Histogram, Gauge } from 'prom-client'; -import { trace, SpanStatusCode } from '@opentelemetry/api'; -``` - ## Resources - [OpenTelemetry](https://opentelemetry.io/docs/) - Distributed tracing standard - [Prometheus](https://prometheus.io/docs/) - Metrics and alerting - [Grafana](https://grafana.com/docs/) - Visualization +- [Detailed Code Examples](./references/REFERENCE.md) - [Deployment Kubernetes](../deployment-kubernetes/SKILL.md) - K8s health probes - [Resilience Patterns](../resilience-patterns/SKILL.md) - Circuit breaker metrics -- [Project Rules](../project-rules/SKILL.md) - GoodGo standards \ No newline at end of file diff --git a/.cursor/skills/observability-monitoring/references/REFERENCE.md b/.cursor/skills/observability-monitoring/references/REFERENCE.md new file mode 100644 index 00000000..be47ff05 --- /dev/null +++ b/.cursor/skills/observability-monitoring/references/REFERENCE.md @@ -0,0 +1,437 @@ +# Observability & Monitoring - Detailed Reference + +This reference contains detailed code examples for observability and monitoring patterns. + +## Structured Logging + +```typescript +// src/lib/logger.ts +import winston from 'winston'; + +const logFormat = winston.format.combine( + winston.format.timestamp(), + winston.format.errors({ stack: true }), + winston.format.json() +); + +export const logger = winston.createLogger({ + level: process.env.LOG_LEVEL || 'info', + format: logFormat, + defaultMeta: { + service: process.env.SERVICE_NAME || 'unknown', + environment: process.env.NODE_ENV || 'development' + }, + transports: [ + new winston.transports.Console({ + format: process.env.NODE_ENV === 'development' + ? winston.format.combine( + winston.format.colorize(), + winston.format.simple() + ) + : logFormat + }), + // Production: Send to log aggregation service + ...(process.env.NODE_ENV === 'production' + ? [new winston.transports.Http({ + host: 'logs.example.com', + path: '/collect', + ssl: true + })] + : []) + ] +}); + +// Request logger middleware +export const requestLogger = (req: Request, res: Response, next: NextFunction) => { + const start = Date.now(); + + res.on('finish', () => { + const duration = Date.now() - start; + + logger.info('HTTP Request', { + method: req.method, + url: req.url, + status: res.statusCode, + duration, + ip: req.ip, + userAgent: req.get('user-agent'), + correlationId: req.headers['x-correlation-id'] + }); + }); + + next(); +}; +``` + +## Metrics Collection + +```typescript +// src/lib/metrics.ts +import { Registry, Counter, Histogram, Gauge } from 'prom-client'; + +export const register = new Registry(); + +// HTTP metrics +export const httpRequestDuration = new Histogram({ + name: 'http_request_duration_seconds', + help: 'Duration of HTTP requests in seconds', + labelNames: ['method', 'route', 'status'], + buckets: [0.1, 0.3, 0.5, 0.7, 1, 3, 5, 7, 10] +}); + +export const httpRequestTotal = new Counter({ + name: 'http_requests_total', + help: 'Total number of HTTP requests', + labelNames: ['method', 'route', 'status'] +}); + +// Business metrics +export const userRegistrations = new Counter({ + name: 'user_registrations_total', + help: 'Total number of user registrations', + labelNames: ['type'] +}); + +export const activeUsers = new Gauge({ + name: 'active_users', + help: 'Number of active users', + labelNames: ['status'] +}); + +// Register metrics +register.registerMetric(httpRequestDuration); +register.registerMetric(httpRequestTotal); +register.registerMetric(userRegistrations); +register.registerMetric(activeUsers); + +// Metrics middleware +export const metricsMiddleware = (req: Request, res: Response, next: NextFunction) => { + const start = Date.now(); + + res.on('finish', () => { + const duration = (Date.now() - start) / 1000; + const route = req.route?.path || req.path; + + httpRequestDuration + .labels(req.method, route, res.statusCode.toString()) + .observe(duration); + + httpRequestTotal + .labels(req.method, route, res.statusCode.toString()) + .inc(); + }); + + next(); +}; + +// Metrics endpoint +export const metricsHandler = async (req: Request, res: Response) => { + res.set('Content-Type', register.contentType); + res.end(await register.metrics()); +}; +``` + +## Distributed Tracing + +```typescript +// src/lib/tracing.ts +import { NodeSDK } from '@opentelemetry/sdk-node'; +import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node'; +import { Resource } from '@opentelemetry/resources'; +import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions'; +import { JaegerExporter } from '@opentelemetry/exporter-jaeger'; + +export const initTracing = () => { + const jaegerExporter = new JaegerExporter({ + endpoint: process.env.JAEGER_ENDPOINT || 'http://localhost:14268/api/traces', + }); + + const sdk = new NodeSDK({ + resource: new Resource({ + [SemanticResourceAttributes.SERVICE_NAME]: process.env.SERVICE_NAME || 'unknown', + [SemanticResourceAttributes.SERVICE_VERSION]: process.env.SERVICE_VERSION || '1.0.0', + }), + traceExporter: jaegerExporter, + instrumentations: [getNodeAutoInstrumentations()] + }); + + sdk.start(); + + process.on('SIGTERM', () => { + sdk.shutdown() + .then(() => console.log('Tracing terminated')) + .catch((error) => console.log('Error terminating tracing', error)) + .finally(() => process.exit(0)); + }); +}; + +// Custom span creation +import { trace, SpanStatusCode } from '@opentelemetry/api'; + +export const tracedOperation = async (name: string, fn: Function) => { + const tracer = trace.getTracer('application'); + const span = tracer.startSpan(name); + + try { + const result = await fn(); + span.setStatus({ code: SpanStatusCode.OK }); + return result; + } catch (error) { + span.setStatus({ + code: SpanStatusCode.ERROR, + message: error.message + }); + span.recordException(error); + throw error; + } finally { + span.end(); + } +}; +``` + +## Health Checks + +```typescript +// src/modules/health/health.controller.ts +export class HealthController { + constructor( + private prisma: PrismaClient, + private redis: Redis + ) {} + + // Liveness probe - is the service running? + async liveness(req: Request, res: Response) { + res.json({ + status: 'ok', + timestamp: new Date().toISOString() + }); + } + + // Readiness probe - is the service ready for traffic? + async readiness(req: Request, res: Response) { + const checks = await this.runHealthChecks(); + const isHealthy = Object.values(checks).every(check => check.status === 'healthy'); + + res.status(isHealthy ? 200 : 503).json({ + status: isHealthy ? 'ready' : 'not ready', + checks, + timestamp: new Date().toISOString() + }); + } + + // Detailed health check + async health(req: Request, res: Response) { + const checks = await this.runHealthChecks(); + const isHealthy = Object.values(checks).every(check => check.status === 'healthy'); + + res.status(isHealthy ? 200 : 503).json({ + status: isHealthy ? 'healthy' : 'unhealthy', + version: process.env.SERVICE_VERSION || '1.0.0', + uptime: process.uptime(), + checks, + timestamp: new Date().toISOString() + }); + } + + private async runHealthChecks() { + const checks: Record = {}; + + // Database check + try { + const start = Date.now(); + await this.prisma.$queryRaw`SELECT 1`; + checks.database = { + status: 'healthy', + responseTime: Date.now() - start + }; + } catch (error) { + checks.database = { + status: 'unhealthy', + error: error.message + }; + } + + // Redis check + try { + const start = Date.now(); + await this.redis.ping(); + checks.redis = { + status: 'healthy', + responseTime: Date.now() - start + }; + } catch (error) { + checks.redis = { + status: 'unhealthy', + error: error.message + }; + } + + // Memory check + const memUsage = process.memoryUsage(); + checks.memory = { + status: memUsage.heapUsed < 500 * 1024 * 1024 ? 'healthy' : 'warning', + heapUsed: Math.round(memUsage.heapUsed / 1024 / 1024), + heapTotal: Math.round(memUsage.heapTotal / 1024 / 1024), + rss: Math.round(memUsage.rss / 1024 / 1024) + }; + + return checks; + } +} +``` + +## Error Tracking + +```typescript +// src/lib/error-tracking.ts +import * as Sentry from '@sentry/node'; + +export const initErrorTracking = () => { + if (process.env.SENTRY_DSN) { + Sentry.init({ + dsn: process.env.SENTRY_DSN, + environment: process.env.NODE_ENV, + tracesSampleRate: 0.1, + beforeSend(event, hint) { + // Filter sensitive data + if (event.request?.cookies) { + delete event.request.cookies; + } + return event; + } + }); + } +}; + +// Error handler middleware +export const errorHandler = ( + err: Error, + req: Request, + res: Response, + next: NextFunction +) => { + // Log error + logger.error('Unhandled error', { + error: err.message, + stack: err.stack, + url: req.url, + method: req.method, + correlationId: req.headers['x-correlation-id'] + }); + + // Report to Sentry + Sentry.captureException(err, { + tags: { + service: process.env.SERVICE_NAME + }, + user: { + id: req.user?.id + } + }); + + // Send response + res.status(500).json({ + success: false, + error: { + code: 'INTERNAL_ERROR', + message: process.env.NODE_ENV === 'production' + ? 'Internal server error' + : err.message + } + }); +}; +``` + +## Performance Monitoring + +```typescript +// src/middlewares/performance.middleware.ts +export const performanceMiddleware = (req: Request, res: Response, next: NextFunction) => { + const start = process.hrtime.bigint(); + + res.on('finish', () => { + const end = process.hrtime.bigint(); + const duration = Number(end - start) / 1000000; // Convert to milliseconds + + // Log slow requests + if (duration > 1000) { + logger.warn('Slow request detected', { + method: req.method, + url: req.url, + duration, + threshold: 1000 + }); + } + + // Add to response header + res.set('X-Response-Time', `${duration}ms`); + }); + + next(); +}; +``` + +## Grafana Dashboard Config + +```json +{ + "dashboard": { + "title": "Service Metrics", + "panels": [ + { + "title": "Request Rate", + "targets": [{ + "expr": "rate(http_requests_total[5m])" + }] + }, + { + "title": "Request Duration", + "targets": [{ + "expr": "histogram_quantile(0.95, http_request_duration_seconds)" + }] + }, + { + "title": "Error Rate", + "targets": [{ + "expr": "rate(http_requests_total{status=~\"5..\"}[5m])" + }] + }, + { + "title": "Active Users", + "targets": [{ + "expr": "active_users" + }] + } + ] + } +} +``` + +## Alerting Rules + +```yaml +# prometheus/alerts.yml +groups: + - name: service_alerts + rules: + - alert: HighErrorRate + expr: rate(http_requests_total{status=~"5.."}[5m]) > 0.05 + for: 5m + annotations: + summary: "High error rate detected" + description: "Error rate is above 5% for 5 minutes" + + - alert: HighLatency + expr: histogram_quantile(0.95, http_request_duration_seconds) > 1 + for: 5m + annotations: + summary: "High latency detected" + description: "95th percentile latency is above 1s" + + - alert: ServiceDown + expr: up{job="service"} == 0 + for: 1m + annotations: + summary: "Service is down" + description: "Service has been down for 1 minute" +``` diff --git a/.cursor/skills/security/SKILL.md b/.cursor/skills/security/SKILL.md index 47e74dff..2ca96b5f 100644 --- a/.cursor/skills/security/SKILL.md +++ b/.cursor/skills/security/SKILL.md @@ -1,7 +1,7 @@ --- name: security description: Security patterns for GoodGo platform. Use for authentication, authorization, data protection, input validation, rate limiting, or secrets management. -dependencies: "bcrypt>=5, helmet>=7, zod>=3, jsonwebtoken" +compatibility: "bcrypt>=5, helmet>=7, zod>=3, jsonwebtoken" --- # Security Patterns for GoodGo Microservices @@ -37,658 +37,240 @@ Use this skill when: ### JWT Token Validation -```typescript -// src/middlewares/auth.middleware.ts -import { Request, Response, NextFunction } from 'express'; -import { jwtService } from '@goodgo/auth-sdk'; -import { logger } from '@goodgo/logger'; - -export const authenticate = () => { - return async (req: Request, res: Response, next: NextFunction) => { - try { - // Extract token from Authorization header or cookie - let token: string | null = null; - - const authHeader = req.headers.authorization; - if (authHeader?.startsWith('Bearer ')) { - token = authHeader.substring(7); - } else if (req.cookies?.access_token) { - token = req.cookies.access_token; - } - - if (!token) { - return res.status(401).json({ - success: false, - error: { code: 'AUTH_REQUIRED', message: 'Authentication required' } - }); - } - - // Verify token - const payload = await jwtService.verifyAccessToken(token); - - // Attach user to request - req.user = { - id: payload.sub, - userId: payload.sub, - email: payload.email, - roles: payload.roles || [], - permissions: payload.permissions || [] - }; - - next(); - } catch (error) { - logger.warn('Authentication failed', { error: error.message }); - return res.status(401).json({ - success: false, - error: { code: 'INVALID_TOKEN', message: 'Invalid or expired token' } - }); - } - }; -}; -``` - -### Role-Based Authorization +Extract tokens from Authorization header or cookies, verify with `jwtService.verifyAccessToken()`, and attach user payload to request. Return 401 for missing/invalid tokens. ```typescript -// src/middlewares/rbac.middleware.ts -export const requireRole = (...allowedRoles: string[]) => { - return (req: Request, res: Response, next: NextFunction) => { - if (!req.user) { - return res.status(401).json({ - success: false, - error: { code: 'AUTH_REQUIRED', message: 'Authentication required' } - }); - } - - const userRoles = req.user.roles || []; - const hasRole = userRoles.some(role => allowedRoles.includes(role)); - - if (!hasRole) { - logger.warn('Access denied - insufficient role', { - userId: req.user.id, - userRoles, - requiredRoles: allowedRoles - }); - - return res.status(403).json({ - success: false, - error: { code: 'FORBIDDEN', message: 'Insufficient permissions' } - }); - } - - next(); - }; -}; - -// Permission-based authorization -export const requirePermission = (resource: string, action: string) => { - return async (req: Request, res: Response, next: NextFunction) => { - if (!req.user) { - return res.status(401).json({ - success: false, - error: { code: 'AUTH_REQUIRED', message: 'Authentication required' } - }); - } - - const permission = `${resource}:${action}`; - const hasPermission = req.user.permissions?.includes(permission); - - if (!hasPermission) { - logger.warn('Access denied - insufficient permission', { - userId: req.user.id, - required: permission - }); - - return res.status(403).json({ - success: false, - error: { code: 'FORBIDDEN', message: 'Insufficient permissions' } - }); - } - - next(); - }; -}; - -// Usage in routes -router.post( - '/api/v1/users', - authenticate(), - requirePermission('users', 'create'), - userController.create -); +// Key pattern - see references/REFERENCE.md for full implementation +const authHeader = req.headers.authorization; +const token = authHeader?.startsWith('Bearer ') ? authHeader.substring(7) : req.cookies?.access_token; +const payload = await jwtService.verifyAccessToken(token); +req.user = { id: payload.sub, roles: payload.roles || [], permissions: payload.permissions || [] }; ``` -### Resource Ownership Validation +### Role-Based Authorization (RBAC) + +Use `requireRole()` middleware for role checks and `requirePermission()` for fine-grained access control with `resource:action` format. ```typescript -// Ensure users can only access their own resources -export const requireOwnership = (resourceIdParam: string = 'id') => { - return (req: Request, res: Response, next: NextFunction) => { - const resourceId = req.params[resourceIdParam]; - const userId = req.user?.id; - - if (resourceId !== userId) { - logger.warn('Access denied - resource ownership mismatch', { - userId, - resourceId - }); - - return res.status(403).json({ - success: false, - error: { code: 'FORBIDDEN', message: 'Access denied' } - }); - } - - next(); - }; -}; +// Usage pattern +router.post('/api/v1/users', authenticate(), requirePermission('users', 'create'), userController.create); +router.delete('/api/v1/admin', authenticate(), requireRole('admin', 'superadmin'), adminController.delete); ``` +### Resource Ownership + +Validate users can only access their own resources using `requireOwnership()` middleware. + ## Data Protection -### Encryption Service +### Encryption -```typescript -// src/core/security/encryption.service.ts -import crypto from 'crypto'; - -const ALGORITHM = 'aes-256-gcm'; -const IV_LENGTH = 16; -const TAG_LENGTH = 16; - -export class EncryptionService { - private getKey(): Buffer { - const secret = process.env.ENCRYPTION_KEY; - if (!secret || secret.length < 32) { - throw new Error('ENCRYPTION_KEY must be at least 32 characters'); - } - return crypto.scryptSync(secret, 'salt', 32); - } - - encrypt(text: string): string { - const key = this.getKey(); - const iv = crypto.randomBytes(IV_LENGTH); - const cipher = crypto.createCipheriv(ALGORITHM, key, iv); - - let encrypted = cipher.update(text, 'utf8', 'hex'); - encrypted += cipher.final('hex'); - const tag = cipher.getAuthTag(); - - return `${iv.toString('hex')}:${tag.toString('hex')}:${encrypted}`; - } - - decrypt(encryptedText: string): string { - const [ivHex, tagHex, encrypted] = encryptedText.split(':'); - const iv = Buffer.from(ivHex, 'hex'); - const tag = Buffer.from(tagHex, 'hex'); - - const key = this.getKey(); - const decipher = crypto.createDecipheriv(ALGORITHM, key, iv); - decipher.setAuthTag(tag); - - let decrypted = decipher.update(encrypted, 'hex', 'utf8'); - decrypted += decipher.final('utf8'); - return decrypted; - } -} - -// Usage: Encrypt PII before storing -const encryption = new EncryptionService(); -const encryptedPhone = encryption.encrypt(user.phone); -``` +- Use AES-256-GCM for encrypting PII at rest +- Store encrypted data as `iv:tag:ciphertext` format +- Require 32+ character ENCRYPTION_KEY ### Password Hashing -```typescript -// Always use bcrypt with appropriate cost factor -import bcrypt from 'bcrypt'; +- Always use bcrypt with cost factor 12 in production +- Never log passwords - sanitize before logging -const SALT_ROUNDS = 12; // Production: 12, Development: 10 +### Token Storage -export class PasswordService { - async hash(password: string): Promise { - return bcrypt.hash(password, SALT_ROUNDS); - } - - async verify(password: string, hash: string): Promise { - return bcrypt.compare(password, hash); - } - - // Never log passwords - sanitizeForLogging(data: any): any { - const sanitized = { ...data }; - if (sanitized.password) sanitized.password = '[REDACTED]'; - if (sanitized.passwordHash) sanitized.passwordHash = '[REDACTED]'; - return sanitized; - } -} -``` - -### Token Hashing - -```typescript -// Hash tokens before storing in database -import crypto from 'crypto'; - -export class TokenService { - hashToken(token: string): string { - const salt = process.env.TOKEN_SALT || 'default-salt-change-in-production'; - return crypto - .createHash('sha256') - .update(token + salt) - .digest('hex'); - } - - generateSecureToken(length: number = 32): string { - return crypto.randomBytes(length).toString('hex'); - } -} -``` +- Hash tokens with SHA-256 before database storage +- Use `crypto.randomBytes(32)` for secure token generation ## Input Validation ### Zod Schema Validation -```typescript -// Always validate inputs with Zod -import { z } from 'zod'; +Always validate inputs with Zod schemas before processing: -// DTO with validation -export const CreateUserDto = z.object({ - email: z.string().email('Invalid email format'), - password: z.string() - .min(8, 'Password must be at least 8 characters') - .regex(/[A-Z]/, 'Password must contain uppercase letter') - .regex(/[a-z]/, 'Password must contain lowercase letter') - .regex(/[0-9]/, 'Password must contain number') - .regex(/[^A-Za-z0-9]/, 'Password must contain special character'), - phone: z.string() - .regex(/^\+[1-9]\d{1,14}$/, 'Invalid phone format (E.164)') - .optional(), +```typescript +const CreateUserDto = z.object({ + email: z.string().email(), + password: z.string().min(8).regex(/[A-Z]/).regex(/[a-z]/).regex(/[0-9]/).regex(/[^A-Za-z0-9]/), + phone: z.string().regex(/^\+[1-9]\d{1,14}$/).optional(), name: z.string().min(1).max(255) }); // In controller -export class UserController { - async create(req: Request, res: Response) { - try { - const dto = CreateUserDto.parse(req.body); - const user = await this.service.create(dto); - res.status(201).json({ success: true, data: user }); - } catch (error) { - if (error instanceof z.ZodError) { - return res.status(400).json({ - success: false, - error: { - code: 'VALIDATION_ERROR', - message: 'Invalid input data', - details: error.errors - } - }); - } - throw error; - } - } -} +const dto = CreateUserDto.parse(req.body); ``` ### File Upload Validation -```typescript -// Validate file uploads -import fileType from 'file-type'; - -export class FileValidationService { - private readonly MAX_FILE_SIZE = 10 * 1024 * 1024; // 10MB - private readonly ALLOWED_TYPES = ['image/jpeg', 'image/png', 'application/pdf']; - - async validateFile(file: Express.Multer.File): Promise { - // Size check - if (file.size > this.MAX_FILE_SIZE) { - throw new HttpError(400, 'FILE_TOO_LARGE', 'File exceeds maximum size'); - } - - // Type check - if (!this.ALLOWED_TYPES.includes(file.mimetype)) { - throw new HttpError(400, 'INVALID_FILE_TYPE', 'File type not allowed'); - } - - // Content validation (prevent MIME type spoofing) - const type = await fileType.fromBuffer(file.buffer); - if (!type || !this.ALLOWED_TYPES.includes(type.mime)) { - throw new HttpError(400, 'INVALID_FILE_CONTENT', 'File content mismatch'); - } - - // TODO: Add virus scanning for production - } -} -``` +- Check file size (max 10MB default) +- Validate MIME type against whitelist +- Verify content with `file-type` library to prevent MIME spoofing ### SQL Injection Prevention -```typescript -// Always use Prisma parameterized queries (automatic) -// Never use string concatenation for queries - -// ❌ BAD - Never do this -const query = `SELECT * FROM users WHERE email = '${email}'`; - -// ✅ GOOD - Use Prisma -const user = await prisma.user.findUnique({ - where: { email } -}); - -// ✅ GOOD - For dynamic queries -const where: any = {}; -if (email) where.email = email; -if (status) where.status = status; - -const users = await prisma.user.findMany({ where }); -``` +Always use Prisma parameterized queries - never string concatenation for queries. ## Rate Limiting +Configure Redis-backed rate limiting for all endpoints: + +| Limiter Type | Window | Max Requests | Use Case | +|-------------|--------|--------------|----------| +| Standard | 15 min | 100 | General API endpoints | +| Strict | 1 hour | 10 | Sensitive operations | +| Login | 15 min | 5 | Authentication endpoints | + ```typescript -// Implement rate limiting for all endpoints -import rateLimit from 'express-rate-limit'; -import RedisStore from 'rate-limit-redis'; -import Redis from 'ioredis'; - -const redis = new Redis(process.env.REDIS_URL); - -// Standard rate limit -export const standardLimiter = rateLimit({ - store: new RedisStore({ - client: redis, - prefix: 'rl:standard:' - }), - windowMs: 15 * 60 * 1000, // 15 minutes - max: 100, // 100 requests per window - message: 'Too many requests, please try again later', - standardHeaders: true, - legacyHeaders: false -}); - -// Strict rate limit for sensitive operations -export const strictLimiter = rateLimit({ - store: new RedisStore({ - client: redis, - prefix: 'rl:strict:' - }), - windowMs: 60 * 60 * 1000, // 1 hour - max: 10, - message: 'Rate limit exceeded for this operation' -}); - -// Login-specific rate limit -export const loginLimiter = rateLimit({ - store: new RedisStore({ - client: redis, - prefix: 'rl:login:' - }), - windowMs: 15 * 60 * 1000, - max: 5, // 5 login attempts per 15 minutes - skipSuccessfulRequests: true, - message: 'Too many login attempts, please try again later' -}); - -// Usage router.post('/api/v1/auth/login', loginLimiter, authController.login); -router.post('/api/v1/users', authenticate(), strictLimiter, userController.create); -``` - -## Error Handling Security - -```typescript -// Sanitize error messages to prevent information disclosure -export class SecureErrorHandler { - handleError(error: Error, req: Request, res: Response) { - const isDev = process.env.NODE_ENV === 'development'; - const isProd = process.env.NODE_ENV === 'production'; - - // Log full error internally - logger.error('Request error', { - error: error.message, - stack: error.stack, - path: req.path, - method: req.method, - userId: req.user?.id - }); - - // Don't expose user existence - if (error.message.includes('user not found') || - error.message.includes('invalid credentials')) { - return res.status(401).json({ - success: false, - error: { - code: 'INVALID_CREDENTIALS', - message: 'Invalid email or password' - } - }); - } - - // Validation errors - safe to expose - if (error instanceof z.ZodError) { - return res.status(400).json({ - success: false, - error: { - code: 'VALIDATION_ERROR', - message: 'Invalid input data', - details: error.errors - } - }); - } - - // Generic errors for production - if (isProd) { - return res.status(500).json({ - success: false, - error: { - code: 'INTERNAL_ERROR', - message: 'An error occurred. Please try again later.' - } - }); - } - - // Detailed errors only in development - return res.status(500).json({ - success: false, - error: { - code: 'INTERNAL_ERROR', - message: error.message, - stack: isDev ? error.stack : undefined - } - }); - } -} -``` - -## Secrets Management - -```typescript -// Never hardcode secrets -// Always use environment variables with validation -import { z } from 'zod'; - -const secretsSchema = z.object({ - JWT_SECRET: z.string().min(32, 'JWT_SECRET must be at least 32 characters'), - JWT_REFRESH_SECRET: z.string().min(32), - DATABASE_URL: z.string().url(), - REDIS_URL: z.string().url().optional(), - ENCRYPTION_KEY: z.string().min(32).optional() -}); - -export const secrets = secretsSchema.parse(process.env); - -// For production, use secret management: -// - AWS Secrets Manager -// - HashiCorp Vault -// - Kubernetes Secrets -// - Azure Key Vault - -// Rotate secrets regularly (quarterly recommended) -``` - -## Audit Logging - -```typescript -// Log all security-relevant events -export class AuditService { - async logSecurityEvent( - event: string, - userId: string | null, - details: Record, - req?: Request - ) { - await this.prisma.auditLog.create({ - data: { - event, - userId, - type: 'SECURITY', - details: this.sanitizeDetails(details), - ipAddress: req?.ip || details.ipAddress, - userAgent: req?.get('user-agent'), - timestamp: new Date() - } - }); - } - - // Sanitize PII from logs - private sanitizeDetails(details: Record): Record { - const sensitive = ['password', 'token', 'secret', 'ssn', 'creditCard']; - const sanitized = { ...details }; - - for (const key of sensitive) { - if (sanitized[key]) { - sanitized[key] = '[REDACTED]'; - } - } - - return sanitized; - } -} - -// Usage -await auditService.logSecurityEvent('LOGIN_SUCCESS', user.id, { - email: user.email, - ipAddress: req.ip -}, req); - -await auditService.logSecurityEvent('PERMISSION_DENIED', user.id, { - resource: 'users', - action: 'delete', - targetId: targetUserId -}, req); ``` ## Security Headers -```typescript -// Add security headers middleware -import helmet from 'helmet'; - -app.use(helmet({ - contentSecurityPolicy: { - directives: { - defaultSrc: ["'self'"], - styleSrc: ["'self'", "'unsafe-inline'"], - scriptSrc: ["'self'"], - imgSrc: ["'self'", "data:", "https:"] - } - }, - hsts: { - maxAge: 31536000, - includeSubDomains: true, - preload: true - } -})); - -// Additional headers -app.use((req, res, next) => { - res.setHeader('X-Content-Type-Options', 'nosniff'); - res.setHeader('X-Frame-Options', 'DENY'); - res.setHeader('X-XSS-Protection', '1; mode=block'); - res.setHeader('Referrer-Policy', 'strict-origin-when-cross-origin'); - next(); -}); -``` +Use Helmet middleware with CSP, HSTS, and additional headers: +- `X-Content-Type-Options: nosniff` +- `X-Frame-Options: DENY` +- `X-XSS-Protection: 1; mode=block` +- `Referrer-Policy: strict-origin-when-cross-origin` ## CORS Configuration +- Whitelist allowed origins from environment variables +- Enable credentials for authenticated requests +- Set `maxAge: 86400` (24 hours) for preflight caching + +## Secrets Management + +- Never hardcode secrets - use environment variables +- Validate secrets with Zod schema at startup +- Use secret managers in production (AWS Secrets Manager, Vault, etc.) +- Rotate secrets quarterly + ```typescript -// Configure CORS securely -import cors from 'cors'; - -const allowedOrigins = process.env.CORS_ORIGIN?.split(',') || []; - -app.use(cors({ - origin: (origin, callback) => { - if (!origin || allowedOrigins.includes(origin)) { - callback(null, true); - } else { - callback(new Error('Not allowed by CORS')); - } - }, - credentials: true, - methods: ['GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'OPTIONS'], - allowedHeaders: ['Content-Type', 'Authorization'], - exposedHeaders: ['X-Request-ID'], - maxAge: 86400 // 24 hours -})); +const secretsSchema = z.object({ + JWT_SECRET: z.string().min(32), + DATABASE_URL: z.string().url(), + ENCRYPTION_KEY: z.string().min(32).optional() +}); ``` +## Audit Logging + +Log all security-relevant events with sanitized details: + +```typescript +await auditService.logSecurityEvent('LOGIN_SUCCESS', user.id, { email: user.email }, req); +await auditService.logSecurityEvent('PERMISSION_DENIED', user.id, { resource, action }, req); +``` + +Sanitize sensitive fields: password, token, secret, ssn, creditCard. + +## Error Handling + +- Log full errors internally +- Never expose user existence (use generic "Invalid credentials") +- Show stack traces only in development +- Return sanitized error codes in production + ## Security Testing +Write tests for: +- SQL injection prevention +- XSS attack prevention +- Authentication enforcement +- Authorization enforcement +- Rate limiting effectiveness + +## Best Practices + +- All endpoints require authentication (except public) +- Authorization checks at every protected route +- Input validation with Zod on all user input +- Rate limiting on all endpoints +- Error messages sanitized in production +- PII encrypted at rest with AES-256-GCM +- Passwords hashed with bcrypt (cost 12+) +- Tokens hashed before database storage +- HTTPS enforced (TLS 1.2+) +- Security headers via Helmet +- Audit logging for security events +- Dependencies scanned for vulnerabilities +- File uploads validated (size, type, content) + +## Common Mistakes + +### 1. Weak Password Hashing + ```typescript -// Security test patterns -describe('Security Tests', () => { - it('should prevent SQL injection', async () => { - const maliciousInput = "'; DROP TABLE users; --"; - const response = await request(app) - .get(`/api/v1/users?search=${encodeURIComponent(maliciousInput)}`) - .set('Authorization', `Bearer ${token}`); - - expect(response.status).not.toBe(500); - // Should return 400 or empty results, not crash - }); +// BAD: Low cost factor +const hash = await bcrypt.hash(password, 8); - it('should prevent XSS attacks', async () => { - const xssPayload = ''; - const response = await request(app) - .post('/api/v1/users') - .send({ email: xssPayload, password: 'test123' }); - - // Response should sanitize or reject - expect(response.body.data?.email).not.toContain(''; + const response = await request(app) + .post('/api/v1/users') + .send({ email: xssPayload, password: 'test123' }); + + // Response should sanitize or reject + expect(response.body.data?.email).not.toContain('