Migrate
This commit is contained in:
Ho Ngoc Hai
@@ -0,0 +1,509 @@
|
||||
# Outbox Pattern - Reference Examples
|
||||
|
||||
## Complete Implementation Examples
|
||||
|
||||
### 1. Complete Outbox Entity Configuration
|
||||
|
||||
```csharp
|
||||
/// <summary>
|
||||
/// EN: EF Core configuration for OutboxMessage.
|
||||
/// VI: Cấu hình EF Core cho OutboxMessage.
|
||||
/// </summary>
|
||||
public class OutboxMessageConfiguration : IEntityTypeConfiguration<OutboxMessage>
|
||||
{
|
||||
public void Configure(EntityTypeBuilder<OutboxMessage> builder)
|
||||
{
|
||||
builder.ToTable("OutboxMessages");
|
||||
|
||||
builder.HasKey(e => e.Id);
|
||||
|
||||
builder.Property(e => e.EventType)
|
||||
.HasMaxLength(500)
|
||||
.IsRequired();
|
||||
|
||||
builder.Property(e => e.Payload)
|
||||
.HasColumnType("jsonb")
|
||||
.IsRequired();
|
||||
|
||||
builder.Property(e => e.CreatedAt)
|
||||
.IsRequired();
|
||||
|
||||
builder.Property(e => e.Error)
|
||||
.HasMaxLength(2000);
|
||||
|
||||
// EN: Index for fast retrieval of unprocessed messages
|
||||
// VI: Index để truy vấn nhanh messages chưa xử lý
|
||||
builder.HasIndex(e => new { e.IsProcessed, e.CreatedAt })
|
||||
.HasFilter("\"IsProcessed\" = false")
|
||||
.HasDatabaseName("IX_OutboxMessages_Pending");
|
||||
|
||||
// EN: Index for retry tracking
|
||||
// VI: Index để theo dõi retry
|
||||
builder.HasIndex(e => e.RetryCount)
|
||||
.HasFilter("\"IsProcessed\" = false AND \"RetryCount\" >= 3")
|
||||
.HasDatabaseName("IX_OutboxMessages_FailedRetries");
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 2. Inbox Table for Idempotency
|
||||
|
||||
```csharp
|
||||
/// <summary>
|
||||
/// EN: Inbox message for tracking processed events.
|
||||
/// VI: Inbox message để theo dõi events đã xử lý.
|
||||
/// </summary>
|
||||
public class InboxMessage
|
||||
{
|
||||
public Guid Id { get; set; } // Same as incoming MessageId
|
||||
public string ConsumerType { get; set; } = default!;
|
||||
public DateTime ProcessedAt { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// EN: Inbox repository for idempotency.
|
||||
/// VI: Repository inbox cho idempotency.
|
||||
/// </summary>
|
||||
public class InboxRepository : IInboxRepository
|
||||
{
|
||||
private readonly AppDbContext _context;
|
||||
|
||||
public InboxRepository(AppDbContext context)
|
||||
{
|
||||
_context = context;
|
||||
}
|
||||
|
||||
public async Task<bool> ExistsAsync(Guid messageId, string consumerType, CancellationToken ct)
|
||||
{
|
||||
return await _context.InboxMessages
|
||||
.AnyAsync(m => m.Id == messageId && m.ConsumerType == consumerType, ct);
|
||||
}
|
||||
|
||||
public async Task MarkAsProcessedAsync(Guid messageId, string consumerType, CancellationToken ct)
|
||||
{
|
||||
_context.InboxMessages.Add(new InboxMessage
|
||||
{
|
||||
Id = messageId,
|
||||
ConsumerType = consumerType,
|
||||
ProcessedAt = DateTime.UtcNow
|
||||
});
|
||||
await _context.SaveChangesAsync(ct);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 3. Idempotent Consumer Base Class
|
||||
|
||||
```csharp
|
||||
/// <summary>
|
||||
/// EN: Base class for idempotent message consumers.
|
||||
/// VI: Lớp cơ sở cho consumer idempotent.
|
||||
/// </summary>
|
||||
public abstract class IdempotentConsumer<TMessage> : IConsumer<TMessage>
|
||||
where TMessage : class
|
||||
{
|
||||
private readonly IInboxRepository _inbox;
|
||||
private readonly ILogger _logger;
|
||||
|
||||
protected IdempotentConsumer(
|
||||
IInboxRepository inbox,
|
||||
ILogger logger)
|
||||
{
|
||||
_inbox = inbox;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public async Task Consume(ConsumeContext<TMessage> context)
|
||||
{
|
||||
var messageId = context.MessageId ?? Guid.NewGuid();
|
||||
var consumerType = GetType().Name;
|
||||
|
||||
// EN: Check if already processed
|
||||
// VI: Kiểm tra đã xử lý chưa
|
||||
if (await _inbox.ExistsAsync(messageId, consumerType, context.CancellationToken))
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Message {MessageId} already processed by {Consumer}, skipping",
|
||||
messageId, consumerType);
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await HandleAsync(context.Message, context.CancellationToken);
|
||||
|
||||
// EN: Mark as processed
|
||||
// VI: Đánh dấu đã xử lý
|
||||
await _inbox.MarkAsProcessedAsync(messageId, consumerType, context.CancellationToken);
|
||||
|
||||
_logger.LogInformation(
|
||||
"Processed message {MessageId} by {Consumer}",
|
||||
messageId, consumerType);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex,
|
||||
"Failed to process message {MessageId} by {Consumer}",
|
||||
messageId, consumerType);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
protected abstract Task HandleAsync(TMessage message, CancellationToken ct);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// EN: Example idempotent consumer.
|
||||
/// VI: Ví dụ consumer idempotent.
|
||||
/// </summary>
|
||||
public class OrderCreatedConsumer : IdempotentConsumer<OrderCreatedIntegrationEvent>
|
||||
{
|
||||
private readonly IInventoryService _inventory;
|
||||
|
||||
public OrderCreatedConsumer(
|
||||
IInboxRepository inbox,
|
||||
IInventoryService inventory,
|
||||
ILogger<OrderCreatedConsumer> logger)
|
||||
: base(inbox, logger)
|
||||
{
|
||||
_inventory = inventory;
|
||||
}
|
||||
|
||||
protected override async Task HandleAsync(
|
||||
OrderCreatedIntegrationEvent message,
|
||||
CancellationToken ct)
|
||||
{
|
||||
foreach (var item in message.Items)
|
||||
{
|
||||
await _inventory.ReserveAsync(
|
||||
item.ProductId,
|
||||
item.Quantity,
|
||||
message.OrderId,
|
||||
ct);
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 4. Enhanced Outbox Processor with Parallel Processing
|
||||
|
||||
```csharp
|
||||
/// <summary>
|
||||
/// EN: Enhanced outbox processor with parallel processing.
|
||||
/// VI: Outbox processor nâng cao với xử lý song song.
|
||||
/// </summary>
|
||||
public class EnhancedOutboxProcessor : BackgroundService
|
||||
{
|
||||
private readonly IServiceScopeFactory _scopeFactory;
|
||||
private readonly ILogger<EnhancedOutboxProcessor> _logger;
|
||||
private readonly OutboxOptions _options;
|
||||
|
||||
public EnhancedOutboxProcessor(
|
||||
IServiceScopeFactory scopeFactory,
|
||||
IOptions<OutboxOptions> options,
|
||||
ILogger<EnhancedOutboxProcessor> logger)
|
||||
{
|
||||
_scopeFactory = scopeFactory;
|
||||
_options = options.Value;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken ct)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Outbox Processor started with {Interval}s interval, batch size {BatchSize}",
|
||||
_options.PollingIntervalSeconds,
|
||||
_options.BatchSize);
|
||||
|
||||
using var timer = new PeriodicTimer(
|
||||
TimeSpan.FromSeconds(_options.PollingIntervalSeconds));
|
||||
|
||||
while (await timer.WaitForNextTickAsync(ct))
|
||||
{
|
||||
try
|
||||
{
|
||||
var processedCount = await ProcessBatchAsync(ct);
|
||||
|
||||
if (processedCount > 0)
|
||||
_logger.LogInformation("Processed {Count} outbox messages", processedCount);
|
||||
}
|
||||
catch (Exception ex) when (ex is not OperationCanceledException)
|
||||
{
|
||||
_logger.LogError(ex, "Error processing outbox batch");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<int> ProcessBatchAsync(CancellationToken ct)
|
||||
{
|
||||
using var scope = _scopeFactory.CreateScope();
|
||||
var context = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||||
var publishEndpoint = scope.ServiceProvider.GetRequiredService<IPublishEndpoint>();
|
||||
|
||||
// EN: Use SKIP LOCKED for concurrent processing
|
||||
// VI: Dùng SKIP LOCKED cho xử lý đồng thời
|
||||
var messages = await context.OutboxMessages
|
||||
.FromSqlRaw(@"
|
||||
SELECT * FROM ""OutboxMessages""
|
||||
WHERE ""IsProcessed"" = false
|
||||
AND ""RetryCount"" < @p0
|
||||
ORDER BY ""CreatedAt""
|
||||
LIMIT @p1
|
||||
FOR UPDATE SKIP LOCKED",
|
||||
_options.MaxRetries,
|
||||
_options.BatchSize)
|
||||
.ToListAsync(ct);
|
||||
|
||||
if (!messages.Any())
|
||||
return 0;
|
||||
|
||||
// EN: Process in parallel with semaphore
|
||||
// VI: Xử lý song song với semaphore
|
||||
using var semaphore = new SemaphoreSlim(_options.MaxParallelism);
|
||||
var tasks = messages.Select(async message =>
|
||||
{
|
||||
await semaphore.WaitAsync(ct);
|
||||
try
|
||||
{
|
||||
await ProcessMessageAsync(message, publishEndpoint, ct);
|
||||
}
|
||||
finally
|
||||
{
|
||||
semaphore.Release();
|
||||
}
|
||||
});
|
||||
|
||||
await Task.WhenAll(tasks);
|
||||
await context.SaveChangesAsync(ct);
|
||||
|
||||
return messages.Count;
|
||||
}
|
||||
|
||||
private async Task ProcessMessageAsync(
|
||||
OutboxMessage message,
|
||||
IPublishEndpoint publishEndpoint,
|
||||
CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
var eventType = Type.GetType(message.EventType);
|
||||
if (eventType == null)
|
||||
{
|
||||
message.Error = $"Unknown event type: {message.EventType}";
|
||||
message.IsProcessed = true; // Dead letter
|
||||
return;
|
||||
}
|
||||
|
||||
var @event = JsonSerializer.Deserialize(message.Payload, eventType);
|
||||
await publishEndpoint.Publish(@event!, eventType, ct);
|
||||
|
||||
message.IsProcessed = true;
|
||||
message.ProcessedAt = DateTime.UtcNow;
|
||||
message.Error = null;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
message.RetryCount++;
|
||||
message.Error = ex.Message.Length > 2000
|
||||
? ex.Message[..2000]
|
||||
: ex.Message;
|
||||
|
||||
_logger.LogWarning(ex,
|
||||
"Failed to publish message {Id}, retry {Retry}/{Max}",
|
||||
message.Id, message.RetryCount, _options.MaxRetries);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public class OutboxOptions
|
||||
{
|
||||
public int PollingIntervalSeconds { get; set; } = 5;
|
||||
public int BatchSize { get; set; } = 100;
|
||||
public int MaxRetries { get; set; } = 5;
|
||||
public int MaxParallelism { get; set; } = 4;
|
||||
}
|
||||
```
|
||||
|
||||
### 5. MassTransit Outbox Integration
|
||||
|
||||
```csharp
|
||||
/// <summary>
|
||||
/// EN: Configure MassTransit with built-in outbox.
|
||||
/// VI: Cấu hình MassTransit với outbox tích hợp.
|
||||
/// </summary>
|
||||
public static class MassTransitOutboxExtensions
|
||||
{
|
||||
public static IServiceCollection AddMassTransitWithOutbox(
|
||||
this IServiceCollection services,
|
||||
IConfiguration configuration)
|
||||
{
|
||||
services.AddMassTransit(x =>
|
||||
{
|
||||
x.AddConsumer<OrderCreatedConsumer>();
|
||||
x.AddConsumer<PaymentProcessedConsumer>();
|
||||
|
||||
// EN: Add Entity Framework Outbox
|
||||
// VI: Thêm Entity Framework Outbox
|
||||
x.AddEntityFrameworkOutbox<AppDbContext>(o =>
|
||||
{
|
||||
o.UsePostgres();
|
||||
o.UseBusOutbox();
|
||||
|
||||
// EN: Configure delivery service
|
||||
// VI: Cấu hình delivery service
|
||||
o.QueryDelay = TimeSpan.FromSeconds(5);
|
||||
o.QueryMessageLimit = 100;
|
||||
o.QueryTimeout = TimeSpan.FromSeconds(30);
|
||||
});
|
||||
|
||||
x.UsingRabbitMq((context, cfg) =>
|
||||
{
|
||||
cfg.Host(configuration["RabbitMQ:Host"], "/", h =>
|
||||
{
|
||||
h.Username(configuration["RabbitMQ:Username"]!);
|
||||
h.Password(configuration["RabbitMQ:Password"]!);
|
||||
});
|
||||
|
||||
cfg.ConfigureEndpoints(context);
|
||||
});
|
||||
});
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// EN: Using MassTransit outbox in handler.
|
||||
/// VI: Sử dụng MassTransit outbox trong handler.
|
||||
/// </summary>
|
||||
public class CreateOrderHandler : IRequestHandler<CreateOrderCommand, OrderResult>
|
||||
{
|
||||
private readonly AppDbContext _context;
|
||||
private readonly IPublishEndpoint _publishEndpoint;
|
||||
|
||||
public CreateOrderHandler(
|
||||
AppDbContext context,
|
||||
IPublishEndpoint publishEndpoint)
|
||||
{
|
||||
_context = context;
|
||||
_publishEndpoint = publishEndpoint;
|
||||
}
|
||||
|
||||
public async Task<OrderResult> Handle(CreateOrderCommand request, CancellationToken ct)
|
||||
{
|
||||
var order = new Order(request.UserId, request.Address);
|
||||
|
||||
_context.Orders.Add(order);
|
||||
|
||||
// EN: MassTransit automatically saves to outbox table
|
||||
// VI: MassTransit tự động lưu vào outbox table
|
||||
await _publishEndpoint.Publish(new OrderCreatedIntegrationEvent
|
||||
{
|
||||
OrderId = order.Id,
|
||||
UserId = order.UserId
|
||||
}, ct);
|
||||
|
||||
// EN: Both order and outbox message saved in same transaction
|
||||
// VI: Cả order và outbox message được lưu trong cùng transaction
|
||||
await _context.SaveChangesAsync(ct);
|
||||
|
||||
return new OrderResult(order.Id);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 6. Cleanup Job for Processed Messages
|
||||
|
||||
```csharp
|
||||
/// <summary>
|
||||
/// EN: Background job to clean up processed outbox messages.
|
||||
/// VI: Job background để dọn dẹp outbox messages đã xử lý.
|
||||
/// </summary>
|
||||
public class OutboxCleanupJob : BackgroundService
|
||||
{
|
||||
private readonly IServiceScopeFactory _scopeFactory;
|
||||
private readonly ILogger<OutboxCleanupJob> _logger;
|
||||
private readonly TimeSpan _cleanupInterval = TimeSpan.FromHours(1);
|
||||
private readonly TimeSpan _retentionPeriod = TimeSpan.FromDays(7);
|
||||
|
||||
public OutboxCleanupJob(
|
||||
IServiceScopeFactory scopeFactory,
|
||||
ILogger<OutboxCleanupJob> logger)
|
||||
{
|
||||
_scopeFactory = scopeFactory;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken ct)
|
||||
{
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
await Task.Delay(_cleanupInterval, ct);
|
||||
|
||||
try
|
||||
{
|
||||
await CleanupAsync(ct);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Error during outbox cleanup");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task CleanupAsync(CancellationToken ct)
|
||||
{
|
||||
using var scope = _scopeFactory.CreateScope();
|
||||
var context = scope.ServiceProvider.GetRequiredService<AppDbContext>();
|
||||
|
||||
var cutoffDate = DateTime.UtcNow - _retentionPeriod;
|
||||
|
||||
var deletedCount = await context.OutboxMessages
|
||||
.Where(m => m.IsProcessed && m.ProcessedAt < cutoffDate)
|
||||
.ExecuteDeleteAsync(ct);
|
||||
|
||||
if (deletedCount > 0)
|
||||
_logger.LogInformation("Cleaned up {Count} processed outbox messages", deletedCount);
|
||||
|
||||
// EN: Clean up inbox messages too
|
||||
// VI: Dọn dẹp inbox messages
|
||||
var inboxDeletedCount = await context.InboxMessages
|
||||
.Where(m => m.ProcessedAt < cutoffDate)
|
||||
.ExecuteDeleteAsync(ct);
|
||||
|
||||
if (inboxDeletedCount > 0)
|
||||
_logger.LogInformation("Cleaned up {Count} inbox messages", inboxDeletedCount);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Database Migrations
|
||||
|
||||
```sql
|
||||
-- EN: Create OutboxMessages table / VI: Tạo bảng OutboxMessages
|
||||
CREATE TABLE "OutboxMessages" (
|
||||
"Id" uuid PRIMARY KEY,
|
||||
"EventType" varchar(500) NOT NULL,
|
||||
"Payload" jsonb NOT NULL,
|
||||
"CreatedAt" timestamp with time zone NOT NULL,
|
||||
"ProcessedAt" timestamp with time zone,
|
||||
"IsProcessed" boolean NOT NULL DEFAULT false,
|
||||
"RetryCount" integer NOT NULL DEFAULT 0,
|
||||
"Error" varchar(2000)
|
||||
);
|
||||
|
||||
CREATE INDEX "IX_OutboxMessages_Pending"
|
||||
ON "OutboxMessages" ("IsProcessed", "CreatedAt")
|
||||
WHERE "IsProcessed" = false;
|
||||
|
||||
-- EN: Create InboxMessages table / VI: Tạo bảng InboxMessages
|
||||
CREATE TABLE "InboxMessages" (
|
||||
"Id" uuid NOT NULL,
|
||||
"ConsumerType" varchar(500) NOT NULL,
|
||||
"ProcessedAt" timestamp with time zone NOT NULL,
|
||||
PRIMARY KEY ("Id", "ConsumerType")
|
||||
);
|
||||
|
||||
CREATE INDEX "IX_InboxMessages_ProcessedAt"
|
||||
ON "InboxMessages" ("ProcessedAt");
|
||||
```
|
||||
Reference in New Issue
Block a user