pos-system/microservices/.agent/skills/dotnet-senior-tester/guidelines/resilience-test.md

Resilience Test Rules / Quy Tắc Resilience Testing

Hướng dẫn chi tiết cho việc kiểm thử khả năng phục hồi với Polly policies trong .NET Microservices.

1. Overview / Tổng Quan

Resilience testing đảm bảo hệ thống hoạt động ổn định khi gặp:

• Transient failures: Lỗi tạm thời (network timeout, 503 errors)
• Partial outages: Một phần hệ thống ngừng hoạt động
• Cascading failures: Lỗi lan truyền giữa các services

Polly Policies Cần Test

Policy	Mục đích	Scenario
Retry	Thử lại khi gặp lỗi tạm thời	500, 503, timeout
Circuit Breaker	Ngắt mạch khi lỗi liên tục	Nhiều request thất bại
Timeout	Giới hạn thời gian chờ	Slow responses
Bulkhead	Cô lập resources	Concurrent request limit
Fallback	Giá trị dự phòng	Graceful degradation

2. Mock HTTP Handler / Handler HTTP Giả Lập

2.1 Basic Mock Handler

/// <summary>
/// EN: Mock HTTP handler for injecting specific responses.
/// VI: Mock HTTP handler để inject responses cụ thể.
/// </summary>
public class MockHttpMessageHandler : HttpMessageHandler
{
    private readonly Func<HttpRequestMessage, CancellationToken, HttpResponseMessage> _handler;
    public List<HttpRequestMessage> ReceivedRequests { get; } = new();

    public MockHttpMessageHandler(
        Func<HttpRequestMessage, CancellationToken, HttpResponseMessage> handler)
    {
        _handler = handler;
    }

    protected override Task<HttpResponseMessage> SendAsync(
        HttpRequestMessage request, CancellationToken cancellationToken)
    {
        ReceivedRequests.Add(request);
        return Task.FromResult(_handler(request, cancellationToken));
    }
}

2.2 Configurable Mock Handler

/// <summary>
/// EN: Handler that returns different responses based on call count.
/// VI: Handler trả về responses khác nhau dựa trên số lần gọi.
/// </summary>
public class SequentialResponseHandler : HttpMessageHandler
{
    private readonly Queue<HttpResponseMessage> _responses;
    private int _callCount = 0;

    public int CallCount => _callCount;

    public SequentialResponseHandler(params HttpResponseMessage[] responses)
    {
        _responses = new Queue<HttpResponseMessage>(responses);
    }

    protected override Task<HttpResponseMessage> SendAsync(
        HttpRequestMessage request, CancellationToken cancellationToken)
    {
        Interlocked.Increment(ref _callCount);
        
        if (_responses.Count > 0)
            return Task.FromResult(_responses.Dequeue());
        
        // Default: return 200 OK when queue is empty
        return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK));
    }
}

2.3 Exception Throwing Handler

/// <summary>
/// EN: Handler that throws exceptions for testing error handling.
/// VI: Handler throw exceptions để test error handling.
/// </summary>
public class ExceptionThrowingHandler : HttpMessageHandler
{
    private readonly Func<int, Exception?> _exceptionFactory;
    private int _callCount = 0;

    public int CallCount => _callCount;

    public ExceptionThrowingHandler(Func<int, Exception?> exceptionFactory)
    {
        _exceptionFactory = exceptionFactory;
    }

    protected override Task<HttpResponseMessage> SendAsync(
        HttpRequestMessage request, CancellationToken cancellationToken)
    {
        _callCount++;
        
        var exception = _exceptionFactory(_callCount);
        if (exception != null)
            throw exception;

        return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK));
    }
}

// Usage: Throw exception on first 2 calls, succeed on 3rd
var handler = new ExceptionThrowingHandler(callCount =>
    callCount <= 2 ? new HttpRequestException("Network error") : null);

3. Retry Policy Testing / Test Retry Policy

3.1 Basic Retry Test

/// <summary>
/// EN: Tests for retry policy behavior.
/// VI: Kiểm thử hành vi của retry policy.
/// </summary>
public class RetryPolicyTests
{
    [Fact]
    public async Task RetryPolicy_TransientFailure_RetriesAndSucceeds()
    {
        // Arrange
        var handler = new SequentialResponseHandler(
            new HttpResponseMessage(HttpStatusCode.ServiceUnavailable),
            new HttpResponseMessage(HttpStatusCode.ServiceUnavailable),
            new HttpResponseMessage(HttpStatusCode.OK)
        );

        var retryPolicy = Policy<HttpResponseMessage>
            .Handle<HttpRequestException>()
            .OrResult(r => (int)r.StatusCode >= 500)
            .WaitAndRetryAsync(3, attempt => TimeSpan.FromMilliseconds(10));

        var httpClient = new HttpClient(handler);

        // Act
        var response = await retryPolicy.ExecuteAsync(() =>
            httpClient.GetAsync("http://test-api/orders"));

        // Assert
        response.StatusCode.Should().Be(HttpStatusCode.OK);
        handler.CallCount.Should().Be(3); // 2 retries + 1 success
    }

    [Fact]
    public async Task RetryPolicy_AllRetriesFail_ThrowsException()
    {
        // Arrange
        var handler = new SequentialResponseHandler(
            new HttpResponseMessage(HttpStatusCode.ServiceUnavailable),
            new HttpResponseMessage(HttpStatusCode.ServiceUnavailable),
            new HttpResponseMessage(HttpStatusCode.ServiceUnavailable),
            new HttpResponseMessage(HttpStatusCode.ServiceUnavailable)
        );

        var retryPolicy = Policy<HttpResponseMessage>
            .Handle<HttpRequestException>()
            .OrResult(r => (int)r.StatusCode >= 500)
            .WaitAndRetryAsync(3, _ => TimeSpan.FromMilliseconds(10));

        var httpClient = new HttpClient(handler);

        // Act
        var response = await retryPolicy.ExecuteAsync(() =>
            httpClient.GetAsync("http://test-api/orders"));

        // Assert - After all retries, returns last failed response
        response.StatusCode.Should().Be(HttpStatusCode.ServiceUnavailable);
        handler.CallCount.Should().Be(4); // 1 initial + 3 retries
    }
}

3.2 Testing Retry with Jitter

/// <summary>
/// EN: Tests for retry policy with exponential backoff and jitter.
/// VI: Kiểm thử retry policy với exponential backoff và jitter.
/// </summary>
public class RetryWithJitterTests
{
    [Fact]
    public async Task RetryPolicy_WithJitter_DelaysIncreaseExponentially()
    {
        // Arrange
        var delays = new List<TimeSpan>();
        
        var retryPolicy = Policy<HttpResponseMessage>
            .Handle<HttpRequestException>()
            .OrResult(r => (int)r.StatusCode >= 500)
            .WaitAndRetryAsync(
                retryCount: 3,
                sleepDurationProvider: (attempt, context) =>
                {
                    var delay = TimeSpan.FromSeconds(Math.Pow(2, attempt)) 
                        + TimeSpan.FromMilliseconds(Random.Shared.Next(0, 100));
                    delays.Add(delay);
                    return delay;
                });

        var handler = new SequentialResponseHandler(
            new HttpResponseMessage(HttpStatusCode.ServiceUnavailable),
            new HttpResponseMessage(HttpStatusCode.ServiceUnavailable),
            new HttpResponseMessage(HttpStatusCode.ServiceUnavailable),
            new HttpResponseMessage(HttpStatusCode.OK)
        );

        var httpClient = new HttpClient(handler);

        // Act
        await retryPolicy.ExecuteAsync(() => httpClient.GetAsync("http://test-api/orders"));

        // Assert
        delays.Should().HaveCount(3);
        delays[1].Should().BeGreaterThan(delays[0]); // Exponential increase
        delays[2].Should().BeGreaterThan(delays[1]);
    }
}

4. Circuit Breaker Testing / Test Circuit Breaker

4.1 Basic Circuit Breaker Test

/// <summary>
/// EN: Tests for circuit breaker policy behavior.
/// VI: Kiểm thử hành vi của circuit breaker policy.
/// </summary>
public class CircuitBreakerPolicyTests
{
    [Fact]
    public async Task CircuitBreaker_ConsecutiveFailures_OpensCircuit()
    {
        // Arrange
        var callCount = 0;
        var circuitBreaker = Policy
            .Handle<HttpRequestException>()
            .CircuitBreakerAsync(
                exceptionsAllowedBeforeBreaking: 2,
                durationOfBreak: TimeSpan.FromSeconds(30));

        var failingAction = new Func<Task>(() =>
        {
            callCount++;
            throw new HttpRequestException("Service unavailable");
        });

        // Act - Trigger failures to open circuit
        await Assert.ThrowsAsync<HttpRequestException>(() => 
            circuitBreaker.ExecuteAsync(failingAction));
        await Assert.ThrowsAsync<HttpRequestException>(() => 
            circuitBreaker.ExecuteAsync(failingAction));

        // Assert - Circuit should now be open
        circuitBreaker.CircuitState.Should().Be(CircuitState.Open);
        callCount.Should().Be(2);

        // Further calls should fail immediately with BrokenCircuitException
        await Assert.ThrowsAsync<BrokenCircuitException>(() =>
            circuitBreaker.ExecuteAsync(failingAction));
        callCount.Should().Be(2); // No additional calls made
    }

    [Fact]
    public async Task CircuitBreaker_Success_ResetsFailureCount()
    {
        // Arrange
        var callCount = 0;
        var shouldFail = true;

        var circuitBreaker = Policy
            .Handle<HttpRequestException>()
            .CircuitBreakerAsync(
                exceptionsAllowedBeforeBreaking: 3,
                durationOfBreak: TimeSpan.FromSeconds(30));

        var action = new Func<Task>(() =>
        {
            callCount++;
            if (shouldFail)
                throw new HttpRequestException("Failed");
            return Task.CompletedTask;
        });

        // Act
        // First failure
        await Assert.ThrowsAsync<HttpRequestException>(() => 
            circuitBreaker.ExecuteAsync(action));
        
        // Successful call
        shouldFail = false;
        await circuitBreaker.ExecuteAsync(action);
        
        // Another failure - should not open circuit (counter reset)
        shouldFail = true;
        await Assert.ThrowsAsync<HttpRequestException>(() => 
            circuitBreaker.ExecuteAsync(action));

        // Assert
        circuitBreaker.CircuitState.Should().Be(CircuitState.Closed);
    }
}

4.2 Testing Circuit Half-Open State

/// <summary>
/// EN: Tests for circuit breaker half-open state behavior.
/// VI: Kiểm thử hành vi trạng thái half-open của circuit breaker.
/// </summary>
public class CircuitBreakerHalfOpenTests
{
    [Fact]
    public async Task CircuitBreaker_AfterBreakDuration_TransitionsToHalfOpen()
    {
        // Arrange
        var breakDuration = TimeSpan.FromMilliseconds(100);
        var circuitBreaker = Policy
            .Handle<HttpRequestException>()
            .CircuitBreakerAsync(
                exceptionsAllowedBeforeBreaking: 1,
                durationOfBreak: breakDuration);

        // Act - Open the circuit
        await Assert.ThrowsAsync<HttpRequestException>(() =>
            circuitBreaker.ExecuteAsync(() => throw new HttpRequestException("Fail")));

        circuitBreaker.CircuitState.Should().Be(CircuitState.Open);

        // Wait for break duration
        await Task.Delay(breakDuration + TimeSpan.FromMilliseconds(50));

        // Assert - Circuit should now be Half-Open
        circuitBreaker.CircuitState.Should().Be(CircuitState.HalfOpen);
    }

    [Fact]
    public async Task CircuitBreaker_HalfOpenSuccess_ClosesCircuit()
    {
        // Arrange
        var breakDuration = TimeSpan.FromMilliseconds(100);
        var shouldFail = true;

        var circuitBreaker = Policy
            .Handle<HttpRequestException>()
            .CircuitBreakerAsync(
                exceptionsAllowedBeforeBreaking: 1,
                durationOfBreak: breakDuration);

        var action = new Func<Task>(() =>
        {
            if (shouldFail) throw new HttpRequestException("Fail");
            return Task.CompletedTask;
        });

        // Open the circuit
        await Assert.ThrowsAsync<HttpRequestException>(() =>
            circuitBreaker.ExecuteAsync(action));

        // Wait for half-open
        await Task.Delay(breakDuration + TimeSpan.FromMilliseconds(50));

        // Act - Success in half-open state
        shouldFail = false;
        await circuitBreaker.ExecuteAsync(action);

        // Assert
        circuitBreaker.CircuitState.Should().Be(CircuitState.Closed);
    }
}

5. Timeout Policy Testing / Test Timeout Policy

5.1 Basic Timeout Test

/// <summary>
/// EN: Tests for timeout policy behavior.
/// VI: Kiểm thử hành vi của timeout policy.
/// </summary>
public class TimeoutPolicyTests
{
    [Fact]
    public async Task TimeoutPolicy_SlowOperation_ThrowsTimeoutRejectedException()
    {
        // Arrange
        var timeoutPolicy = Policy
            .TimeoutAsync<HttpResponseMessage>(TimeSpan.FromMilliseconds(100));

        var slowOperation = new Func<CancellationToken, Task<HttpResponseMessage>>(async ct =>
        {
            await Task.Delay(TimeSpan.FromSeconds(5), ct);
            return new HttpResponseMessage(HttpStatusCode.OK);
        });

        // Act & Assert
        await Assert.ThrowsAsync<TimeoutRejectedException>(() =>
            timeoutPolicy.ExecuteAsync(slowOperation, CancellationToken.None));
    }

    [Fact]
    public async Task TimeoutPolicy_FastOperation_Succeeds()
    {
        // Arrange
        var timeoutPolicy = Policy
            .TimeoutAsync<HttpResponseMessage>(TimeSpan.FromSeconds(5));

        var fastOperation = new Func<CancellationToken, Task<HttpResponseMessage>>(async ct =>
        {
            await Task.Delay(TimeSpan.FromMilliseconds(50), ct);
            return new HttpResponseMessage(HttpStatusCode.OK);
        });

        // Act
        var response = await timeoutPolicy.ExecuteAsync(fastOperation, CancellationToken.None);

        // Assert
        response.StatusCode.Should().Be(HttpStatusCode.OK);
    }

    [Fact]
    public async Task TimeoutPolicy_OptimisticTimeout_CancelsToken()
    {
        // Arrange
        var tokenWasCancelled = false;
        var timeoutPolicy = Policy
            .TimeoutAsync(TimeSpan.FromMilliseconds(100), TimeoutStrategy.Optimistic);

        var operation = new Func<CancellationToken, Task>(async ct =>
        {
            try
            {
                await Task.Delay(TimeSpan.FromSeconds(5), ct);
            }
            catch (OperationCanceledException)
            {
                tokenWasCancelled = true;
                throw;
            }
        });

        // Act
        try
        {
            await timeoutPolicy.ExecuteAsync(operation, CancellationToken.None);
        }
        catch (TimeoutRejectedException)
        {
            // Expected
        }

        // Assert
        tokenWasCancelled.Should().BeTrue();
    }
}

6. Fallback Policy Testing / Test Fallback Policy

6.1 Basic Fallback Test

/// <summary>
/// EN: Tests for fallback policy behavior.
/// VI: Kiểm thử hành vi của fallback policy.
/// </summary>
public class FallbackPolicyTests
{
    [Fact]
    public async Task FallbackPolicy_OnException_ReturnsFallbackValue()
    {
        // Arrange
        var fallbackOrder = new OrderDto { Id = Guid.Empty, Status = "Fallback" };

        var fallbackPolicy = Policy<OrderDto?>
            .Handle<HttpRequestException>()
            .FallbackAsync(fallbackOrder);

        var failingOperation = new Func<Task<OrderDto?>>(() =>
            throw new HttpRequestException("Service unavailable"));

        // Act
        var result = await fallbackPolicy.ExecuteAsync(failingOperation);

        // Assert
        result.Should().BeEquivalentTo(fallbackOrder);
    }

    [Fact]
    public async Task FallbackPolicy_WithFallbackAction_ExecutesAction()
    {
        // Arrange
        var fallbackWasCalled = false;

        var fallbackPolicy = Policy<OrderDto?>
            .Handle<HttpRequestException>()
            .FallbackAsync(
                fallbackAction: async (ctx, ct) =>
                {
                    fallbackWasCalled = true;
                    return await Task.FromResult<OrderDto?>(new OrderDto { Status = "Cached" });
                });

        // Act
        var result = await fallbackPolicy.ExecuteAsync(() =>
            throw new HttpRequestException("Failed"));

        // Assert
        fallbackWasCalled.Should().BeTrue();
        result!.Status.Should().Be("Cached");
    }
}

7. Combined Policies Testing / Test Policies Kết Hợp

7.1 PolicyWrap Testing

/// <summary>
/// EN: Tests for combined policies (PolicyWrap).
/// VI: Kiểm thử các policies kết hợp (PolicyWrap).
/// </summary>
public class PolicyWrapTests
{
    [Fact]
    public async Task PolicyWrap_CombinedPolicies_AppliesInCorrectOrder()
    {
        // Arrange
        var callCount = 0;
        var executionLog = new List<string>();

        // Outer policy: Circuit Breaker
        var circuitBreaker = Policy
            .Handle<HttpRequestException>()
            .CircuitBreakerAsync(
                exceptionsAllowedBeforeBreaking: 5,
                durationOfBreak: TimeSpan.FromSeconds(30),
                onBreak: (_, _) => executionLog.Add("CircuitBreak"),
                onReset: () => executionLog.Add("CircuitReset"));

        // Inner policy: Retry
        var retry = Policy
            .Handle<HttpRequestException>()
            .RetryAsync(3, onRetry: (_, retryCount) =>
            {
                executionLog.Add($"Retry-{retryCount}");
            });

        // Wrap: Retry inside CircuitBreaker
        var policyWrap = Policy.WrapAsync(circuitBreaker, retry);

        var handler = new SequentialResponseHandler(
            CreateFailure(), CreateFailure(), CreateFailure(),
            new HttpResponseMessage(HttpStatusCode.OK)
        );

        var httpClient = new HttpClient(handler);

        var operation = new Func<Task<HttpResponseMessage>>(async () =>
        {
            callCount++;
            var response = await httpClient.GetAsync("http://test/api");
            if (!response.IsSuccessStatusCode)
                throw new HttpRequestException($"Status: {response.StatusCode}");
            return response;
        });

        // Act
        var result = await policyWrap.ExecuteAsync(operation);

        // Assert
        result.StatusCode.Should().Be(HttpStatusCode.OK);
        executionLog.Should().Contain("Retry-1");
        executionLog.Should().Contain("Retry-2");
        executionLog.Should().Contain("Retry-3");
    }

    private static HttpResponseMessage CreateFailure() =>
        new(HttpStatusCode.ServiceUnavailable);
}

8. Best Practices / Thực Hành Tốt

Test Time Sensitivity

/// <summary>
/// EN: Use short timeouts in tests to avoid slow test suites.
/// VI: Dùng timeout ngắn trong tests để tránh test suite chậm.
/// </summary>
public class TimeoutBestPracticeTests
{
    // ❌ BAD: Long timeouts slow down tests
    private static readonly TimeSpan BadTimeout = TimeSpan.FromSeconds(30);

    // ✅ GOOD: Short timeouts for fast feedback
    private static readonly TimeSpan GoodTimeout = TimeSpan.FromMilliseconds(100);
}

Policy Factory for Testing

/// <summary>
/// EN: Factory to create policies with testable parameters.
/// VI: Factory tạo policies với parameters có thể test.
/// </summary>
public class ResiliencePolicyFactory
{
    private readonly ILogger _logger;

    public ResiliencePolicyFactory(ILogger logger)
    {
        _logger = logger;
    }

    public IAsyncPolicy<HttpResponseMessage> CreateRetryPolicy(
        int retryCount = 3,
        TimeSpan? delay = null)
    {
        return Policy<HttpResponseMessage>
            .Handle<HttpRequestException>()
            .OrResult(r => (int)r.StatusCode >= 500)
            .WaitAndRetryAsync(
                retryCount,
                attempt => delay ?? TimeSpan.FromMilliseconds(100 * Math.Pow(2, attempt)),
                onRetry: (outcome, timespan, retryAttempt, context) =>
                {
                    _logger.LogWarning(
                        "Retry {RetryAttempt} after {Delay}ms due to {Reason}",
                        retryAttempt, timespan.TotalMilliseconds,
                        outcome.Exception?.Message ?? outcome.Result.StatusCode.ToString());
                });
    }
}

Verifying Retry Delays

/// <summary>
/// EN: Custom policy with observable delays for testing.
/// VI: Custom policy với delays có thể quan sát để test.
/// </summary>
public class ObservableRetryPolicy
{
    public List<TimeSpan> ObservedDelays { get; } = new();

    public IAsyncPolicy<HttpResponseMessage> CreatePolicy(int retryCount)
    {
        return Policy<HttpResponseMessage>
            .Handle<HttpRequestException>()
            .OrResult(r => !r.IsSuccessStatusCode)
            .WaitAndRetryAsync(
                retryCount,
                sleepDurationProvider: attempt =>
                {
                    var delay = TimeSpan.FromMilliseconds(100 * attempt);
                    ObservedDelays.Add(delay);
                    return delay;
                });
    }
}

9. Common Pitfalls / Lỗi Thường Gặp

❌ Testing Against Real Services

// ❌ BAD: Flaky tests due to real service availability
var client = new HttpClient { BaseAddress = new Uri("https://real-api.com") };
await policy.ExecuteAsync(() => client.GetAsync("/orders"));

// ✅ GOOD: Mock handler for deterministic behavior
var mockHandler = new MockHttpMessageHandler(...)
var client = new HttpClient(mockHandler);
await policy.ExecuteAsync(() => client.GetAsync("/orders"));

❌ Not Testing Circuit State Transitions

// ❌ BAD: Only test open state
circuitBreaker.CircuitState.Should().Be(CircuitState.Open);

// ✅ GOOD: Test all state transitions
circuitBreaker.CircuitState.Should().Be(CircuitState.Closed);
// ... trigger failures
circuitBreaker.CircuitState.Should().Be(CircuitState.Open);
// ... wait for break duration
circuitBreaker.CircuitState.Should().Be(CircuitState.HalfOpen);
// ... success
circuitBreaker.CircuitState.Should().Be(CircuitState.Closed);

❌ Ignoring Policy Order in Wrap

// ❌ BAD: Wrong order - retry outside circuit breaker
var wrong = Policy.WrapAsync(retry, circuitBreaker);

// ✅ GOOD: Retry should be inside circuit breaker
// Circuit breaker tracks failures INCLUDING retried calls
var correct = Policy.WrapAsync(circuitBreaker, retry);