first commit
This commit is contained in:
@@ -0,0 +1,383 @@
|
||||
package errors
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"math/rand"
|
||||
"service/pkg/logger"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// RetryConfig configures retry behavior
|
||||
type RetryConfig struct {
|
||||
MaxAttempts int
|
||||
Delay time.Duration
|
||||
MaxDelay time.Duration
|
||||
Backoff BackoffStrategy
|
||||
RetryIf func(Error) bool
|
||||
OnRetry func(attempt int, err Error)
|
||||
}
|
||||
|
||||
// BackoffStrategy defines backoff strategy
|
||||
type BackoffStrategy int
|
||||
|
||||
const (
|
||||
LinearBackoff BackoffStrategy = iota
|
||||
ExponentialBackoff
|
||||
ExponentialBackoffWithJitter
|
||||
FixedBackoff
|
||||
)
|
||||
|
||||
// DefaultRetryConfig returns default retry configuration
|
||||
func DefaultRetryConfig() *RetryConfig {
|
||||
return &RetryConfig{
|
||||
MaxAttempts: 3,
|
||||
Delay: time.Second,
|
||||
MaxDelay: 30 * time.Second,
|
||||
Backoff: ExponentialBackoff,
|
||||
RetryIf: DefaultRetryCondition,
|
||||
OnRetry: nil,
|
||||
}
|
||||
}
|
||||
|
||||
// DefaultRetryCondition determines if error is retryable
|
||||
func DefaultRetryCondition(err Error) bool {
|
||||
return IsRetryable(err.Code())
|
||||
}
|
||||
|
||||
// Retry executes function with retry logic
|
||||
func Retry(fn func() (interface{}, error), config *RetryConfig) (interface{}, error) {
|
||||
if config == nil {
|
||||
config = DefaultRetryConfig()
|
||||
}
|
||||
|
||||
var lastErr Error
|
||||
log := logger.Default()
|
||||
|
||||
for attempt := 0; attempt < config.MaxAttempts; attempt++ {
|
||||
result, err := fn()
|
||||
if err == nil {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
appErr := FromError(err)
|
||||
if appErr == nil {
|
||||
appErr = NewWithCode(ErrCodeInternalError, err.Error())
|
||||
}
|
||||
|
||||
// Check if error is retryable
|
||||
if config.RetryIf != nil && !config.RetryIf(appErr) {
|
||||
return nil, appErr
|
||||
}
|
||||
|
||||
lastErr = appErr
|
||||
|
||||
// Don't wait after last attempt
|
||||
if attempt < config.MaxAttempts-1 {
|
||||
delay := calculateDelay(attempt, config)
|
||||
|
||||
if config.OnRetry != nil {
|
||||
config.OnRetry(attempt+1, appErr)
|
||||
}
|
||||
|
||||
log.Warn(appErr.Error(),
|
||||
logger.Int("attempt", attempt+1),
|
||||
logger.Int("max_attempts", config.MaxAttempts),
|
||||
logger.String("retry_delay", delay.String()),
|
||||
logger.Bool("will_retry", attempt < config.MaxAttempts-1),
|
||||
)
|
||||
|
||||
time.Sleep(delay)
|
||||
}
|
||||
}
|
||||
|
||||
return nil, lastErr
|
||||
}
|
||||
|
||||
// RetryWithContext executes function with retry logic and context
|
||||
func RetryWithContext(ctx context.Context, fn func() (interface{}, error), config *RetryConfig) (interface{}, error) {
|
||||
if config == nil {
|
||||
config = DefaultRetryConfig()
|
||||
}
|
||||
|
||||
var lastErr Error
|
||||
log := logger.Default()
|
||||
|
||||
for attempt := 0; attempt < config.MaxAttempts; attempt++ {
|
||||
// Check context cancellation
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, Wrap(ctx.Err(), ErrCodeTimeout, "Context cancelled during retry")
|
||||
default:
|
||||
}
|
||||
|
||||
result, err := fn()
|
||||
if err == nil {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
appErr := FromError(err)
|
||||
if appErr == nil {
|
||||
appErr = NewWithCode(ErrCodeInternalError, err.Error())
|
||||
}
|
||||
|
||||
// Check if error is retryable
|
||||
if config.RetryIf != nil && !config.RetryIf(appErr) {
|
||||
return nil, appErr
|
||||
}
|
||||
|
||||
lastErr = appErr
|
||||
|
||||
// Don't wait after last attempt
|
||||
if attempt < config.MaxAttempts-1 {
|
||||
delay := calculateDelay(attempt, config)
|
||||
|
||||
if config.OnRetry != nil {
|
||||
config.OnRetry(attempt+1, appErr)
|
||||
}
|
||||
|
||||
log.Warn(appErr.Error(),
|
||||
logger.Int("attempt", attempt+1),
|
||||
logger.Int("max_attempts", config.MaxAttempts),
|
||||
logger.String("retry_delay", delay.String()),
|
||||
logger.Bool("will_retry", attempt < config.MaxAttempts-1),
|
||||
)
|
||||
|
||||
// Wait with context cancellation
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, Wrap(ctx.Err(), ErrCodeTimeout, "Context cancelled during retry delay")
|
||||
case <-time.After(delay):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil, lastErr
|
||||
}
|
||||
|
||||
// calculateDelay calculates delay based on backoff strategy
|
||||
func calculateDelay(attempt int, config *RetryConfig) time.Duration {
|
||||
var delay time.Duration
|
||||
|
||||
switch config.Backoff {
|
||||
case LinearBackoff:
|
||||
delay = time.Duration(attempt+1) * config.Delay
|
||||
case ExponentialBackoff:
|
||||
delay = config.Delay * time.Duration(math.Pow(2, float64(attempt)))
|
||||
case ExponentialBackoffWithJitter:
|
||||
delay = config.Delay * time.Duration(math.Pow(2, float64(attempt)))
|
||||
// Add jitter
|
||||
jitter := time.Duration(rand.Float64() * float64(delay) * 0.1)
|
||||
delay += jitter
|
||||
case FixedBackoff:
|
||||
delay = config.Delay
|
||||
default:
|
||||
delay = config.Delay
|
||||
}
|
||||
|
||||
// Apply max delay limit
|
||||
if delay > config.MaxDelay {
|
||||
delay = config.MaxDelay
|
||||
}
|
||||
|
||||
return delay
|
||||
}
|
||||
|
||||
// CircuitBreaker implements circuit breaker pattern
|
||||
type CircuitBreaker struct {
|
||||
name string
|
||||
maxFailures int
|
||||
resetTimeout time.Duration
|
||||
state CircuitState
|
||||
failures int
|
||||
lastFailTime time.Time
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
// CircuitState represents circuit breaker state
|
||||
type CircuitState int
|
||||
|
||||
const (
|
||||
CircuitClosed CircuitState = iota
|
||||
CircuitOpen
|
||||
CircuitHalfOpen
|
||||
)
|
||||
|
||||
// NewCircuitBreaker creates new circuit breaker
|
||||
func NewCircuitBreaker(name string, maxFailures int, resetTimeout time.Duration) *CircuitBreaker {
|
||||
return &CircuitBreaker{
|
||||
name: name,
|
||||
maxFailures: maxFailures,
|
||||
resetTimeout: resetTimeout,
|
||||
state: CircuitClosed,
|
||||
}
|
||||
}
|
||||
|
||||
// Execute executes function with circuit breaker protection
|
||||
func (cb *CircuitBreaker) Execute(fn func() (interface{}, error)) (interface{}, error) {
|
||||
cb.mu.Lock()
|
||||
defer cb.mu.Unlock()
|
||||
|
||||
log := logger.Default()
|
||||
|
||||
// Check if circuit is open
|
||||
if cb.state == CircuitOpen {
|
||||
if time.Since(cb.lastFailTime) > cb.resetTimeout {
|
||||
cb.state = CircuitHalfOpen
|
||||
log.Info("Circuit breaker transitioning to half-open", logger.String("circuit", cb.name))
|
||||
} else {
|
||||
return nil, NewWithCode(ErrCodeServiceUnavailable,
|
||||
fmt.Sprintf("Circuit breaker '%s' is open", cb.name))
|
||||
}
|
||||
}
|
||||
|
||||
// Execute function
|
||||
result, err := fn()
|
||||
if err != nil {
|
||||
cb.onFailure()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cb.onSuccess()
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// onSuccess handles successful execution
|
||||
func (cb *CircuitBreaker) onSuccess() {
|
||||
cb.failures = 0
|
||||
if cb.state == CircuitHalfOpen {
|
||||
cb.state = CircuitClosed
|
||||
log := logger.Default()
|
||||
log.Info("Circuit breaker closed", logger.String("circuit", cb.name))
|
||||
}
|
||||
}
|
||||
|
||||
// onFailure handles failed execution
|
||||
func (cb *CircuitBreaker) onFailure() {
|
||||
cb.failures++
|
||||
cb.lastFailTime = time.Now()
|
||||
|
||||
if cb.failures >= cb.maxFailures {
|
||||
cb.state = CircuitOpen
|
||||
log := logger.Default()
|
||||
log.Warn("Circuit breaker opened",
|
||||
logger.String("circuit", cb.name),
|
||||
logger.Int("failures", cb.failures),
|
||||
logger.Int("max_failures", cb.maxFailures),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// GetState returns current circuit breaker state
|
||||
func (cb *CircuitBreaker) GetState() CircuitState {
|
||||
cb.mu.RLock()
|
||||
defer cb.mu.RUnlock()
|
||||
return cb.state
|
||||
}
|
||||
|
||||
// GetFailures returns current failure count
|
||||
func (cb *CircuitBreaker) GetFailures() int {
|
||||
cb.mu.RLock()
|
||||
defer cb.mu.RUnlock()
|
||||
return cb.failures
|
||||
}
|
||||
|
||||
// Reset resets circuit breaker
|
||||
func (cb *CircuitBreaker) Reset() {
|
||||
cb.mu.Lock()
|
||||
defer cb.mu.Unlock()
|
||||
|
||||
cb.state = CircuitClosed
|
||||
cb.failures = 0
|
||||
cb.lastFailTime = time.Time{}
|
||||
|
||||
log := logger.Default()
|
||||
log.Info("Circuit breaker reset", logger.String("circuit", cb.name))
|
||||
}
|
||||
|
||||
// RecoveryManager manages error recovery strategies
|
||||
type RecoveryManager struct {
|
||||
circuitBreakers map[string]*CircuitBreaker
|
||||
retryConfigs map[string]*RetryConfig
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
// NewRecoveryManager creates new recovery manager
|
||||
func NewRecoveryManager() *RecoveryManager {
|
||||
return &RecoveryManager{
|
||||
circuitBreakers: make(map[string]*CircuitBreaker),
|
||||
retryConfigs: make(map[string]*RetryConfig),
|
||||
}
|
||||
}
|
||||
|
||||
// RegisterCircuitBreaker registers circuit breaker
|
||||
func (rm *RecoveryManager) RegisterCircuitBreaker(name string, maxFailures int, resetTimeout time.Duration) {
|
||||
rm.mu.Lock()
|
||||
defer rm.mu.Unlock()
|
||||
rm.circuitBreakers[name] = NewCircuitBreaker(name, maxFailures, resetTimeout)
|
||||
}
|
||||
|
||||
// RegisterRetryConfig registers retry configuration
|
||||
func (rm *RecoveryManager) RegisterRetryConfig(name string, config *RetryConfig) {
|
||||
rm.mu.Lock()
|
||||
defer rm.mu.Unlock()
|
||||
rm.retryConfigs[name] = config
|
||||
}
|
||||
|
||||
// ExecuteWithRecovery executes function with recovery strategies
|
||||
func (rm *RecoveryManager) ExecuteWithRecovery(operationName string, fn func() (interface{}, error)) (interface{}, error) {
|
||||
rm.mu.RLock()
|
||||
cb, hasCB := rm.circuitBreakers[operationName]
|
||||
config, hasConfig := rm.retryConfigs[operationName]
|
||||
rm.mu.RUnlock()
|
||||
|
||||
// Execute with circuit breaker if available
|
||||
if hasCB {
|
||||
return cb.Execute(func() (interface{}, error) {
|
||||
// Execute with retry if available
|
||||
if hasConfig {
|
||||
return Retry(fn, config)
|
||||
}
|
||||
return fn()
|
||||
})
|
||||
}
|
||||
|
||||
// Execute with retry only
|
||||
if hasConfig {
|
||||
return Retry(fn, config)
|
||||
}
|
||||
|
||||
// Execute normally
|
||||
return fn()
|
||||
}
|
||||
|
||||
// GetCircuitBreaker returns circuit breaker by name
|
||||
func (rm *RecoveryManager) GetCircuitBreaker(name string) (*CircuitBreaker, bool) {
|
||||
rm.mu.RLock()
|
||||
defer rm.mu.RUnlock()
|
||||
cb, exists := rm.circuitBreakers[name]
|
||||
return cb, exists
|
||||
}
|
||||
|
||||
// GetRetryConfig returns retry config by name
|
||||
func (rm *RecoveryManager) GetRetryConfig(name string) (*RetryConfig, bool) {
|
||||
rm.mu.RLock()
|
||||
defer rm.mu.RUnlock()
|
||||
config, exists := rm.retryConfigs[name]
|
||||
return config, exists
|
||||
}
|
||||
|
||||
// Global recovery manager
|
||||
var globalRecoveryManager = NewRecoveryManager()
|
||||
|
||||
// GetRecoveryManager returns global recovery manager
|
||||
func GetRecoveryManager() *RecoveryManager {
|
||||
return globalRecoveryManager
|
||||
}
|
||||
|
||||
// ExecuteWithGlobalRecovery executes with global recovery manager
|
||||
func ExecuteWithGlobalRecovery(operationName string, fn func() (interface{}, error)) (interface{}, error) {
|
||||
return globalRecoveryManager.ExecuteWithRecovery(operationName, fn)
|
||||
}
|
||||
Reference in New Issue
Block a user