Khi tích hợp AI API vào production system, SLA (Service Level Agreement) không chỉ là con số trên giấy — đó là cam kết về uptime, latency và khả năng phục hồi. Trong bài viết này, tôi sẽ chia sẻ kinh nghiệm thực chiến khi triển khai hệ thống AI API với HolySheep AI — nền tảng cung cấp SLA 99.9% uptime với chi phí tối ưu (chỉ từ $0.42/MTok với DeepSeek V3.2).
Tại Sao SLA Quan Trọng Trong AI API Integration
Trong quá trình vận hành hệ thống AI tại production, tôi đã gặp nhiều trường hợp downtime không lường trước gây ra cascade failure. HolyShehe AI cung cấp:
- 99.9% SLA — tương đương downtime tối đa 8.76 giờ/năm
- Latency trung bình <50ms — đảm bảo response time nhanh
- Geographic redundancy — backup infrastructure đa vùng
- Tỷ giá ¥1 = $1 — tiết kiệm 85%+ chi phí so với provider khác
Kiến Trúc Production-Grade Với HolySheep AI
1. Client SDK Với Built-in Retry Logic
"""
HolySheep AI Production Client với Exponential Backoff
Author: HolySheep AI Technical Team
"""
import time
import asyncio
import logging
from typing import Optional, Dict, Any
from dataclasses import dataclass, field
from enum import Enum
import aiohttp
from aiohttp import ClientTimeout
logger = logging.getLogger(__name__)
class RetryStrategy(Enum):
EXPONENTIAL = "exponential"
LINEAR = "linear"
FIBONACCI = "fibonacci"
@dataclass
class HolySheepConfig:
"""Cấu hình HolySheep AI Client"""
api_key: str
base_url: str = "https://api.holysheep.ai/v1"
max_retries: int = 5
base_delay: float = 1.0
max_delay: float = 60.0
timeout: int = 30
retry_on_status: tuple = (429, 500, 502, 503, 504)
retry_strategy: RetryStrategy = RetryStrategy.EXPONENTIAL
circuit_breaker_threshold: int = 5
circuit_breaker_timeout: float = 60.0
@dataclass
class RequestMetrics:
"""Metrics cho monitoring"""
total_requests: int = 0
successful_requests: int = 0
failed_requests: int = 0
total_latency_ms: float = 0.0
retry_count: int = 0
last_error: Optional[str] = None
class CircuitBreaker:
"""Circuit Breaker Pattern Implementation"""
def __init__(self, threshold: int, timeout: float):
self.threshold = threshold
self.timeout = timeout
self.failure_count = 0
self.last_failure_time: Optional[float] = None
self.state = "CLOSED" # CLOSED, OPEN, HALF_OPEN
def record_success(self):
self.failure_count = 0
self.state = "CLOSED"
def record_failure(self):
self.failure_count += 1
self.last_failure_time = time.time()
if self.failure_count >= self.threshold:
self.state = "OPEN"
logger.warning(f"Circuit breaker OPENED after {self.failure_count} failures")
def can_attempt(self) -> bool:
if self.state == "CLOSED":
return True
if self.state == "OPEN":
if time.time() - self.last_failure_time >= self.timeout:
self.state = "HALF_OPEN"
logger.info("Circuit breaker transitioning to HALF_OPEN")
return True
return False
return True # HALF_OPEN allows one attempt
class HolySheepAIClient:
"""
Production-grade AI API Client với:
- Automatic retry với exponential backoff
- Circuit breaker pattern
- Request queuing và rate limiting
- Comprehensive metrics
"""
def __init__(self, config: HolySheepConfig):
self.config = config
self.metrics = RequestMetrics()
self.circuit_breaker = CircuitBreaker(
config.circuit_breaker_threshold,
config.circuit_breaker_timeout
)
self._session: Optional[aiohttp.ClientSession] = None
self._rate_limiter = asyncio.Semaphore(100) # Max concurrent requests
self._sla_start_time = time.time()
async def __aenter__(self):
timeout = ClientTimeout(total=self.config.timeout)
self._session = aiohttp.ClientSession(timeout=timeout)
return self
async def __aexit__(self, *args):
if self._session:
await self._session.close()
def _calculate_delay(self, attempt: int) -> float:
"""Tính toán delay theo retry strategy"""
if self.config.retry_strategy == RetryStrategy.EXPONENTIAL:
delay = self.config.base_delay * (2 ** attempt)
elif self.config.retry_strategy == RetryStrategy.FIBONACCI:
delay = self.config.base_delay * self._fibonacci(attempt + 1)
else:
delay = self.config.base_delay * attempt
# Add jitter để tránh thundering herd
jitter = delay * 0.1 * (hash(str(time.time())) % 10 / 10)
return min(delay + jitter, self.config.max_delay)
def _fibonacci(self, n: int) -> int:
"""Fibonacci sequence helper"""
if n <= 1:
return n
a, b = 0, 1
for _ in range(n - 1):
a, b = b, a + b
return b
async def _make_request(
self,
method: str,
endpoint: str,
data: Optional[Dict] = None,
headers: Optional[Dict] = None
) -> Dict[str, Any]:
"""Internal request method với retry logic"""
if not self.circuit_breaker.can_attempt():
raise Exception("Circuit breaker is OPEN - request blocked")
headers = headers or {}
headers["Authorization"] = f"Bearer {self.config.api_key}"
headers["Content-Type"] = "application/json"
url = f"{self.config.base_url}/{endpoint.lstrip('/')}"
async with self._rate_limiter:
for attempt in range(self.config.max_retries):
self.metrics.total_requests += 1
start_time = time.time()
try:
if method.upper() == "POST":
async with self._session.post(url, json=data, headers=headers) as response:
status = response.status
response_data = await response.json()
else:
async with self._session.get(url, params=data, headers=headers) as response:
status = response.status
response_data = await response.json()
latency = (time.time() - start_time) * 1000
self.metrics.total_latency_ms += latency
if status == 200:
self.metrics.successful_requests += 1
self.circuit_breaker.record_success()
response_data["_metrics"] = {
"latency_ms": latency,
"attempt": attempt + 1
}
return response_data
elif status in self.config.retry_on_status:
self.metrics.retry_count += 1
delay = self._calculate_delay(attempt)
logger.warning(
f"Request failed with status {status}, "
f"retrying in {delay:.2f}s (attempt {attempt + 1}/{self.config.max_retries})"
)
await asyncio.sleep(delay)
else:
self.metrics.failed_requests += 1
self.circuit_breaker.record_failure()
self.metrics.last_error = f"Status {status}: {response_data.get('error', 'Unknown')}"
raise Exception(f"Request failed: {status}")
except aiohttp.ClientError as e:
self.metrics.retry_count += 1
self.metrics.last_error = str(e)
delay = self._calculate_delay(attempt)
logger.error(f"Network error: {e}, retrying in {delay:.2f}s")
await asyncio.sleep(delay)
self.metrics.failed_requests += 1
self.circuit_breaker.record_failure()
raise Exception(f"Max retries ({self.config.max_retries}) exceeded")
async def chat_completion(
self,
messages: list,
model: str = "gpt-4",
**kwargs
) -> Dict[str, Any]:
"""Chat Completion API với full retry support"""
data = {
"model": model,
"messages": messages,
**kwargs
}
return await self._make_request("POST", "chat/completions", data)
async def embedding(
self,
input_text: str,
model: str = "text-embedding-3-small"
) -> Dict[str, Any]:
"""Embedding API"""
data = {
"model": model,
"input": input_text
}
return await self._make_request("POST", "embeddings", data)
def get_sla_report(self) -> Dict[str, Any]:
"""Generate SLA compliance report"""
uptime_seconds = time.time() - self._sla_start_time
success_rate = (
self.metrics.successful_requests / self.metrics.total_requests * 100
if self.metrics.total_requests > 0 else 0
)
avg_latency = (
self.metrics.total_latency_ms / self.metrics.successful_requests
if self.metrics.successful_requests > 0 else 0
)
return {
"uptime_seconds": uptime_seconds,
"total_requests": self.metrics.total_requests,
"successful_requests": self.metrics.successful_requests,
"failed_requests": self.metrics.failed_requests,
"success_rate_percent": round(success_rate, 3),
"average_latency_ms": round(avg_latency, 2),
"total_retries": self.metrics.retry_count,
"last_error": self.metrics.last_error,
"circuit_breaker_state": self.circuit_breaker.state,
"sla_compliance": success_rate >= 99.9
}
Usage Example
async def main():
config = HolySheepConfig(
api_key="YOUR_HOLYSHEEP_API_KEY",
max_retries=5,
base_delay=1.0,
max_delay=60.0,
retry_strategy=RetryStrategy.EXPONENTIAL
)
async with HolySheepAIClient(config) as client:
# Chat completion
response = await client.chat_completion(
messages=[
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "Explain SLA in AI APIs"}
],
model="deepseek-v3.2",
temperature=0.7
)
print(f"Response: {response['choices'][0]['message']['content']}")
print(f"Latency: {response['_metrics']['latency_ms']:.2f}ms")
# Get SLA report
report = client.get_sla_report()
print(f"SLA Compliance: {report['sla_compliance']}")
print(f"Success Rate: {report['success_rate_percent']}%")
if __name__ == "__main__":
asyncio.run(main())2. Benchmark Tool Đo Lường SLA Thực Tế
"""
HolySheep AI SLA Benchmark Tool
Đo lường và xác minh SLA compliance với dữ liệu thực tế
"""
import asyncio
import time
import statistics
import json
from datetime import datetime, timedelta
from typing import List, Dict, Tuple
from dataclasses import dataclass, asdict
from concurrent.futures import ThreadPoolExecutor
import aiohttp
@dataclass
class BenchmarkResult:
"""Kết quả benchmark cho một request"""
request_id: int
timestamp: float
latency_ms: float
success: bool
error_type: Optional[str] = None
model: str = ""
@dataclass
class SLAReport:
"""Báo cáo SLA tổng hợp"""
test_duration_seconds: float
total_requests: int
successful_requests: int
failed_requests: int
uptime_percent: float
availability_sla: float # 99.9, 99.95, etc.
sla_met: bool
latency_p50_ms: float
latency_p95_ms: float
latency_p99_ms: float
latency_avg_ms: float
latency_min_ms: float
latency_max_ms: float
cost_estimate_usd: float
cost_per_1k_requests_usd: float
class SLAProfiler:
"""
Comprehensive SLA Profiling Tool
- Concurrent request testing
- Latency distribution analysis
- Cost estimation
- SLA compliance verification
"""
# HolySheep AI Pricing (2026)
PRICING = {
"gpt-4.1": 8.00, # $8/MTok
"claude-sonnet-4.5": 15.00, # $15/MTok
"gemini-2.5-flash": 2.50, # $2.50/MTok
"deepseek-v3.2": 0.42, # $0.42/MTok
}
# Token estimation per request
TOKENS_PER_REQUEST = 500 # Average input + output
def __init__(
self,
api_key: str,
base_url: str = "https://api.holysheep.ai/v1",
target_sla: float = 99.9
):
self.api_key = api_key
self.base_url = base_url
self.target_sla = target_sla
self.results: List[BenchmarkResult] = []
self._session: Optional[aiohttp.ClientSession] = None
async def __aenter__(self):
self._session = aiohttp.ClientSession()
return self
async def __aexit__(self, *args):
if self._session:
await self._session.close()
async def _send_request(
self,
request_id: int,
model: str,
prompt: str
) -> BenchmarkResult:
"""Gửi single request và đo latency"""
headers = {
"Authorization": f"Bearer {self.api_key}",
"Content-Type": "application/json"
}
data = {
"model": model,
"messages": [{"role": "user", "content": prompt}],
"temperature": 0.7,
"max_tokens": 500
}
start_time = time.time()
error_type = None
success = False
try:
async with self._session.post(
f"{self.base_url}/chat/completions",
json=data,
headers=headers,
timeout=aiohttp.ClientTimeout(total=30)
) as response:
if response.status == 200:
await response.json()
success = True
else:
error_type = f"HTTP_{response.status}"
except asyncio.TimeoutError:
error_type = "TIMEOUT"
except Exception as e:
error_type = f"ERROR_{type(e).__name__}"
latency_ms = (time.time() - start_time) * 1000
return BenchmarkResult(
request_id=request_id,
timestamp=time.time(),
latency_ms=latency_ms,
success=success,
error_type=error_type,
model=model
)
async def run_load_test(
self,
model: str,
num_requests: int = 1000,
concurrency: int = 50,
prompt: str = "Explain the concept of microservices architecture in detail."
) -> SLAReport:
"""
Chạy load test với specified concurrency
Args:
model: Model ID (e.g., 'deepseek-v3.2')
num_requests: Total requests to send
concurrency: Concurrent connections
prompt: Test prompt
Returns:
SLAReport: Comprehensive SLA metrics
"""
print(f"Starting SLA load test for {model}")
print(f"Requests: {num_requests}, Concurrency: {concurrency}")
print("-" * 50)
start_time = time.time()
semaphore = asyncio.Semaphore(concurrency)
async def bounded_request(req_id: int):
async with semaphore:
return await self._send_request(req_id, model, prompt)
tasks = [bounded_request(i) for i in range(num_requests)]
self.results = await asyncio.gather(*tasks)
end_time = time.time()
duration = end_time - start_time
return self._generate_report(duration, model)
def _generate_report(self, duration: float, model: str) -> SLAReport:
"""Generate comprehensive SLA report từ results"""
successful = [r for r in self.results if r.success]
failed = [r for r in self.results if not r.success]
if successful:
latencies = sorted([r.latency_ms for r in successful])
p50_idx = int(len(latencies) * 0.50)
p95_idx = int(len(latencies) * 0.95)
p99_idx = int(len(latencies) * 0.99)
latencies_p50 = latencies[p50_idx]
latencies_p95 = latencies[p95_idx]
latencies_p99 = latencies[p99_idx]
latencies_avg = statistics.mean(latencies)
latencies_min = min(latencies)
latencies_max = max(latencies)
else:
latencies_p50 = latencies_p95 = latencies_p99 = 0
latencies_avg = latencies_min = latencies_max = 0
total = len(self.results)
uptime_percent = (len(successful) / total * 100) if total > 0 else 0
# Cost estimation
price_per_mtok = self.PRICING.get(model, 0.42)
tokens_per_mtok = 1_000_000
tokens_used = total * self.TOKENS_PER_REQUEST
cost_estimate = (tokens_used / tokens_per_mtok) * price_per_mtok
cost_per_request = cost_estimate / total if total > 0 else 0
report = SLAReport(
test_duration_seconds=duration,
total_requests=total,
successful_requests=len(successful),
failed_requests=len(failed),
uptime_percent=round(uptime_percent, 4),
availability_sla=self.target_sla,
sla_met=uptime_percent >= self.target_sla,
latency_p50_ms=round(latencies_p50, 2),
latency_p95_ms=round(latencies_p95, 2),
latency_p99_ms=round(latencies_p99, 2),
latency_avg_ms=round(latencies_avg, 2),
latency_min_ms=round(latencies_min, 2),
latency_max_ms=round(latencies_max, 2),
cost_estimate_usd=round(cost_estimate, 4),
cost_per_1k_requests_usd=round(cost_per_request * 1000, 4)
)
return report
def print_report(self, report: SLAReport):
"""In báo cáo SLA đẹp mắt"""
print("\n" + "=" * 60)
print("HOLYSHEEP AI - SLA BENCHMARK REPORT")
print("=" * 60)
print(f"\n📊 REQUEST STATISTICS")
print(f" Total Requests: {report.total_requests:,}")
print(f" Successful: {report.successful_requests:,}")
print(f" Failed: {report.failed_requests:,}")
print(f" Duration: {report.test_duration_seconds:.2f}s")
print(f"\n⏱️ LATENCY DISTRIBUTION")
print(f" Min: {report.latency_min_ms:.2f}ms")
print(f" Average: {report.latency_avg_ms:.2f}ms")
print(f" P50 (Median): {report.latency_p50_ms:.2f}ms")
print(f" P95: {report.latency_p95_ms:.2f}ms")
print(f" P99: {report.latency_p99_ms:.2f}ms")
print(f" Max: {report.latency_max_ms:.2f}ms")
print(f"\n🎯 SLA COMPLIANCE")
print(f" Target SLA: {report.availability_sla}%")
print(f" Actual Uptime: {report.uptime_percent}%")
status = "✅ MET" if report.sla_met else "❌ NOT MET"
print(f" Status: {status}")
print(f"\n💰 COST ESTIMATION")
print(f" Per Request: ${report.cost_per_1k_requests_usd / 1000:.6f}")
print(f" Per 1K Requests: ${report.cost_per_1k_requests_usd:.4f}")
print(f" Total Estimate: ${report.cost_estimate_usd:.4f}")
print("\n" + "=" * 60)
Multi-model comparison benchmark
async def run_model_comparison():
"""So sánh hiệu suất giữa các model"""
api_key = "YOUR_HOLYSHEEP_API_KEY"
models_to_test = [
("deepseek-v3.2", "deepseek-v3.2"),
("gemini-2.5-flash", "gemini-2.5-flash"),
("gpt-4.1", "gpt-4.1"),
]
results_summary = []
async with SLAProfiler(api_key) as profiler:
for model_id, model_name in models_to_test:
print(f"\n🔄 Testing {model_name}...")
report = await profiler.run_load_test(
model=model_id,
num_requests=200,
concurrency=20
)
profiler.print_report(report)
results_summary.append({
"model": model_name,
"uptime_percent": report.uptime_percent,
"latency_avg_ms": report.latency_avg_ms,
"latency_p99_ms": report.latency_p99_ms,
"cost_per_1k": report.cost_per_1k_requests_usd
})
# Summary comparison
print("\n" + "=" * 60)
print("MODEL COMPARISON SUMMARY")
print("=" * 60)
print(f"{'Model':<20} {'Uptime':<10} {'P99 Latency':<15} {'Cost/1K':<15}")
print("-" * 60)
for r in results_summary:
print(f"{r['model']:<20} {r['uptime_percent']:.3f}% "
f"{r['latency_p99_ms']:.2f}ms ${r['cost_per_1k']:.4f}")
if __name__ == "__main__":
asyncio.run(run_model_comparison())Chiến Lược Fault Recovery & High Availability
3. Multi-Provider Fallback Với HolySheep AI
/**
* HolySheep AI Multi-Provider Fallback System
* Production-ready implementation với automatic failover
*/
// Provider Configuration với HolySheep làm primary
interface ProviderConfig {
name: string;
baseUrl: string;
apiKey: string;
priority: number;
maxRetries: number;
timeout: number;
circuitBreakerThreshold: number;
isHealthy: boolean;
lastHealthCheck: number;
currentFailures: number;
}
interface AIRequest {
model: string;
messages: Array<{ role: string; content: string }>;
temperature?: number;
maxTokens?: number;
stream?: boolean;
}
interface AIResponse {
provider: string;
content: string;
latencyMs: number;
tokensUsed: number;
cached: boolean;
fallback: boolean;
}
interface HealthMetrics {
provider: string;
uptime: number;
avgLatency: number;
successRate: number;
totalRequests: number;
queueDepth: number;
}
class CircuitBreaker {
private state: 'CLOSED' | 'OPEN' | 'HALF_OPEN' = 'CLOSED';
private failureCount = 0;
private lastFailureTime = 0;
private successCount = 0;
constructor(
private threshold: number = 5,
private timeout: number = 60000,
private successThreshold: number = 3
) {}
canExecute(): boolean {
if (this.state === 'CLOSED') return true;
if (this.state === 'OPEN') {
if (Date.now() - this.lastFailureTime >= this.timeout) {
this.state = 'HALF_OPEN';
this.successCount = 0;
return true;
}
return false;
}
return true; // HALF_OPEN
}
recordSuccess(): void {
if (this.state === 'HALF_OPEN') {
this.successCount++;
if (this.successCount >= this.successThreshold) {
this.state = 'CLOSED';
this.failureCount = 0;
}
} else {
this.failureCount = 0;
}
}
recordFailure(): void {
this.failureCount++;
this.lastFailureTime = Date.now();
if (this.state === 'HALF_OPEN' || this.failureCount >= this.threshold) {
this.state = 'OPEN';
}
}
getState(): string {
return this.state;
}
}
class HealthChecker {
private checkInterval: number = 30000; // 30s
private providers: Map = new Map();
private onHealthChange: (provider: string, healthy: boolean) => void;
constructor(
providers: ProviderConfig[],
onHealthChange: (provider: string, healthy: boolean) => void
) {
providers.forEach(p => this.providers.set(p.name, p));
this.onHealthChange = onHealthChange;
}
async checkProvider(provider: ProviderConfig): Promise {
const startTime = Date.now();
try {
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), 5000);
const response = await fetch(${provider.baseUrl}/models, {
method: 'GET',
headers: {
'Authorization': Bearer ${provider.apiKey},
'Content-Type': 'application/json'
},
signal: controller.signal
});
clearTimeout(timeoutId);
const healthy = response.ok;
if (provider.isHealthy !== healthy) {
provider.isHealthy = healthy;
provider.lastHealthCheck = Date.now();
this.onHealthChange(provider.name, healthy);
}
return healthy;
} catch (error) {
provider.isHealthy = false;
provider.lastHealthCheck = Date.now();
this.onHealthChange(provider.name, false);
return false;
}
}
async checkAllProviders(): Promise {
const checks = Array.from(this.providers.values()).map(p =>
this.checkProvider(p)
);
await Promise.all(checks);
}
startPeriodicChecks(): NodeJS.Timer {
return setInterval(() => this.checkAllProviders(), this.checkInterval);
}
getHealthyProviders(): ProviderConfig[] {
return Array.from(this.providers.values())
.filter(p => p.isHealthy)
.sort((a, b) => a.priority - b.priority);
}
}
class RateLimiter {
private tokens: number;
private lastRefill: number;
constructor(
private maxTokens: number,
private refillRate: number, // tokens per second
private refillInterval: number = 1000
) {
this.tokens = maxTokens;
this.lastRefill = Date.now();
}
async acquire(tokens: number = 1): Promise {
while (this.tokens < tokens) {
this.refill();
if (this.tokens < tokens) {
await new Promise(resolve => setTimeout(resolve, 100));
}
}
this.tokens -= tokens;
}
private refill(): void {
const now = Date.now();
const elapsed = now - this.lastRefill;
const refillAmount = (elapsed / this.refillInterval) * this.refillRate;
this.tokens = Math.min(this.maxTokens, this.tokens + refillAmount);
this.lastRefill = now;
}
getAvailableTokens(): number {
this.refill();
return this.tokens;
}
}
class MultiProviderAIGateway {
private providers: Map = new Map();
private circuitBreakers: Map = new Map();
private healthChecker: HealthChecker;
private requestQueue: Array<{
request: AIRequest;
resolve: (response: AIResponse) => void;
reject: (error: Error) => void;
priority: number;
timestamp: number;
}> = [];
private isProcessing = false;
private maxQueueSize = 10000;
private processingInterval: number = 100;
private metrics = {
totalRequests: 0,
successfulRequests: 0,
failedRequests: 0,
fallbackRequests: 0,
totalLatencyMs: 0,
providerLatencies: new Map(),
providerErrors: new Map()
};
constructor() {
// Initialize HolySheep AI as primary provider
this.registerProvider({
name: 'holysheep',
baseUrl: 'https://api.holysheep.ai/v1',
apiKey: 'YOUR_HOLYSHEEP_API_KEY',
priority: 1,
maxRetries: 3,
timeout: 30000,
circuitBreakerThreshold: 5,
isHealthy: true,
lastHealthCheck: Date.now(),
currentFailures: 0
});
// Fallback providers
this.registerProvider({
name: 'holysheep-fallback',
baseUrl: 'https://api.holysheep.ai/v1',
apiKey: 'YOUR_HOLYSHEEP_API_KEY',
priority: 2,
maxRetries: 2,
timeout: 45000,
circuitBreakerThreshold: 10,
isHealthy: true,
lastHealthCheck: Date.now(),
currentFailures: 0
});
this.healthChecker = new HealthChecker(
Array.from(this.providers.values()),
(name, healthy) => {
console.log(Provider ${name} health changed: ${healthy ? 'HEALTHY' : 'UNHEALTHY'});
}
);
// Start health checks
this.healthChecker.startPeriodicChecks();
// Start queue processing
this.startQueueProcessing();
}
private registerProvider(config: ProviderConfig): void {
this.providers.set(config.name, config);
this.circuitBreakers.set(
config.name,
new CircuitBreaker(config.circuitBreakerThreshold)
);
}
async chatCompletion(request: AIRequest): Promise {
this.metrics.totalRequests++;
const healthyProviders = this.healthChecker.getHealthyProviders();
if (healthyProviders.length === 0) {
throw new Error('No healthy providers available');
}
let lastError: Error | null = null;
let usedFallback = false;
for (const provider of healthyProviders) {
const breaker = this.circuitBreakers.get(provider.name)!;
if (!breaker.canExecute()) {
continue;
}
try {
const response = await this.executeRequest(provider, request);
if (usedFallback) {
this.metrics.fallbackRequests++;
}
breaker.recordSuccess();
return response;
} catch (error) {
lastError = error as Error;
breaker.recordFailure();
const currentErrors = this.metrics.providerErrors.get(provider.name) || 0;
this.metrics.providerErrors.set(provider.name, currentErrors + 1);
usedFallback = true;
console.warn(Provider ${provider.name} failed:, error);
continue;
}
}
this.metrics.failedRequests++;
throw lastError || new Error('All providers failed');
}
private async executeRequest(
provider: ProviderConfig,
request: AIRequest
): Promise {
const startTime = Date.now();
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), provider.timeout);
try {
const response = await fetch(${provider.baseUrl}/chat/completions, {
method: 'POST',
headers: {
'Authorization': Bearer ${provider.apiKey},
'Content-Type': 'application/json'
},
body: JSON.stringify({
model: request.model,
messages: request.messages,
temperature: request.temperature ?? 0.7,
max_tokens: request.maxTokens ?? 1000,
stream: request.stream ?? false
}),
signal: controller.signal
});
clearTimeout(timeoutId);
if (!response.ok) {
const error = await response.json();
throw new Error(Provider error: ${response.status} - ${JSON.stringify(error)});
}
const data = await response.json();
const latencyMs = Date.now() - startTime;
// Record metrics
this.metrics.totalLatencyMs += latencyMs;
this.metrics.successfulRequests++;
const latencies = this.metrics.providerLatencies.get(provider.name) || [];
latencies.push(latencyMs);
this.metrics.providerLatencies.set(provider.name, latencies);
return {
provider: provider.name,
content: data.choices?.[0]?.message?.content || '',
latencyMs,
tokensUsed: data.usage?.total_tokens || 0,
cached: data.cached || false,
fallback: provider.priority > 1
};
} finally {
clearTimeout(timeoutId);
}
}
private startQueueProcessing(): void {
setInterval(() => this.processQueue(), this.processingInterval);
}
private async processQueue(): Promise {
if (this.isProcessing || this.requestQueue.length === 0) return;
this.isProcessing = true;
while (this.requestQueue.length > 0) {
const item = this.requestQueue.shift()!;
try {
const response = await this.chatCompletion(item.request);
item.resolve(response);
} catch (error) {
item.reject(error as Error);
}
}
this.is