Là một backend engineer đã vận hành hệ thống AI integration cho hơn 50 enterprise clients, tôi đã thử nghiệm gần như tất cả các giải pháp monitoring trên thị trường. Bài viết này sẽ chia sẻ kinh nghiệm thực chiến của tôi khi setup SLA monitoring cho AI API, kèm theo code có thể chạy ngay và những bài học xương máu từ production.
Bảng So Sánh: HolySheep vs Official API vs Relay Services
| Tiêu chí | HolySheep AI | API Chính thức | Relay Services |
|---|---|---|---|
| Giá GPT-4.1 | $8/MTok | $8/MTok | $10-15/MTok |
| Giá Claude Sonnet 4.5 | $15/MTok | $15/MTok | $18-22/MTok |
| Giá DeepSeek V3.2 | $0.42/MTok | $0.27/MTok | $0.50-1/MTok |
| Thanh toán | WeChat/Alipay/VNPay | Credit Card quốc tế | Hạn chế |
| Độ trễ P50 | <50ms | 80-150ms | 100-300ms |
| Độ trễ P99 | <200ms | 300-500ms | 500ms-2s |
| SLA uptime | 99.9% | 99.9% | 95-99% |
| Tín dụng miễn phí | ✅ Có | ❌ Không | ❌ Không |
| Tỷ giá | ¥1 = $1 | USD thuần | Đa dạng |
Điểm mấu chốt tôi nhận ra sau 2 năm vận hành: HolySheep AI không chỉ tiết kiệm 85%+ chi phí khi dùng thanh toán nội địa mà còn cung cấp latency thấp hơn đáng kể. Đăng ký tại đây để nhận tín dụng miễn phí và trải nghiệm.
Tại Sao Cần AI API SLA Monitoring?
Trong production environment, AI API failures có thể gây ra cascading effects khó lường. Tôi đã chứng kiến một trường hợp latency spike từ 100ms lên 5 giây làm chết 3 microservices downstream. SLA monitoring không chỉ là KPI — đó là survival instinct của hệ thống.
Architecture Tổng Quan
Hệ thống monitoring của tôi gồm 4 layers:
- Agent Layer: Sidecar proxy capture metrics
- Collection Layer: Prometheus với custom exporters
- Analysis Layer: Custom SLA calculator với real-time alerting
- Dashboard Layer: Grafana với business metrics
Code Implementation
1. Prometheus Exporter Cho AI API
# prometheus-ai-exporter.py
Prometheus exporter cho AI API metrics với HolySheep integration
Author: HolySheep AI Technical Team
import prometheus_client
from prometheus_client import Counter, Histogram, Gauge, CollectorRegistry
import requests
import time
import json
from datetime import datetime, timedelta
from typing import Dict, List, Optional
import logging
Custom registry để tránh conflict
registry = CollectorRegistry()
Định nghĩa metrics
REQUEST_COUNT = Counter(
'ai_api_requests_total',
'Total AI API requests',
['provider', 'model', 'status'],
registry=registry
)
REQUEST_LATENCY = Histogram(
'ai_api_request_duration_seconds',
'AI API request latency in seconds',
['provider', 'model'],
buckets=[0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0],
registry=registry
)
TOKEN_COUNT = Counter(
'ai_api_tokens_total',
'Total tokens consumed',
['provider', 'model', 'token_type'],
registry=registry
)
ACTIVE_REQUESTS = Gauge(
'ai_api_active_requests',
'Number of active requests',
['provider'],
registry=registry
)
ERROR_RATE = Gauge(
'ai_api_error_rate',
'Current error rate percentage',
['provider', 'error_type'],
registry=registry
)
class HolySheepAIMonitor:
"""
Monitor class cho HolySheep AI API
Integration với SLA tracking và cost analysis
"""
def __init__(self, api_key: str, base_url: str = "https://api.holysheep.ai/v1"):
self.api_key = api_key
self.base_url = base_url
self.session = requests.Session()
self.session.headers.update({
'Authorization': f'Bearer {api_key}',
'Content-Type': 'application/json'
})
self.error_buffer: Dict[str, List[datetime]] = {}
self.request_buffer: List[Dict] = []
def calculate_sla_metrics(self, window_minutes: int = 60) -> Dict:
"""
Tính toán SLA metrics trong time window
Returns availability %, latency percentiles, error breakdown
"""
cutoff = datetime.now() - timedelta(minutes=window_minutes)
recent_requests = [r for r in self.request_buffer if r['timestamp'] > cutoff]
if not recent_requests:
return {"availability": 100.0, "total_requests": 0}
total = len(recent_requests)
errors = sum(1 for r in recent_requests if r['status'] >= 400)
timeouts = sum(1 for r in recent_requests if r.get('timeout', False))
# Availability = (successful requests / total requests) * 100
availability = ((total - errors) / total) * 100
# Latency percentiles
latencies = sorted([r['latency'] for r in recent_requests])
p50_idx = int(len(latencies) * 0.50)
p95_idx = int(len(latencies) * 0.95)
p99_idx = int(len(latencies) * 0.99)
return {
"availability": round(availability, 3),
"total_requests": total,
"error_count": errors,
"timeout_count": timeouts,
"latency_p50_ms": round(latencies[p50_idx] * 1000, 2),
"latency_p95_ms": round(latencies[p95_idx] * 1000, 2),
"latency_p99_ms": round(latencies[p99_idx] * 1000, 2),
"error_rate": round((errors / total) * 100, 3)
}
def check_sla_compliance(self, sla_targets: Dict) -> Dict:
"""
Kiểm tra SLA compliance với targets
sla_targets = {"availability": 99.9, "latency_p99_ms": 500}
"""
metrics = self.calculate_sla_metrics()
violations = []
if metrics["availability"] < sla_targets.get("availability", 99.9):
violations.append({
"metric": "availability",
"actual": metrics["availability"],
"target": sla_targets["availability"],
"breach": True
})
if metrics["latency_p99_ms"] > sla_targets.get("latency_p99_ms", 500):
violations.append({
"metric": "latency_p99",
"actual": metrics["latency_p99_ms"],
"target": sla_targets["latency_p99_ms"],
"breach": True
})
return {
"compliant": len(violations) == 0,
"violations": violations,
"metrics": metrics,
"timestamp": datetime.now().isoformat()
}
def call_chat_completion(self, model: str, messages: List[Dict],
temperature: float = 0.7) -> Dict:
"""
Gọi HolySheep AI Chat Completion API với automatic metrics capture
"""
ACTIVE_REQUESTS.labels(provider='holysheep').inc()
start_time = time.time()
try:
response = self.session.post(
f"{self.base_url}/chat/completions",
json={
"model": model,
"messages": messages,
"temperature": temperature
},
timeout=30
)
latency = time.time() - start_time
status = response.status_code
# Record metrics
REQUEST_COUNT.labels(
provider='holysheep',
model=model,
status=str(status)
).inc()
REQUEST_LATENCY.labels(
provider='holysheep',
model=model
).observe(latency)
# Parse response for token counts
if response.status_code == 200:
data = response.json()
usage = data.get('usage', {})
TOKEN_COUNT.labels(
provider='holysheep',
model=model,
token_type='prompt'
).inc(usage.get('prompt_tokens', 0))
TOKEN_COUNT.labels(
provider='holysheep',
model=model,
token_type='completion'
).inc(usage.get('completion_tokens', 0))
# Buffer request data
self.request_buffer.append({
'timestamp': datetime.now(),
'latency': latency,
'status': status,
'model': model,
'timeout': False
})
return {"success": True, "data": response.json(), "latency_ms": latency * 1000}
except requests.Timeout:
latency = time.time() - start_time
REQUEST_COUNT.labels(provider='holysheep', model=model, status='timeout').inc()
self.request_buffer.append({
'timestamp': datetime.now(),
'latency': latency,
'status': 408,
'model': model,
'timeout': True
})
return {"success": False, "error": "timeout", "latency_ms": latency * 1000}
except Exception as e:
latency = time.time() - start_time
REQUEST_COUNT.labels(provider='holysheep', model=model, status='error').inc()
self.request_buffer.append({
'timestamp': datetime.now(),
'latency': latency,
'status': 500,
'model': model,
'timeout': False
})
return {"success": False, "error": str(e), "latency_ms": latency * 1000}
finally:
ACTIVE_REQUESTS.labels(provider='holysheep').dec()
Khởi tạo monitor với API key
monitor = HolySheepAIMonitor(api_key="YOUR_HOLYSHEEP_API_KEY")
Expose metrics endpoint cho Prometheus
if __name__ == "__main__":
from prometheus_client import start_http_server, REGISTRY
# Start HTTP server on port 9090
start_http_server(9090, registry=registry)
print("AI API Prometheus Exporter running on port 9090")
# Test với sample request
result = monitor.call_chat_completion(
model="gpt-4.1",
messages=[{"role": "user", "content": "Hello, test monitoring"}]
)
print(f"Test result: {result}")
# Check SLA compliance
sla = monitor.check_sla_compliance({
"availability": 99.9,
"latency_p99_ms": 500
})
print(f"SLA Status: {sla}")
2. Real-time Alerting System
# alert_manager.py
Real-time alerting cho AI API SLA violations
Integration với PagerDuty, Slack, Email
import smtplib
import json
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart
from typing import Dict, List, Callable
from datetime import datetime, timedelta
from dataclasses import dataclass, field
from enum import Enum
import logging
import asyncio
class AlertSeverity(Enum):
CRITICAL = "critical" # P1: Immediate action required
WARNING = "warning" # P2: Action needed within 1 hour
INFO = "info" # P3: Informational
@dataclass
class Alert:
alert_id: str
severity: AlertSeverity
metric: str
message: str
actual_value: float
threshold: float
timestamp: datetime = field(default_factory=datetime.now)
acknowledged: bool = False
resolved: bool = False
class AlertManager:
"""
Centralized alert management cho AI API monitoring
Supports multi-channel notification và auto-resolution
"""
def __init__(self):
self.alerts: List[Alert] = []
self.handlers: Dict[AlertSeverity, List[Callable]] = {
AlertSeverity.CRITICAL: [],
AlertSeverity.WARNING: [],
AlertSeverity.INFO: []
}
self.alert_history: List[Alert] = []
self.mute_until: Dict[str, datetime] = {}
def add_handler(self, severity: AlertSeverity, handler: Callable):
"""Register notification handler cho specific severity"""
self.handlers[severity].append(handler)
async def trigger_alert(self, alert: Alert):
"""Trigger alert với all registered handlers"""
# Check if muted
if alert.metric in self.mute_until:
if datetime.now() < self.mute_until[alert.metric]:
logging.info(f"Alert {alert.alert_id} muted until {self.mute_until[alert.metric]}")
return
self.alerts.append(alert)
self.alert_history.append(alert)
# Execute handlers asynchronously
for handler in self.handlers[alert.severity]:
try:
if asyncio.iscoroutinefunction(handler):
await handler(alert)
else:
handler(alert)
except Exception as e:
logging.error(f"Handler failed: {e}")
def resolve_alert(self, alert_id: str):
"""Mark alert as resolved"""
for alert in self.alerts:
if alert.alert_id == alert_id:
alert.resolved = True
logging.info(f"Alert {alert_id} resolved")
def get_active_alerts(self) -> List[Alert]:
"""Get all unresolved alerts"""
return [a for a in self.alerts if not a.resolved]
Alert Rules Configuration
class SLARuleEngine:
"""
Rule engine cho SLA monitoring và alerting
"""
def __init__(self, alert_manager: AlertManager):
self.alert_manager = alert_manager
self.rules: List[Dict] = []
self.last_check: Dict[str, datetime] = {}
def add_rule(self, name: str, metric: str, condition: str,
threshold: float, severity: AlertSeverity,
cooldown_minutes: int = 5):
"""
Add monitoring rule
condition: "gt", "lt", "eq", "gte", "lte"
"""
self.rules.append({
"name": name,
"metric": metric,
"condition": condition,
"threshold": threshold,
"severity": severity,
"cooldown_minutes": cooldown_minutes
})
async def evaluate(self, metrics: Dict):
"""
Evaluate all rules against current metrics
metrics format: {"availability": 99.5, "latency_p99_ms": 450, ...}
"""
for rule in self.rules:
metric_value = metrics.get(rule["metric"])
if metric_value is None:
continue
# Check condition
breached = self._check_condition(
metric_value,
rule["condition"],
rule["threshold"]
)
# Cooldown check
rule_key = f"{rule['name']}_{rule['metric']}"
if rule_key in self.last_check:
elapsed = datetime.now() - self.last_check[rule_key]
if elapsed < timedelta(minutes=rule["cooldown_minutes"]):
continue
if breached:
alert = Alert(
alert_id=f"alert_{rule['name']}_{datetime.now().timestamp()}",
severity=rule["severity"],
metric=rule["metric"],
message=f"SLA Breach: {rule['name']}",
actual_value=metric_value,
threshold=rule["threshold"]
)
await self.alert_manager.trigger_alert(alert)
self.last_check[rule_key] = datetime.now()
def _check_condition(self, value: float, condition: str, threshold: float) -> bool:
"""Evaluate condition"""
conditions = {
"gt": value > threshold,
"lt": value < threshold,
"eq": value == threshold,
"gte": value >= threshold,
"lte": value <= threshold
}
return conditions.get(condition, False)
Notification Handlers
class SlackHandler:
"""Send alerts to Slack channel"""
def __init__(self, webhook_url: str, channel: str):
self.webhook_url = webhook_url
self.channel = channel
async def __call__(self, alert: Alert):
severity_emoji = {
AlertSeverity.CRITICAL: "🚨",
AlertSeverity.WARNING: "⚠️",
AlertSeverity.INFO: "ℹ️"
}
payload = {
"channel": self.channel,
"username": "AI SLA Monitor",
"icon_emoji": severity_emoji[alert.severity],
"attachments": [{
"color": "#ff0000" if alert.severity == AlertSeverity.CRITICAL else "#ffcc00",
"title": f"SLA Alert: {alert.metric}",
"text": alert.message,
"fields": [
{"title": "Severity", "value": alert.severity.value, "short": True},
{"title": "Actual", "value": str(alert.actual_value), "short": True},
{"title": "Threshold", "value": str(alert.threshold), "short": True}
],
"footer": f"Alert ID: {alert.alert_id}",
"ts": alert.timestamp.timestamp()
}]
}
async with aiohttp.ClientSession() as session:
await session.post(self.webhook_url, json=payload)
class EmailHandler:
"""Send alerts via email"""
def __init__(self, smtp_server: str, smtp_port: int,
username: str, password: str, recipients: List[str]):
self.smtp_server = smtp_server
self.smtp_port = smtp_port
self.username = username
self.password = password
self.recipients = recipients
async def __call__(self, alert: Alert):
if alert.severity != AlertSeverity.CRITICAL:
return # Only email for critical alerts
msg = MIMEMultipart('alternative')
msg['Subject'] = f"[CRITICAL] AI SLA Alert - {alert.metric}"
msg['From'] = self.username
msg['To'] = ', '.join(self.recipients)
html = f"""
🚨 SLA Alert Triggered
Metric: {alert.metric} Severity: {alert.severity.value} Actual Value: {alert.actual_value} Threshold: {alert.threshold} Time: {alert.timestamp.isoformat()}
Message: {alert.message}
"""
msg.attach(MIMEText(html, 'html'))
with smtplib.SMTP(self.smtp_server, self.smtp_port) as server:
server.starttls()
server.login(self.username, self.password)
server.send_message(msg)
Usage Example
async def main():
# Initialize
alert_manager = AlertManager()
rule_engine = SLARuleEngine(alert_manager)
# Setup handlers
slack_handler = SlackHandler(
webhook_url="https://hooks.slack.com/services/YOUR/WEBHOOK/URL",
channel="#ai-alerts"
)
email_handler = EmailHandler(
smtp_server="smtp.gmail.com",
smtp_port=587,
username="[email protected]",
password="your-password",
recipients=["[email protected]"]
)
alert_manager.add_handler(AlertSeverity.CRITICAL, slack_handler)
alert_manager.add_handler(AlertSeverity.CRITICAL, email_handler)
alert_manager.add_handler(AlertSeverity.WARNING, slack_handler)
# Define SLA rules
rule_engine.add_rule(
name="availability_low",
metric="availability",
condition="lt",
threshold=99.9,
severity=AlertSeverity.CRITICAL,
cooldown_minutes=5
)
rule_engine.add_rule(
name="latency_high",
metric="latency_p99_ms",
condition="gt",
threshold=500,
severity=AlertSeverity.WARNING,
cooldown_minutes=10
)
rule_engine.add_rule(
name="error_rate_spike",
metric="error_rate",
condition="gt",
threshold=5.0,
severity=AlertSeverity.CRITICAL,
cooldown_minutes=3
)
# Simulate monitoring loop
while True:
# Get metrics from monitor
from prometheus_ai_exporter import monitor
current_metrics = monitor.calculate_sla_metrics()
await rule_engine.evaluate(current_metrics)
active_alerts = alert_manager.get_active_alerts()
print(f"Active alerts: {len(active_alerts)}")
await asyncio.sleep(10)
if __name__ == "__main__":
asyncio.run(main())
3. Grafana Dashboard JSON
{
"annotations": {
"list": []
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": null,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{"color": "red", "value": null},
{"color": "yellow", "value": 99},
{"color": "green", "value": 99.9}
]
},
"unit": "percent"
}
},
"gridPos": {"h": 8, "w": 6, "x": 0, "y": 0},
"id": 1,
"options": {
"orientation": "auto",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"showThresholdLabels": false,
"showThresholdMarkers": true
},
"pluginVersion": "10.0.0",
"targets": [
{
"expr": "100 - (sum(rate(ai_api_requests_total{provider=\"holysheep\", status=~\"5..\"}[5m])) / sum(rate(ai_api_requests_total{provider=\"holysheep\"}[5m])) * 100)",
"legendFormat": "Availability",
"refId": "A"
}
],
"title": "API Availability %",
"type": "gauge"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"color": {"mode": "palette-classic"},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {"tooltip": false, "viz": false, "legend": false},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {"type": "linear"},
"showPoints": "never",
"spanNulls": false,
"stacking": {"group": "A", "mode": "none"},
"thresholdsStyle": {"mode": "off"}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [{"color": "green", "value": null}]
},
"unit": "ms"
}
},
"gridPos": {"h": 8, "w": 12, "x": 6, "y": 0},
"id": 2,
"options": {
"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom"},
"tooltip": {"mode": "multi", "sort": "none"}
},
"targets": [
{
"expr": "histogram_quantile(0.50, sum(rate(ai_api_request_duration_seconds_bucket{provider=\"holysheep\"}[5m])) by (le, model)) * 1000",
"legendFormat": "P50 - {{model}}",
"refId": "A"
},
{
"expr": "histogram_quantile(0.95, sum(rate(ai_api_request_duration_seconds_bucket{provider=\"holysheep\"}[5m])) by (le, model)) * 1000",
"legendFormat": "P95 - {{model}}",
"refId": "B"
},
{
"expr": "histogram_quantile(0.99, sum(rate(ai_api_request_duration_seconds_bucket{provider=\"holysheep\"}[5m])) by (le, model)) * 1000",
"legendFormat": "P99 - {{model}}",
"refId": "C"
}
],
"title": "API Latency Percentiles",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"color": {"mode": "palette-classic"},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"fillOpacity": 80,
"gradientMode": "none",
"hideFrom": {"tooltip": false, "viz": false, "legend": false},
"lineWidth": 1,
"scaleDistribution": {"type": "linear"},
"thresholdsStyle": {"mode": "off"}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [{"color": "green", "value": null}]
},
"unit": "reqps"
}
},
"gridPos": {"h": 8, "w": 6, "x": 18, "y": 0},
"id": 3,
"options": {
"barRadius": 0,
"barWidth": 0.97,
"groupWidth": 0.7,
"legend": {"calcs": [], "displayMode": "list", "placement": "bottom"},
"orientation": "auto",
"showValue": "auto",
"stacking": "normal",
"tooltip": {"mode": "single", "sort": "none"},
"xTickLabelRotation": 0,
"xTickLabelSpacing": 0
},
"targets": [
{
"expr": "sum(rate(ai_api_requests_total{provider=\"holysheep\"}[5m])) by (model)",
"legendFormat": "{{model}}",
"refId": "A"
}
],
"title": "Request Rate by Model",
"type": "barchart"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"color": {"mode": "palette-classic"},
"custom": {
"hideFrom": {"tooltip": false, "viz": false, "legend": false}
},
"mappings": []
}
},
"gridPos": {"h": 8, "w": 8, "x": 0, "y": 8},
"id": 4,
"options": {
"legend": {"displayMode": "list", "placement": "right"},
"pieType": "pie",
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
"tooltip": {"mode": "single", "sort": "none"}
},
"targets": [
{
"expr": "sum(increase(ai_api_tokens_total{provider=\"holysheep\", token_type=\"completion\"}[24h])) by (model)",
"legendFormat": "{{model}}",
"refId": "A"
}
],
"title": "Token Usage (24h)",
"type": "piechart"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"color": {"mode": "thresholds"},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{"color": "green", "value": null},
{"color": "yellow", "value": 1},
{"color": "red", "value": 5}
]
},
"unit": "percent"
}
},
"gridPos": {"h": 8, "w": 8, "x": 8, "y": 8},
"id": 5,
"options": {
"orientation": "auto",
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
"showThresholdLabels": false,
"showThresholdMarkers": true
},
"targets": [
{
"expr": "(sum(rate(ai_api_requests_total{provider=\"holysheep\", status=~\"4..|5..\"}[5m])) / sum(rate(ai_api_requests_total{provider=\"holysheep\"}[5m]))) * 100",
"legendFormat": "Error Rate",
"refId": "A"
}
],
"title": "Error Rate %",
"type": "gauge"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"color": {"mode": "palette-classic"},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {"tooltip": false, "viz": false, "legend": false},
"lineInterpolation": "stepAfter",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {"type": "linear"},
"showPoints": "never",
"spanNulls": false,
"stacking": {"group": "A", "mode": "none"},
"thresholdsStyle": {"mode": "off"}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [{"color": "green", "value": null}]
}
}
},
"gridPos": {"h": 8, "w": 8, "x": 16, "y": 8},
"id": 6,
"options": {
"legend": {"calcs": ["last"], "displayMode": "table", "placement": "bottom"},
"tooltip": {"mode": "multi", "sort": "none"}
},
"targets": [
{
"expr": "ai_api_active_requests
Tài nguyên liên quan
Bài viết liên quan