Trong bài viết này, tôi sẽ chia sẻ kinh nghiệm thực chiến khi thiết kế hệ thống Feedback Learning cho AI Agent — từ việc thu thập dữ liệu phản hồi, fine-tune model, đến deployment. Đặc biệt, tôi sẽ hướng dẫn cách di chuyển infrastructure sang HolySheep AI để tối ưu chi phí lên đến 85% với độ trễ dưới 50ms.
Vì Sao Cần Feedback Learning Cho AI Agent?
Khi triển khai AI Agent vào production, model foundation không thể hiểu hết ngữ cảnh domain của bạn. Feedback Learning giúp:
- Cải thiện độ chính xác — Agent học từ các phản hồi của người dùng để trả lời đúng hơn
- Giảm hallucination — Fine-tuned model ít bịa đặt thông tin
- Tối ưu chi phí inference — Model nhỏ hơn, nhanh hơn sau khi fine-tune
Kiến Trúc Tổng Quan: Feedback Loop System
┌─────────────────────────────────────────────────────────────────┐
│ FEEDBACK LEARNING ARCHITECTURE │
├─────────────────────────────────────────────────────────────────┤
│ │
│ ┌──────────┐ ┌──────────┐ ┌──────────┐ ┌──────────┐ │
│ │ User │───▶│ Agent │───▶│ Response │◀───│ Feedback │ │
│ │ Input │ │ Engine │ │ Store │ │ Handler │ │
│ └──────────┘ └────┬─────┘ └──────────┘ └────┬─────┘ │
│ │ │ │
│ ▼ ▼ │
│ ┌──────────┐ ┌──────────┐ │
│ │ HolySheep│ │ Fine-tune│ │
│ │ API │ │ Pipeline │ │
│ │ <50ms │ │ Process │ │
│ └──────────┘ └──────────┘ │
│ │ │
│ ▼ │
│ ┌──────────┐ │
│ │ Updated │ │
│ │ Model │ │
│ └──────────┘ │
└─────────────────────────────────────────────────────────────────┘
Phase 1: Thu Thập Dữ Liệu Feedback
Đầu tiên, chúng ta cần hệ thống thu thập phản hồi từ người dùng một cách có cấu trúc. Dưới đây là implementation hoàn chỉnh:
import httpx
import json
from datetime import datetime
from typing import Optional, List, Dict
from dataclasses import dataclass, asdict
from enum import Enum
class FeedbackType(Enum):
"""Các loại feedback từ người dùng"""
UPVOTE = "upvote"
DOWNVOTE = "downvote"
CORRECTION = "correction"
RATING = "rating" # 1-5 stars
@dataclass
class FeedbackEntry:
"""Cấu trúc một feedback entry"""
session_id: str
user_id: str
prompt: str
response: str
feedback_type: str
rating: Optional[int] = None
corrected_response: Optional[str] = None
metadata: Optional[Dict] = None
latency_ms: float
model_name: str
tokens_used: int
timestamp: str = ""
def __post_init__(self):
if not self.timestamp:
self.timestamp = datetime.utcnow().isoformat()
class FeedbackCollector:
"""
Hệ thống thu thập feedback cho AI Agent
Lưu trữ local + sync lên cloud storage
"""
def __init__(
self,
api_key: str,
base_url: str = "https://api.holysheep.ai/v1",
storage_path: str = "./feedback_data/"
):
self.api_key = api_key
self.base_url = base_url
self.storage_path = storage_path
self._client = httpx.Client(
timeout=30.0,
headers={
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json"
}
)
self._feedback_buffer: List[FeedbackEntry] = []
self._buffer_size = 100 # Flush sau 100 entries
def log_interaction(
self,
session_id: str,
user_id: str,
prompt: str,
response: str,
model_name: str,
latency_ms: float,
tokens_used: int,
metadata: Optional[Dict] = None
) -> str:
"""Log một interaction để track"""
entry = FeedbackEntry(
session_id=session_id,
user_id=user_id,
prompt=prompt,
response=response,
feedback_type="interaction", # pending feedback
model_name=model_name,
latency_ms=latency_ms,
tokens_used=tokens_used,
metadata=metadata or {}
)
self._feedback_buffer.append(entry)
# Auto-flush khi buffer đầy
if len(self._feedback_buffer) >= self._buffer_size:
self._flush_buffer()
return entry.timestamp
def submit_feedback(
self,
session_id: str,
feedback_type: str,
rating: Optional[int] = None,
corrected_response: Optional[str] = None
) -> bool:
"""Submit feedback từ phía người dùng"""
# Tìm interaction tương ứng
for entry in reversed(self._feedback_buffer):
if entry.session_id == session_id:
entry.feedback_type = feedback_type
entry.rating = rating
entry.corrected_response = corrected_response
return True
return False
def _flush_buffer(self):
"""Flush buffer ra disk để training"""
if not self._feedback_buffer:
return
filename = f"feedback_{datetime.now().strftime('%Y%m%d_%H%M%S')}.jsonl"
filepath = f"{self.storage_path}{filename}"
with open(filepath, 'w', encoding='utf-8') as f:
for entry in self._feedback_buffer:
f.write(json.dumps(asdict(entry), ensure_ascii=False) + '\n')
print(f"✅ Flushed {len(self._feedback_buffer)} entries to {filepath}")
self._feedback_buffer.clear()
def get_training_data(self, min_rating: int = 4) -> List[Dict]:
"""Lấy data cho fine-tuning từ feedback tích cực"""
training_data = []
import glob
for filepath in glob.glob(f"{self.storage_path}*.jsonl"):
with open(filepath, 'r', encoding='utf-8') as f:
for line in f:
entry = json.loads(line)
# Chỉ lấy feedback positive hoặc có correction
if entry['feedback_type'] in ['upvote', 'correction'] or \
(entry.get('rating') and entry['rating'] >= min_rating):
training_data.append({
"messages": [
{"role": "user", "content": entry['prompt']},
{"role": "assistant", "content": entry.get('corrected_response') or entry['response']}
]
})
return training_data
=== USAGE EXAMPLE ===
if __name__ == "__main__":
collector = FeedbackCollector(
api_key="YOUR_HOLYSHEEP_API_KEY",
base_url="https://api.holysheep.ai/v1"
)
# Log interaction
start = datetime.now()
# Gọi API thực tế
response = collector._client.post(
"https://api.holysheep.ai/v1/chat/completions",
json={
"model": "gpt-4.1",
"messages": [{"role": "user", "content": "Tính BMI của tôi"}],
"temperature": 0.7
}
)
latency = (datetime.now() - start).total_seconds() * 1000
result = response.json()
collector.log_interaction(
session_id="sess_001",
user_id="user_123",
prompt="Tính BMI của tôi",
response=result['choices'][0]['message']['content'],
model_name="gpt-4.1",
latency_ms=latency,
tokens_used=result.get('usage', {}).get('total_tokens', 0)
)
Phase 2: Pipeline Fine-tuning Với HolySheep AI
Sau khi thu thập đủ dữ liệu (tối thiểu 100-500 examples), chúng ta tiến hành fine-tune. HolySheep hỗ trợ OpenAI-compatible fine-tuning API:
import os
import time
import json
from pathlib import Path
from typing import Optional, Dict, List
import requests
from datetime import datetime
class FineTunePipeline:
"""
Pipeline hoàn chỉnh cho AI Agent Fine-tuning
Sử dụng HolySheep AI endpoint - tương thích OpenAI format
"""
def __init__(
self,
api_key: str,
base_url: str = "https://api.holysheep.ai/v1",
organization_id: Optional[str] = None
):
self.api_key = api_key
self.base_url = base_url
self.organization_id = organization_id
self._session = requests.Session()
self._session.headers.update({
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json"
})
# Pricing reference (2026) - HolySheep AI
self.pricing = {
"gpt-4.1": {"input": 8.00, "output": 8.00, "currency": "USD"},
"claude-sonnet-4.5": {"input": 15.00, "output": 15.00, "currency": "USD"},
"gemini-2.5-flash": {"input": 2.50, "output": 2.50, "currency": "USD"},
"deepseek-v3.2": {"input": 0.42, "output": 0.42, "currency": "USD"},
# Fine-tune pricing
"gpt-4.1-finetune": {"training": 64.00, "input": 8.00, "output": 8.00}
}
def prepare_training_file(self, feedback_data: List[Dict], output_path: str) -> str:
"""
Chuẩn bị file training JSONL format
Input: List of {"messages": [{"role": "...", "content": "..."}]}
"""
output_file = Path(output_path)
output_file.parent.mkdir(parents=True, exist_ok=True)
with open(output_file, 'w', encoding='utf-8') as f:
for item in feedback_data:
f.write(json.dumps(item, ensure_ascii=False) + '\n')
# Upload lên HolySheep
with open(output_file, 'rb') as f:
files = {'file': (output_file.name, f, 'application/jsonl')}
response = self._session.post(
f"{self.base_url}/files",
files=files
)
if response.status_code != 200:
raise Exception(f"Upload failed: {response.text}")
file_id = response.json()['id']
print(f"✅ Training file uploaded: {file_id}")
return file_id
def create_fine_tune_job(
self,
training_file_id: str,
model: str = "gpt-4.1",
epochs: int = 3,
batch_size: int = 4,
learning_rate_multiplier: float = 2.0,
validation_file_id: Optional[str] = None
) -> str:
"""
Tạo fine-tune job
Returns: job_id để track tiến trình
"""
payload = {
"training_file": training_file_id,
"model": model,
"hyperparameters": {
"n_epochs": epochs,
"batch_size": batch_size,
"learning_rate_multiplier": learning_rate_multiplier
},
"suffix": "agent-v1"
}
if validation_file_id:
payload["validation_file"] = validation_file_id
response = self._session.post(
f"{self.base_url}/fine-tunes",
json=payload
)
if response.status_code != 200:
raise Exception(f"Fine-tune creation failed: {response.text}")
job = response.json()
print(f"🚀 Fine-tune job created: {job['id']}")
print(f" Model: {job['model']}")
print(f" Status: {job['status']}")
return job['id']
def estimate_cost(
self,
num_examples: int,
avg_tokens_per_example: int,
model: str = "gpt-4.1",
epochs: int = 3
) -> Dict:
"""
Ước tính chi phí fine-tune
"""
training_tokens = num_examples * avg_tokens_per_example * epochs
# Tính theo pricing HolySheep (đã bao gồm discount)
base_cost = self.pricing.get(model, {}).get("training", 64.00)
estimated_cost = (training_tokens / 1_000_000) * base_cost
return {
"training_tokens": training_tokens,
"epochs": epochs,
"estimated_cost_usd": round(estimated_cost, 2),
"vs_openai_savings": round(estimated_cost * 0.85, 2), # Tiết kiệm 85%
"cost_breakdown": {
"per_1m_tokens": base_cost,
"num_examples": num_examples,
"avg_tokens_each": avg_tokens_per_example
}
}
def monitor_job(self, job_id: str, poll_interval: int = 60) -> Dict:
"""
Monitor fine-tune job status
"""
start_time = time.time()
while True:
response = self._session.get(f"{self.base_url}/fine-tunes/{job_id}")
job = response.json()
elapsed = time.time() - start_time
print(f"[{elapsed:.0f}s] Status: {job['status']}")
if job['status'] == 'succeeded':
print(f"✅ Fine-tune completed!")
print(f" Model ID: {job.get('fine_tuned_model')}")
return job
elif job['status'] == 'failed':
print(f"❌ Fine-tune failed: {job.get('error', {}).get('message')}")
return job
time.sleep(poll_interval)
def deploy_model(self, fine_tuned_model: str) -> str:
"""
Deploy model để inference
"""
# Verify model available
response = self._session.get(f"{self.base_url}/models/{fine_tuned_model}")
if response.status_code == 200:
print(f"✅ Model {fine_tuned_model} ready for inference")
return fine_tuned_model
raise Exception(f"Model deployment failed: {response.text}")
=== FULL PIPELINE EXAMPLE ===
def run_full_pipeline():
"""Chạy toàn bộ pipeline từ A-Z"""
pipeline = FineTunePipeline(
api_key="YOUR_HOLYSHEEP_API_KEY",
base_url="https://api.holysheep.ai/v1"
)
# Step 1: Lấy training data từ collector
from feedback_collector import FeedbackCollector
collector = FeedbackCollector(api_key="YOUR_HOLYSHEEP_API_KEY")
training_data = collector.get_training_data(min_rating=4)
print(f"📊 Collected {len(training_data)} training examples")
# Step 2: Ước tính chi phí
cost_estimate = pipeline.estimate_cost(
num_examples=len(training_data),
avg_tokens_per_example=500,
model="gpt-4.1",
epochs=3
)
print(f"💰 Cost estimate: ${cost_estimate['estimated_cost_usd']}")
print(f" Savings vs OpenAI: ${cost_estimate['vs_openai_savings']}")
# Step 3: Prepare & Upload training file
training_file_id = pipeline.prepare_training_file(
feedback_data=training_data,
output_path="./training_data/agent_training.jsonl"
)
# Step 4: Create fine-tune job
job_id = pipeline.create_fine_tune_job(
training_file_id=training_file_id,
model="gpt-4.1",
epochs=3,
batch_size=4,
learning_rate_multiplier=2.0
)
# Step 5: Monitor & wait for completion
result = pipeline.monitor_job(job_id, poll_interval=30)
if result['status'] == 'succeeded':
fine_tuned_model = result['fine_tuned_model']
pipeline.deploy_model(fine_tuned_model)
print(f"🎉 Pipeline completed! Model: {fine_tuned_model}")
return fine_tuned_model
return None
if __name__ == "__main__":
run_full_pipeline()
Phase 3: Production Deployment và A/B Testing
Sau khi có fine-tuned model, cần implement deployment strategy với rollback plan:
import asyncio
import random
from typing import Optional, Callable
from dataclasses import dataclass
from datetime import datetime
import httpx
@dataclass
class DeploymentConfig:
"""Cấu hình deployment với traffic splitting"""
model_a: str # Current production model
model_b: str # New fine-tuned model
traffic_split: float = 0.1 # 10% traffic đi vào model B
rollback_threshold: float = 0.15 # Rollback nếu error rate > 15%
sample_size: int = 1000 # Minimum samples trước khi đánh giá
class ProductionDeployment:
"""
Deployment manager với automatic rollback
Sử dụng HolySheep AI endpoint cho inference
"""
def __init__(
self,
api_key: str,
base_url: str = "https://api.holysheep.ai/v1"
):
self.api_key = api_key
self.base_url = base_url
self.client = httpx.Client(
timeout=60.0,
headers={
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json"
}
)
# Metrics tracking
self.metrics = {
"model_a": {"requests": 0, "errors": 0, "latencies": []},
"model_b": {"requests": 0, "errors": 0, "latencies": []}
}
async def infer(
self,
prompt: str,
config: DeploymentConfig,
use_new_model: bool = None
) -> dict:
"""
Inference với traffic splitting
"""
# Determine which model to use
if use_new_model is None:
use_new_model = random.random() < config.traffic_split
model = config.model_b if use_new_model else config.model_a
model_key = "model_b" if use_new_model else "model_a"
start = datetime.now()
try:
response = self.client.post(
f"{self.base_url}/chat/completions",
json={
"model": model,
"messages": [{"role": "user", "content": prompt}],
"temperature": 0.7,
"max_tokens": 2000
}
)
latency = (datetime.now() - start).total_seconds() * 1000
if response.status_code == 200:
result = response.json()
self.metrics[model_key]["requests"] += 1
self.metrics[model_key]["latencies"].append(latency)
return {
"success": True,
"response": result['choices'][0]['message']['content'],
"model": model,
"latency_ms": latency,
"tokens": result.get('usage', {}).get('total_tokens', 0)
}
else:
raise Exception(f"API error: {response.status_code}")
except Exception as e:
self.metrics[model_key]["errors"] += 1
self.metrics[model_key]["requests"] += 1
# Fallback sang model A
if model_key == "model_b":
return await self._fallback_to_model_a(prompt)
raise
async def _fallback_to_model_a(self, prompt: str) -> dict:
"""Fallback mechanism"""
start = datetime.now()
response = self.client.post(
f"{self.base_url}/chat/completions",
json={
"model": "gpt-4.1", # Fallback base model
"messages": [{"role": "user", "content": prompt}]
}
)
latency = (datetime.now() - start).total_seconds() * 1000
result = response.json()
return {
"success": True,
"response": result['choices'][0]['message']['content'],
"model": "gpt-4.1-fallback",
"latency_ms": latency,
"fallback": True
}
def evaluate_deployment(self, config: DeploymentConfig) -> dict:
"""
Đánh giá deployment và quyết định rollback hay promote
"""
metrics_a = self.metrics["model_a"]
metrics_b = self.metrics["model_b"]
# Calculate metrics
error_rate_a = metrics_a["errors"] / max(metrics_a["requests"], 1)
error_rate_b = metrics_b["errors"] / max(metrics_b["requests"], 1)
avg_latency_a = sum(metrics_a["latencies"]) / max(len(metrics_a["latencies"]), 1)
avg_latency_b = sum(metrics_b["latencies"]) / max(len(metrics_b["latencies"]), 1)
total_requests = metrics_a["requests"] + metrics_b["requests"]
# Decision logic
decision = "continue"
reason = ""
if metrics_b["requests"] >= config.sample_size:
# Đủ sample để đánh giá
if error_rate_b > config.rollback_threshold:
decision = "rollback"
reason = f"Error rate {error_rate_b:.2%} exceeds threshold {config.rollback_threshold:.2%}"
elif error_rate_b < error_rate_a and avg_latency_b < avg_latency_a:
decision = "promote"
reason = f"Better metrics: error {error_rate_b:.2%} vs {error_rate_a:.2%}, latency {avg_latency_b:.0f}ms vs {avg_latency_a:.0f}ms"
return {
"decision": decision,
"reason": reason,
"metrics_a": {
"requests": metrics_a["requests"],
"error_rate": error_rate_a,
"avg_latency_ms": avg_latency_a
},
"metrics_b": {
"requests": metrics_b["requests"],
"error_rate": error_rate_b,
"avg_latency_ms": avg_latency_b
},
"total_requests": total_requests
}
def rollback(self, config: DeploymentConfig) -> bool:
"""
Thực hiện rollback - chuyển 100% traffic về model A
"""
config.traffic_split = 0.0
config.model_b = config.model_a # Stop using model B
print(f"🔄 Rolled back. Now using: {config.model_a} for 100% traffic")
return True
def promote(self, config: DeploymentConfig) -> bool:
"""
Promote model B lên production (100% traffic)
"""
config.traffic_split = 1.0
config.model_a = config.model_b
print(f"🚀 Promoted {config.model_b} to production")
return True
=== USAGE ===
async def main():
deployment = ProductionDeployment(
api_key="YOUR_HOLYSHEEP_API_KEY"
)
config = DeploymentConfig(
model_a="gpt-4.1",
model_b="gpt-4.1-agent-v1", # Fine-tuned model
traffic_split=0.1, # 10% thử nghiệm
rollback_threshold=0.15,
sample_size=1000
)
# Simulate production traffic
for i in range(100):
result = await deployment.infer(
prompt=f"Tìm kiếm sản phẩm #{i}",
config=config
)
print(f"Request {i}: {result['model']} - {result.get('latency_ms', 0):.0f}ms")
# Evaluate sau mỗi 50 requests
if (i + 1) % 50 == 0:
evaluation = deployment.evaluate_deployment(config)
print(f"\n📊 Evaluation: {evaluation['decision']}")
print(f" Reason: {evaluation['reason']}")
if evaluation['decision'] == 'rollback':
deployment.rollback(config)
elif evaluation['decision'] == 'promote':
deployment.promote(config)
if __name__ == "__main__":
asyncio.run(main())
So Sánh Chi Phí: HolySheep vs OpenAI Direct
| Model | OpenAI (USD/1M tok) | HolySheep (USD/1M tok) | Tiết kiệm |
|---|---|---|---|
| GPT-4.1 | $60 | $8 | 86.7% |
| Claude Sonnet 4.5 | $90 | $15 | 83.3% |
| Gemini 2.5 Flash | $15 | $2.50 | 83.3% |
| DeepSeek V3.2 | $2.80 | $0.42 | 85% |
ROI Calculator: Feedback Learning System
Dựa trên kinh nghiệm triển khai thực tế, đây là ROI calculation cho một hệ thống AI Agent vừa:
=== ROI CALCULATION EXAMPLE ===
Giả sử: 100,000 requests/tháng, 500 tokens/request
monthly_volume = 100_000 # requests
avg_tokens = 500 # tokens/request
monthly_tokens = monthly_volume * avg_tokens # 50M tokens
Chi phí OpenAI Direct
openai_cost = (monthly_tokens / 1_000_000) * 60 # $60/1M for GPT-4
print(f"OpenAI Monthly: ${openai_cost:,.2f}") # $3,000
Chi phí HolySheep AI
holysheep_cost = (monthly_tokens / 1_000_000) * 8 # $8/1M với 85% discount
print(f"HolySheep Monthly: ${holysheep_cost:,.2f}") # $400
Tiết kiệm hàng năm
annual_savings = (openai_cost - holysheep_cost) * 12
print(f"Annual Savings: ${annual_savings:,.2f}") # $31,200
Fine-tune cost (one-time)
fine_tune_examples = 1000
fine_tune_tokens = fine_tune_examples * 500 * 3 # 3 epochs
fine_tune_cost = (fine_tune_tokens / 1_000_000) * 64 # $64/1M training
print(f"One-time Fine-tune: ${fine_tune_cost:,.2f}") # ~$96
ROI với fine-tune
roi_months = fine_tune_cost / (openai_cost - holysheep_cost)
print(f"ROI Timeline: {roi_months:.2f} months") # Payback sau 0.3 tháng!
Performance gains (ước tính)
Sau fine-tune: +15% accuracy → giảm 20% retry → tiết kiệm thêm
retry_savings = monthly_volume * 0.20 * (avg_tokens / 1_000_000) * 8
print(f"Additional savings (fewer retries): ${retry_savings:,.2f}/month")
Lỗi Thường Gặp và Cách Khắc Phục
1. Lỗi: "Invalid API Key" hoặc Authentication Failed
❌ SAI - Cách dùng key không đúng
response = requests.get(
"https://api.holysheep.ai/v1/models",
headers={"api-key": "YOUR_KEY"} # Sai header name!
)
✅ ĐÚNG - Sử dụng Authorization Bearer header
response = requests.get(
"https://api.holysheep.ai/v1/models",
headers={"Authorization": "Bearer YOUR_HOLYSHEEP_API_KEY"}
)
Nguyên nhân: HolySheep AI dùng chuẩn OAuth 2.0 Bearer token, không phải API key header đơn giản. Đảm bảo prefix "Bearer " được thêm vào.
2. Lỗi: "Model Not Found" Khi Sử Dụng Fine-tuned Model
❌ SAI - Hardcode model name không đúng format
payload = {
"model": "my-fine-tuned-model", # Thiếu organization prefix
"messages": [...]
}
✅ ĐÚNG - Kiểm tra model name từ response khi create fine-tune
Response từ fine-tune job sẽ có format:
"fine_tuned_model": "ft-holysheep-gpt-4.1-agent-v1-xxxx"
Hoặc list available models trước
models_response = requests.get(
"https://api.holysheep.ai/v1/models",
headers={"Authorization": f"Bearer {api_key}"}
)
available_models = [m['id'] for m in models_response.json()['data']]
print(f"Available models: {available_models}")
Sử dụng model chính xác
payload = {
"model": "ft-holysheep-gpt-4.1-agent-v1-abc123",
"messages": [...]
}
Nguyên nhân: Fine-tuned models có ID riêng được gán khi job hoàn thành. Phải dùng đúng ID từ response.
3. Lỗi: Timeout Khi Fine-tuning Job Chạy Quá lâu
❌ SAI - Timeout quá ngắn hoặc không có retry logic
response = requests.post(
f"{BASE_URL}/fine-tunes",
json=payload,
timeout=30 # Timeout sau 30s - không đủ cho fine-tune!
)
✅ ĐÚNG - Async polling với exponential backoff
import time
def create_and_monitor_finetune(api_key, payload, max_retries=10):
"""Tạo job và monitor với retry logic"""
# Create job (thường nhanh, có thể set timeout ngắn)
response = requests.post(
f"{BASE_URL}/fine-tunes",
json=payload,
headers={"Authorization": f"Bearer {api_key}"}
)
job_id = response.json()['id']
# Monitor với polling (không timeout ở đây)
for attempt in range(max_retries):
status_response = requests.get(
f"{BASE_URL}/fine-tunes/{job_id}",
headers={"Authorization": f"Bearer {api_key}"}
)
job = status_response.json()
if job['status'] == 'succeeded':
return job['fine_tuned_model']
if job['status'] == 'failed':
raise Exception(f"Fine-tune failed: {job.get('error')}")
# Exponential backoff: 30s, 60s, 120s...
wait_time = min(30 * (2 ** attempt), 300)
print(f"Waiting {wait_time}s before retry {attempt + 1}/{max_ret