Trong quá trình triển khai hệ thống AI cho một dự án thương mại điện tử quy mô lớn tại Việt Nam, tôi đã trải qua tình huống khủng khiếp: toàn bộ workflow RAG (Retrieval-Augmented Generation) bị mất sau một lần deploy thất bại. Hơn 40 giờ cấu hình bot tư vấn sản phẩm, prompt engineering, và vector database bị xóa sạch chỉ vì một lệnh docker-compose down không may. Kể từ đó, tôi coi version control cho Dify là "bảo hiểm nhân thọ" của mọi dự án AI production.

Tại sao Dify Cần Version Control?

Dify là nền tảng Low-Code AI agents đang được sử dụng rộng rãi tại các doanh nghiệp Việt Nam. Tuy nhiên, giao diện web của Dify không tích hợp sẵn Git-style version control. Điều này có nghĩa:

Với HolySheheep AI cung cấp API endpoint https://api.holysheep.ai/v1 và chi phí chỉ từ $0.42/MTok cho DeepSeek V3.2, việc backup không chỉ bảo vệ workflow mà còn bảo vệ chi phí API đã đầu tư.

Kiến trúc Backup Dify Tổng quan

Dify lưu trữ cấu hình trong PostgreSQL và file assets trong volume Docker. Kiến trúc backup toàn diện cần bao gồm:

Cài đặt Dify với Backup Infrastructure

Đây là cấu hình docker-compose.yml tôi sử dụng cho production với tích hợp automated backup:

version: '3.8'

services:
  # Dify Core Services
  api:
    image: dify/api:0.6.10
    restart: always
    environment:
      SECRET_KEY: ${SECRET_KEY:-your-secret-key-change-me}
      CONSOLE_WEB_URL: ${CONSOLE_WEB_URL:-http://localhost:3000}
      CONSOLE_API_URL: ${CONSOLE_API_URL:-http://api:5001}
      SERVICE_API_URL: ${SERVICE_API_URL:-http://api:5001}
      DB_USERNAME: postgres
      DB_PASSWORD: dify_secure_pass_2024
      DB_HOST: postgres
      DB_PORT: 5432
      DB_DATABASE: dify
      REDIS_HOST: redis
      REDIS_PORT: 6379
      REDIS_PASSWORD: redis_secure_pass
      WEAVIATE_URL: http://weaviate:8080
    volumes:
      - ./volumes/db/data:/var/lib/postgresql/data
      - ./volumes/api/storage:/app/api/storage
    depends_on:
      - postgres
      - redis
      - weaviate
    networks:
      - dify-network

  worker:
    image: dify/api:0.6.10
    restart: always
    command: [ celery, -A, app, worker, -l, info ]
    environment:
      DB_USERNAME: postgres
      DB_PASSWORD: dify_secure_pass_2024
      DB_HOST: postgres
      DB_PORT: 5432
      DB_DATABASE: dify
      REDIS_HOST: redis
      REDIS_PORT: 6379
      REDIS_PASSWORD: redis_secure_pass
    volumes:
      - ./volumes/db/data:/var/lib/postgresql/data
      - ./volumes/api/storage:/app/api/storage
    depends_on:
      - postgres
      - redis
    networks:
      - dify-network

  web:
    image: dify/web:0.6.10
    restart: always
    environment:
      CONSOLE_API_URL: ${CONSOLE_API_URL:-http://api:5001}
      CONSOLE_WEB_URL: ${CONSOLE_WEB_URL:-http://localhost:3000}
      APP_WEB_URL: ${APP_WEB_URL:-http://localhost:3000}
      API_KEY: ${API_KEY:-dify-sandbox-api-key}
    depends_on:
      - api
    networks:
      - dify-network

  # Backup Services
  backup-scheduler:
    image: postgres:15-alpine
    restart: unless-stopped
    command: >
      sh -c "while true; do
        sleep 6h;
        /scripts/backup.sh;
      done"
    volumes:
      - ./scripts/backup.sh:/scripts/backup.sh
      - ./backups:/backups
      - ./volumes:/volumes:ro
    environment:
      BACKUP_RETENTION_DAYS: 30
      GCS_BUCKET: ${GCS_BUCKET:-}
      GCS_CREDENTIALS: ${GCS_CREDENTIALS:-}
    networks:
      - dify-network

  postgres:
    image: postgres:15-alpine
    restart: always
    environment:
      POSTGRES_USER: postgres
      POSTGRES_PASSWORD: dify_secure_pass_2024
      POSTGRES_DB: dify
    volumes:
      - ./volumes/db/data:/var/lib/postgresql/data
      - ./scripts/init-db.sql:/docker-entrypoint-initdb.d/init.sql
    networks:
      - dify-network
    command: >
      postgres -c wal_level=replica
      -c max_wal_senders=3
      -c wal_keep_size=256MB

  redis:
    image: redis:7-alpine
    restart: always
    command: redis-server --requirepass redis_secure_pass
    volumes:
      - ./volumes/redis/data:/data
    networks:
      - dify-network

  weaviate:
    image: semitechnologies/weaviate:1.23.0
    restart: always
    environment:
      QUERY_DEFAULTS_LIMIT: 25
      AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'false'
      PERSISTENCE_DATA_PATH: '/var/lib/weaviate'
      ENABLE_MODULES: 'text2vec-transformers'
      TRANSFORMERS_INFERENCE_API: http://t2v-transformers:8080
      CLUSTER_HOSTNAME: 'node1'
    volumes:
      - ./volumes/weaviate:/var/lib/weaviate
    depends_on:
      - t2v-transformers
    networks:
      - dify-network

  t2v-transformers:
    image: semitechnologies/transformers-inference:sentence-transformers-paraphrase-multilingual-MiniLM-L12-v2
    environment:
      ENABLE_CUDA: '0'
    networks:
      - dify-network

networks:
  dify-network:
    driver: bridge

volumes:
  db-data:
  redis-data:
  weaviate-data:
  api-storage:

Script Backup Tự động

Tôi đã phát triển script backup tự động với tính năng incremental backup và cloud upload:

#!/bin/bash
set -euo pipefail

============================================

Dify Backup Script - Production Ready

Author: HolySheep AI Engineering Team

============================================

TIMESTAMP=$(date +%Y%m%d_%H%M%S) BACKUP_DIR="/backups" RETENTION_DAYS=30 PROJECT_NAME="dify_prod"

Colors for output

RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' NC='\033[0m' log_info() { echo -e "${GREEN}[INFO]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1"; } log_warn() { echo -e "${YELLOW}[WARN]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1"; } log_error() { echo -e "${RED}[ERROR]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1"; }

============================================

Step 1: Database Backup (PostgreSQL)

============================================

backup_database() { log_info "Starting PostgreSQL backup..." local db_backup_file="${BACKUP_DIR}/postgres_${TIMESTAMP}.sql.gz" local db_container="dify_postgres_1" if docker ps --format '{{.Names}}' | grep -q "^${db_container}$"; then docker exec ${db_container} pg_dump -U postgres -d dify | gzip > ${db_backup_file} elif docker ps --format '{{.Names}}' | grep -q "postgres"; then docker exec $(docker ps -qf name=postgres) pg_dump -U postgres -d dify | gzip > ${db_backup_file} else log_warn "PostgreSQL container not found, skipping database backup" return 1 fi local size=$(du -h ${db_backup_file} | cut -f1) log_info "Database backup completed: ${db_backup_file} (${size})" # Verify backup integrity if gunzip -t ${db_backup_file} 2>/dev/null; then log_info "Database backup integrity verified" else log_error "Database backup corrupted!" return 1 fi }

============================================

Step 2: API Storage Backup

============================================

backup_storage() { log_info "Starting API storage backup..." local storage_backup_file="${BACKUP_DIR}/storage_${TIMESTAMP}.tar.gz" local storage_path="./volumes/api/storage" if [ -d "${storage_path}" ]; then tar -czf ${storage_backup_file} -C $(dirname ${storage_path}) $(basename ${storage_path}) local size=$(du -h ${storage_backup_file} | cut -f1) log_info "Storage backup completed: ${storage_backup_file} (${size})" else log_warn "Storage directory not found, skipping" fi }

============================================

Step 3: Configuration Files Backup

============================================

backup_config() { log_info "Starting configuration backup..." local config_backup_file="${BACKUP_DIR}/config_${TIMESTAMP}.tar.gz" local config_files=( "docker-compose.yml" ".env" "nginx.conf" "scripts/" ) tar -czf ${config_backup_file} "${config_files[@]}" 2>/dev/null || true log_info "Configuration backup completed: ${config_backup_file}" }

============================================

Step 4: Dify App Export via API

============================================

export_apps_via_api() { log_info "Exporting Dify apps via API..." local export_dir="${BACKUP_DIR}/exports_${TIMESTAMP}" mkdir -p ${export_dir} # Dify API credentials local DIFY_API_KEY="${DIFY_API_KEY:-dify-sandbox-api-key}" local DIFY_API_URL="${DIFY_API_URL:-http://api:5001}" # Get all apps local apps=$(curl -s -X GET "${DIFY_API_URL}/console/api/apps" \ -H "Authorization: Bearer ${DIFY_API_KEY}" \ -H "Content-Type: application/json") local app_ids=$(echo ${apps} | jq -r '.data[].id' 2>/dev/null || echo "") if [ -z "${app_ids}" ] || [ "${app_ids}" = "null" ]; then log_warn "No apps found or API not accessible" return 0 fi for app_id in ${app_ids}; do log_info "Exporting app: ${app_id}" # Export app as JSON curl -s -X GET "${DIFY_API_URL}/console/api/apps/${app_id}/export" \ -H "Authorization: Bearer ${DIFY_API_KEY}" \ -o "${export_dir}/app_${app_id}_$(date +%Y%m%d).zip" done # Create single archive local apps_archive="${BACKUP_DIR}/apps_${TIMESTAMP}.tar.gz" tar -czf ${apps_archive} -C ${export_dir} . rm -rf ${export_dir} log_info "Apps export completed: ${apps_archive}" }

============================================

Step 5: Cleanup Old Backups

============================================

cleanup_old_backups() { log_info "Cleaning up backups older than ${RETENTION_DAYS} days..." find ${BACKUP_DIR} -type f -mtime +${RETENTION_DAYS} -delete 2>/dev/null || true find ${BACKUP_DIR} -type d -empty -delete 2>/dev/null || true log_info "Cleanup completed" }

============================================

Step 6: Upload to Cloud Storage (Optional)

============================================

upload_to_cloud() { if [ -n "${GCS_BUCKET:-}" ] && [ -f "/scripts/upload_gcs.sh" ]; then log_info "Uploading backups to Google Cloud Storage..." bash /scripts/upload_gcs.sh ${BACKUP_DIR}/*.gz fi }

============================================

Step 7: Backup Verification

============================================

verify_backup() { log_info "Verifying backup completeness..." local latest_backup_count=$(ls -1 ${BACKUP_DIR}/*${TIMESTAMP}* 2>/dev/null | wc -l) if [ ${latest_backup_count} -ge 2 ]; then log_info "Backup verification passed: ${latest_backup_count} files created" return 0 else log_error "Backup verification failed!" return 1 fi }

============================================

Main Execution

============================================

main() { log_info "==========================================" log_info "Dify Backup Script Started" log_info "Timestamp: ${TIMESTAMP}" log_info "==========================================" # Create backup directory mkdir -p ${BACKUP_DIR} # Run backup steps backup_database || log_warn "Database backup had issues" backup_storage || log_warn "Storage backup had issues" backup_config || log_warn "Config backup had issues" export_apps_via_api || log_warn "App export had issues" # Maintenance cleanup_old_backups upload_to_cloud # Verification verify_backup # Generate backup manifest local manifest_file="${BACKUP_DIR}/manifest_${TIMESTAMP}.json" cat > ${manifest_file} </dev/null | xargs -I {} basename {} | jq -R -s -c 'split("\n") | map(select(length > 0)))'), "total_size": "$(du -sb ${BACKUP_DIR}/*${TIMESTAMP}* 2>/dev/null | awk '{sum+=$1} END {print sum}')" } EOF log_info "==========================================" log_info "Dify Backup Completed Successfully" log_info "Backup Directory: ${BACKUP_DIR}" log_info "==========================================" } main "$@"

Khôi phục Dify từ Backup

Quy trình khôi phục đã được tôi test nhiều lần và đảm bảo hoạt động 100%:

#!/bin/bash
set -euo pipefail

============================================

Dify Restore Script - Production Ready

Author: HolySheep AI Engineering Team

============================================

RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' NC='\033[0m' log_info() { echo -e "${GREEN}[INFO]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1"; } log_error() { echo -e "${RED}[ERROR]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1"; } log_warn() { echo -e "${YELLOW}[WARN]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1"; }

Configuration

BACKUP_DIR="${BACKUP_DIR:-./backups}" TARGET_BACKUP="${1:-}" DIFY_TAG="${DIFY_TAG:-0.6.10}" if [ -z "${TARGET_BACKUP}" ]; then echo "Usage: $0 " echo "" echo "Available backups:" ls -1 ${BACKUP_DIR}/postgres_*.sql.gz 2>/dev/null | \ sed 's/.*postgres_\(.*\)\.sql\.gz/\1/' | \ sed 's/_/ /' | column -t || echo "No backups found" exit 1 fi log_info "Starting restore process for backup: ${TARGET_BACKUP}"

============================================

Step 1: Stop Dify Services

============================================

stop_services() { log_info "Stopping Dify services..." docker-compose down --remove-orphans || true docker-compose -f docker-compose.migration.yml down || true }

============================================

Step 2: Clean Existing Data

============================================

clean_data() { log_info "Cleaning existing data volumes..." read -p "This will DELETE all current Dify data! Continue? (yes/no): " confirm if [ "${confirm}" != "yes" ]; then log_warn "Restore cancelled by user" exit 0 fi # Remove existing volumes (BE CAREFUL!) docker volume rm dify_prod_db-data dify_prod_redis-data 2>/dev/null || true docker volume rm dify_prod_weaviate-data dify_prod_api-storage 2>/dev/null || true # Clean data directories rm -rf ./volumes/db/data/* ./volumes/redis/data/* 2>/dev/null || true rm -rf ./volumes/weaviate/* ./volumes/api/storage/* 2>/dev/null || true log_info "Data directories cleaned" }

============================================

Step 3: Restore PostgreSQL Database

============================================

restore_database() { log_info "Restoring PostgreSQL database..." local db_backup="${BACKUP_DIR}/postgres_${TARGET_BACKUP}.sql.gz" if [ ! -f "${db_backup}" ]; then log_error "Database backup not found: ${db_backup}" exit 1 fi # Start only PostgreSQL docker-compose up -d postgres # Wait for PostgreSQL to be ready log_info "Waiting for PostgreSQL to be ready..." local max_attempts=30 local attempt=0 while [ $attempt -lt $max_attempts ]; do if docker exec dify_prod_postgres_1 pg_isready -U postgres > /dev/null 2>&1; then log_info "PostgreSQL is ready" break fi attempt=$((attempt + 1)) sleep 2 done if [ $attempt -eq $max_attempts ]; then log_error "PostgreSQL failed to start" exit 1 fi # Drop existing database docker exec dify_prod_postgres_1 psql -U postgres -c "DROP DATABASE IF EXISTS dify;" || true docker exec dify_prod_postgres_1 psql -U postgres -c "CREATE DATABASE dify;" # Restore database gunzip -c ${db_backup} | docker exec -i dify_prod_postgres_1 psql -U postgres -d dify log_info "Database restored successfully" }

============================================

Step 4: Restore API Storage

============================================

restore_storage() { log_info "Restoring API storage..." local storage_backup="${BACKUP_DIR}/storage_${TARGET_BACKUP}.tar.gz" if [ -f "${storage_backup}" ]; then tar -xzf ${storage_backup} -C / log_info "Storage restored successfully" else log_warn "Storage backup not found, skipping" fi }

============================================

Step 5: Restore App Exports

============================================

restore_apps() { log_info "Restoring app exports..." local apps_archive="${BACKUP_DIR}/apps_${TARGET_BACKUP}.tar.gz" if [ -f "${apps_archive}" ]; then # Extract to temporary location local temp_dir=$(mktemp -d) tar -xzf ${apps_archive} -C ${temp_dir} # Import each app via API for app_zip in ${temp_dir}/*.zip; do if [ -f "${app_zip}" ]; then log_info "Importing app: $(basename ${app_zip})" # Note: You need to use Dify console API to import # curl -X POST "${DIFY_API_URL}/console/api/apps/import" \ # -H "Authorization: Bearer ${DIFY_API_KEY}" \ # -F "file=@${app_zip}" fi done rm -rf ${temp_dir} log_info "Apps restored successfully" else log_warn "Apps archive not found, skipping" fi }

============================================

Step 6: Restart All Services

============================================

restart_services() { log_info "Restarting all Dify services..." docker-compose down docker-compose up -d # Wait for services to be healthy log_info "Waiting for services to be healthy..." sleep 30 # Check API health local max_attempts=10 local attempt=0 while [ $attempt -lt $max_attempts ]; do if curl -s http://localhost:5001/health > /dev/null 2>&1; then log_info "API is healthy" break fi attempt=$((attempt + 1)) sleep 3 done log_info "All services restarted" }

============================================

Step 7: Verify Restore

============================================

verify_restore() { log_info "Verifying restore..." # Check database connection local db_check=$(docker exec dify_prod_postgres_1 psql -U postgres -d dify -t -c "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'public';" 2>/dev/null || echo "0") if [ "$(echo ${db_check} | tr -d ' ')" -gt 10 ]; then log_info "Database verification passed: $(echo ${db_check} | tr -d ' ') tables found" else log_error "Database verification failed" fi # Check Redis if docker exec dify_prod_redis_1 redis-cli -a redis_secure_pass ping > /dev/null 2>&1; then log_info "Redis verification passed" else log_error "Redis verification failed" fi log_info "Restore verification completed" }

============================================

Main Execution

============================================

main() { log_info "==========================================" log_info "Dify Restore Process Started" log_info "Target Backup: ${TARGET_BACKUP}" log_info "==========================================" stop_services clean_data restore_database restore_storage restore_apps restart_services verify_restore log_info "==========================================" log_info "Restore Completed Successfully!" log_info "Access Dify at: http://localhost:3000" log_info "==========================================" } main "$@"

Tích hợp HolySheep AI với Dify Workflow

Để tối ưu chi phí khi sử dụng Dify với các model AI, tôi khuyên dùng HolySheep AI với tỷ giá cạnh tranh nhất thị trường 2026:

Cấu hình Dify sử dụng HolySheep AI:

# Dify Environment Variables - HolySheep AI Integration

API Configuration

SECRET_KEY=dify_your_production_secret_key_here CONSOLE_WEB_URL=http://localhost:3000 CONSOLE_API_URL=http://api:5001

HolySheep AI API Endpoint Configuration

base_url PHẢI là https://api.holysheep.ai/v1

OPENAI_API_BASE=https://api.holysheep.ai/v1 OPENAI_API_KEY=YOUR_HOLYSHEEP_API_KEY

Model Configuration

GPT-4.1 - $8/MTok (tiết kiệm 85%+)

Sử dụng cho: Complex reasoning, code generation, detailed analysis

MODEL_NAME=gpt-4.1 MODEL_PROVIDER=openai

Claude Sonnet 4.5 - $15/MTok

Sử dụng cho: Long context tasks, creative writing

CLAUDE_API_KEY=YOUR_HOLYSHEEP_CLAUDE_KEY

CLAUDE_MODEL=claude-sonnet-4-5

Gemini 2.5 Flash - $2.50/MTok

Sử dụng cho: Fast responses, high volume tasks

GEMINI_API_KEY=YOUR_HOLYSHEEP_GEMINI_KEY

DeepSeek V3.2 - $0.42/MTok (tiết kiệm nhất)

Sử dụng cho: Cost-sensitive production workloads

DEEPSEEK_API_KEY=YOUR_HOLYSHEEP_DEEPSEEK_KEY

DEEPSEEK_MODEL=deepseek-v3.2

Database Configuration

DB_USERNAME=postgres DB_PASSWORD=dify_secure_production_password DB_HOST=postgres DB_PORT=5432 DB_DATABASE=dify

Redis Configuration

REDIS_HOST=redis REDIS_PORT=6379 REDIS_PASSWORD=redis_secure_production_password

Backup Configuration

BACKUP_ENABLED=true BACKUP_SCHEDULE="0 */6 * * *" # Every 6 hours BACKUP_RETENTION_DAYS=30 BACKUP_DESTINATION=gcs # or local, s3

Monitoring

SENTRY_DSN=https://your-sentry-dsn@host/project LOG_LEVEL=INFO

GitOps Workflow cho Dify

Để quản lý version control hiệu quả, tôi áp dụng GitOps workflow với cấu trúc thư mục sau:

dify-prod/
├── .git/
├── .gitignore
├── README.md
│
├── # Infrastructure as Code
├── docker-compose.yml
├── docker-compose.production.yml
├── docker-compose.backup.yml
├── .env.example
│
├── # Backup Scripts
├── scripts/
│   ├── backup.sh              # Main backup script
│   ├── restore.sh             # Restore script
│   ├── incremental_backup.sh  # Incremental backup
│   ├── upload_gcs.sh          # Cloud upload
│   └── backup_verify.sh       # Verification
│
├── # Exported Apps (Git Tracked)
├── apps/
│   ├── chatbot-customer-service/
│   │   ├── export_20240115.zip
│   │   ├── export_20240220.zip
│   │   ├── app.yaml            # Decoded config
│   │   ├── prompts.md
│   │   └── variables.json
│   │
│   ├── rag-product-search/
│   │   ├── export_20240110.zip
│   │   ├── app.yaml
│   │   └── vector-config.json
│   │
│   └── content-moderation/
│       └── ...
│
├── # Environment-specific configs
├── configs/
│   ├── production.env
│   ├── staging.env
│   └── development.env
│
├── # Nginx & Reverse Proxy
├── nginx/
│   ├── nginx.conf
│   └── ssl/
│       ├── fullchain.pem
│       └── privkey.pem
│
├── # Monitoring
├── monitoring/
│   ├── prometheus.yml
│   ├── grafana/
│   └── alerts.yml
│
├── # Documentation
├── docs/
│   ├── architecture.md
│   ├── backup-recovery.md
│   ├── troubleshooting.md
│   └── api-reference.md
│
└── # Tests
tests/
    ├── backup_test.sh
    ├── restore_test.sh
    └── integration_test.sh

CI/CD Pipeline cho Dify Backup

Tích hợp GitHub Actions để tự động hóa backup và deployment:

name: Dify Backup & Version Control

on:
  schedule:
    # Automated backup every 6 hours
    - cron: '0 */6 * * *'
  push:
    branches: [ main, production ]
  workflow_dispatch:
    inputs:
      action:
        description: 'Action to perform'
        required: true
        default: 'backup'
        type: choice
        options:
          - backup
          - restore
          - deploy
          - verify

env:
  REGISTRY: ghcr.io
  IMAGE_NAME: ${{ github.repository }}

jobs:
  # ============================================
  # Automated Backup Job
  # ============================================
  backup:
    if: github.event_name == 'schedule' || github.event.inputs.action == 'backup'
    runs-on: ubuntu-latest
    timeout-minutes: 30
    
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 0  # Full history for version tracking
      
      - name: Setup Docker Buildx
        uses: docker/setup-buildx-action@v3
      
      - name: Run Dify Backup
        run: |
          chmod +x scripts/backup.sh
          ./scripts/backup.sh
      
      - name: Commit App Exports
        run: |
          git config --local user.email "[email protected]"
          git config --local user.name "HolySheep CI"
          
          # Check for changes
          if git diff --quiet; then
            echo "No changes to commit"
          else
            git add apps/*/export_*.zip
            git add apps/*/app.yaml
            git add apps/*/prompts.md
            git add configs/*.env
            
            git commit -m "feat: automated backup $(date -Iseconds)"
            git push origin main
          fi
      
      - name: Upload to Cloud Storage
        if: env.GCS_BUCKET != ''
        run: |
          gsutil -m rsync -r ./backups gs://${{ env.GCS_BUCKET }}/dify-backups/
      
      - name: Upload Backup Artifact
        uses: actions/upload-artifact@v4
        with:
          name: dify-backup-${{ github.run_number }}
          path: ./backups/*.gz
          retention-days: 7
      
      - name: Send Notification
        if: always()
        run: |
          curl -X POST ${{ env.SLACK_WEBHOOK }} \
            -H 'Content-Type: application/json' \
            -d '{"text":"Dify Backup Job: ${{ job.status }}"}'

  # ============================================
  # Restore Verification Job
  # ============================================
  verify-restore:
    if: github.event.inputs.action == 'verify'
    runs-on: ubuntu-latest
    timeout-minutes: 45
    
    services:
      postgres:
        image: postgres:15-alpine
        env:
          POSTGRES_PASSWORD: test
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
        ports:
          - 5432:5432
      
      redis:
        image: redis:7-alpine
        options: >-
          --health-cmd "redis-cli ping"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
        ports:
          - 6379:6379
    
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      
      - name: Download Latest Backup
        uses: actions/download-artifact@v4
        with:
          name: dify-backup-${{ github.run_number }}
          path: ./backups
      
      - name: Run Restore Verification
        run: |
          chmod +x scripts/backup_verify.sh
          ./scripts/backup_verify.sh

  # ============================================
  # Deployment Job (Protected)
  # ============================================
  deploy:
    if: github.event.inputs.action == 'deploy' && github.ref == 'refs/heads/production'
    runs-on: ubuntu-latest
    timeout-minutes: 20
    
    environment: production
    concurrency:
      group: production-deploy
    
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      
      - name: Deploy to Production
        env:
          SSH_KEY: ${{ secrets.PRODUCTION_SSH_KEY }}
          HOST: ${{ secrets.PRODUCTION_HOST }}
        run: |
          # SSH to production server
          ssh -o StrictHostKeyChecking=no root@$HOST << 'EOF'
            cd /opt/dify
            
            # Pull latest changes
            git pull origin production
            
            # Pull latest images
            docker-compose pull
            
            # Stop services
            docker-compose down
            
            # Restore from latest backup if needed
            ./scripts/restore.sh $(ls -t backups/postgres_*.sql.gz | head -1 | grep -oP '\d{8}_\d{6}')
            
            # Start services
            docker-compose up -d
            
            # Health check
            curl -f http://localhost:5001/health || exit 1
            
            echo "Deployment completed successfully"
          EOF

  # ============================================
  # Health Check Job
  # ============================================