WHMCS Backup Security: Protecting Your Customer Data

#!/bin/bash
# WHMCS Secure Backup System
# File: /usr/local/bin/whmcs-secure-backup.sh
# Implements AES-256 encryption, integrity checks, and secure transmission

# Configuration
WHMCS_PATH="/var/www/html/whmcs"
BACKUP_DIR="/backup/whmcs"
OFFSITE_SERVER="backup.yourcompany.com"
OFFSITE_USER="backup-user"
ENCRYPTION_KEY_FILE="/etc/whmcs-backup/encryption.key"
LOG_FILE="/var/log/whmcs-backup.log"
RETENTION_DAYS=90
GPG_RECIPIENT="[email protected]"

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Logging function
log_message() {
    local level="$1"
    local message="$2"
    echo "$(date '+%Y-%m-%d %H:%M:%S') [$level] $message" >> "$LOG_FILE"
    
    case $level in
        "INFO")  echo -e "${BLUE}[INFO]${NC} $message" ;;
        "WARN")  echo -e "${YELLOW}[WARN]${NC} $message" ;;
        "ERROR") echo -e "${RED}[ERROR]${NC} $message" ;;
        "SUCCESS") echo -e "${GREEN}[SUCCESS]${NC} $message" ;;
    esac
}

# Generate or load encryption key
setup_encryption() {
    if [[ ! -f "$ENCRYPTION_KEY_FILE" ]]; then
        log_message "INFO" "Generating new encryption key"
        mkdir -p "$(dirname "$ENCRYPTION_KEY_FILE")"
        
        # Generate 256-bit key
        openssl rand -base64 32 > "$ENCRYPTION_KEY_FILE"
        chmod 600 "$ENCRYPTION_KEY_FILE"
        chown root:root "$ENCRYPTION_KEY_FILE"
        
        log_message "SUCCESS" "Encryption key generated"
    fi
}

# Database backup with encryption
backup_database() {
    local timestamp="$1"
    local backup_file="$BACKUP_DIR/database/whmcs-db-${timestamp}.sql"
    local encrypted_file="${backup_file}.enc"
    
    log_message "INFO" "Starting database backup"
    
    # Create directory
    mkdir -p "$(dirname "$backup_file")"
    
    # Get database credentials from WHMCS configuration
    local db_host=$(grep 'db_host' "$WHMCS_PATH/configuration.php" | cut -d"'" -f4)
    local db_name=$(grep 'db_name' "$WHMCS_PATH/configuration.php" | cut -d"'" -f4)
    local db_username=$(grep 'db_username' "$WHMCS_PATH/configuration.php" | cut -d"'" -f4)
    local db_password=$(grep 'db_password' "$WHMCS_PATH/configuration.php" | cut -d"'" -f4)
    
    # Create MySQL dump with compression
    mysqldump --host="$db_host" \
              --user="$db_username" \
              --password="$db_password" \
              --single-transaction \
              --routines \
              --triggers \
              --add-drop-table \
              --disable-keys \
              --extended-insert \
              --quick \
              --lock-tables=false \
              "$db_name" | gzip > "${backup_file}.gz"
    
    if [[ $? -ne 0 ]]; then
        log_message "ERROR" "Database backup failed"
        return 1
    fi
    
    # Encrypt backup
    openssl enc -aes-256-cbc \
                -salt \
                -in "${backup_file}.gz" \
                -out "$encrypted_file" \
                -pass file:"$ENCRYPTION_KEY_FILE"
    
    if [[ $? -ne 0 ]]; then
        log_message "ERROR" "Database encryption failed"
        return 1
    fi
    
    # Generate integrity hash
    sha256sum "$encrypted_file" > "${encrypted_file}.sha256"
    
    # Clean up unencrypted file
    rm -f "${backup_file}.gz"
    
    log_message "SUCCESS" "Database backup completed: $encrypted_file"
    echo "$encrypted_file"
}

# File system backup with encryption
backup_files() {
    local timestamp="$1"
    local backup_file="$BACKUP_DIR/files/whmcs-files-${timestamp}.tar.gz"
    local encrypted_file="${backup_file}.enc"
    
    log_message "INFO" "Starting file system backup"
    
    # Create directory
    mkdir -p "$(dirname "$backup_file")"
    
    # Create compressed archive excluding sensitive files
    tar -czf "$backup_file" \
        --exclude="$WHMCS_PATH/downloads/cache/*" \
        --exclude="$WHMCS_PATH/templates_c/*" \
        --exclude="$WHMCS_PATH/attachments/downloads/*" \
        --exclude="*.log" \
        --exclude="*.tmp" \
        -C "$(dirname "$WHMCS_PATH")" \
        "$(basename "$WHMCS_PATH")"
    
    if [[ $? -ne 0 ]]; then
        log_message "ERROR" "File system backup failed"
        return 1
    fi
    
    # Encrypt backup
    openssl enc -aes-256-cbc \
                -salt \
                -in "$backup_file" \
                -out "$encrypted_file" \
                -pass file:"$ENCRYPTION_KEY_FILE"
    
    if [[ $? -ne 0 ]]; then
        log_message "ERROR" "File encryption failed"
        return 1
    fi
    
    # Generate integrity hash
    sha256sum "$encrypted_file" > "${encrypted_file}.sha256"
    
    # Clean up unencrypted file
    rm -f "$backup_file"
    
    log_message "SUCCESS" "File system backup completed: $encrypted_file"
    echo "$encrypted_file"
}

# Configuration backup
backup_configuration() {
    local timestamp="$1"
    local config_backup="$BACKUP_DIR/config/whmcs-config-${timestamp}.tar.gz.enc"
    
    log_message "INFO" "Starting configuration backup"
    
    mkdir -p "$(dirname "$config_backup")"
    
    # Backup configuration files
    tar -czf - \
        "$WHMCS_PATH/configuration.php" \
        "$WHMCS_PATH/includes/hooks/" \
        "$WHMCS_PATH/modules/servers/" \
        "$WHMCS_PATH/modules/gateways/" \
        "$WHMCS_PATH/modules/addons/" \
        "$WHMCS_PATH/modules/reports/" 2>/dev/null | \
    openssl enc -aes-256-cbc \
                -salt \
                -out "$config_backup" \
                -pass file:"$ENCRYPTION_KEY_FILE"
    
    # Generate integrity hash
    sha256sum "$config_backup" > "${config_backup}.sha256"
    
    log_message "SUCCESS" "Configuration backup completed: $config_backup"
    echo "$config_backup"
}

# Secure offsite transfer
transfer_to_offsite() {
    local file="$1"
    local remote_path="/secure-backups/$(basename "$file")"
    
    log_message "INFO" "Transferring to offsite: $(basename "$file")"
    
    # Use rsync over SSH with compression
    rsync -avz \
          --progress \
          --partial \
          --compress \
          --timeout=3600 \
          -e "ssh -o StrictHostKeyChecking=no -i /root/.ssh/backup_key" \
          "$file" \
          "$file.sha256" \
          "${OFFSITE_USER}@${OFFSITE_SERVER}:${remote_path%/*}/"
    
    if [[ $? -eq 0 ]]; then
        log_message "SUCCESS" "Offsite transfer completed for $(basename "$file")"
        return 0
    else
        log_message "ERROR" "Offsite transfer failed for $(basename "$file")"
        return 1
    fi
}

# Verify backup integrity
verify_backup_integrity() {
    local file="$1"
    local hash_file="${file}.sha256"
    
    if [[ ! -f "$hash_file" ]]; then
        log_message "ERROR" "Hash file not found: $hash_file"
        return 1
    fi
    
    # Verify hash
    local current_hash=$(sha256sum "$file" | cut -d' ' -f1)
    local stored_hash=$(cut -d' ' -f1 "$hash_file")
    
    if [[ "$current_hash" == "$stored_hash" ]]; then
        log_message "SUCCESS" "Integrity check passed for $(basename "$file")"
        return 0
    else
        log_message "ERROR" "Integrity check failed for $(basename "$file")"
        return 1
    fi
}

# Test backup decryption
test_backup_decryption() {
    local encrypted_file="$1"
    local test_dir="/tmp/backup-test-$$"
    
    log_message "INFO" "Testing backup decryption for $(basename "$encrypted_file")"
    
    mkdir -p "$test_dir"
    
    # Test decryption
    if openssl enc -d -aes-256-cbc \
                  -in "$encrypted_file" \
                  -pass file:"$ENCRYPTION_KEY_FILE" | \
       head -c 100 > "$test_dir/test" 2>/dev/null; then
        
        if [[ -s "$test_dir/test" ]]; then
            log_message "SUCCESS" "Decryption test passed for $(basename "$encrypted_file")"
            rm -rf "$test_dir"
            return 0
        fi
    fi
    
    log_message "ERROR" "Decryption test failed for $(basename "$encrypted_file")"
    rm -rf "$test_dir"
    return 1
}

# Cleanup old backups
cleanup_old_backups() {
    log_message "INFO" "Cleaning up backups older than $RETENTION_DAYS days"
    
    # Local cleanup
    find "$BACKUP_DIR" -type f -mtime +$RETENTION_DAYS -delete
    
    # Remote cleanup
    ssh -i /root/.ssh/backup_key "${OFFSITE_USER}@${OFFSITE_SERVER}" \
        "find /secure-backups -type f -mtime +$RETENTION_DAYS -delete" 2>/dev/null
    
    log_message "SUCCESS" "Cleanup completed"
}

# Send notification
send_notification() {
    local status="$1"
    local message="$2"
    
    if [[ "$status" == "SUCCESS" ]]; then
        local subject=" WHMCS Backup Completed Successfully"
    else
        local subject=" WHMCS Backup Failed"
    fi
    
    # Email notification
    echo "$message" | mail -s "$subject" [email protected]
    
    # Slack notification (if webhook configured)
    if [[ -n "$SLACK_WEBHOOK" ]]; then
        curl -X POST -H 'Content-type: application/json' \
             --data "{\"text\":\"$subject\n$message\"}" \
             "$SLACK_WEBHOOK"
    fi
}

# Main backup function
main() {
    local timestamp=$(date +"%Y%m%d_%H%M%S")
    local backup_summary=""
    local errors=0
    
    log_message "INFO" "Starting WHMCS secure backup process"
    
    # Setup encryption
    setup_encryption
    
    # Database backup
    local db_backup=$(backup_database "$timestamp")
    if [[ $? -eq 0 ]]; then
        if verify_backup_integrity "$db_backup" && 
           test_backup_decryption "$db_backup" && 
           transfer_to_offsite "$db_backup"; then
            backup_summary+=" Database backup: SUCCESS\n"
        else
            backup_summary+=" Database backup: FAILED\n"
            ((errors++))
        fi
    else
        backup_summary+=" Database backup: FAILED\n"
        ((errors++))
    fi
    
    # Files backup
    local files_backup=$(backup_files "$timestamp")
    if [[ $? -eq 0 ]]; then
        if verify_backup_integrity "$files_backup" && 
           test_backup_decryption "$files_backup" && 
           transfer_to_offsite "$files_backup"; then
            backup_summary+=" Files backup: SUCCESS\n"
        else
            backup_summary+=" Files backup: FAILED\n"
            ((errors++))
        fi
    else
        backup_summary+=" Files backup: FAILED\n"
        ((errors++))
    fi
    
    # Configuration backup
    local config_backup=$(backup_configuration "$timestamp")
    if [[ $? -eq 0 ]]; then
        if verify_backup_integrity "$config_backup" && 
           test_backup_decryption "$config_backup" && 
           transfer_to_offsite "$config_backup"; then
            backup_summary+=" Configuration backup: SUCCESS\n"
        else
            backup_summary+=" Configuration backup: FAILED\n"
            ((errors++))
        fi
    else
        backup_summary+=" Configuration backup: FAILED\n"
        ((errors++))
    fi
    
    # Cleanup old backups
    cleanup_old_backups
    
    # Final summary
    backup_summary+="\nTimestamp: $timestamp\n"
    backup_summary+="Local backups: $BACKUP_DIR\n"
    backup_summary+="Offsite server: $OFFSITE_SERVER\n"
    backup_summary+="Retention: $RETENTION_DAYS days\n"
    
    if [[ $errors -eq 0 ]]; then
        log_message "SUCCESS" "Backup process completed successfully"
        send_notification "SUCCESS" "$backup_summary"
    else
        log_message "ERROR" "Backup process completed with $errors errors"
        send_notification "FAILED" "$backup_summary"
        exit 1
    fi
}

# Run main function
main "$@"

#!/bin/bash
# WHMCS Backup Monitor
# File: /usr/local/bin/whmcs-backup-monitor.sh
# Monitors backup health, integrity, and availability

BACKUP_DIR="/backup/whmcs"
LOG_FILE="/var/log/whmcs-backup-monitor.log"
ALERT_EMAIL="[email protected]"
SLACK_WEBHOOK="https://hooks.slack.com/services/YOUR/WEBHOOK/URL"

# Check backup freshness
check_backup_freshness() {
    local max_age=25 # hours
    local latest_backup=$(find "$BACKUP_DIR" -name "*.enc" -type f -printf '%T@ %p\n' | sort -n | tail -1 | cut -d' ' -f2-)
    
    if [[ -z "$latest_backup" ]]; then
        echo "CRITICAL: No backup files found"
        return 2
    fi
    
    local backup_age=$(( ($(date +%s) - $(stat -c %Y "$latest_backup")) / 3600 ))
    
    if [[ $backup_age -gt $max_age ]]; then
        echo "WARNING: Latest backup is $backup_age hours old (max: $max_age)"
        return 1
    else
        echo "OK: Latest backup is $backup_age hours old"
        return 0
    fi
}

# Check backup integrity
check_backup_integrity() {
    local errors=0
    
    # Check recent backups (last 7 days)
    find "$BACKUP_DIR" -name "*.enc" -mtime -7 -type f | while read -r backup_file; do
        local hash_file="${backup_file}.sha256"
        
        if [[ ! -f "$hash_file" ]]; then
            echo "ERROR: Missing hash file for $(basename "$backup_file")"
            ((errors++))
            continue
        fi
        
        # Verify integrity
        if ! sha256sum -c "$hash_file" >/dev/null 2>&1; then
            echo "CRITICAL: Integrity check failed for $(basename "$backup_file")"
            ((errors++))
        fi
    done
    
    if [[ $errors -eq 0 ]]; then
        echo "OK: All backup integrity checks passed"
        return 0
    else
        echo "CRITICAL: $errors backup integrity failures"
        return 2
    fi
}

# Check offsite backup sync
check_offsite_sync() {
    local offsite_server="backup.yourcompany.com"
    local offsite_user="backup-user"
    
    # Get latest local backup
    local latest_local=$(find "$BACKUP_DIR" -name "*.enc" -type f -printf '%T@ %p\n' | sort -n | tail -1 | cut -d' ' -f2- | xargs basename)
    
    if [[ -z "$latest_local" ]]; then
        echo "ERROR: No local backup found"
        return 1
    fi
    
    # Check if exists on remote server
    if ssh -i /root/.ssh/backup_key "${offsite_user}@${offsite_server}" "test -f /secure-backups/$latest_local" 2>/dev/null; then
        echo "OK: Latest backup synced to offsite"
        return 0
    else
        echo "WARNING: Latest backup not found on offsite server"
        return 1
    fi
}

# Check backup storage space
check_storage_space() {
    local usage=$(df "$BACKUP_DIR" | tail -1 | awk '{print $5}' | sed 's/%//')
    local threshold=85
    
    if [[ $usage -gt $threshold ]]; then
        echo "WARNING: Backup storage ${usage}% full (threshold: ${threshold}%)"
        return 1
    else
        echo "OK: Backup storage ${usage}% used"
        return 0
    fi
}

# Check encryption key accessibility
check_encryption_key() {
    local key_file="/etc/whmcs-backup/encryption.key"
    
    if [[ ! -f "$key_file" ]]; then
        echo "CRITICAL: Encryption key file not found"
        return 2
    fi
    
    if [[ ! -r "$key_file" ]]; then
        echo "CRITICAL: Encryption key file not readable"
        return 2
    fi
    
    # Test key by creating a small encrypted test
    if echo "test" | openssl enc -aes-256-cbc -salt -pass file:"$key_file" >/dev/null 2>&1; then
        echo "OK: Encryption key accessible and functional"
        return 0
    else
        echo "CRITICAL: Encryption key test failed"
        return 2
    fi
}

# Check WHMCS database connectivity
check_database_connectivity() {
    local whmcs_path="/var/www/html/whmcs"
    local config_file="$whmcs_path/configuration.php"
    
    if [[ ! -f "$config_file" ]]; then
        echo "ERROR: WHMCS configuration file not found"
        return 1
    fi
    
    # Extract database credentials
    local db_host=$(grep 'db_host' "$config_file" | cut -d"'" -f4)
    local db_name=$(grep 'db_name' "$config_file" | cut -d"'" -f4)
    local db_username=$(grep 'db_username' "$config_file" | cut -d"'" -f4)
    local db_password=$(grep 'db_password' "$config_file" | cut -d"'" -f4)
    
    # Test connection
    if mysql -h "$db_host" -u "$db_username" -p"$db_password" -e "USE $db_name; SELECT 1;" >/dev/null 2>&1; then
        echo "OK: Database connection successful"
        return 0
    else
        echo "CRITICAL: Database connection failed"
        return 2
    fi
}

# Generate monitoring report
generate_report() {
    local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
    local report=""
    local overall_status="OK"
    local critical_issues=0
    local warnings=0
    
    echo "=== WHMCS Backup Health Report ==="
    echo "Generated: $timestamp"
    echo ""
    
    # Run all checks
    local checks=(
        "Backup Freshness:check_backup_freshness"
        "Backup Integrity:check_backup_integrity"
        "Offsite Sync:check_offsite_sync"
        "Storage Space:check_storage_space"
        "Encryption Key:check_encryption_key"
        "Database Connectivity:check_database_connectivity"
    )
    
    for check in "${checks[@]}"; do
        local check_name="${check%%:*}"
        local check_function="${check##*:}"
        
        echo "Checking: $check_name"
        local result=$($check_function)
        local status=$?
        
        case $status in
            0) echo " $result" ;;
            1) 
                echo "  $result"
                ((warnings++))
                if [[ "$overall_status" == "OK" ]]; then
                    overall_status="WARNING"
                fi
                ;;
            2) 
                echo " $result"
                ((critical_issues++))
                overall_status="CRITICAL"
                ;;
        esac
        echo ""
    done
    
    echo "=== Summary ==="
    echo "Overall Status: $overall_status"
    echo "Critical Issues: $critical_issues"
    echo "Warnings: $warnings"
    echo "================================"
    
    # Send alerts if needed
    if [[ $critical_issues -gt 0 ]] || [[ $warnings -gt 0 ]]; then
        send_alert "$overall_status" "$critical_issues" "$warnings"
    fi
}

# Send alert notifications
send_alert() {
    local status="$1"
    local critical="$2"
    local warnings="$3"
    
    local subject="🚨 WHMCS Backup Health Alert - $status"
    local message="WHMCS Backup monitoring detected issues:\n\n"
    message+="Critical Issues: $critical\n"
    message+="Warnings: $warnings\n\n"
    message+="Please check the backup system immediately.\n\n"
    message+="Server: $(hostname)\n"
    message+="Time: $(date)\n"
    
    # Email alert
    echo -e "$message" | mail -s "$subject" "$ALERT_EMAIL"
    
    # Slack alert
    if [[ -n "$SLACK_WEBHOOK" ]]; then
        local slack_payload="{
            \"text\": \"$subject\",
            \"color\": \"$([ "$status" == "CRITICAL" ] && echo "danger" || echo "warning")\",
            \"fields\": [
                {\"title\": \"Critical Issues\", \"value\": \"$critical\", \"short\": true},
                {\"title\": \"Warnings\", \"value\": \"warnings\", \"short\": true},
                {\"title\": \"Server\", \"value\": \"$(hostname)\", \"short\": true},
                {\"title\": \"Time\", \"value\": \"$(date)\", \"short\": true}
            ]
        }"
        
        curl -X POST -H 'Content-type: application/json' \
             --data "$slack_payload" \
             "$SLACK_WEBHOOK"
    fi
}

# Main execution
generate_report

#!/bin/bash
# WHMCS Disaster Recovery System
# File: /usr/local/bin/whmcs-disaster-recovery.sh
# Automates complete system recovery from encrypted backups

BACKUP_DIR="/backup/whmcs"
WHMCS_PATH="/var/www/html/whmcs"
ENCRYPTION_KEY_FILE="/etc/whmcs-backup/encryption.key"
OFFSITE_SERVER="backup.yourcompany.com"
OFFSITE_USER="backup-user"
RECOVERY_LOG="/var/log/whmcs-recovery.log"

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

log_recovery() {
    local level="$1"
    local message="$2"
    echo "$(date '+%Y-%m-%d %H:%M:%S') [$level] $message" | tee -a "$RECOVERY_LOG"
    
    case $level in
        "INFO")  echo -e "${BLUE}[INFO]${NC} $message" ;;
        "WARN")  echo -e "${YELLOW}[WARN]${NC} $message" ;;
        "ERROR") echo -e "${RED}[ERROR]${NC} $message" ;;
        "SUCCESS") echo -e "${GREEN}[SUCCESS]${NC} $message" ;;
    esac
}

# List available backups
list_backups() {
    echo "Available backup sets:"
    echo "====================="
    
    # Local backups
    if [[ -d "$BACKUP_DIR" ]]; then
        echo "Local backups:"
        find "$BACKUP_DIR" -name "*-db-*.enc" -type f | sort -r | head -10 | while read -r db_backup; do
            local timestamp=$(basename "$db_backup" | sed 's/.*-db-\(.*\)\.sql\.enc/\1/')
            local size=$(du -h "$db_backup" | cut -f1)
            echo "  $timestamp ($size)"
        done
        echo ""
    fi
    
    # Remote backups
    echo "Offsite backups:"
    ssh -i /root/.ssh/backup_key "${OFFSITE_USER}@${OFFSITE_SERVER}" \
        "find /secure-backups -name '*-db-*.enc' -type f | sort -r | head -10" 2>/dev/null | while read -r remote_backup; do
        local timestamp=$(basename "$remote_backup" | sed 's/.*-db-\(.*\)\.sql\.enc/\1/')
        echo "  $timestamp (offsite)"
    done
}

# Download backup from offsite
download_backup() {
    local timestamp="$1"
    local backup_type="$2" # db, files, or config
    
    log_recovery "INFO" "Downloading $backup_type backup for $timestamp from offsite"
    
    local remote_file="/secure-backups/whmcs-${backup_type}-${timestamp}.*"
    local local_dir="$BACKUP_DIR/${backup_type}/"
    
    mkdir -p "$local_dir"
    
    rsync -avz \
          --progress \
          -e "ssh -i /root/.ssh/backup_key" \
          "${OFFSITE_USER}@${OFFSITE_SERVER}:${remote_file}" \
          "$local_dir/"
    
    if [[ $? -eq 0 ]]; then
        log_recovery "SUCCESS" "$backup_type backup downloaded successfully"
        return 0
    else
        log_recovery "ERROR" "Failed to download $backup_type backup"
        return 1
    fi
}

# Verify backup integrity before restoration
verify_before_restore() {
    local backup_file="$1"
    local hash_file="${backup_file}.sha256"
    
    log_recovery "INFO" "Verifying backup integrity: $(basename "$backup_file")"
    
    if [[ ! -f "$hash_file" ]]; then
        log_recovery "ERROR" "Hash file not found: $hash_file"
        return 1
    fi
    
    if sha256sum -c "$hash_file" >/dev/null 2>&1; then
        log_recovery "SUCCESS" "Integrity verification passed"
        return 0
    else
        log_recovery "ERROR" "Integrity verification failed"
        return 1
    fi
}

# Restore database
restore_database() {
    local db_backup="$1"
    
    log_recovery "INFO" "Starting database restoration"
    
    # Verify backup integrity
    if ! verify_before_restore "$db_backup"; then
        return 1
    fi
    
    # Get database credentials
    local db_host=$(grep 'db_host' "$WHMCS_PATH/configuration.php" | cut -d"'" -f4)
    local db_name=$(grep 'db_name' "$WHMCS_PATH/configuration.php" | cut -d"'" -f4)
    local db_username=$(grep 'db_username' "$WHMCS_PATH/configuration.php" | cut -d"'" -f4)
    local db_password=$(grep 'db_password' "$WHMCS_PATH/configuration.php" | cut -d"'" -f4)
    
    # Create database backup of current state
    local current_backup="/tmp/current-db-$(date +%s).sql"
    log_recovery "INFO" "Creating backup of current database"
    mysqldump --host="$db_host" --user="$db_username" --password="$db_password" "$db_name" > "$current_backup"
    
    # Decrypt and restore
    log_recovery "INFO" "Decrypting and restoring database"
    if openssl enc -d -aes-256-cbc \
                  -in "$db_backup" \
                  -pass file:"$ENCRYPTION_KEY_FILE" | \
       gunzip | \
       mysql --host="$db_host" --user="$db_username" --password="$db_password" "$db_name"; then
        
        log_recovery "SUCCESS" "Database restored successfully"
        rm -f "$current_backup"
        return 0
    else
        log_recovery "ERROR" "Database restoration failed, attempting rollback"
        
        # Rollback to previous state
        mysql --host="$db_host" --user="$db_username" --password="$db_password" "$db_name" < "$current_backup"
        rm -f "$current_backup"
        return 1
    fi
}

# Restore files
restore_files() {
    local files_backup="$1"
    
    log_recovery "INFO" "Starting file system restoration"
    
    # Verify backup integrity
    if ! verify_before_restore "$files_backup"; then
        return 1
    fi
    
    # Create backup of current files
    local current_backup="/tmp/current-files-$(date +%s).tar.gz"
    log_recovery "INFO" "Creating backup of current files"
    tar -czf "$current_backup" -C "$(dirname "$WHMCS_PATH")" "$(basename "$WHMCS_PATH")"
    
    # Decrypt and extract
    log_recovery "INFO" "Decrypting and extracting files"
    if openssl enc -d -aes-256-cbc \
                  -in "$files_backup" \
                  -pass file:"$ENCRYPTION_KEY_FILE" | \
       tar -xzf - -C "$(dirname "$WHMCS_PATH")"; then
        
        log_recovery "SUCCESS" "Files restored successfully"
        
        # Set proper permissions
        chown -R www-data:www-data "$WHMCS_PATH"
        find "$WHMCS_PATH" -type f -exec chmod 644 {} \;
        find "$WHMCS_PATH" -type d -exec chmod 755 {} \;
        chmod 644 "$WHMCS_PATH/configuration.php"
        
        rm -f "$current_backup"
        return 0
    else
        log_recovery "ERROR" "File restoration failed, attempting rollback"
        
        # Rollback to previous state
        rm -rf "$WHMCS_PATH"
        tar -xzf "$current_backup" -C "$(dirname "$WHMCS_PATH")"
        rm -f "$current_backup"
        return 1
    fi
}

# Restore configuration
restore_configuration() {
    local config_backup="$1"
    
    log_recovery "INFO" "Starting configuration restoration"
    
    # Verify backup integrity
    if ! verify_before_restore "$config_backup"; then
        return 1
    fi
    
    # Decrypt and extract configuration
    if openssl enc -d -aes-256-cbc \
                  -in "$config_backup" \
                  -pass file:"$ENCRYPTION_KEY_FILE" | \
       tar -xzf - -C /; then
        
        log_recovery "SUCCESS" "Configuration restored successfully"
        
        # Set proper permissions
        chmod 644 "$WHMCS_PATH/configuration.php"
        chown -R www-data:www-data "$WHMCS_PATH/includes/hooks/"
        chown -R www-data:www-data "$WHMCS_PATH/modules/"
        
        return 0
    else
        log_recovery "ERROR" "Configuration restoration failed"
        return 1
    fi
}

# Post-recovery validation
post_recovery_validation() {
    log_recovery "INFO" "Running post-recovery validation"
    
    local errors=0
    
    # Check WHMCS files
    if [[ ! -f "$WHMCS_PATH/configuration.php" ]]; then
        log_recovery "ERROR" "WHMCS configuration file missing"
        ((errors++))
    fi
    
    if [[ ! -f "$WHMCS_PATH/index.php" ]]; then
        log_recovery "ERROR" "WHMCS index file missing"
        ((errors++))
    fi
    
    # Check database connectivity
    local db_host=$(grep 'db_host' "$WHMCS_PATH/configuration.php" | cut -d"'" -f4)
    local db_name=$(grep 'db_name' "$WHMCS_PATH/configuration.php" | cut -d"'" -f4)
    local db_username=$(grep 'db_username' "$WHMCS_PATH/configuration.php" | cut -d"'" -f4)
    local db_password=$(grep 'db_password' "$WHMCS_PATH/configuration.php" | cut -d"'" -f4)
    
    if ! mysql -h "$db_host" -u "$db_username" -p"$db_password" -e "USE $db_name; SELECT COUNT(*) FROM tblclients;" >/dev/null 2>&1; then
        log_recovery "ERROR" "Database connectivity test failed"
        ((errors++))
    fi
    
    # Check web server response
    if command -v curl >/dev/null; then
        local response_code=$(curl -s -o /dev/null -w "%{http_code}" http://localhost/whmcs/ || echo "000")
        if [[ "$response_code" != "200" ]]; then
            log_recovery "WARN" "Web server returned HTTP $response_code"
        else
            log_recovery "SUCCESS" "Web server responding correctly"
        fi
    fi
    
    if [[ $errors -eq 0 ]]; then
        log_recovery "SUCCESS" "Post-recovery validation passed"
        return 0
    else
        log_recovery "ERROR" "Post-recovery validation failed with $errors errors"
        return 1
    fi
}

# Full system recovery
full_recovery() {
    local timestamp="$1"
    
    if [[ -z "$timestamp" ]]; then
        echo "Usage: $0 --full-recovery TIMESTAMP"
        echo "Example: $0 --full-recovery 20240101_120000"
        exit 1
    fi
    
    log_recovery "INFO" "Starting full WHMCS recovery for timestamp: $timestamp"
    
    # Check if backups exist locally, if not download from offsite
    local db_backup="$BACKUP_DIR/database/whmcs-db-${timestamp}.sql.enc"
    local files_backup="$BACKUP_DIR/files/whmcs-files-${timestamp}.tar.gz.enc"
    local config_backup="$BACKUP_DIR/config/whmcs-config-${timestamp}.tar.gz.enc"
    
    # Download missing backups
    if [[ ! -f "$db_backup" ]]; then
        download_backup "$timestamp" "database" || exit 1
    fi
    
    if [[ ! -f "$files_backup" ]]; then
        download_backup "$timestamp" "files" || exit 1
    fi
    
    if [[ ! -f "$config_backup" ]]; then
        download_backup "$timestamp" "config" || exit 1
    fi
    
    # Stop web server
    log_recovery "INFO" "Stopping web server"
    systemctl stop apache2 2>/dev/null || systemctl stop nginx 2>/dev/null
    
    # Restore components
    if restore_database "$db_backup" && 
       restore_files "$files_backup" && 
       restore_configuration "$config_backup"; then
        
        log_recovery "SUCCESS" "All components restored successfully"
        
        # Start web server
        log_recovery "INFO" "Starting web server"
        systemctl start apache2 2>/dev/null || systemctl start nginx 2>/dev/null
        
        # Run validation
        if post_recovery_validation; then
            log_recovery "SUCCESS" "Full recovery completed successfully"
            
            # Send success notification
            echo "WHMCS recovery completed successfully at $(date)" | \
            mail -s " WHMCS Recovery Success" [email protected]
            
        else
            log_recovery "ERROR" "Recovery validation failed"
            exit 1
        fi
    else
        log_recovery "ERROR" "Recovery failed"
        
        # Send failure notification
        echo "WHMCS recovery failed at $(date). Please investigate immediately." | \
        mail -s " WHMCS Recovery Failed" [email protected]
        
        exit 1
    fi
}

# Parse command line arguments
case "$1" in
    --list-backups)
        list_backups
        ;;
    --full-recovery)
        full_recovery "$2"
        ;;
    --download-backup)
        download_backup "$2" "$3"
        ;;
    --restore-database)
        restore_database "$2"
        ;;
    --restore-files)
        restore_files "$2"
        ;;
    --restore-config)
        restore_configuration "$2"
        ;;
    --validate)
        post_recovery_validation
        ;;
    *)
        echo "WHMCS Disaster Recovery System"
        echo "Usage: $0 [OPTION] [ARGUMENTS]"
        echo ""
        echo "Options:"
        echo "  --list-backups                 List available backups"
        echo "  --full-recovery TIMESTAMP      Perform full system recovery"
        echo "  --download-backup TS TYPE      Download backup from offsite"
        echo "  --restore-database FILE        Restore database only"
        echo "  --restore-files FILE           Restore files only"
        echo "  --restore-config FILE          Restore configuration only"
        echo "  --validate                     Run post-recovery validation"
        echo ""
        echo "Examples:"
        echo "  $0 --list-backups"
        echo "  $0 --full-recovery 20240101_120000"
        echo "  $0 --download-backup 20240101_120000 database"
        exit 1
        ;;
esac