innercontext/deploy.sh

#!/usr/bin/env bash
# Usage: ./deploy.sh [frontend|backend|all|rollback|list]

set -eEuo pipefail

SERVER="${DEPLOY_SERVER:-innercontext}"
REMOTE_ROOT="${DEPLOY_ROOT:-/opt/innercontext}"
RELEASES_DIR="$REMOTE_ROOT/releases"
CURRENT_LINK="$REMOTE_ROOT/current"
REMOTE_SCRIPTS_DIR="$REMOTE_ROOT/scripts"
LOCK_FILE="$REMOTE_ROOT/.deploy.lock"
LOG_FILE="$REMOTE_ROOT/deploy.log"
KEEP_RELEASES="${KEEP_RELEASES:-5}"
SERVICE_TIMEOUT="${SERVICE_TIMEOUT:-60}"

SCOPE="${1:-all}"
TIMESTAMP="$(date +%Y%m%d_%H%M%S)"
RELEASE_DIR="$RELEASES_DIR/$TIMESTAMP"

LOCK_ACQUIRED=0
PROMOTED=0
DEPLOY_SUCCESS=0
PREVIOUS_RELEASE=""

RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'

log() {
    echo -e "${GREEN}==>${NC} $*"
}

warn() {
    echo -e "${YELLOW}WARN:${NC} $*"
}

error() {
    echo -e "${RED}ERROR:${NC} $*" >&2
}

remote() {
    ssh "$SERVER" "$@"
}

log_deployment() {
    local status="$1"
    remote "mkdir -p '$REMOTE_ROOT'"
    remote "{
        echo '---'
        echo 'timestamp: $(date -u +%Y-%m-%dT%H:%M:%SZ)'
        echo 'deployer: $(whoami)@$(hostname)'
        echo 'commit: $(git rev-parse HEAD 2>/dev/null || echo unknown)'
        echo 'branch: $(git branch --show-current 2>/dev/null || echo unknown)'
        echo 'scope: $SCOPE'
        echo 'release: $TIMESTAMP'
        echo 'status: $status'
    } >> '$LOG_FILE'" || true
}

release_lock() {
    if [[ "$LOCK_ACQUIRED" -eq 1 ]]; then
        remote "rm -f '$LOCK_FILE'" || true
    fi
}

cleanup_on_exit() {
    release_lock
}

rollback_to_release() {
    local target_release="$1"
    local reason="$2"

    if [[ -z "$target_release" ]]; then
        error "Rollback skipped: no target release"
        return 1
    fi

    warn "Rolling back to $(basename "$target_release") ($reason)"
    remote "ln -sfn '$target_release' '$CURRENT_LINK'"
    remote "sudo systemctl restart innercontext && sudo systemctl restart innercontext-node && sudo systemctl restart innercontext-pricing-worker"

    if wait_for_service innercontext "$SERVICE_TIMEOUT" \
        && wait_for_service innercontext-node "$SERVICE_TIMEOUT" \
        && wait_for_service innercontext-pricing-worker "$SERVICE_TIMEOUT" \
        && check_backend_health \
        && check_frontend_health; then
        log "Rollback succeeded"
        log_deployment "ROLLBACK_SUCCESS:$reason"
        return 0
    fi

    error "Rollback failed"
    log_deployment "ROLLBACK_FAILED:$reason"
    return 1
}

on_error() {
    local exit_code="$?"
    trap - ERR

    error "Deployment failed (exit $exit_code)"

    if [[ "$PROMOTED" -eq 1 && "$DEPLOY_SUCCESS" -eq 0 ]]; then
        rollback_to_release "$PREVIOUS_RELEASE" "deploy_error" || true
    elif [[ -n "${RELEASE_DIR:-}" ]]; then
        remote "rm -rf '$RELEASE_DIR'" || true
    fi

    log_deployment "FAILED"
    exit "$exit_code"
}

trap cleanup_on_exit EXIT
trap on_error ERR

validate_local() {
    log "Running local validation"

    if [[ "${DEPLOY_ALLOW_DIRTY:-0}" != "1" ]]; then
        if ! git diff-index --quiet HEAD -- 2>/dev/null; then
            error "Working tree has uncommitted changes"
            error "Commit/stash changes or run with DEPLOY_ALLOW_DIRTY=1"
            exit 1
        fi
    else
        warn "Skipping clean working tree check (DEPLOY_ALLOW_DIRTY=1)"
    fi

    if [[ "$SCOPE" == "all" || "$SCOPE" == "backend" ]]; then
        log "Backend checks"
        (cd backend && uv run ruff check .)
        (cd backend && uv run black --check .)
        (cd backend && uv run isort --check-only .)
    fi

    if [[ "$SCOPE" == "all" || "$SCOPE" == "frontend" ]]; then
        log "Frontend checks"
        (cd frontend && pnpm check)
        (cd frontend && pnpm lint)
        log "Building frontend artifact"
        (cd frontend && pnpm build)
    fi
}

acquire_lock() {
    log "Acquiring deployment lock"
    local lock_payload
    lock_payload="$(date -u +%Y-%m-%dT%H:%M:%SZ) $(whoami)@$(hostname) $(git rev-parse --short HEAD 2>/dev/null || echo unknown)"

    if ! remote "( set -o noclobber; echo '$lock_payload' > '$LOCK_FILE' ) 2>/dev/null"; then
        error "Deployment lock exists: $LOCK_FILE"
        remote "cat '$LOCK_FILE'" || true
        exit 1
    fi

    LOCK_ACQUIRED=1
}

ensure_remote_structure() {
    log "Ensuring remote directory structure"
    remote "mkdir -p '$RELEASES_DIR' '$REMOTE_ROOT/shared/backend' '$REMOTE_ROOT/shared/frontend' '$REMOTE_SCRIPTS_DIR'"
}

capture_previous_release() {
    PREVIOUS_RELEASE="$(remote "readlink -f '$CURRENT_LINK' 2>/dev/null || true")"
    if [[ -n "$PREVIOUS_RELEASE" ]]; then
        log "Previous release: $(basename "$PREVIOUS_RELEASE")"
    else
        warn "No previous release detected"
    fi
}

create_release_directory() {
    log "Creating release directory: $(basename "$RELEASE_DIR")"
    remote "rm -rf '$RELEASE_DIR' && mkdir -p '$RELEASE_DIR'"
}

upload_backend() {
    log "Uploading backend"
    remote "mkdir -p '$RELEASE_DIR/backend'"
    rsync -az --delete \
        --exclude='.venv/' \
        --exclude='__pycache__/' \
        --exclude='*.pyc' \
        --exclude='.env' \
        backend/ "$SERVER:$RELEASE_DIR/backend/"

    log "Linking backend shared env"
    remote "ln -sfn ../../../shared/backend/.env '$RELEASE_DIR/backend/.env'"
}

upload_frontend() {
    log "Uploading frontend build artifact"
    remote "mkdir -p '$RELEASE_DIR/frontend'"
    rsync -az --delete frontend/build/ "$SERVER:$RELEASE_DIR/frontend/build/"
    rsync -az frontend/package.json frontend/pnpm-lock.yaml "$SERVER:$RELEASE_DIR/frontend/"

    log "Installing frontend production dependencies on server"
    remote "cd '$RELEASE_DIR/frontend' && pnpm install --prod --frozen-lockfile --ignore-scripts"

    log "Linking frontend shared env"
    remote "ln -sfn ../../../shared/frontend/.env.production '$RELEASE_DIR/frontend/.env.production'"
}

validate_remote_env_files() {
    if [[ "$SCOPE" == "all" || "$SCOPE" == "backend" ]]; then
        log "Validating remote backend env file"
        remote "test -f '$REMOTE_ROOT/shared/backend/.env'"
    fi

    if [[ "$SCOPE" == "all" || "$SCOPE" == "frontend" ]]; then
        log "Validating remote frontend env file"
        remote "test -f '$REMOTE_ROOT/shared/frontend/.env.production'"
    fi
}

validate_remote_sudo_permissions() {
    local sudo_rules
    local sudo_rules_compact
    local required=()
    local missing=0
    local rule

    log "Validating remote sudo permissions"

    if ! sudo_rules="$(remote "sudo -n -l 2>/dev/null")"; then
        error "Remote user cannot run sudo non-interactively"
        error "Configure /etc/sudoers.d/innercontext-deploy for user 'innercontext'"
        exit 1
    fi

    case "$SCOPE" in
        frontend)
            required+=("/usr/bin/systemctl restart innercontext-node")
            required+=("/usr/bin/systemctl is-active innercontext-node")
            ;;
        backend)
            required+=("/usr/bin/systemctl restart innercontext")
            required+=("/usr/bin/systemctl restart innercontext-pricing-worker")
            required+=("/usr/bin/systemctl is-active innercontext")
            required+=("/usr/bin/systemctl is-active innercontext-pricing-worker")
            ;;
        all|rollback)
            required+=("/usr/bin/systemctl restart innercontext")
            required+=("/usr/bin/systemctl restart innercontext-node")
            required+=("/usr/bin/systemctl restart innercontext-pricing-worker")
            required+=("/usr/bin/systemctl is-active innercontext")
            required+=("/usr/bin/systemctl is-active innercontext-node")
            required+=("/usr/bin/systemctl is-active innercontext-pricing-worker")
            ;;
    esac

    sudo_rules_compact="$(printf '%s' "$sudo_rules" | tr '\n' ' ' | tr -s ' ')"

    for rule in "${required[@]}"; do
        if [[ "$sudo_rules_compact" != *"$rule"* ]]; then
            error "Missing sudo permission: $rule"
            missing=1
        fi
    done

    if [[ "$missing" -eq 1 ]]; then
        error "Update /etc/sudoers.d/innercontext-deploy and verify with: sudo -u innercontext sudo -n -l"
        exit 1
    fi
}

upload_ops_files() {
    log "Uploading operational files"
    remote "mkdir -p '$RELEASE_DIR/scripts' '$RELEASE_DIR/systemd' '$RELEASE_DIR/nginx'"
    rsync -az scripts/ "$SERVER:$RELEASE_DIR/scripts/"
    rsync -az systemd/ "$SERVER:$RELEASE_DIR/systemd/"
    rsync -az nginx/ "$SERVER:$RELEASE_DIR/nginx/"
    rsync -az scripts/ "$SERVER:$REMOTE_SCRIPTS_DIR/"
    remote "chmod +x '$REMOTE_SCRIPTS_DIR'/*.sh || true"
}

sync_backend_dependencies() {
    log "Syncing backend dependencies"
    remote "cd '$RELEASE_DIR/backend' && UV_PROJECT_ENVIRONMENT=.venv uv sync --frozen --no-dev --no-editable"
}

run_db_migrations() {
    log "Running database migrations"
    remote "cd '$RELEASE_DIR/backend' && UV_PROJECT_ENVIRONMENT=.venv uv run alembic upgrade head"
}

promote_release() {
    log "Promoting release $(basename "$RELEASE_DIR")"
    remote "ln -sfn '$RELEASE_DIR' '$CURRENT_LINK'"
    PROMOTED=1
}

restart_services() {
    case "$SCOPE" in
        frontend)
            log "Restarting frontend service"
            remote "sudo systemctl restart innercontext-node"
            ;;
        backend)
            log "Restarting backend services"
            remote "sudo systemctl restart innercontext && sudo systemctl restart innercontext-pricing-worker"
            ;;
        all)
            log "Restarting all services"
            remote "sudo systemctl restart innercontext && sudo systemctl restart innercontext-node && sudo systemctl restart innercontext-pricing-worker"
            ;;
    esac
}

wait_for_service() {
    local service="$1"
    local timeout="$2"
    local i

    for ((i = 1; i <= timeout; i++)); do
        if remote "[ \"\$(sudo systemctl is-active '$service' 2>/dev/null)\" = 'active' ]"; then
            log "$service is active"
            return 0
        fi
        sleep 1
    done

    error "$service did not become active within ${timeout}s"
    remote "sudo journalctl -u '$service' -n 50" || true
    return 1
}

check_backend_health() {
    local i
    for ((i = 1; i <= 30; i++)); do
        if remote "curl -sf http://127.0.0.1:8000/health-check >/dev/null"; then
            log "Backend health check passed"
            return 0
        fi
        sleep 2
    done

    error "Backend health check failed"
    remote "sudo journalctl -u innercontext -n 50" || true
    return 1
}

check_frontend_health() {
    local i
    for ((i = 1; i <= 30; i++)); do
        # Allow 200 OK or 302/303/307 Redirect (to login)
        if remote "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:3000/ | grep -qE '^(200|302|303|307)$'"; then
            log "Frontend health check passed"
            return 0
        fi
        sleep 2
    done

    error "Frontend health check failed"
    remote "sudo journalctl -u innercontext-node -n 50" || true
    return 1
}

verify_deployment() {
    case "$SCOPE" in
        frontend)
            wait_for_service innercontext-node "$SERVICE_TIMEOUT"
            check_frontend_health
            ;;
        backend)
            wait_for_service innercontext "$SERVICE_TIMEOUT"
            wait_for_service innercontext-pricing-worker "$SERVICE_TIMEOUT"
            check_backend_health
            ;;
        all)
            wait_for_service innercontext "$SERVICE_TIMEOUT"
            wait_for_service innercontext-node "$SERVICE_TIMEOUT"
            wait_for_service innercontext-pricing-worker "$SERVICE_TIMEOUT"
            check_backend_health
            check_frontend_health
            ;;
    esac
}

cleanup_old_releases() {
    log "Cleaning old releases (keeping $KEEP_RELEASES)"
    remote "
        cd '$RELEASES_DIR' && \
        ls -1dt [0-9]* 2>/dev/null | tail -n +$((KEEP_RELEASES + 1)) | xargs -r rm -rf
    " || true
}

list_releases() {
    log "Current release"
    remote "readlink -f '$CURRENT_LINK' 2>/dev/null || echo 'none'"
    log "Recent releases"
    remote "ls -1dt '$RELEASES_DIR'/* 2>/dev/null | head -10" || true
}

rollback_to_previous() {
    local previous_release
    previous_release="$(remote "
        current=\$(readlink -f '$CURRENT_LINK' 2>/dev/null || true)
        for r in \$(ls -1dt '$RELEASES_DIR'/* 2>/dev/null); do
            if [ \"\$r\" != \"\$current\" ]; then
                echo \"\$r\"
                break
            fi
        done
    ")"

    if [[ -z "$previous_release" ]]; then
        error "No previous release found"
        exit 1
    fi

    rollback_to_release "$previous_release" "manual"
}

run_deploy() {
    validate_local
    acquire_lock
    ensure_remote_structure
    validate_remote_sudo_permissions
    capture_previous_release
    create_release_directory
    validate_remote_env_files

    if [[ "$SCOPE" == "all" || "$SCOPE" == "backend" ]]; then
        upload_backend
        sync_backend_dependencies
        run_db_migrations
    fi

    if [[ "$SCOPE" == "all" || "$SCOPE" == "frontend" ]]; then
        upload_frontend
    fi

    upload_ops_files
    promote_release
    restart_services
    verify_deployment
    cleanup_old_releases

    DEPLOY_SUCCESS=1
    log_deployment "SUCCESS"
    log "Deployment complete"
}

case "$SCOPE" in
    frontend|backend|all)
        run_deploy
        ;;
    rollback)
        acquire_lock
        validate_remote_sudo_permissions
        rollback_to_previous
        ;;
    list)
        list_releases
        ;;
    *)
        echo "Usage: $0 [frontend|backend|all|rollback|list]"
        exit 1
        ;;
esac