diff --git a/.forgejo/workflows/deploy-manual.yml b/.forgejo/workflows/deploy-manual.yml new file mode 100644 index 0000000..db4a7a7 --- /dev/null +++ b/.forgejo/workflows/deploy-manual.yml @@ -0,0 +1,73 @@ +name: Deploy (Manual) + +on: + workflow_dispatch: + inputs: + scope: + description: "Deployment scope" + required: true + default: "all" + type: choice + options: + - all + - backend + - frontend + - rollback + - list + +jobs: + deploy: + name: Manual deployment to LXC + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install uv + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: "24" + + - name: Install pnpm + run: npm install -g pnpm + + - name: Configure SSH key + env: + DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }} + run: | + mkdir -p "$HOME/.ssh" + chmod 700 "$HOME/.ssh" + printf '%s\n' "$DEPLOY_SSH_KEY" > "$HOME/.ssh/id_ed25519" + chmod 600 "$HOME/.ssh/id_ed25519" + + - name: Configure known hosts + env: + DEPLOY_KNOWN_HOSTS: ${{ secrets.DEPLOY_KNOWN_HOSTS }} + run: | + if [ -z "$DEPLOY_KNOWN_HOSTS" ]; then + echo "DEPLOY_KNOWN_HOSTS secret is required" + exit 1 + fi + printf '%s\n' "$DEPLOY_KNOWN_HOSTS" > "$HOME/.ssh/known_hosts" + chmod 644 "$HOME/.ssh/known_hosts" + + - name: Run deployment + env: + DEPLOY_SERVER: ${{ secrets.DEPLOY_SERVER }} + run: | + if [ -z "$DEPLOY_SERVER" ]; then + echo "DEPLOY_SERVER secret is required" + exit 1 + fi + chmod +x ./deploy.sh + ./deploy.sh "${{ inputs.scope }}" diff --git a/README.md b/README.md index c3c683d..498dac3 100644 --- a/README.md +++ b/README.md @@ -100,4 +100,7 @@ uv run pytest ## Deployment -See [docs/DEPLOYMENT.md](docs/DEPLOYMENT.md) for a step-by-step guide for a Proxmox LXC setup (Debian 13, nginx, systemd services). +Deployments are push-based from an external machine (laptop/CI runner) to the LXC host over SSH. + +- Canonical runbook: [docs/DEPLOYMENT.md](docs/DEPLOYMENT.md) +- Operator checklist: [docs/DEPLOYMENT-QUICKSTART.md](docs/DEPLOYMENT-QUICKSTART.md) diff --git a/deploy.sh b/deploy.sh index 46119fb..39ef614 100755 --- a/deploy.sh +++ b/deploy.sh @@ -1,63 +1,463 @@ #!/usr/bin/env bash -# Usage: ./deploy.sh [frontend|backend|all] -# default: all -# -# SSH config (~/.ssh/config) — recommended: -# Host innercontext -# HostName -# User innercontext -# -# The innercontext user needs passwordless sudo for systemctl only: -# /etc/sudoers.d/innercontext-deploy: -# innercontext ALL=(root) NOPASSWD: /usr/bin/systemctl restart innercontext, /usr/bin/systemctl restart innercontext-node, /usr/bin/systemctl restart innercontext-pricing-worker, /usr/bin/systemctl is-active innercontext, /usr/bin/systemctl is-active innercontext-node, /usr/bin/systemctl is-active innercontext-pricing-worker -set -euo pipefail +# Usage: ./deploy.sh [frontend|backend|all|rollback|list] + +set -eEuo pipefail + +SERVER="${DEPLOY_SERVER:-innercontext}" +REMOTE_ROOT="${DEPLOY_ROOT:-/opt/innercontext}" +RELEASES_DIR="$REMOTE_ROOT/releases" +CURRENT_LINK="$REMOTE_ROOT/current" +REMOTE_SCRIPTS_DIR="$REMOTE_ROOT/scripts" +LOCK_FILE="$REMOTE_ROOT/.deploy.lock" +LOG_FILE="$REMOTE_ROOT/deploy.log" +KEEP_RELEASES="${KEEP_RELEASES:-5}" +SERVICE_TIMEOUT="${SERVICE_TIMEOUT:-60}" -SERVER="${DEPLOY_SERVER:-innercontext}" # ssh host alias or user@host -REMOTE="/opt/innercontext" SCOPE="${1:-all}" +TIMESTAMP="$(date +%Y%m%d_%H%M%S)" +RELEASE_DIR="$RELEASES_DIR/$TIMESTAMP" -# ── Frontend ─────────────────────────────────────────────────────────────── -deploy_frontend() { - echo "==> [frontend] Building locally..." - (cd frontend && pnpm run build) +LOCK_ACQUIRED=0 +PROMOTED=0 +DEPLOY_SUCCESS=0 +PREVIOUS_RELEASE="" - echo "==> [frontend] Uploading build/ and package files..." - rsync -az --delete frontend/build/ "$SERVER:$REMOTE/frontend/build/" - rsync -az frontend/package.json frontend/pnpm-lock.yaml "$SERVER:$REMOTE/frontend/" +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' - echo "==> [frontend] Installing production dependencies on server..." - ssh "$SERVER" "cd $REMOTE/frontend && pnpm install --prod --frozen-lockfile --ignore-scripts" - - echo "==> [frontend] Restarting service..." - ssh "$SERVER" "sudo systemctl restart innercontext-node && echo OK" +log() { + echo -e "${GREEN}==>${NC} $*" } -# ── Backend ──────────────────────────────────────────────────────────────── -deploy_backend() { - echo "==> [backend] Uploading source..." +warn() { + echo -e "${YELLOW}WARN:${NC} $*" +} + +error() { + echo -e "${RED}ERROR:${NC} $*" >&2 +} + +remote() { + ssh "$SERVER" "$@" +} + +log_deployment() { + local status="$1" + remote "mkdir -p '$REMOTE_ROOT'" + remote "{ + echo '---' + echo 'timestamp: $(date -u +%Y-%m-%dT%H:%M:%SZ)' + echo 'deployer: $(whoami)@$(hostname)' + echo 'commit: $(git rev-parse HEAD 2>/dev/null || echo unknown)' + echo 'branch: $(git branch --show-current 2>/dev/null || echo unknown)' + echo 'scope: $SCOPE' + echo 'release: $TIMESTAMP' + echo 'status: $status' + } >> '$LOG_FILE'" || true +} + +release_lock() { + if [[ "$LOCK_ACQUIRED" -eq 1 ]]; then + remote "rm -f '$LOCK_FILE'" || true + fi +} + +cleanup_on_exit() { + release_lock +} + +rollback_to_release() { + local target_release="$1" + local reason="$2" + + if [[ -z "$target_release" ]]; then + error "Rollback skipped: no target release" + return 1 + fi + + warn "Rolling back to $(basename "$target_release") ($reason)" + remote "ln -sfn '$target_release' '$CURRENT_LINK'" + remote "sudo systemctl restart innercontext && sudo systemctl restart innercontext-node && sudo systemctl restart innercontext-pricing-worker" + + if wait_for_service innercontext "$SERVICE_TIMEOUT" \ + && wait_for_service innercontext-node "$SERVICE_TIMEOUT" \ + && wait_for_service innercontext-pricing-worker "$SERVICE_TIMEOUT" \ + && check_backend_health \ + && check_frontend_health; then + log "Rollback succeeded" + log_deployment "ROLLBACK_SUCCESS:$reason" + return 0 + fi + + error "Rollback failed" + log_deployment "ROLLBACK_FAILED:$reason" + return 1 +} + +on_error() { + local exit_code="$?" + trap - ERR + + error "Deployment failed (exit $exit_code)" + + if [[ "$PROMOTED" -eq 1 && "$DEPLOY_SUCCESS" -eq 0 ]]; then + rollback_to_release "$PREVIOUS_RELEASE" "deploy_error" || true + elif [[ -n "${RELEASE_DIR:-}" ]]; then + remote "rm -rf '$RELEASE_DIR'" || true + fi + + log_deployment "FAILED" + exit "$exit_code" +} + +trap cleanup_on_exit EXIT +trap on_error ERR + +validate_local() { + log "Running local validation" + + if [[ "${DEPLOY_ALLOW_DIRTY:-0}" != "1" ]]; then + if ! git diff-index --quiet HEAD -- 2>/dev/null; then + error "Working tree has uncommitted changes" + error "Commit/stash changes or run with DEPLOY_ALLOW_DIRTY=1" + exit 1 + fi + else + warn "Skipping clean working tree check (DEPLOY_ALLOW_DIRTY=1)" + fi + + if [[ "$SCOPE" == "all" || "$SCOPE" == "backend" ]]; then + log "Backend checks" + (cd backend && uv run ruff check .) + (cd backend && uv run black --check .) + (cd backend && uv run isort --check-only .) + fi + + if [[ "$SCOPE" == "all" || "$SCOPE" == "frontend" ]]; then + log "Frontend checks" + (cd frontend && pnpm check) + (cd frontend && pnpm lint) + log "Building frontend artifact" + (cd frontend && pnpm build) + fi +} + +acquire_lock() { + log "Acquiring deployment lock" + local lock_payload + lock_payload="$(date -u +%Y-%m-%dT%H:%M:%SZ) $(whoami)@$(hostname) $(git rev-parse --short HEAD 2>/dev/null || echo unknown)" + + if ! remote "( set -o noclobber; echo '$lock_payload' > '$LOCK_FILE' ) 2>/dev/null"; then + error "Deployment lock exists: $LOCK_FILE" + remote "cat '$LOCK_FILE'" || true + exit 1 + fi + + LOCK_ACQUIRED=1 +} + +ensure_remote_structure() { + log "Ensuring remote directory structure" + remote "mkdir -p '$RELEASES_DIR' '$REMOTE_ROOT/shared/backend' '$REMOTE_ROOT/shared/frontend' '$REMOTE_SCRIPTS_DIR'" +} + +capture_previous_release() { + PREVIOUS_RELEASE="$(remote "readlink -f '$CURRENT_LINK' 2>/dev/null || true")" + if [[ -n "$PREVIOUS_RELEASE" ]]; then + log "Previous release: $(basename "$PREVIOUS_RELEASE")" + else + warn "No previous release detected" + fi +} + +create_release_directory() { + log "Creating release directory: $(basename "$RELEASE_DIR")" + remote "rm -rf '$RELEASE_DIR' && mkdir -p '$RELEASE_DIR'" +} + +upload_backend() { + log "Uploading backend" + remote "mkdir -p '$RELEASE_DIR/backend'" rsync -az --delete \ --exclude='.venv/' \ --exclude='__pycache__/' \ --exclude='*.pyc' \ --exclude='.env' \ - backend/ "$SERVER:$REMOTE/backend/" + backend/ "$SERVER:$RELEASE_DIR/backend/" - echo "==> [backend] Syncing dependencies..." - ssh "$SERVER" "cd $REMOTE/backend && uv sync --frozen --no-dev --no-editable" - - echo "==> [backend] Restarting services (alembic runs on API start)..." - ssh "$SERVER" "sudo systemctl restart innercontext && sudo systemctl restart innercontext-pricing-worker && echo OK" + log "Linking backend shared env" + remote "ln -sfn ../../../shared/backend/.env '$RELEASE_DIR/backend/.env'" +} + +upload_frontend() { + log "Uploading frontend build artifact" + remote "mkdir -p '$RELEASE_DIR/frontend'" + rsync -az --delete frontend/build/ "$SERVER:$RELEASE_DIR/frontend/build/" + rsync -az frontend/package.json frontend/pnpm-lock.yaml "$SERVER:$RELEASE_DIR/frontend/" + + log "Installing frontend production dependencies on server" + remote "cd '$RELEASE_DIR/frontend' && pnpm install --prod --frozen-lockfile --ignore-scripts" + + log "Linking frontend shared env" + remote "ln -sfn ../../../shared/frontend/.env.production '$RELEASE_DIR/frontend/.env.production'" +} + +validate_remote_env_files() { + if [[ "$SCOPE" == "all" || "$SCOPE" == "backend" ]]; then + log "Validating remote backend env file" + remote "test -f '$REMOTE_ROOT/shared/backend/.env'" + fi + + if [[ "$SCOPE" == "all" || "$SCOPE" == "frontend" ]]; then + log "Validating remote frontend env file" + remote "test -f '$REMOTE_ROOT/shared/frontend/.env.production'" + fi +} + +validate_remote_sudo_permissions() { + local sudo_rules + local sudo_rules_compact + local required=() + local missing=0 + local rule + + log "Validating remote sudo permissions" + + if ! sudo_rules="$(remote "sudo -n -l 2>/dev/null")"; then + error "Remote user cannot run sudo non-interactively" + error "Configure /etc/sudoers.d/innercontext-deploy for user 'innercontext'" + exit 1 + fi + + case "$SCOPE" in + frontend) + required+=("/usr/bin/systemctl restart innercontext-node") + required+=("/usr/bin/systemctl is-active innercontext-node") + ;; + backend) + required+=("/usr/bin/systemctl restart innercontext") + required+=("/usr/bin/systemctl restart innercontext-pricing-worker") + required+=("/usr/bin/systemctl is-active innercontext") + required+=("/usr/bin/systemctl is-active innercontext-pricing-worker") + ;; + all|rollback) + required+=("/usr/bin/systemctl restart innercontext") + required+=("/usr/bin/systemctl restart innercontext-node") + required+=("/usr/bin/systemctl restart innercontext-pricing-worker") + required+=("/usr/bin/systemctl is-active innercontext") + required+=("/usr/bin/systemctl is-active innercontext-node") + required+=("/usr/bin/systemctl is-active innercontext-pricing-worker") + ;; + esac + + sudo_rules_compact="$(printf '%s' "$sudo_rules" | tr '\n' ' ' | tr -s ' ')" + + for rule in "${required[@]}"; do + if [[ "$sudo_rules_compact" != *"$rule"* ]]; then + error "Missing sudo permission: $rule" + missing=1 + fi + done + + if [[ "$missing" -eq 1 ]]; then + error "Update /etc/sudoers.d/innercontext-deploy and verify with: sudo -u innercontext sudo -n -l" + exit 1 + fi +} + +upload_ops_files() { + log "Uploading operational files" + remote "mkdir -p '$RELEASE_DIR/scripts' '$RELEASE_DIR/systemd' '$RELEASE_DIR/nginx'" + rsync -az scripts/ "$SERVER:$RELEASE_DIR/scripts/" + rsync -az systemd/ "$SERVER:$RELEASE_DIR/systemd/" + rsync -az nginx/ "$SERVER:$RELEASE_DIR/nginx/" + rsync -az scripts/ "$SERVER:$REMOTE_SCRIPTS_DIR/" + remote "chmod +x '$REMOTE_SCRIPTS_DIR'/*.sh || true" +} + +sync_backend_dependencies() { + log "Syncing backend dependencies" + remote "cd '$RELEASE_DIR/backend' && UV_PROJECT_ENVIRONMENT=.venv uv sync --frozen --no-dev --no-editable" +} + +run_db_migrations() { + log "Running database migrations" + remote "cd '$RELEASE_DIR/backend' && UV_PROJECT_ENVIRONMENT=.venv uv run alembic upgrade head" +} + +promote_release() { + log "Promoting release $(basename "$RELEASE_DIR")" + remote "ln -sfn '$RELEASE_DIR' '$CURRENT_LINK'" + PROMOTED=1 +} + +restart_services() { + case "$SCOPE" in + frontend) + log "Restarting frontend service" + remote "sudo systemctl restart innercontext-node" + ;; + backend) + log "Restarting backend services" + remote "sudo systemctl restart innercontext && sudo systemctl restart innercontext-pricing-worker" + ;; + all) + log "Restarting all services" + remote "sudo systemctl restart innercontext && sudo systemctl restart innercontext-node && sudo systemctl restart innercontext-pricing-worker" + ;; + esac +} + +wait_for_service() { + local service="$1" + local timeout="$2" + local i + + for ((i = 1; i <= timeout; i++)); do + if remote "[ \"\$(sudo systemctl is-active '$service' 2>/dev/null)\" = 'active' ]"; then + log "$service is active" + return 0 + fi + sleep 1 + done + + error "$service did not become active within ${timeout}s" + remote "sudo journalctl -u '$service' -n 50" || true + return 1 +} + +check_backend_health() { + local i + for ((i = 1; i <= 30; i++)); do + if remote "curl -sf http://127.0.0.1:8000/health-check >/dev/null"; then + log "Backend health check passed" + return 0 + fi + sleep 2 + done + + error "Backend health check failed" + remote "sudo journalctl -u innercontext -n 50" || true + return 1 +} + +check_frontend_health() { + local i + for ((i = 1; i <= 30; i++)); do + if remote "curl -sf http://127.0.0.1:3000/ >/dev/null"; then + log "Frontend health check passed" + return 0 + fi + sleep 2 + done + + error "Frontend health check failed" + remote "sudo journalctl -u innercontext-node -n 50" || true + return 1 +} + +verify_deployment() { + case "$SCOPE" in + frontend) + wait_for_service innercontext-node "$SERVICE_TIMEOUT" + check_frontend_health + ;; + backend) + wait_for_service innercontext "$SERVICE_TIMEOUT" + wait_for_service innercontext-pricing-worker "$SERVICE_TIMEOUT" + check_backend_health + ;; + all) + wait_for_service innercontext "$SERVICE_TIMEOUT" + wait_for_service innercontext-node "$SERVICE_TIMEOUT" + wait_for_service innercontext-pricing-worker "$SERVICE_TIMEOUT" + check_backend_health + check_frontend_health + ;; + esac +} + +cleanup_old_releases() { + log "Cleaning old releases (keeping $KEEP_RELEASES)" + remote " + cd '$RELEASES_DIR' && \ + ls -1dt [0-9]* 2>/dev/null | tail -n +$((KEEP_RELEASES + 1)) | xargs -r rm -rf + " || true +} + +list_releases() { + log "Current release" + remote "readlink -f '$CURRENT_LINK' 2>/dev/null || echo 'none'" + log "Recent releases" + remote "ls -1dt '$RELEASES_DIR'/* 2>/dev/null | head -10" || true +} + +rollback_to_previous() { + local previous_release + previous_release="$(remote " + current=\$(readlink -f '$CURRENT_LINK' 2>/dev/null || true) + for r in \$(ls -1dt '$RELEASES_DIR'/* 2>/dev/null); do + if [ \"\$r\" != \"\$current\" ]; then + echo \"\$r\" + break + fi + done + ")" + + if [[ -z "$previous_release" ]]; then + error "No previous release found" + exit 1 + fi + + rollback_to_release "$previous_release" "manual" +} + +run_deploy() { + validate_local + acquire_lock + ensure_remote_structure + validate_remote_sudo_permissions + capture_previous_release + create_release_directory + validate_remote_env_files + + if [[ "$SCOPE" == "all" || "$SCOPE" == "backend" ]]; then + upload_backend + sync_backend_dependencies + run_db_migrations + fi + + if [[ "$SCOPE" == "all" || "$SCOPE" == "frontend" ]]; then + upload_frontend + fi + + upload_ops_files + promote_release + restart_services + verify_deployment + cleanup_old_releases + + DEPLOY_SUCCESS=1 + log_deployment "SUCCESS" + log "Deployment complete" } -# ── Dispatch ─────────────────────────────────────────────────────────────── case "$SCOPE" in - frontend) deploy_frontend ;; - backend) deploy_backend ;; - all) deploy_frontend; deploy_backend ;; + frontend|backend|all) + run_deploy + ;; + rollback) + acquire_lock + validate_remote_sudo_permissions + rollback_to_previous + ;; + list) + list_releases + ;; *) - echo "Usage: $0 [frontend|backend|all]" + echo "Usage: $0 [frontend|backend|all|rollback|list]" exit 1 ;; esac - -echo "==> Done." diff --git a/docs/DEPLOYMENT-QUICKSTART.md b/docs/DEPLOYMENT-QUICKSTART.md new file mode 100644 index 0000000..fb30168 --- /dev/null +++ b/docs/DEPLOYMENT-QUICKSTART.md @@ -0,0 +1,97 @@ +# Deployment Quickstart + +This is the short operator checklist. Full details are in `docs/DEPLOYMENT.md`. + +Canonical env file locations (and only these): + +- `/opt/innercontext/shared/backend/.env` +- `/opt/innercontext/shared/frontend/.env.production` + +## 1) Server prerequisites (once) + +```bash +mkdir -p /opt/innercontext/releases +mkdir -p /opt/innercontext/shared/backend +mkdir -p /opt/innercontext/shared/frontend +mkdir -p /opt/innercontext/scripts +chown -R innercontext:innercontext /opt/innercontext +``` + +Create shared env files: + +```bash +cat > /opt/innercontext/shared/backend/.env <<'EOF' +DATABASE_URL=postgresql+psycopg://innercontext:change-me@/innercontext +GEMINI_API_KEY=your-key +EOF + +cat > /opt/innercontext/shared/frontend/.env.production <<'EOF' +PUBLIC_API_BASE=http://127.0.0.1:8000 +ORIGIN=http://innercontext.lan +EOF + +chmod 600 /opt/innercontext/shared/backend/.env +chmod 600 /opt/innercontext/shared/frontend/.env.production +chown innercontext:innercontext /opt/innercontext/shared/backend/.env +chown innercontext:innercontext /opt/innercontext/shared/frontend/.env.production +``` + +Deploy sudoers: + +```bash +cat > /etc/sudoers.d/innercontext-deploy << 'EOF' +innercontext ALL=(root) NOPASSWD: \ + /usr/bin/systemctl restart innercontext, \ + /usr/bin/systemctl restart innercontext-node, \ + /usr/bin/systemctl restart innercontext-pricing-worker, \ + /usr/bin/systemctl is-active innercontext, \ + /usr/bin/systemctl is-active innercontext-node, \ + /usr/bin/systemctl is-active innercontext-pricing-worker +EOF +chmod 440 /etc/sudoers.d/innercontext-deploy +visudo -c -f /etc/sudoers.d/innercontext-deploy +sudo -u innercontext sudo -n -l +``` + +## 2) Local SSH config + +`~/.ssh/config`: + +``` +Host innercontext + HostName + User innercontext +``` + +## 3) Deploy from your machine + +```bash +./deploy.sh +./deploy.sh backend +./deploy.sh frontend +./deploy.sh list +./deploy.sh rollback +``` + +## 4) Verify + +```bash +curl -sf http://innercontext.lan/api/health-check +curl -sf http://innercontext.lan/ +``` + +## 5) Common fixes + +Lock stuck: + +```bash +rm -f /opt/innercontext/.deploy.lock +``` + +Show service logs: + +```bash +journalctl -u innercontext -n 100 +journalctl -u innercontext-node -n 100 +journalctl -u innercontext-pricing-worker -n 100 +``` diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 9f14878..ea1089c 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -1,376 +1,259 @@ -# Deployment guide — Proxmox LXC (home network) +# Deployment Guide (LXC + systemd + nginx) -Target architecture: +This project deploys from an external machine (developer laptop or CI runner) to a Debian LXC host over SSH. + +Deployments are push-based, release-based, and atomic: + +- Build and validate locally +- Upload to `/opt/innercontext/releases/` +- Run backend dependency sync and migrations in that release directory +- Promote once by switching `/opt/innercontext/current` +- Restart services and run health checks +- Auto-rollback on failure + +Environment files have exactly two persistent locations on the server: + +- `/opt/innercontext/shared/backend/.env` +- `/opt/innercontext/shared/frontend/.env.production` + +Each release links to those files from: + +- `/opt/innercontext/current/backend/.env` -> `../../../shared/backend/.env` +- `/opt/innercontext/current/frontend/.env.production` -> `../../../shared/frontend/.env.production` + +## Architecture ``` -Reverse proxy (existing) innercontext LXC (new, Debian 13) -┌──────────────────────┐ ┌────────────────────────────────────┐ -│ reverse proxy │────────────▶│ nginx :80 │ -│ innercontext.lan → * │ │ /api/* → uvicorn :8000/* │ -└──────────────────────┘ │ /* → SvelteKit Node :3000 │ - └────────────────────────────────────┘ - │ │ - FastAPI SvelteKit Node +external machine (manual now, CI later) + | + | ssh + rsync + v +LXC host + /opt/innercontext/ + current -> releases/ + releases/ + shared/backend/.env + shared/frontend/.env.production + scripts/ ``` -> **Frontend is never built on the server.** The `vite build` + `adapter-node` -> esbuild step is CPU/RAM-intensive and will hang on a small LXC. Build locally, -> deploy the `build/` artifact via `deploy.sh`. +Services: -## 1. Prerequisites +- `innercontext` (FastAPI, localhost:8000) +- `innercontext-node` (SvelteKit Node, localhost:3000) +- `innercontext-pricing-worker` (background worker) -- Proxmox VE host with an existing PostgreSQL LXC and a reverse proxy -- LAN hostname `innercontext.lan` resolvable on the network (via router DNS or `/etc/hosts`) -- The PostgreSQL LXC must accept connections from the innercontext LXC IP +nginx routes: ---- +- `/api/*` -> `http://127.0.0.1:8000/*` +- `/*` -> `http://127.0.0.1:3000/*` -## 2. Create the LXC container +## Run Model -In the Proxmox UI (or via CLI): +- Manual deploy: run `./deploy.sh ...` from repo root on your laptop. +- Optional CI deploy: run the same script from a manual workflow (`workflow_dispatch`). +- The server never builds frontend assets. -```bash -# CLI example — adjust storage, bridge, IP to your environment -pct create 200 local:vztmpl/debian-13-standard_13.0-1_amd64.tar.zst \ - --hostname innercontext \ - --cores 2 \ - --memory 1024 \ - --swap 512 \ - --rootfs local-lvm:8 \ - --net0 name=eth0,bridge=vmbr0,ip=dhcp \ - --unprivileged 1 \ - --start 1 -``` +## One-Time Server Setup -Note the container's IP address after it starts (`pct exec 200 -- ip -4 a`). +Run on the LXC host as root. ---- - -## 3. Container setup - -```bash -pct enter 200 # or SSH into the container -``` - -### System packages +### 1) Install runtime dependencies ```bash apt update && apt upgrade -y -apt install -y git nginx curl ca-certificates gnupg lsb-release libpq5 rsync -``` +apt install -y git nginx curl ca-certificates libpq5 rsync python3 python3-venv -### Python 3.12+ + uv - -```bash -apt install -y python3 python3-venv curl -LsSf https://astral.sh/uv/install.sh | UV_INSTALL_DIR=/usr/local/bin sh -``` -Installing to `/usr/local/bin` makes `uv` available system-wide (required for `sudo -u innercontext uv sync`). - -### Node.js 24 LTS + pnpm - -The server needs Node.js to **run** the pre-built frontend bundle, and pnpm to -**install production runtime dependencies** (`clsx`, `bits-ui`, etc. — -`adapter-node` bundles the SvelteKit framework but leaves these external). -The frontend is never **built** on the server. - -```bash curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.4/install.sh | bash . "$HOME/.nvm/nvm.sh" nvm install 24 -``` - -Copy `node` to `/usr/local/bin` so it is accessible system-wide -(required for `sudo -u innercontext` and for systemd). -Use `--remove-destination` to replace any existing symlink with a real file: - -```bash cp --remove-destination "$(nvm which current)" /usr/local/bin/node -``` -Install pnpm as a standalone binary — self-contained, no wrapper scripts, -works system-wide: - -```bash curl -fsSL "https://github.com/pnpm/pnpm/releases/latest/download/pnpm-linux-x64" \ -o /usr/local/bin/pnpm chmod 755 /usr/local/bin/pnpm ``` -### Application user +### 2) Create app user and directories ```bash useradd --system --create-home --shell /bin/bash innercontext -``` ---- - -## 4. Create the database on the PostgreSQL LXC - -Run on the **PostgreSQL LXC**: - -```bash -psql -U postgres <<'SQL' -CREATE USER innercontext WITH PASSWORD 'change-me'; -CREATE DATABASE innercontext OWNER innercontext; -SQL -``` - -Edit `/etc/postgresql/18/main/pg_hba.conf` and add (replace `` with the innercontext container IP): - -``` -host innercontext innercontext /32 scram-sha-256 -``` - -Then reload: - -```bash -systemctl reload postgresql -``` - ---- - -## 5. Clone the repository - -```bash -mkdir -p /opt/innercontext -git clone https://github.com/your-user/innercontext.git /opt/innercontext +mkdir -p /opt/innercontext/releases +mkdir -p /opt/innercontext/shared/backend +mkdir -p /opt/innercontext/shared/frontend +mkdir -p /opt/innercontext/scripts chown -R innercontext:innercontext /opt/innercontext ``` ---- - -## 6. Backend setup +### 3) Create shared env files ```bash -cd /opt/innercontext/backend -``` - -### Install dependencies - -```bash -sudo -u innercontext uv sync -``` - -### Create `.env` - -```bash -cat > /opt/innercontext/backend/.env <<'EOF' -DATABASE_URL=postgresql+psycopg://innercontext:change-me@/innercontext -GEMINI_API_KEY=your-gemini-api-key -# GEMINI_MODEL=gemini-flash-latest # optional, this is the default +cat > /opt/innercontext/shared/backend/.env <<'EOF' +DATABASE_URL=postgresql+psycopg://innercontext:change-me@/innercontext +GEMINI_API_KEY=your-key EOF -chmod 600 /opt/innercontext/backend/.env -chown innercontext:innercontext /opt/innercontext/backend/.env -``` -### Run database migrations - -```bash -sudo -u innercontext bash -c ' - cd /opt/innercontext/backend - uv run alembic upgrade head -' -``` - -This creates all tables on first run. On subsequent deploys it applies only the new migrations. - -> **Existing database (tables already created by `create_db_and_tables`):** -> Run `uv run alembic stamp head` instead to mark the current schema as migrated without re-running DDL. - -### Test - -```bash -sudo -u innercontext bash -c ' - cd /opt/innercontext/backend - uv run uvicorn main:app --host 127.0.0.1 --port 8000 -' -# Ctrl-C after confirming it starts -``` - -### Install systemd service - -```bash -cp /opt/innercontext/systemd/innercontext.service /etc/systemd/system/ -systemctl daemon-reload -systemctl enable --now innercontext -systemctl status innercontext -``` - ---- - -## 7. Frontend setup - -The frontend is **built locally and uploaded** via `deploy.sh` — never built on the server. -This section only covers the one-time server-side configuration. - -### Create `.env.production` - -```bash -cat > /opt/innercontext/frontend/.env.production <<'EOF' -PUBLIC_API_BASE=http://innercontext.lan/api +cat > /opt/innercontext/shared/frontend/.env.production <<'EOF' +PUBLIC_API_BASE=http://127.0.0.1:8000 ORIGIN=http://innercontext.lan EOF -chmod 600 /opt/innercontext/frontend/.env.production -chown innercontext:innercontext /opt/innercontext/frontend/.env.production + +chmod 600 /opt/innercontext/shared/backend/.env +chmod 600 /opt/innercontext/shared/frontend/.env.production +chown innercontext:innercontext /opt/innercontext/shared/backend/.env +chown innercontext:innercontext /opt/innercontext/shared/frontend/.env.production ``` -### Grant `innercontext` passwordless sudo for service restarts +### 4) Grant deploy sudo permissions ```bash cat > /etc/sudoers.d/innercontext-deploy << 'EOF' innercontext ALL=(root) NOPASSWD: \ /usr/bin/systemctl restart innercontext, \ /usr/bin/systemctl restart innercontext-node, \ - /usr/bin/systemctl restart innercontext-pricing-worker + /usr/bin/systemctl restart innercontext-pricing-worker, \ + /usr/bin/systemctl is-active innercontext, \ + /usr/bin/systemctl is-active innercontext-node, \ + /usr/bin/systemctl is-active innercontext-pricing-worker EOF + chmod 440 /etc/sudoers.d/innercontext-deploy +visudo -c -f /etc/sudoers.d/innercontext-deploy + +# Must work without password or TTY prompt: +sudo -u innercontext sudo -n -l ``` -### Install systemd services +If `sudo -n -l` fails, deployments will fail during restart/rollback with: +`sudo: a terminal is required` or `sudo: a password is required`. + +### 5) Install systemd and nginx configs + +After first deploy (or after copying repo content to `/opt/innercontext/current`), install configs: ```bash -cp /opt/innercontext/systemd/innercontext-node.service /etc/systemd/system/ -cp /opt/innercontext/systemd/innercontext-pricing-worker.service /etc/systemd/system/ +cp /opt/innercontext/current/systemd/innercontext.service /etc/systemd/system/ +cp /opt/innercontext/current/systemd/innercontext-node.service /etc/systemd/system/ +cp /opt/innercontext/current/systemd/innercontext-pricing-worker.service /etc/systemd/system/ systemctl daemon-reload +systemctl enable innercontext systemctl enable innercontext-node -systemctl enable --now innercontext-pricing-worker -# Do NOT start yet — build/ is empty until the first deploy.sh run -``` +systemctl enable innercontext-pricing-worker ---- - -## 8. nginx setup - -```bash -cp /opt/innercontext/nginx/innercontext.conf /etc/nginx/sites-available/innercontext -ln -s /etc/nginx/sites-available/innercontext /etc/nginx/sites-enabled/ +cp /opt/innercontext/current/nginx/innercontext.conf /etc/nginx/sites-available/innercontext +ln -sf /etc/nginx/sites-available/innercontext /etc/nginx/sites-enabled/innercontext rm -f /etc/nginx/sites-enabled/default -nginx -t -systemctl reload nginx +nginx -t && systemctl reload nginx ``` ---- +## Local Machine Setup -## 9. Reverse proxy configuration - -Point your existing reverse proxy at the innercontext LXC's nginx (`:80`). - -Example — Caddy: - -``` -innercontext.lan { - reverse_proxy :80 -} -``` - -Example — nginx upstream: - -```nginx -server { - listen 80; - server_name innercontext.lan; - location / { - proxy_pass http://:80; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } -} -``` - -Reload your reverse proxy after applying the change. - ---- - -## 10. First deploy from local machine - -All subsequent deploys (including the first one) use `deploy.sh` from your local machine. - -### SSH config - -Add to `~/.ssh/config` on your local machine: +`~/.ssh/config`: ``` Host innercontext - HostName + HostName User innercontext ``` -Make sure your SSH public key is in `/home/innercontext/.ssh/authorized_keys` on the server. +Ensure your public key is in `/home/innercontext/.ssh/authorized_keys`. -### Run the first deploy +## Deploy Commands + +From repository root on external machine: ```bash -# From the repo root on your local machine: -./deploy.sh +./deploy.sh # full deploy (default = all) +./deploy.sh all +./deploy.sh backend +./deploy.sh frontend +./deploy.sh list +./deploy.sh rollback ``` -This will: -1. Build the frontend locally (`pnpm run build`) -2. Upload `frontend/build/` to the server via rsync -3. Restart `innercontext-node` -4. Upload `backend/` source to the server -5. Run `uv sync --frozen` on the server -6. Restart `innercontext` (runs alembic migrations on start) -7. Restart `innercontext-pricing-worker` - ---- - -## 11. Verification +Optional overrides: ```bash -# From any machine on the LAN: -curl http://innercontext.lan/api/health-check # {"status":"ok"} -curl http://innercontext.lan/api/products # [] -curl http://innercontext.lan/ # SvelteKit HTML shell +DEPLOY_SERVER=innercontext ./deploy.sh all +DEPLOY_ROOT=/opt/innercontext ./deploy.sh backend +DEPLOY_ALLOW_DIRTY=1 ./deploy.sh frontend ``` -The web UI should be accessible at `http://innercontext.lan`. +## What `deploy.sh` Does ---- +For `backend` / `frontend` / `all`: -## 12. Updating the application +1. Local checks (strict, fail-fast) +2. Acquire `/opt/innercontext/.deploy.lock` +3. Create `` release directory +4. Upload selected component(s) +5. Link shared env files in the release directory +6. `uv sync` + `alembic upgrade head` (backend scope) +7. Upload `scripts/`, `systemd/`, `nginx/` +8. Switch `current` to the prepared release +9. Restart affected services +10. Run health checks +11. Remove old releases (keep last 5) +12. Write deploy entry to `/opt/innercontext/deploy.log` + +If anything fails after promotion, script auto-rolls back to previous release. + +## Health Checks + +- Backend: `http://127.0.0.1:8000/health-check` +- Frontend: `http://127.0.0.1:3000/` +- Worker: `systemctl is-active innercontext-pricing-worker` + +Manual checks: ```bash -# From the repo root on your local machine: -./deploy.sh # full deploy (frontend + backend) -./deploy.sh frontend # frontend only -./deploy.sh backend # backend only +curl -sf http://127.0.0.1:8000/health-check +curl -sf http://127.0.0.1:3000/ +systemctl is-active innercontext +systemctl is-active innercontext-node +systemctl is-active innercontext-pricing-worker ``` ---- +## Troubleshooting -## 13. Troubleshooting - -### 502 Bad Gateway on `/api/*` +### Lock exists ```bash -systemctl status innercontext -journalctl -u innercontext -n 50 -# Check .env DATABASE_URL is correct and PG LXC accepts connections +cat /opt/innercontext/.deploy.lock +rm -f /opt/innercontext/.deploy.lock ``` -### Product prices stay empty / stale +Only remove the lock if no deployment is running. + +### Sudo password prompt during deploy + +Re-check `/etc/sudoers.d/innercontext-deploy` and run: ```bash -systemctl status innercontext-pricing-worker -journalctl -u innercontext-pricing-worker -n 50 -# Ensure worker is running and can connect to PostgreSQL +visudo -c -f /etc/sudoers.d/innercontext-deploy +sudo -u innercontext sudo systemctl is-active innercontext ``` -### 502 Bad Gateway on `/` +### Backend migration failure + +Validate env file and DB connectivity: ```bash -systemctl status innercontext-node -journalctl -u innercontext-node -n 50 -# Verify /opt/innercontext/frontend/build/index.js exists (deploy.sh ran successfully) +ls -la /opt/innercontext/shared/backend/.env +grep '^DATABASE_URL=' /opt/innercontext/shared/backend/.env ``` -### Database connection refused +### Service fails after deploy ```bash -# From innercontext LXC: -psql postgresql+psycopg://innercontext:change-me@/innercontext -c "SELECT 1" -# If it fails, check pg_hba.conf on the PG LXC and verify the IP matches +journalctl -u innercontext -n 100 +journalctl -u innercontext-node -n 100 +journalctl -u innercontext-pricing-worker -n 100 ``` + +## Manual CI Deploy (Optional) + +Use the manual Forgejo workflow (`workflow_dispatch`) to run the same `./deploy.sh all` path from CI once server secrets and SSH trust are configured. diff --git a/scripts/backup-database.sh b/scripts/backup-database.sh new file mode 100755 index 0000000..d44b078 --- /dev/null +++ b/scripts/backup-database.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# +# Database backup script for innercontext PostgreSQL database +# Should be run daily via cron on the PostgreSQL LXC: +# 0 2 * * * /opt/innercontext/scripts/backup-database.sh >> /opt/innercontext/backup.log 2>&1 +# +# Note: This script should be copied to the PostgreSQL LXC container +# and run there (not on the app LXC) +# + +set -euo pipefail + +# Configuration +BACKUP_DIR="/opt/innercontext/backups" +DB_NAME="innercontext" +DB_USER="innercontext" +KEEP_DAYS=7 +TIMESTAMP=$(date '+%Y%m%d_%H%M%S') +BACKUP_FILE="$BACKUP_DIR/innercontext_${TIMESTAMP}.sql.gz" + +# Color codes +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +log() { + echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" +} + +# Create backup directory if it doesn't exist +mkdir -p "$BACKUP_DIR" + +# Create backup +log "Starting database backup..." +if pg_dump -U "$DB_USER" -d "$DB_NAME" | gzip > "$BACKUP_FILE"; then + BACKUP_SIZE=$(du -h "$BACKUP_FILE" | cut -f1) + log "${GREEN}✓${NC} Backup created: $BACKUP_FILE ($BACKUP_SIZE)" +else + log "${RED}✗${NC} Backup failed" + exit 1 +fi + +# Clean up old backups +log "Cleaning up backups older than $KEEP_DAYS days..." +find "$BACKUP_DIR" -name "innercontext_*.sql.gz" -type f -mtime +$KEEP_DAYS -delete +REMAINING=$(find "$BACKUP_DIR" -name "innercontext_*.sql.gz" -type f | wc -l) +log "${GREEN}✓${NC} Cleanup complete. $REMAINING backup(s) remaining" + +# Verify backup can be read +if gunzip -t "$BACKUP_FILE" 2>/dev/null; then + log "${GREEN}✓${NC} Backup integrity verified" +else + log "${RED}✗${NC} Backup integrity check failed" + exit 1 +fi + +log "${GREEN}✓${NC} Database backup completed successfully" +exit 0 diff --git a/scripts/healthcheck.sh b/scripts/healthcheck.sh new file mode 100755 index 0000000..9d370ff --- /dev/null +++ b/scripts/healthcheck.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# +# Health check script for innercontext services +# Should be run via cron every 5 minutes: +# */5 * * * * /opt/innercontext/scripts/healthcheck.sh >> /opt/innercontext/healthcheck.log 2>&1 +# + +set -euo pipefail + +BACKEND_URL="http://127.0.0.1:8000/health-check" +FRONTEND_URL="http://127.0.0.1:3000/" +TIMEOUT=10 +TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S') + +# Color codes +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +log() { + echo "[$TIMESTAMP] $1" +} + +check_service() { + local service_name=$1 + local url=$2 + + if systemctl is-active --quiet "$service_name"; then + if curl -sf --max-time "$TIMEOUT" "$url" > /dev/null 2>&1; then + log "${GREEN}✓${NC} $service_name is healthy" + return 0 + else + log "${YELLOW}⚠${NC} $service_name is running but not responding at $url" + return 1 + fi + else + log "${RED}✗${NC} $service_name is not running" + return 1 + fi +} + +# Check all services +backend_ok=0 +frontend_ok=0 +worker_ok=0 + +check_service "innercontext" "$BACKEND_URL" || backend_ok=1 +check_service "innercontext-node" "$FRONTEND_URL" || frontend_ok=1 + +# Worker doesn't have HTTP endpoint, just check if it's running +if systemctl is-active --quiet "innercontext-pricing-worker"; then + log "${GREEN}✓${NC} innercontext-pricing-worker is running" +else + log "${RED}✗${NC} innercontext-pricing-worker is not running" + worker_ok=1 +fi + +# If any service is unhealthy, exit with error code +if [ $backend_ok -ne 0 ] || [ $frontend_ok -ne 0 ] || [ $worker_ok -ne 0 ]; then + log "${RED}Health check failed${NC}" + exit 1 +else + log "${GREEN}All services healthy${NC}" + exit 0 +fi diff --git a/scripts/validate-env.sh b/scripts/validate-env.sh new file mode 100755 index 0000000..3ca90a4 --- /dev/null +++ b/scripts/validate-env.sh @@ -0,0 +1,157 @@ +#!/bin/bash +# +# Validate environment variables for innercontext deployment +# Checks both shared directory (persistent config) and current release (symlinks) +# + +set -euo pipefail + +# Color codes +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +# Shared directory (persistent configuration) +SHARED_BACKEND_ENV="/opt/innercontext/shared/backend/.env" +SHARED_FRONTEND_ENV="/opt/innercontext/shared/frontend/.env.production" + +# Current release (should be symlinks to shared) +CURRENT_BACKEND_ENV="/opt/innercontext/current/backend/.env" +CURRENT_FRONTEND_ENV="/opt/innercontext/current/frontend/.env.production" + +errors=0 +warnings=0 + +log_error() { + echo -e "${RED}✗${NC} $1" + ((errors++)) +} + +log_success() { + echo -e "${GREEN}✓${NC} $1" +} + +log_warning() { + echo -e "${YELLOW}⚠${NC} $1" + ((warnings++)) +} + +check_symlink() { + local symlink_path=$1 + local expected_target=$2 + + if [ ! -L "$symlink_path" ]; then + log_error "Not a symlink: $symlink_path" + return 1 + fi + + local actual_target=$(readlink "$symlink_path") + if [ "$actual_target" != "$expected_target" ]; then + log_warning "Symlink target mismatch: $symlink_path -> $actual_target (expected: $expected_target)" + else + log_success "Symlink correct: $symlink_path -> $actual_target" + fi +} + +check_var() { + local file=$1 + local var_name=$2 + local optional=${3:-false} + + if [ ! -f "$file" ]; then + log_error "File not found: $file" + return 1 + fi + + # Check if variable exists and is not empty + if grep -q "^${var_name}=" "$file"; then + local value=$(grep "^${var_name}=" "$file" | cut -d'=' -f2-) + if [ -z "$value" ]; then + if [ "$optional" = true ]; then + log_warning "$var_name is empty in $file (optional)" + else + log_error "$var_name is empty in $file" + fi + else + log_success "$var_name is set" + fi + else + if [ "$optional" = true ]; then + log_warning "$var_name not found in $file (optional)" + else + log_error "$var_name not found in $file" + fi + fi +} + +echo "=== Validating Shared Directory Structure ===" + +# Check shared directory exists +if [ -d "/opt/innercontext/shared" ]; then + log_success "Shared directory exists: /opt/innercontext/shared" +else + log_error "Shared directory not found: /opt/innercontext/shared" +fi + +# Check shared backend .env +if [ -f "$SHARED_BACKEND_ENV" ]; then + log_success "Shared backend .env exists: $SHARED_BACKEND_ENV" +else + log_error "Shared backend .env not found: $SHARED_BACKEND_ENV" +fi + +# Check shared frontend .env.production +if [ -f "$SHARED_FRONTEND_ENV" ]; then + log_success "Shared frontend .env.production exists: $SHARED_FRONTEND_ENV" +else + log_error "Shared frontend .env.production not found: $SHARED_FRONTEND_ENV" +fi + +echo "" +echo "=== Validating Symlinks in Current Release ===" + +# Check current release symlinks point to shared directory +if [ -e "$CURRENT_BACKEND_ENV" ]; then + check_symlink "$CURRENT_BACKEND_ENV" "../../../shared/backend/.env" +else + log_error "Current backend .env not found: $CURRENT_BACKEND_ENV" +fi + +if [ -e "$CURRENT_FRONTEND_ENV" ]; then + check_symlink "$CURRENT_FRONTEND_ENV" "../../../shared/frontend/.env.production" +else + log_error "Current frontend .env.production not found: $CURRENT_FRONTEND_ENV" +fi + +echo "" +echo "=== Validating Backend Environment Variables ===" +if [ -f "$SHARED_BACKEND_ENV" ]; then + check_var "$SHARED_BACKEND_ENV" "DATABASE_URL" + check_var "$SHARED_BACKEND_ENV" "GEMINI_API_KEY" + check_var "$SHARED_BACKEND_ENV" "LOG_LEVEL" true + check_var "$SHARED_BACKEND_ENV" "CORS_ORIGINS" true +fi + +echo "" +echo "=== Validating Frontend Environment Variables ===" +if [ -f "$SHARED_FRONTEND_ENV" ]; then + check_var "$SHARED_FRONTEND_ENV" "PUBLIC_API_BASE" + check_var "$SHARED_FRONTEND_ENV" "ORIGIN" +fi + +echo "" +if [ $errors -eq 0 ]; then + if [ $warnings -eq 0 ]; then + echo -e "${GREEN}✓ All environment checks passed${NC}" + else + echo -e "${YELLOW}⚠ Environment validation passed with $warnings warning(s)${NC}" + fi + exit 0 +else + echo -e "${RED}✗ Found $errors error(s) in environment configuration${NC}" + if [ $warnings -gt 0 ]; then + echo -e "${YELLOW} And $warnings warning(s)${NC}" + fi + exit 1 +fi