fix(deploy): make LXC deploys atomic and fail-fast
Rebuild the deployment flow to prepare releases remotely, validate env/sudo prerequisites, run migrations in-release, and auto-rollback on health failures. Consolidate deployment docs and add a manual CI workflow so laptop and CI use the same push-based deploy path.
This commit is contained in:
parent
d228b44209
commit
2efdb2b785
8 changed files with 1057 additions and 319 deletions
73
.forgejo/workflows/deploy-manual.yml
Normal file
73
.forgejo/workflows/deploy-manual.yml
Normal file
|
|
@ -0,0 +1,73 @@
|
|||
name: Deploy (Manual)
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
scope:
|
||||
description: "Deployment scope"
|
||||
required: true
|
||||
default: "all"
|
||||
type: choice
|
||||
options:
|
||||
- all
|
||||
- backend
|
||||
- frontend
|
||||
- rollback
|
||||
- list
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
name: Manual deployment to LXC
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install uv
|
||||
run: |
|
||||
curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "24"
|
||||
|
||||
- name: Install pnpm
|
||||
run: npm install -g pnpm
|
||||
|
||||
- name: Configure SSH key
|
||||
env:
|
||||
DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
|
||||
run: |
|
||||
mkdir -p "$HOME/.ssh"
|
||||
chmod 700 "$HOME/.ssh"
|
||||
printf '%s\n' "$DEPLOY_SSH_KEY" > "$HOME/.ssh/id_ed25519"
|
||||
chmod 600 "$HOME/.ssh/id_ed25519"
|
||||
|
||||
- name: Configure known hosts
|
||||
env:
|
||||
DEPLOY_KNOWN_HOSTS: ${{ secrets.DEPLOY_KNOWN_HOSTS }}
|
||||
run: |
|
||||
if [ -z "$DEPLOY_KNOWN_HOSTS" ]; then
|
||||
echo "DEPLOY_KNOWN_HOSTS secret is required"
|
||||
exit 1
|
||||
fi
|
||||
printf '%s\n' "$DEPLOY_KNOWN_HOSTS" > "$HOME/.ssh/known_hosts"
|
||||
chmod 644 "$HOME/.ssh/known_hosts"
|
||||
|
||||
- name: Run deployment
|
||||
env:
|
||||
DEPLOY_SERVER: ${{ secrets.DEPLOY_SERVER }}
|
||||
run: |
|
||||
if [ -z "$DEPLOY_SERVER" ]; then
|
||||
echo "DEPLOY_SERVER secret is required"
|
||||
exit 1
|
||||
fi
|
||||
chmod +x ./deploy.sh
|
||||
./deploy.sh "${{ inputs.scope }}"
|
||||
|
|
@ -100,4 +100,7 @@ uv run pytest
|
|||
|
||||
## Deployment
|
||||
|
||||
See [docs/DEPLOYMENT.md](docs/DEPLOYMENT.md) for a step-by-step guide for a Proxmox LXC setup (Debian 13, nginx, systemd services).
|
||||
Deployments are push-based from an external machine (laptop/CI runner) to the LXC host over SSH.
|
||||
|
||||
- Canonical runbook: [docs/DEPLOYMENT.md](docs/DEPLOYMENT.md)
|
||||
- Operator checklist: [docs/DEPLOYMENT-QUICKSTART.md](docs/DEPLOYMENT-QUICKSTART.md)
|
||||
|
|
|
|||
486
deploy.sh
486
deploy.sh
|
|
@ -1,63 +1,463 @@
|
|||
#!/usr/bin/env bash
|
||||
# Usage: ./deploy.sh [frontend|backend|all]
|
||||
# default: all
|
||||
#
|
||||
# SSH config (~/.ssh/config) — recommended:
|
||||
# Host innercontext
|
||||
# HostName <IP_LXC>
|
||||
# User innercontext
|
||||
#
|
||||
# The innercontext user needs passwordless sudo for systemctl only:
|
||||
# /etc/sudoers.d/innercontext-deploy:
|
||||
# innercontext ALL=(root) NOPASSWD: /usr/bin/systemctl restart innercontext, /usr/bin/systemctl restart innercontext-node, /usr/bin/systemctl restart innercontext-pricing-worker, /usr/bin/systemctl is-active innercontext, /usr/bin/systemctl is-active innercontext-node, /usr/bin/systemctl is-active innercontext-pricing-worker
|
||||
set -euo pipefail
|
||||
# Usage: ./deploy.sh [frontend|backend|all|rollback|list]
|
||||
|
||||
set -eEuo pipefail
|
||||
|
||||
SERVER="${DEPLOY_SERVER:-innercontext}"
|
||||
REMOTE_ROOT="${DEPLOY_ROOT:-/opt/innercontext}"
|
||||
RELEASES_DIR="$REMOTE_ROOT/releases"
|
||||
CURRENT_LINK="$REMOTE_ROOT/current"
|
||||
REMOTE_SCRIPTS_DIR="$REMOTE_ROOT/scripts"
|
||||
LOCK_FILE="$REMOTE_ROOT/.deploy.lock"
|
||||
LOG_FILE="$REMOTE_ROOT/deploy.log"
|
||||
KEEP_RELEASES="${KEEP_RELEASES:-5}"
|
||||
SERVICE_TIMEOUT="${SERVICE_TIMEOUT:-60}"
|
||||
|
||||
SERVER="${DEPLOY_SERVER:-innercontext}" # ssh host alias or user@host
|
||||
REMOTE="/opt/innercontext"
|
||||
SCOPE="${1:-all}"
|
||||
TIMESTAMP="$(date +%Y%m%d_%H%M%S)"
|
||||
RELEASE_DIR="$RELEASES_DIR/$TIMESTAMP"
|
||||
|
||||
# ── Frontend ───────────────────────────────────────────────────────────────
|
||||
deploy_frontend() {
|
||||
echo "==> [frontend] Building locally..."
|
||||
(cd frontend && pnpm run build)
|
||||
LOCK_ACQUIRED=0
|
||||
PROMOTED=0
|
||||
DEPLOY_SUCCESS=0
|
||||
PREVIOUS_RELEASE=""
|
||||
|
||||
echo "==> [frontend] Uploading build/ and package files..."
|
||||
rsync -az --delete frontend/build/ "$SERVER:$REMOTE/frontend/build/"
|
||||
rsync -az frontend/package.json frontend/pnpm-lock.yaml "$SERVER:$REMOTE/frontend/"
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m'
|
||||
|
||||
echo "==> [frontend] Installing production dependencies on server..."
|
||||
ssh "$SERVER" "cd $REMOTE/frontend && pnpm install --prod --frozen-lockfile --ignore-scripts"
|
||||
|
||||
echo "==> [frontend] Restarting service..."
|
||||
ssh "$SERVER" "sudo systemctl restart innercontext-node && echo OK"
|
||||
log() {
|
||||
echo -e "${GREEN}==>${NC} $*"
|
||||
}
|
||||
|
||||
# ── Backend ────────────────────────────────────────────────────────────────
|
||||
deploy_backend() {
|
||||
echo "==> [backend] Uploading source..."
|
||||
warn() {
|
||||
echo -e "${YELLOW}WARN:${NC} $*"
|
||||
}
|
||||
|
||||
error() {
|
||||
echo -e "${RED}ERROR:${NC} $*" >&2
|
||||
}
|
||||
|
||||
remote() {
|
||||
ssh "$SERVER" "$@"
|
||||
}
|
||||
|
||||
log_deployment() {
|
||||
local status="$1"
|
||||
remote "mkdir -p '$REMOTE_ROOT'"
|
||||
remote "{
|
||||
echo '---'
|
||||
echo 'timestamp: $(date -u +%Y-%m-%dT%H:%M:%SZ)'
|
||||
echo 'deployer: $(whoami)@$(hostname)'
|
||||
echo 'commit: $(git rev-parse HEAD 2>/dev/null || echo unknown)'
|
||||
echo 'branch: $(git branch --show-current 2>/dev/null || echo unknown)'
|
||||
echo 'scope: $SCOPE'
|
||||
echo 'release: $TIMESTAMP'
|
||||
echo 'status: $status'
|
||||
} >> '$LOG_FILE'" || true
|
||||
}
|
||||
|
||||
release_lock() {
|
||||
if [[ "$LOCK_ACQUIRED" -eq 1 ]]; then
|
||||
remote "rm -f '$LOCK_FILE'" || true
|
||||
fi
|
||||
}
|
||||
|
||||
cleanup_on_exit() {
|
||||
release_lock
|
||||
}
|
||||
|
||||
rollback_to_release() {
|
||||
local target_release="$1"
|
||||
local reason="$2"
|
||||
|
||||
if [[ -z "$target_release" ]]; then
|
||||
error "Rollback skipped: no target release"
|
||||
return 1
|
||||
fi
|
||||
|
||||
warn "Rolling back to $(basename "$target_release") ($reason)"
|
||||
remote "ln -sfn '$target_release' '$CURRENT_LINK'"
|
||||
remote "sudo systemctl restart innercontext && sudo systemctl restart innercontext-node && sudo systemctl restart innercontext-pricing-worker"
|
||||
|
||||
if wait_for_service innercontext "$SERVICE_TIMEOUT" \
|
||||
&& wait_for_service innercontext-node "$SERVICE_TIMEOUT" \
|
||||
&& wait_for_service innercontext-pricing-worker "$SERVICE_TIMEOUT" \
|
||||
&& check_backend_health \
|
||||
&& check_frontend_health; then
|
||||
log "Rollback succeeded"
|
||||
log_deployment "ROLLBACK_SUCCESS:$reason"
|
||||
return 0
|
||||
fi
|
||||
|
||||
error "Rollback failed"
|
||||
log_deployment "ROLLBACK_FAILED:$reason"
|
||||
return 1
|
||||
}
|
||||
|
||||
on_error() {
|
||||
local exit_code="$?"
|
||||
trap - ERR
|
||||
|
||||
error "Deployment failed (exit $exit_code)"
|
||||
|
||||
if [[ "$PROMOTED" -eq 1 && "$DEPLOY_SUCCESS" -eq 0 ]]; then
|
||||
rollback_to_release "$PREVIOUS_RELEASE" "deploy_error" || true
|
||||
elif [[ -n "${RELEASE_DIR:-}" ]]; then
|
||||
remote "rm -rf '$RELEASE_DIR'" || true
|
||||
fi
|
||||
|
||||
log_deployment "FAILED"
|
||||
exit "$exit_code"
|
||||
}
|
||||
|
||||
trap cleanup_on_exit EXIT
|
||||
trap on_error ERR
|
||||
|
||||
validate_local() {
|
||||
log "Running local validation"
|
||||
|
||||
if [[ "${DEPLOY_ALLOW_DIRTY:-0}" != "1" ]]; then
|
||||
if ! git diff-index --quiet HEAD -- 2>/dev/null; then
|
||||
error "Working tree has uncommitted changes"
|
||||
error "Commit/stash changes or run with DEPLOY_ALLOW_DIRTY=1"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
warn "Skipping clean working tree check (DEPLOY_ALLOW_DIRTY=1)"
|
||||
fi
|
||||
|
||||
if [[ "$SCOPE" == "all" || "$SCOPE" == "backend" ]]; then
|
||||
log "Backend checks"
|
||||
(cd backend && uv run ruff check .)
|
||||
(cd backend && uv run black --check .)
|
||||
(cd backend && uv run isort --check-only .)
|
||||
fi
|
||||
|
||||
if [[ "$SCOPE" == "all" || "$SCOPE" == "frontend" ]]; then
|
||||
log "Frontend checks"
|
||||
(cd frontend && pnpm check)
|
||||
(cd frontend && pnpm lint)
|
||||
log "Building frontend artifact"
|
||||
(cd frontend && pnpm build)
|
||||
fi
|
||||
}
|
||||
|
||||
acquire_lock() {
|
||||
log "Acquiring deployment lock"
|
||||
local lock_payload
|
||||
lock_payload="$(date -u +%Y-%m-%dT%H:%M:%SZ) $(whoami)@$(hostname) $(git rev-parse --short HEAD 2>/dev/null || echo unknown)"
|
||||
|
||||
if ! remote "( set -o noclobber; echo '$lock_payload' > '$LOCK_FILE' ) 2>/dev/null"; then
|
||||
error "Deployment lock exists: $LOCK_FILE"
|
||||
remote "cat '$LOCK_FILE'" || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
LOCK_ACQUIRED=1
|
||||
}
|
||||
|
||||
ensure_remote_structure() {
|
||||
log "Ensuring remote directory structure"
|
||||
remote "mkdir -p '$RELEASES_DIR' '$REMOTE_ROOT/shared/backend' '$REMOTE_ROOT/shared/frontend' '$REMOTE_SCRIPTS_DIR'"
|
||||
}
|
||||
|
||||
capture_previous_release() {
|
||||
PREVIOUS_RELEASE="$(remote "readlink -f '$CURRENT_LINK' 2>/dev/null || true")"
|
||||
if [[ -n "$PREVIOUS_RELEASE" ]]; then
|
||||
log "Previous release: $(basename "$PREVIOUS_RELEASE")"
|
||||
else
|
||||
warn "No previous release detected"
|
||||
fi
|
||||
}
|
||||
|
||||
create_release_directory() {
|
||||
log "Creating release directory: $(basename "$RELEASE_DIR")"
|
||||
remote "rm -rf '$RELEASE_DIR' && mkdir -p '$RELEASE_DIR'"
|
||||
}
|
||||
|
||||
upload_backend() {
|
||||
log "Uploading backend"
|
||||
remote "mkdir -p '$RELEASE_DIR/backend'"
|
||||
rsync -az --delete \
|
||||
--exclude='.venv/' \
|
||||
--exclude='__pycache__/' \
|
||||
--exclude='*.pyc' \
|
||||
--exclude='.env' \
|
||||
backend/ "$SERVER:$REMOTE/backend/"
|
||||
backend/ "$SERVER:$RELEASE_DIR/backend/"
|
||||
|
||||
echo "==> [backend] Syncing dependencies..."
|
||||
ssh "$SERVER" "cd $REMOTE/backend && uv sync --frozen --no-dev --no-editable"
|
||||
|
||||
echo "==> [backend] Restarting services (alembic runs on API start)..."
|
||||
ssh "$SERVER" "sudo systemctl restart innercontext && sudo systemctl restart innercontext-pricing-worker && echo OK"
|
||||
log "Linking backend shared env"
|
||||
remote "ln -sfn ../../../shared/backend/.env '$RELEASE_DIR/backend/.env'"
|
||||
}
|
||||
|
||||
# ── Dispatch ───────────────────────────────────────────────────────────────
|
||||
case "$SCOPE" in
|
||||
frontend) deploy_frontend ;;
|
||||
backend) deploy_backend ;;
|
||||
all) deploy_frontend; deploy_backend ;;
|
||||
*)
|
||||
echo "Usage: $0 [frontend|backend|all]"
|
||||
upload_frontend() {
|
||||
log "Uploading frontend build artifact"
|
||||
remote "mkdir -p '$RELEASE_DIR/frontend'"
|
||||
rsync -az --delete frontend/build/ "$SERVER:$RELEASE_DIR/frontend/build/"
|
||||
rsync -az frontend/package.json frontend/pnpm-lock.yaml "$SERVER:$RELEASE_DIR/frontend/"
|
||||
|
||||
log "Installing frontend production dependencies on server"
|
||||
remote "cd '$RELEASE_DIR/frontend' && pnpm install --prod --frozen-lockfile --ignore-scripts"
|
||||
|
||||
log "Linking frontend shared env"
|
||||
remote "ln -sfn ../../../shared/frontend/.env.production '$RELEASE_DIR/frontend/.env.production'"
|
||||
}
|
||||
|
||||
validate_remote_env_files() {
|
||||
if [[ "$SCOPE" == "all" || "$SCOPE" == "backend" ]]; then
|
||||
log "Validating remote backend env file"
|
||||
remote "test -f '$REMOTE_ROOT/shared/backend/.env'"
|
||||
fi
|
||||
|
||||
if [[ "$SCOPE" == "all" || "$SCOPE" == "frontend" ]]; then
|
||||
log "Validating remote frontend env file"
|
||||
remote "test -f '$REMOTE_ROOT/shared/frontend/.env.production'"
|
||||
fi
|
||||
}
|
||||
|
||||
validate_remote_sudo_permissions() {
|
||||
local sudo_rules
|
||||
local sudo_rules_compact
|
||||
local required=()
|
||||
local missing=0
|
||||
local rule
|
||||
|
||||
log "Validating remote sudo permissions"
|
||||
|
||||
if ! sudo_rules="$(remote "sudo -n -l 2>/dev/null")"; then
|
||||
error "Remote user cannot run sudo non-interactively"
|
||||
error "Configure /etc/sudoers.d/innercontext-deploy for user 'innercontext'"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
case "$SCOPE" in
|
||||
frontend)
|
||||
required+=("/usr/bin/systemctl restart innercontext-node")
|
||||
required+=("/usr/bin/systemctl is-active innercontext-node")
|
||||
;;
|
||||
backend)
|
||||
required+=("/usr/bin/systemctl restart innercontext")
|
||||
required+=("/usr/bin/systemctl restart innercontext-pricing-worker")
|
||||
required+=("/usr/bin/systemctl is-active innercontext")
|
||||
required+=("/usr/bin/systemctl is-active innercontext-pricing-worker")
|
||||
;;
|
||||
all|rollback)
|
||||
required+=("/usr/bin/systemctl restart innercontext")
|
||||
required+=("/usr/bin/systemctl restart innercontext-node")
|
||||
required+=("/usr/bin/systemctl restart innercontext-pricing-worker")
|
||||
required+=("/usr/bin/systemctl is-active innercontext")
|
||||
required+=("/usr/bin/systemctl is-active innercontext-node")
|
||||
required+=("/usr/bin/systemctl is-active innercontext-pricing-worker")
|
||||
;;
|
||||
esac
|
||||
|
||||
echo "==> Done."
|
||||
sudo_rules_compact="$(printf '%s' "$sudo_rules" | tr '\n' ' ' | tr -s ' ')"
|
||||
|
||||
for rule in "${required[@]}"; do
|
||||
if [[ "$sudo_rules_compact" != *"$rule"* ]]; then
|
||||
error "Missing sudo permission: $rule"
|
||||
missing=1
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ "$missing" -eq 1 ]]; then
|
||||
error "Update /etc/sudoers.d/innercontext-deploy and verify with: sudo -u innercontext sudo -n -l"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
upload_ops_files() {
|
||||
log "Uploading operational files"
|
||||
remote "mkdir -p '$RELEASE_DIR/scripts' '$RELEASE_DIR/systemd' '$RELEASE_DIR/nginx'"
|
||||
rsync -az scripts/ "$SERVER:$RELEASE_DIR/scripts/"
|
||||
rsync -az systemd/ "$SERVER:$RELEASE_DIR/systemd/"
|
||||
rsync -az nginx/ "$SERVER:$RELEASE_DIR/nginx/"
|
||||
rsync -az scripts/ "$SERVER:$REMOTE_SCRIPTS_DIR/"
|
||||
remote "chmod +x '$REMOTE_SCRIPTS_DIR'/*.sh || true"
|
||||
}
|
||||
|
||||
sync_backend_dependencies() {
|
||||
log "Syncing backend dependencies"
|
||||
remote "cd '$RELEASE_DIR/backend' && UV_PROJECT_ENVIRONMENT=.venv uv sync --frozen --no-dev --no-editable"
|
||||
}
|
||||
|
||||
run_db_migrations() {
|
||||
log "Running database migrations"
|
||||
remote "cd '$RELEASE_DIR/backend' && UV_PROJECT_ENVIRONMENT=.venv uv run alembic upgrade head"
|
||||
}
|
||||
|
||||
promote_release() {
|
||||
log "Promoting release $(basename "$RELEASE_DIR")"
|
||||
remote "ln -sfn '$RELEASE_DIR' '$CURRENT_LINK'"
|
||||
PROMOTED=1
|
||||
}
|
||||
|
||||
restart_services() {
|
||||
case "$SCOPE" in
|
||||
frontend)
|
||||
log "Restarting frontend service"
|
||||
remote "sudo systemctl restart innercontext-node"
|
||||
;;
|
||||
backend)
|
||||
log "Restarting backend services"
|
||||
remote "sudo systemctl restart innercontext && sudo systemctl restart innercontext-pricing-worker"
|
||||
;;
|
||||
all)
|
||||
log "Restarting all services"
|
||||
remote "sudo systemctl restart innercontext && sudo systemctl restart innercontext-node && sudo systemctl restart innercontext-pricing-worker"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
wait_for_service() {
|
||||
local service="$1"
|
||||
local timeout="$2"
|
||||
local i
|
||||
|
||||
for ((i = 1; i <= timeout; i++)); do
|
||||
if remote "[ \"\$(sudo systemctl is-active '$service' 2>/dev/null)\" = 'active' ]"; then
|
||||
log "$service is active"
|
||||
return 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
error "$service did not become active within ${timeout}s"
|
||||
remote "sudo journalctl -u '$service' -n 50" || true
|
||||
return 1
|
||||
}
|
||||
|
||||
check_backend_health() {
|
||||
local i
|
||||
for ((i = 1; i <= 30; i++)); do
|
||||
if remote "curl -sf http://127.0.0.1:8000/health-check >/dev/null"; then
|
||||
log "Backend health check passed"
|
||||
return 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
error "Backend health check failed"
|
||||
remote "sudo journalctl -u innercontext -n 50" || true
|
||||
return 1
|
||||
}
|
||||
|
||||
check_frontend_health() {
|
||||
local i
|
||||
for ((i = 1; i <= 30; i++)); do
|
||||
if remote "curl -sf http://127.0.0.1:3000/ >/dev/null"; then
|
||||
log "Frontend health check passed"
|
||||
return 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
error "Frontend health check failed"
|
||||
remote "sudo journalctl -u innercontext-node -n 50" || true
|
||||
return 1
|
||||
}
|
||||
|
||||
verify_deployment() {
|
||||
case "$SCOPE" in
|
||||
frontend)
|
||||
wait_for_service innercontext-node "$SERVICE_TIMEOUT"
|
||||
check_frontend_health
|
||||
;;
|
||||
backend)
|
||||
wait_for_service innercontext "$SERVICE_TIMEOUT"
|
||||
wait_for_service innercontext-pricing-worker "$SERVICE_TIMEOUT"
|
||||
check_backend_health
|
||||
;;
|
||||
all)
|
||||
wait_for_service innercontext "$SERVICE_TIMEOUT"
|
||||
wait_for_service innercontext-node "$SERVICE_TIMEOUT"
|
||||
wait_for_service innercontext-pricing-worker "$SERVICE_TIMEOUT"
|
||||
check_backend_health
|
||||
check_frontend_health
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
cleanup_old_releases() {
|
||||
log "Cleaning old releases (keeping $KEEP_RELEASES)"
|
||||
remote "
|
||||
cd '$RELEASES_DIR' && \
|
||||
ls -1dt [0-9]* 2>/dev/null | tail -n +$((KEEP_RELEASES + 1)) | xargs -r rm -rf
|
||||
" || true
|
||||
}
|
||||
|
||||
list_releases() {
|
||||
log "Current release"
|
||||
remote "readlink -f '$CURRENT_LINK' 2>/dev/null || echo 'none'"
|
||||
log "Recent releases"
|
||||
remote "ls -1dt '$RELEASES_DIR'/* 2>/dev/null | head -10" || true
|
||||
}
|
||||
|
||||
rollback_to_previous() {
|
||||
local previous_release
|
||||
previous_release="$(remote "
|
||||
current=\$(readlink -f '$CURRENT_LINK' 2>/dev/null || true)
|
||||
for r in \$(ls -1dt '$RELEASES_DIR'/* 2>/dev/null); do
|
||||
if [ \"\$r\" != \"\$current\" ]; then
|
||||
echo \"\$r\"
|
||||
break
|
||||
fi
|
||||
done
|
||||
")"
|
||||
|
||||
if [[ -z "$previous_release" ]]; then
|
||||
error "No previous release found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
rollback_to_release "$previous_release" "manual"
|
||||
}
|
||||
|
||||
run_deploy() {
|
||||
validate_local
|
||||
acquire_lock
|
||||
ensure_remote_structure
|
||||
validate_remote_sudo_permissions
|
||||
capture_previous_release
|
||||
create_release_directory
|
||||
validate_remote_env_files
|
||||
|
||||
if [[ "$SCOPE" == "all" || "$SCOPE" == "backend" ]]; then
|
||||
upload_backend
|
||||
sync_backend_dependencies
|
||||
run_db_migrations
|
||||
fi
|
||||
|
||||
if [[ "$SCOPE" == "all" || "$SCOPE" == "frontend" ]]; then
|
||||
upload_frontend
|
||||
fi
|
||||
|
||||
upload_ops_files
|
||||
promote_release
|
||||
restart_services
|
||||
verify_deployment
|
||||
cleanup_old_releases
|
||||
|
||||
DEPLOY_SUCCESS=1
|
||||
log_deployment "SUCCESS"
|
||||
log "Deployment complete"
|
||||
}
|
||||
|
||||
case "$SCOPE" in
|
||||
frontend|backend|all)
|
||||
run_deploy
|
||||
;;
|
||||
rollback)
|
||||
acquire_lock
|
||||
validate_remote_sudo_permissions
|
||||
rollback_to_previous
|
||||
;;
|
||||
list)
|
||||
list_releases
|
||||
;;
|
||||
*)
|
||||
echo "Usage: $0 [frontend|backend|all|rollback|list]"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
|
|
|||
97
docs/DEPLOYMENT-QUICKSTART.md
Normal file
97
docs/DEPLOYMENT-QUICKSTART.md
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
# Deployment Quickstart
|
||||
|
||||
This is the short operator checklist. Full details are in `docs/DEPLOYMENT.md`.
|
||||
|
||||
Canonical env file locations (and only these):
|
||||
|
||||
- `/opt/innercontext/shared/backend/.env`
|
||||
- `/opt/innercontext/shared/frontend/.env.production`
|
||||
|
||||
## 1) Server prerequisites (once)
|
||||
|
||||
```bash
|
||||
mkdir -p /opt/innercontext/releases
|
||||
mkdir -p /opt/innercontext/shared/backend
|
||||
mkdir -p /opt/innercontext/shared/frontend
|
||||
mkdir -p /opt/innercontext/scripts
|
||||
chown -R innercontext:innercontext /opt/innercontext
|
||||
```
|
||||
|
||||
Create shared env files:
|
||||
|
||||
```bash
|
||||
cat > /opt/innercontext/shared/backend/.env <<'EOF'
|
||||
DATABASE_URL=postgresql+psycopg://innercontext:change-me@<pg-ip>/innercontext
|
||||
GEMINI_API_KEY=your-key
|
||||
EOF
|
||||
|
||||
cat > /opt/innercontext/shared/frontend/.env.production <<'EOF'
|
||||
PUBLIC_API_BASE=http://127.0.0.1:8000
|
||||
ORIGIN=http://innercontext.lan
|
||||
EOF
|
||||
|
||||
chmod 600 /opt/innercontext/shared/backend/.env
|
||||
chmod 600 /opt/innercontext/shared/frontend/.env.production
|
||||
chown innercontext:innercontext /opt/innercontext/shared/backend/.env
|
||||
chown innercontext:innercontext /opt/innercontext/shared/frontend/.env.production
|
||||
```
|
||||
|
||||
Deploy sudoers:
|
||||
|
||||
```bash
|
||||
cat > /etc/sudoers.d/innercontext-deploy << 'EOF'
|
||||
innercontext ALL=(root) NOPASSWD: \
|
||||
/usr/bin/systemctl restart innercontext, \
|
||||
/usr/bin/systemctl restart innercontext-node, \
|
||||
/usr/bin/systemctl restart innercontext-pricing-worker, \
|
||||
/usr/bin/systemctl is-active innercontext, \
|
||||
/usr/bin/systemctl is-active innercontext-node, \
|
||||
/usr/bin/systemctl is-active innercontext-pricing-worker
|
||||
EOF
|
||||
chmod 440 /etc/sudoers.d/innercontext-deploy
|
||||
visudo -c -f /etc/sudoers.d/innercontext-deploy
|
||||
sudo -u innercontext sudo -n -l
|
||||
```
|
||||
|
||||
## 2) Local SSH config
|
||||
|
||||
`~/.ssh/config`:
|
||||
|
||||
```
|
||||
Host innercontext
|
||||
HostName <lxc-ip>
|
||||
User innercontext
|
||||
```
|
||||
|
||||
## 3) Deploy from your machine
|
||||
|
||||
```bash
|
||||
./deploy.sh
|
||||
./deploy.sh backend
|
||||
./deploy.sh frontend
|
||||
./deploy.sh list
|
||||
./deploy.sh rollback
|
||||
```
|
||||
|
||||
## 4) Verify
|
||||
|
||||
```bash
|
||||
curl -sf http://innercontext.lan/api/health-check
|
||||
curl -sf http://innercontext.lan/
|
||||
```
|
||||
|
||||
## 5) Common fixes
|
||||
|
||||
Lock stuck:
|
||||
|
||||
```bash
|
||||
rm -f /opt/innercontext/.deploy.lock
|
||||
```
|
||||
|
||||
Show service logs:
|
||||
|
||||
```bash
|
||||
journalctl -u innercontext -n 100
|
||||
journalctl -u innercontext-node -n 100
|
||||
journalctl -u innercontext-pricing-worker -n 100
|
||||
```
|
||||
|
|
@ -1,376 +1,259 @@
|
|||
# Deployment guide — Proxmox LXC (home network)
|
||||
# Deployment Guide (LXC + systemd + nginx)
|
||||
|
||||
Target architecture:
|
||||
This project deploys from an external machine (developer laptop or CI runner) to a Debian LXC host over SSH.
|
||||
|
||||
Deployments are push-based, release-based, and atomic:
|
||||
|
||||
- Build and validate locally
|
||||
- Upload to `/opt/innercontext/releases/<timestamp>`
|
||||
- Run backend dependency sync and migrations in that release directory
|
||||
- Promote once by switching `/opt/innercontext/current`
|
||||
- Restart services and run health checks
|
||||
- Auto-rollback on failure
|
||||
|
||||
Environment files have exactly two persistent locations on the server:
|
||||
|
||||
- `/opt/innercontext/shared/backend/.env`
|
||||
- `/opt/innercontext/shared/frontend/.env.production`
|
||||
|
||||
Each release links to those files from:
|
||||
|
||||
- `/opt/innercontext/current/backend/.env` -> `../../../shared/backend/.env`
|
||||
- `/opt/innercontext/current/frontend/.env.production` -> `../../../shared/frontend/.env.production`
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
Reverse proxy (existing) innercontext LXC (new, Debian 13)
|
||||
┌──────────────────────┐ ┌────────────────────────────────────┐
|
||||
│ reverse proxy │────────────▶│ nginx :80 │
|
||||
│ innercontext.lan → * │ │ /api/* → uvicorn :8000/* │
|
||||
└──────────────────────┘ │ /* → SvelteKit Node :3000 │
|
||||
└────────────────────────────────────┘
|
||||
│ │
|
||||
FastAPI SvelteKit Node
|
||||
external machine (manual now, CI later)
|
||||
|
|
||||
| ssh + rsync
|
||||
v
|
||||
LXC host
|
||||
/opt/innercontext/
|
||||
current -> releases/<timestamp>
|
||||
releases/<timestamp>
|
||||
shared/backend/.env
|
||||
shared/frontend/.env.production
|
||||
scripts/
|
||||
```
|
||||
|
||||
> **Frontend is never built on the server.** The `vite build` + `adapter-node`
|
||||
> esbuild step is CPU/RAM-intensive and will hang on a small LXC. Build locally,
|
||||
> deploy the `build/` artifact via `deploy.sh`.
|
||||
Services:
|
||||
|
||||
## 1. Prerequisites
|
||||
- `innercontext` (FastAPI, localhost:8000)
|
||||
- `innercontext-node` (SvelteKit Node, localhost:3000)
|
||||
- `innercontext-pricing-worker` (background worker)
|
||||
|
||||
- Proxmox VE host with an existing PostgreSQL LXC and a reverse proxy
|
||||
- LAN hostname `innercontext.lan` resolvable on the network (via router DNS or `/etc/hosts`)
|
||||
- The PostgreSQL LXC must accept connections from the innercontext LXC IP
|
||||
nginx routes:
|
||||
|
||||
---
|
||||
- `/api/*` -> `http://127.0.0.1:8000/*`
|
||||
- `/*` -> `http://127.0.0.1:3000/*`
|
||||
|
||||
## 2. Create the LXC container
|
||||
## Run Model
|
||||
|
||||
In the Proxmox UI (or via CLI):
|
||||
- Manual deploy: run `./deploy.sh ...` from repo root on your laptop.
|
||||
- Optional CI deploy: run the same script from a manual workflow (`workflow_dispatch`).
|
||||
- The server never builds frontend assets.
|
||||
|
||||
```bash
|
||||
# CLI example — adjust storage, bridge, IP to your environment
|
||||
pct create 200 local:vztmpl/debian-13-standard_13.0-1_amd64.tar.zst \
|
||||
--hostname innercontext \
|
||||
--cores 2 \
|
||||
--memory 1024 \
|
||||
--swap 512 \
|
||||
--rootfs local-lvm:8 \
|
||||
--net0 name=eth0,bridge=vmbr0,ip=dhcp \
|
||||
--unprivileged 1 \
|
||||
--start 1
|
||||
```
|
||||
## One-Time Server Setup
|
||||
|
||||
Note the container's IP address after it starts (`pct exec 200 -- ip -4 a`).
|
||||
Run on the LXC host as root.
|
||||
|
||||
---
|
||||
|
||||
## 3. Container setup
|
||||
|
||||
```bash
|
||||
pct enter 200 # or SSH into the container
|
||||
```
|
||||
|
||||
### System packages
|
||||
### 1) Install runtime dependencies
|
||||
|
||||
```bash
|
||||
apt update && apt upgrade -y
|
||||
apt install -y git nginx curl ca-certificates gnupg lsb-release libpq5 rsync
|
||||
```
|
||||
apt install -y git nginx curl ca-certificates libpq5 rsync python3 python3-venv
|
||||
|
||||
### Python 3.12+ + uv
|
||||
|
||||
```bash
|
||||
apt install -y python3 python3-venv
|
||||
curl -LsSf https://astral.sh/uv/install.sh | UV_INSTALL_DIR=/usr/local/bin sh
|
||||
```
|
||||
|
||||
Installing to `/usr/local/bin` makes `uv` available system-wide (required for `sudo -u innercontext uv sync`).
|
||||
|
||||
### Node.js 24 LTS + pnpm
|
||||
|
||||
The server needs Node.js to **run** the pre-built frontend bundle, and pnpm to
|
||||
**install production runtime dependencies** (`clsx`, `bits-ui`, etc. —
|
||||
`adapter-node` bundles the SvelteKit framework but leaves these external).
|
||||
The frontend is never **built** on the server.
|
||||
|
||||
```bash
|
||||
curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.4/install.sh | bash
|
||||
. "$HOME/.nvm/nvm.sh"
|
||||
nvm install 24
|
||||
```
|
||||
|
||||
Copy `node` to `/usr/local/bin` so it is accessible system-wide
|
||||
(required for `sudo -u innercontext` and for systemd).
|
||||
Use `--remove-destination` to replace any existing symlink with a real file:
|
||||
|
||||
```bash
|
||||
cp --remove-destination "$(nvm which current)" /usr/local/bin/node
|
||||
```
|
||||
|
||||
Install pnpm as a standalone binary — self-contained, no wrapper scripts,
|
||||
works system-wide:
|
||||
|
||||
```bash
|
||||
curl -fsSL "https://github.com/pnpm/pnpm/releases/latest/download/pnpm-linux-x64" \
|
||||
-o /usr/local/bin/pnpm
|
||||
chmod 755 /usr/local/bin/pnpm
|
||||
```
|
||||
|
||||
### Application user
|
||||
### 2) Create app user and directories
|
||||
|
||||
```bash
|
||||
useradd --system --create-home --shell /bin/bash innercontext
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 4. Create the database on the PostgreSQL LXC
|
||||
|
||||
Run on the **PostgreSQL LXC**:
|
||||
|
||||
```bash
|
||||
psql -U postgres <<'SQL'
|
||||
CREATE USER innercontext WITH PASSWORD 'change-me';
|
||||
CREATE DATABASE innercontext OWNER innercontext;
|
||||
SQL
|
||||
```
|
||||
|
||||
Edit `/etc/postgresql/18/main/pg_hba.conf` and add (replace `<lxc-ip>` with the innercontext container IP):
|
||||
|
||||
```
|
||||
host innercontext innercontext <lxc-ip>/32 scram-sha-256
|
||||
```
|
||||
|
||||
Then reload:
|
||||
|
||||
```bash
|
||||
systemctl reload postgresql
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 5. Clone the repository
|
||||
|
||||
```bash
|
||||
mkdir -p /opt/innercontext
|
||||
git clone https://github.com/your-user/innercontext.git /opt/innercontext
|
||||
mkdir -p /opt/innercontext/releases
|
||||
mkdir -p /opt/innercontext/shared/backend
|
||||
mkdir -p /opt/innercontext/shared/frontend
|
||||
mkdir -p /opt/innercontext/scripts
|
||||
chown -R innercontext:innercontext /opt/innercontext
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 6. Backend setup
|
||||
### 3) Create shared env files
|
||||
|
||||
```bash
|
||||
cd /opt/innercontext/backend
|
||||
```
|
||||
|
||||
### Install dependencies
|
||||
|
||||
```bash
|
||||
sudo -u innercontext uv sync
|
||||
```
|
||||
|
||||
### Create `.env`
|
||||
|
||||
```bash
|
||||
cat > /opt/innercontext/backend/.env <<'EOF'
|
||||
DATABASE_URL=postgresql+psycopg://innercontext:change-me@<pg-lxc-ip>/innercontext
|
||||
GEMINI_API_KEY=your-gemini-api-key
|
||||
# GEMINI_MODEL=gemini-flash-latest # optional, this is the default
|
||||
cat > /opt/innercontext/shared/backend/.env <<'EOF'
|
||||
DATABASE_URL=postgresql+psycopg://innercontext:change-me@<pg-ip>/innercontext
|
||||
GEMINI_API_KEY=your-key
|
||||
EOF
|
||||
chmod 600 /opt/innercontext/backend/.env
|
||||
chown innercontext:innercontext /opt/innercontext/backend/.env
|
||||
```
|
||||
|
||||
### Run database migrations
|
||||
|
||||
```bash
|
||||
sudo -u innercontext bash -c '
|
||||
cd /opt/innercontext/backend
|
||||
uv run alembic upgrade head
|
||||
'
|
||||
```
|
||||
|
||||
This creates all tables on first run. On subsequent deploys it applies only the new migrations.
|
||||
|
||||
> **Existing database (tables already created by `create_db_and_tables`):**
|
||||
> Run `uv run alembic stamp head` instead to mark the current schema as migrated without re-running DDL.
|
||||
|
||||
### Test
|
||||
|
||||
```bash
|
||||
sudo -u innercontext bash -c '
|
||||
cd /opt/innercontext/backend
|
||||
uv run uvicorn main:app --host 127.0.0.1 --port 8000
|
||||
'
|
||||
# Ctrl-C after confirming it starts
|
||||
```
|
||||
|
||||
### Install systemd service
|
||||
|
||||
```bash
|
||||
cp /opt/innercontext/systemd/innercontext.service /etc/systemd/system/
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now innercontext
|
||||
systemctl status innercontext
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 7. Frontend setup
|
||||
|
||||
The frontend is **built locally and uploaded** via `deploy.sh` — never built on the server.
|
||||
This section only covers the one-time server-side configuration.
|
||||
|
||||
### Create `.env.production`
|
||||
|
||||
```bash
|
||||
cat > /opt/innercontext/frontend/.env.production <<'EOF'
|
||||
PUBLIC_API_BASE=http://innercontext.lan/api
|
||||
cat > /opt/innercontext/shared/frontend/.env.production <<'EOF'
|
||||
PUBLIC_API_BASE=http://127.0.0.1:8000
|
||||
ORIGIN=http://innercontext.lan
|
||||
EOF
|
||||
chmod 600 /opt/innercontext/frontend/.env.production
|
||||
chown innercontext:innercontext /opt/innercontext/frontend/.env.production
|
||||
|
||||
chmod 600 /opt/innercontext/shared/backend/.env
|
||||
chmod 600 /opt/innercontext/shared/frontend/.env.production
|
||||
chown innercontext:innercontext /opt/innercontext/shared/backend/.env
|
||||
chown innercontext:innercontext /opt/innercontext/shared/frontend/.env.production
|
||||
```
|
||||
|
||||
### Grant `innercontext` passwordless sudo for service restarts
|
||||
### 4) Grant deploy sudo permissions
|
||||
|
||||
```bash
|
||||
cat > /etc/sudoers.d/innercontext-deploy << 'EOF'
|
||||
innercontext ALL=(root) NOPASSWD: \
|
||||
/usr/bin/systemctl restart innercontext, \
|
||||
/usr/bin/systemctl restart innercontext-node, \
|
||||
/usr/bin/systemctl restart innercontext-pricing-worker
|
||||
/usr/bin/systemctl restart innercontext-pricing-worker, \
|
||||
/usr/bin/systemctl is-active innercontext, \
|
||||
/usr/bin/systemctl is-active innercontext-node, \
|
||||
/usr/bin/systemctl is-active innercontext-pricing-worker
|
||||
EOF
|
||||
|
||||
chmod 440 /etc/sudoers.d/innercontext-deploy
|
||||
visudo -c -f /etc/sudoers.d/innercontext-deploy
|
||||
|
||||
# Must work without password or TTY prompt:
|
||||
sudo -u innercontext sudo -n -l
|
||||
```
|
||||
|
||||
### Install systemd services
|
||||
If `sudo -n -l` fails, deployments will fail during restart/rollback with:
|
||||
`sudo: a terminal is required` or `sudo: a password is required`.
|
||||
|
||||
### 5) Install systemd and nginx configs
|
||||
|
||||
After first deploy (or after copying repo content to `/opt/innercontext/current`), install configs:
|
||||
|
||||
```bash
|
||||
cp /opt/innercontext/systemd/innercontext-node.service /etc/systemd/system/
|
||||
cp /opt/innercontext/systemd/innercontext-pricing-worker.service /etc/systemd/system/
|
||||
cp /opt/innercontext/current/systemd/innercontext.service /etc/systemd/system/
|
||||
cp /opt/innercontext/current/systemd/innercontext-node.service /etc/systemd/system/
|
||||
cp /opt/innercontext/current/systemd/innercontext-pricing-worker.service /etc/systemd/system/
|
||||
systemctl daemon-reload
|
||||
systemctl enable innercontext
|
||||
systemctl enable innercontext-node
|
||||
systemctl enable --now innercontext-pricing-worker
|
||||
# Do NOT start yet — build/ is empty until the first deploy.sh run
|
||||
```
|
||||
systemctl enable innercontext-pricing-worker
|
||||
|
||||
---
|
||||
|
||||
## 8. nginx setup
|
||||
|
||||
```bash
|
||||
cp /opt/innercontext/nginx/innercontext.conf /etc/nginx/sites-available/innercontext
|
||||
ln -s /etc/nginx/sites-available/innercontext /etc/nginx/sites-enabled/
|
||||
cp /opt/innercontext/current/nginx/innercontext.conf /etc/nginx/sites-available/innercontext
|
||||
ln -sf /etc/nginx/sites-available/innercontext /etc/nginx/sites-enabled/innercontext
|
||||
rm -f /etc/nginx/sites-enabled/default
|
||||
nginx -t
|
||||
systemctl reload nginx
|
||||
nginx -t && systemctl reload nginx
|
||||
```
|
||||
|
||||
---
|
||||
## Local Machine Setup
|
||||
|
||||
## 9. Reverse proxy configuration
|
||||
|
||||
Point your existing reverse proxy at the innercontext LXC's nginx (`<innercontext-lxc-ip>:80`).
|
||||
|
||||
Example — Caddy:
|
||||
|
||||
```
|
||||
innercontext.lan {
|
||||
reverse_proxy <innercontext-lxc-ip>:80
|
||||
}
|
||||
```
|
||||
|
||||
Example — nginx upstream:
|
||||
|
||||
```nginx
|
||||
server {
|
||||
listen 80;
|
||||
server_name innercontext.lan;
|
||||
location / {
|
||||
proxy_pass http://<innercontext-lxc-ip>:80;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Reload your reverse proxy after applying the change.
|
||||
|
||||
---
|
||||
|
||||
## 10. First deploy from local machine
|
||||
|
||||
All subsequent deploys (including the first one) use `deploy.sh` from your local machine.
|
||||
|
||||
### SSH config
|
||||
|
||||
Add to `~/.ssh/config` on your local machine:
|
||||
`~/.ssh/config`:
|
||||
|
||||
```
|
||||
Host innercontext
|
||||
HostName <innercontext-lxc-ip>
|
||||
HostName <lxc-ip>
|
||||
User innercontext
|
||||
```
|
||||
|
||||
Make sure your SSH public key is in `/home/innercontext/.ssh/authorized_keys` on the server.
|
||||
Ensure your public key is in `/home/innercontext/.ssh/authorized_keys`.
|
||||
|
||||
### Run the first deploy
|
||||
## Deploy Commands
|
||||
|
||||
From repository root on external machine:
|
||||
|
||||
```bash
|
||||
# From the repo root on your local machine:
|
||||
./deploy.sh
|
||||
./deploy.sh # full deploy (default = all)
|
||||
./deploy.sh all
|
||||
./deploy.sh backend
|
||||
./deploy.sh frontend
|
||||
./deploy.sh list
|
||||
./deploy.sh rollback
|
||||
```
|
||||
|
||||
This will:
|
||||
1. Build the frontend locally (`pnpm run build`)
|
||||
2. Upload `frontend/build/` to the server via rsync
|
||||
3. Restart `innercontext-node`
|
||||
4. Upload `backend/` source to the server
|
||||
5. Run `uv sync --frozen` on the server
|
||||
6. Restart `innercontext` (runs alembic migrations on start)
|
||||
7. Restart `innercontext-pricing-worker`
|
||||
|
||||
---
|
||||
|
||||
## 11. Verification
|
||||
Optional overrides:
|
||||
|
||||
```bash
|
||||
# From any machine on the LAN:
|
||||
curl http://innercontext.lan/api/health-check # {"status":"ok"}
|
||||
curl http://innercontext.lan/api/products # []
|
||||
curl http://innercontext.lan/ # SvelteKit HTML shell
|
||||
DEPLOY_SERVER=innercontext ./deploy.sh all
|
||||
DEPLOY_ROOT=/opt/innercontext ./deploy.sh backend
|
||||
DEPLOY_ALLOW_DIRTY=1 ./deploy.sh frontend
|
||||
```
|
||||
|
||||
The web UI should be accessible at `http://innercontext.lan`.
|
||||
## What `deploy.sh` Does
|
||||
|
||||
---
|
||||
For `backend` / `frontend` / `all`:
|
||||
|
||||
## 12. Updating the application
|
||||
1. Local checks (strict, fail-fast)
|
||||
2. Acquire `/opt/innercontext/.deploy.lock`
|
||||
3. Create `<timestamp>` release directory
|
||||
4. Upload selected component(s)
|
||||
5. Link shared env files in the release directory
|
||||
6. `uv sync` + `alembic upgrade head` (backend scope)
|
||||
7. Upload `scripts/`, `systemd/`, `nginx/`
|
||||
8. Switch `current` to the prepared release
|
||||
9. Restart affected services
|
||||
10. Run health checks
|
||||
11. Remove old releases (keep last 5)
|
||||
12. Write deploy entry to `/opt/innercontext/deploy.log`
|
||||
|
||||
If anything fails after promotion, script auto-rolls back to previous release.
|
||||
|
||||
## Health Checks
|
||||
|
||||
- Backend: `http://127.0.0.1:8000/health-check`
|
||||
- Frontend: `http://127.0.0.1:3000/`
|
||||
- Worker: `systemctl is-active innercontext-pricing-worker`
|
||||
|
||||
Manual checks:
|
||||
|
||||
```bash
|
||||
# From the repo root on your local machine:
|
||||
./deploy.sh # full deploy (frontend + backend)
|
||||
./deploy.sh frontend # frontend only
|
||||
./deploy.sh backend # backend only
|
||||
curl -sf http://127.0.0.1:8000/health-check
|
||||
curl -sf http://127.0.0.1:3000/
|
||||
systemctl is-active innercontext
|
||||
systemctl is-active innercontext-node
|
||||
systemctl is-active innercontext-pricing-worker
|
||||
```
|
||||
|
||||
---
|
||||
## Troubleshooting
|
||||
|
||||
## 13. Troubleshooting
|
||||
|
||||
### 502 Bad Gateway on `/api/*`
|
||||
### Lock exists
|
||||
|
||||
```bash
|
||||
systemctl status innercontext
|
||||
journalctl -u innercontext -n 50
|
||||
# Check .env DATABASE_URL is correct and PG LXC accepts connections
|
||||
cat /opt/innercontext/.deploy.lock
|
||||
rm -f /opt/innercontext/.deploy.lock
|
||||
```
|
||||
|
||||
### Product prices stay empty / stale
|
||||
Only remove the lock if no deployment is running.
|
||||
|
||||
### Sudo password prompt during deploy
|
||||
|
||||
Re-check `/etc/sudoers.d/innercontext-deploy` and run:
|
||||
|
||||
```bash
|
||||
systemctl status innercontext-pricing-worker
|
||||
journalctl -u innercontext-pricing-worker -n 50
|
||||
# Ensure worker is running and can connect to PostgreSQL
|
||||
visudo -c -f /etc/sudoers.d/innercontext-deploy
|
||||
sudo -u innercontext sudo systemctl is-active innercontext
|
||||
```
|
||||
|
||||
### 502 Bad Gateway on `/`
|
||||
### Backend migration failure
|
||||
|
||||
Validate env file and DB connectivity:
|
||||
|
||||
```bash
|
||||
systemctl status innercontext-node
|
||||
journalctl -u innercontext-node -n 50
|
||||
# Verify /opt/innercontext/frontend/build/index.js exists (deploy.sh ran successfully)
|
||||
ls -la /opt/innercontext/shared/backend/.env
|
||||
grep '^DATABASE_URL=' /opt/innercontext/shared/backend/.env
|
||||
```
|
||||
|
||||
### Database connection refused
|
||||
### Service fails after deploy
|
||||
|
||||
```bash
|
||||
# From innercontext LXC:
|
||||
psql postgresql+psycopg://innercontext:change-me@<pg-lxc-ip>/innercontext -c "SELECT 1"
|
||||
# If it fails, check pg_hba.conf on the PG LXC and verify the IP matches
|
||||
journalctl -u innercontext -n 100
|
||||
journalctl -u innercontext-node -n 100
|
||||
journalctl -u innercontext-pricing-worker -n 100
|
||||
```
|
||||
|
||||
## Manual CI Deploy (Optional)
|
||||
|
||||
Use the manual Forgejo workflow (`workflow_dispatch`) to run the same `./deploy.sh all` path from CI once server secrets and SSH trust are configured.
|
||||
|
|
|
|||
59
scripts/backup-database.sh
Executable file
59
scripts/backup-database.sh
Executable file
|
|
@ -0,0 +1,59 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Database backup script for innercontext PostgreSQL database
|
||||
# Should be run daily via cron on the PostgreSQL LXC:
|
||||
# 0 2 * * * /opt/innercontext/scripts/backup-database.sh >> /opt/innercontext/backup.log 2>&1
|
||||
#
|
||||
# Note: This script should be copied to the PostgreSQL LXC container
|
||||
# and run there (not on the app LXC)
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Configuration
|
||||
BACKUP_DIR="/opt/innercontext/backups"
|
||||
DB_NAME="innercontext"
|
||||
DB_USER="innercontext"
|
||||
KEEP_DAYS=7
|
||||
TIMESTAMP=$(date '+%Y%m%d_%H%M%S')
|
||||
BACKUP_FILE="$BACKUP_DIR/innercontext_${TIMESTAMP}.sql.gz"
|
||||
|
||||
# Color codes
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
log() {
|
||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1"
|
||||
}
|
||||
|
||||
# Create backup directory if it doesn't exist
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
|
||||
# Create backup
|
||||
log "Starting database backup..."
|
||||
if pg_dump -U "$DB_USER" -d "$DB_NAME" | gzip > "$BACKUP_FILE"; then
|
||||
BACKUP_SIZE=$(du -h "$BACKUP_FILE" | cut -f1)
|
||||
log "${GREEN}✓${NC} Backup created: $BACKUP_FILE ($BACKUP_SIZE)"
|
||||
else
|
||||
log "${RED}✗${NC} Backup failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Clean up old backups
|
||||
log "Cleaning up backups older than $KEEP_DAYS days..."
|
||||
find "$BACKUP_DIR" -name "innercontext_*.sql.gz" -type f -mtime +$KEEP_DAYS -delete
|
||||
REMAINING=$(find "$BACKUP_DIR" -name "innercontext_*.sql.gz" -type f | wc -l)
|
||||
log "${GREEN}✓${NC} Cleanup complete. $REMAINING backup(s) remaining"
|
||||
|
||||
# Verify backup can be read
|
||||
if gunzip -t "$BACKUP_FILE" 2>/dev/null; then
|
||||
log "${GREEN}✓${NC} Backup integrity verified"
|
||||
else
|
||||
log "${RED}✗${NC} Backup integrity check failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log "${GREEN}✓${NC} Database backup completed successfully"
|
||||
exit 0
|
||||
66
scripts/healthcheck.sh
Executable file
66
scripts/healthcheck.sh
Executable file
|
|
@ -0,0 +1,66 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Health check script for innercontext services
|
||||
# Should be run via cron every 5 minutes:
|
||||
# */5 * * * * /opt/innercontext/scripts/healthcheck.sh >> /opt/innercontext/healthcheck.log 2>&1
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
BACKEND_URL="http://127.0.0.1:8000/health-check"
|
||||
FRONTEND_URL="http://127.0.0.1:3000/"
|
||||
TIMEOUT=10
|
||||
TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
|
||||
# Color codes
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
log() {
|
||||
echo "[$TIMESTAMP] $1"
|
||||
}
|
||||
|
||||
check_service() {
|
||||
local service_name=$1
|
||||
local url=$2
|
||||
|
||||
if systemctl is-active --quiet "$service_name"; then
|
||||
if curl -sf --max-time "$TIMEOUT" "$url" > /dev/null 2>&1; then
|
||||
log "${GREEN}✓${NC} $service_name is healthy"
|
||||
return 0
|
||||
else
|
||||
log "${YELLOW}⚠${NC} $service_name is running but not responding at $url"
|
||||
return 1
|
||||
fi
|
||||
else
|
||||
log "${RED}✗${NC} $service_name is not running"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Check all services
|
||||
backend_ok=0
|
||||
frontend_ok=0
|
||||
worker_ok=0
|
||||
|
||||
check_service "innercontext" "$BACKEND_URL" || backend_ok=1
|
||||
check_service "innercontext-node" "$FRONTEND_URL" || frontend_ok=1
|
||||
|
||||
# Worker doesn't have HTTP endpoint, just check if it's running
|
||||
if systemctl is-active --quiet "innercontext-pricing-worker"; then
|
||||
log "${GREEN}✓${NC} innercontext-pricing-worker is running"
|
||||
else
|
||||
log "${RED}✗${NC} innercontext-pricing-worker is not running"
|
||||
worker_ok=1
|
||||
fi
|
||||
|
||||
# If any service is unhealthy, exit with error code
|
||||
if [ $backend_ok -ne 0 ] || [ $frontend_ok -ne 0 ] || [ $worker_ok -ne 0 ]; then
|
||||
log "${RED}Health check failed${NC}"
|
||||
exit 1
|
||||
else
|
||||
log "${GREEN}All services healthy${NC}"
|
||||
exit 0
|
||||
fi
|
||||
157
scripts/validate-env.sh
Executable file
157
scripts/validate-env.sh
Executable file
|
|
@ -0,0 +1,157 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Validate environment variables for innercontext deployment
|
||||
# Checks both shared directory (persistent config) and current release (symlinks)
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Color codes
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Shared directory (persistent configuration)
|
||||
SHARED_BACKEND_ENV="/opt/innercontext/shared/backend/.env"
|
||||
SHARED_FRONTEND_ENV="/opt/innercontext/shared/frontend/.env.production"
|
||||
|
||||
# Current release (should be symlinks to shared)
|
||||
CURRENT_BACKEND_ENV="/opt/innercontext/current/backend/.env"
|
||||
CURRENT_FRONTEND_ENV="/opt/innercontext/current/frontend/.env.production"
|
||||
|
||||
errors=0
|
||||
warnings=0
|
||||
|
||||
log_error() {
|
||||
echo -e "${RED}✗${NC} $1"
|
||||
((errors++))
|
||||
}
|
||||
|
||||
log_success() {
|
||||
echo -e "${GREEN}✓${NC} $1"
|
||||
}
|
||||
|
||||
log_warning() {
|
||||
echo -e "${YELLOW}⚠${NC} $1"
|
||||
((warnings++))
|
||||
}
|
||||
|
||||
check_symlink() {
|
||||
local symlink_path=$1
|
||||
local expected_target=$2
|
||||
|
||||
if [ ! -L "$symlink_path" ]; then
|
||||
log_error "Not a symlink: $symlink_path"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local actual_target=$(readlink "$symlink_path")
|
||||
if [ "$actual_target" != "$expected_target" ]; then
|
||||
log_warning "Symlink target mismatch: $symlink_path -> $actual_target (expected: $expected_target)"
|
||||
else
|
||||
log_success "Symlink correct: $symlink_path -> $actual_target"
|
||||
fi
|
||||
}
|
||||
|
||||
check_var() {
|
||||
local file=$1
|
||||
local var_name=$2
|
||||
local optional=${3:-false}
|
||||
|
||||
if [ ! -f "$file" ]; then
|
||||
log_error "File not found: $file"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Check if variable exists and is not empty
|
||||
if grep -q "^${var_name}=" "$file"; then
|
||||
local value=$(grep "^${var_name}=" "$file" | cut -d'=' -f2-)
|
||||
if [ -z "$value" ]; then
|
||||
if [ "$optional" = true ]; then
|
||||
log_warning "$var_name is empty in $file (optional)"
|
||||
else
|
||||
log_error "$var_name is empty in $file"
|
||||
fi
|
||||
else
|
||||
log_success "$var_name is set"
|
||||
fi
|
||||
else
|
||||
if [ "$optional" = true ]; then
|
||||
log_warning "$var_name not found in $file (optional)"
|
||||
else
|
||||
log_error "$var_name not found in $file"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
echo "=== Validating Shared Directory Structure ==="
|
||||
|
||||
# Check shared directory exists
|
||||
if [ -d "/opt/innercontext/shared" ]; then
|
||||
log_success "Shared directory exists: /opt/innercontext/shared"
|
||||
else
|
||||
log_error "Shared directory not found: /opt/innercontext/shared"
|
||||
fi
|
||||
|
||||
# Check shared backend .env
|
||||
if [ -f "$SHARED_BACKEND_ENV" ]; then
|
||||
log_success "Shared backend .env exists: $SHARED_BACKEND_ENV"
|
||||
else
|
||||
log_error "Shared backend .env not found: $SHARED_BACKEND_ENV"
|
||||
fi
|
||||
|
||||
# Check shared frontend .env.production
|
||||
if [ -f "$SHARED_FRONTEND_ENV" ]; then
|
||||
log_success "Shared frontend .env.production exists: $SHARED_FRONTEND_ENV"
|
||||
else
|
||||
log_error "Shared frontend .env.production not found: $SHARED_FRONTEND_ENV"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=== Validating Symlinks in Current Release ==="
|
||||
|
||||
# Check current release symlinks point to shared directory
|
||||
if [ -e "$CURRENT_BACKEND_ENV" ]; then
|
||||
check_symlink "$CURRENT_BACKEND_ENV" "../../../shared/backend/.env"
|
||||
else
|
||||
log_error "Current backend .env not found: $CURRENT_BACKEND_ENV"
|
||||
fi
|
||||
|
||||
if [ -e "$CURRENT_FRONTEND_ENV" ]; then
|
||||
check_symlink "$CURRENT_FRONTEND_ENV" "../../../shared/frontend/.env.production"
|
||||
else
|
||||
log_error "Current frontend .env.production not found: $CURRENT_FRONTEND_ENV"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=== Validating Backend Environment Variables ==="
|
||||
if [ -f "$SHARED_BACKEND_ENV" ]; then
|
||||
check_var "$SHARED_BACKEND_ENV" "DATABASE_URL"
|
||||
check_var "$SHARED_BACKEND_ENV" "GEMINI_API_KEY"
|
||||
check_var "$SHARED_BACKEND_ENV" "LOG_LEVEL" true
|
||||
check_var "$SHARED_BACKEND_ENV" "CORS_ORIGINS" true
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=== Validating Frontend Environment Variables ==="
|
||||
if [ -f "$SHARED_FRONTEND_ENV" ]; then
|
||||
check_var "$SHARED_FRONTEND_ENV" "PUBLIC_API_BASE"
|
||||
check_var "$SHARED_FRONTEND_ENV" "ORIGIN"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
if [ $errors -eq 0 ]; then
|
||||
if [ $warnings -eq 0 ]; then
|
||||
echo -e "${GREEN}✓ All environment checks passed${NC}"
|
||||
else
|
||||
echo -e "${YELLOW}⚠ Environment validation passed with $warnings warning(s)${NC}"
|
||||
fi
|
||||
exit 0
|
||||
else
|
||||
echo -e "${RED}✗ Found $errors error(s) in environment configuration${NC}"
|
||||
if [ $warnings -gt 0 ]; then
|
||||
echo -e "${YELLOW} And $warnings warning(s)${NC}"
|
||||
fi
|
||||
exit 1
|
||||
fi
|
||||
Loading…
Add table
Add a link
Reference in a new issue