diff --git a/deploy/zopu-runtime/.env.template b/deploy/zopu-runtime/.env.template new file mode 100644 index 0000000..045209d --- /dev/null +++ b/deploy/zopu-runtime/.env.template @@ -0,0 +1,83 @@ +############################################################################### +# Zopu Runtime Deployment — Environment Template +# +# Copy to .env and fill in real values. Never commit .env to the repository. +# This file documents every environment group required by the single-node +# execution plane. Lines marked REQUIRED must be set before first start. +############################################################################### + +# --------------------------------------------------------------------------- +# 1. Convex (control plane) — REQUIRED +# Convex is already deployed; provide the production deployment URL. +# --------------------------------------------------------------------------- +CONVEX_URL=https://your-deployment.convex.cloud +CONVEX_SITE_URL=https://your-deployment.convex.site +SITE_URL=http://localhost:5173 + +# --------------------------------------------------------------------------- +# 2. Self-hosted Git / Gitea — REQUIRED for issue lifecycle +# The agent daemon clones repos and creates PRs through Gitea. +# --------------------------------------------------------------------------- +GITEA_URL=https://git.openputer.com +GITEA_TOKEN=replace-with-gitea-api-token + +# --------------------------------------------------------------------------- +# 3. Model gateway — REQUIRED +# All model calls route through this OpenAI-compatible endpoint. +# --------------------------------------------------------------------------- +AGENT_MODEL_PROVIDER=cheaptricks +AGENT_MODEL_NAME=glm-5.2 +AGENT_MODEL_API=openai-completions +AGENT_MODEL_BASE_URL=https://ai.example.invalid/v1 +AGENT_MODEL_API_KEY=replace-with-model-gateway-key +AGENT_MODEL_CONTEXT_WINDOW=262000 +AGENT_MODEL_MAX_TOKENS=131072 + +# --------------------------------------------------------------------------- +# 4. AgentOS / RivetKit — OPTIONAL (defaults shown) +# registry.start() in the daemon boots an in-process RivetKit engine +# (envoy mode) backed by a native Rust sidecar. createClient() connects +# back to RIVET_ENDPOINT. No separate Rivet Engine process is required +# for single-node operation. Leave RIVET_ENDPOINT unset to use the +# library default (http://localhost:6420). +# --------------------------------------------------------------------------- +#RIVET_ENDPOINT=http://localhost:6420 + +# --------------------------------------------------------------------------- +# 5. Zopu agent service (Flue) +# FLUE_DB_TOKEN authenticates the Flue persistence adapter. +# PORT is the Flue Node HTTP server listen port. +# --------------------------------------------------------------------------- +FLUE_DB_TOKEN=replace-with-long-random-token +PORT=3583 + +# --------------------------------------------------------------------------- +# 6. Daemon identity +# --------------------------------------------------------------------------- +DAEMON_ID=zopu-dedicated +DAEMON_NAME=Zopu Dedicated Server +DAEMON_VERSION=0.0.0 +DAEMON_HEARTBEAT_MS=15000 +DAEMON_COMMAND_LEASE_MS=60000 + +# --------------------------------------------------------------------------- +# 7. Docker sandbox +# The zopu service user is added to the docker group during bootstrap. +# Orb sandboxes will use Docker for full-system isolation, but the Orb +# lane contract has not landed yet. Docker access is provisioned now so +# the boundary is ready; no Docker-backed sandbox code is wired today. +# --------------------------------------------------------------------------- +# No env vars required; Docker socket access is via group membership. + +# --------------------------------------------------------------------------- +# 8. Service authentication / secrets +# These tokens authenticate inter-service calls. Generate strong randoms. +# --------------------------------------------------------------------------- +# Better Auth / Convex JWT secret (if the agent service needs to mint tokens): +#AUTH_SECRET=replace-with-64-char-hex + +# --------------------------------------------------------------------------- +# 9. Private networking (Tailscale) — OPTIONAL +# When Tailscale is available, set the hostname for private DNS. +# --------------------------------------------------------------------------- +#TAILSCALE_HOSTNAME=zopu-runtime diff --git a/deploy/zopu-runtime/README.md b/deploy/zopu-runtime/README.md new file mode 100644 index 0000000..99e1c67 --- /dev/null +++ b/deploy/zopu-runtime/README.md @@ -0,0 +1,322 @@ +# Zopu Single-Node Runtime Deployment + +Deployment artifacts for the Zopu execution plane on a single Debian dedicated +server. Convex is already deployed as the control plane; this lane deploys only +the execution plane: the daemon (Bun/Effect + AgentOS/RivetKit), the Flue agent +service, Docker Engine for future Orb sandboxes, and supporting infrastructure. + +## Architecture + +``` +┌─────────────────────────────────────────────────────────────┐ +│ Debian Dedicated Server │ +│ ~12 CPU cores · ~40 GB RAM · single-node │ +│ │ +│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ +│ │ zopu-daemon │ │ zopu-agent │ │ Docker │ │ +│ │ (systemd) │ │ (systemd) │ │ Engine │ │ +│ │ │ │ │ │ │ │ +│ │ Effect daemon│ │ Flue Node │ │ Orb sandboxes│ │ +│ │ + RivetKit │ │ server.mjs │ │ (future) │ │ +│ │ in-process │ │ :3583 │ │ │ │ +│ │ engine │ │ │ │ │ │ +│ │ + native │ │ │ │ │ │ +│ │ sidecar │ │ │ │ │ │ +│ │ :6420 │ │ │ │ │ │ +│ └──────┬───────┘ └──────┬───────┘ └──────────────┘ │ +│ │ │ │ +│ └────────┬───────────┘ │ +│ │ │ +│ ┌────────▼─────────┐ │ +│ │ Convex (cloud) │ ← control plane (pre-deployed)│ +│ └──────────────────┘ │ +│ │ +│ systemd timers: health-check (60s), docker-cleanup (daily) │ +│ cron: disk-monitor (daily 06:00) │ +│ ufw: deny-by-default, SSH only │ +│ Tailscale: optional private overlay │ +└─────────────────────────────────────────────────────────────┘ +``` + +### Single-node RivetKit topology + +The daemon calls `registry.start()` from `rivetkit`, which boots an +**in-process RivetKit engine** (envoy mode) backed by a **native Rust +sidecar** binary (`@rivet-dev/agentos-sidecar`, platform-resolved). The engine +listens on `RIVET_ENDPOINT` (default `http://localhost:6420`). The daemon then +calls `createClient(RIVET_ENDPOINT)` to connect back to its own in-process +engine for actor dispatch. + +Evidence: RivetKit source `chunk-YDUQHING.js` line 4751 — +`DEFAULT_ENDPOINT = "http://localhost:6420"`. The `Registry.start()` method +calls `#startEnvoy()` → `runtime.serveRegistry()` for serverful mode (Mode A). +The `createClient()` function reads `RIVET_ENDPOINT` env or defaults to the +same `http://localhost:6420`. + +**No separate Rivet Engine process is required.** The engine, actor envoy, and +sidecar all run inside the daemon process. A future multi-node deployment +would externalize the engine, but that is out of scope. + +### Docker / Orb boundary + +Docker Engine is installed and the `zopu` service user is in the `docker` +group. The daemon's systemd unit includes `SupplementaryGroups=docker`. +However, **no Docker-backed sandbox code is currently wired**. The Orb +sandbox lane contract has not landed; Docker access is provisioned now so the +boundary is ready. The current agent uses the in-process AgentOS VM (Wasm/V8) +sandbox, not Docker. + +## Files + +``` +deploy/zopu-runtime/ +├── bootstrap.sh # One-shot Debian installer +├── .env.template # Environment template (all groups documented) +├── README.md # This file (runbook) +├── systemd/ +│ ├── zopu-daemon.service # Daemon systemd unit +│ ├── zopu-agent.service # Agent systemd unit +│ ├── zopu-health.service # Health check oneshot +│ ├── zopu-health.timer # Health check every 60s +│ ├── zopu-docker-cleanup.service +│ └── zopu-docker-cleanup.timer +├── scripts/ +│ ├── health-check.sh # TCP/process health probes +│ ├── update.sh # Update to branch or commit +│ ├── rollback.sh # Roll back to previous commit +│ ├── docker-cleanup.sh # Prune stopped containers/images/networks +│ └── disk-monitor.sh # Disk usage alerting +└── caddy/ + └── Caddyfile # Optional reverse proxy config (documentation) +``` + +## Fresh install + +```bash +# 1. SSH into the fresh Debian 12 server as root. + +# 2. Set environment overrides (optional): +export ZOPU_REPO_URL="ssh://git@git.openputer.com:2222/puter/zopu-code.git" +export ZOPU_REPO_BRANCH="dogfood/v0" +# export TAILSCALE_AUTHKEY="tskey-..." +# export TAILSCALE_HOSTNAME="zopu-runtime" + +# 3. Run the bootstrap script: +bash bootstrap.sh + +# 4. Edit .env with real values: +nano /opt/zopu/.env + +# 5. Start services: +systemctl start zopu-daemon +sleep 3 +systemctl start zopu-agent + +# 6. Enable timers: +systemctl enable --now zopu-health.timer zopu-docker-cleanup.timer + +# 7. Verify: +/opt/zopu/deploy/zopu-runtime/scripts/health-check.sh +``` + +## Start / stop / restart + +```bash +# Start both services +systemctl start zopu-daemon zopu-agent + +# Stop both services +systemctl stop zopu-agent zopu-daemon + +# Restart (daemon first — it owns the RivetKit engine) +systemctl restart zopu-daemon && sleep 3 && systemctl restart zopu-agent + +# Enable on boot +systemctl enable zopu-daemon zopu-agent + +# Disable on boot +systemctl disable zopu-daemon zopu-agent +``` + +## Log inspection + +All service logs go to journald with `SyslogIdentifier` tags. + +```bash +# Daemon logs (live follow) +journalctl -u zopu-daemon -f + +# Agent logs (live follow) +journalctl -u zopu-agent -f + +# Last 100 lines of daemon +journalctl -u zopu-daemon -n 100 + +# Logs since boot +journalctl -u zopu-daemon -b + +# Health check timer logs +journalctl -u zopu-health.service -n 50 + +# Docker cleanup logs +journalctl -u zopu-docker-cleanup.service -n 50 + +# Disk monitor logs (cron → file) +tail -100 /var/log/zopu/disk-monitor.log + +# All Zopu syslog identifiers +journalctl -t zopu-daemon -t zopu-agent --since "1 hour ago" +``` + +## Health checks + +```bash +# Manual health check (prints all probes) +/opt/zopu/deploy/zopu-runtime/scripts/health-check.sh + +# Quiet mode (exit code only) +/opt/zopu/deploy/zopu-runtime/scripts/health-check.sh --quiet + +# Check systemd timer is running +systemctl status zopu-health.timer +systemctl list-timers zopu-health.timer +``` + +The health check probes: +1. `zopu-daemon` systemd unit is active +2. `zopu-agent` systemd unit is active +3. RivetKit engine port (default 6420) accepts TCP connections +4. Flue agent port (default 3583) accepts TCP connections +5. Docker daemon responds to `docker info` + +No HTTP health endpoints are assumed. Flue does not expose one by design +(per Flue docs: "Flue does not add a health endpoint"). RivetKit's health +route is internal to the registry runtime and not documented as +publicly addressable on the engine endpoint. + +## Update to commit + +```bash +# Update to latest of dogfood/v0 (default) +/opt/zopu/deploy/zopu-runtime/scripts/update.sh + +# Update to a specific branch +/opt/zopu/deploy/zopu-runtime/scripts/update.sh dogfood/runtime-deploy + +# Update to a specific commit +/opt/zopu/deploy/zopu-runtime/scripts/update.sh abc123def456 +``` + +The update script: +1. Records current HEAD to `.last-deployed-sha` +2. Fetches, resolves branch-or-commit, checks out +3. `bun install`, builds daemon and agent +4. Restarts daemon, waits, restarts agent +5. Runs health check; reports failure and rollback instructions + +## Rollback + +```bash +# Roll back to the previously deployed commit +/opt/zopu/deploy/zopu-runtime/scripts/rollback.sh + +# Roll back to a specific commit +/opt/zopu/deploy/zopu-runtime/scripts/rollback.sh abc123def456 +``` + +Rollback reads `.last-deployed-sha` (written by `update.sh`), checks out that +commit, rebuilds, and restarts services. The pre-rollback SHA is saved to +`.pre-rollback-sha` for re-rollback if needed. + +## Docker cleanup + +```bash +# Manual cleanup +/opt/zopu/deploy/zopu-runtime/scripts/docker-cleanup.sh + +# Check Docker disk usage +docker system df + +# Timer runs daily; check its schedule +systemctl list-timers zopu-docker-cleanup.timer +``` + +Cleanup prunes: +- Stopped containers older than 24 hours +- Dangling (untagged) images +- Unused networks + + +Named volumes and running containers are never removed. + +## Disk-space monitoring + +```bash +# Manual check +/opt/zopu/deploy/zopu-runtime/scripts/disk-monitor.sh + +# Custom threshold (90%) +/opt/zopu/deploy/zopu-runtime/scripts/disk-monitor.sh --warn-percent 90 +``` + +A cron job runs at 06:00 daily and writes to `/var/log/zopu/disk-monitor.log`. +Default alert threshold is 80%. + +## Firewall and private networking + +The firewall (`ufw`) is deny-by-default: +- SSH (port 22) is allowed +- All other incoming traffic is denied + +The RivetKit engine (`:6420`) and Flue agent (`:3583`) ports are **not** +exposed publicly. Access is via: + +1. **Tailscale** (recommended): private overlay network. Set + `TAILSCALE_AUTHKEY` before running bootstrap to configure automatically. +2. **Direct private IP**: if the server is on a private network, the ports + are reachable from other machines on the same subnet. +3. **Caddy** (optional): install Caddy and use the annotated Caddyfile in + `caddy/` if you need TLS termination or a public ingress point. + +## Environment groups + +See [`.env.template`](./.env.template) for the full annotated template. The +eight required groups: + +| Group | Variables | +|-------|-----------| +| Convex | `CONVEX_URL`, `CONVEX_SITE_URL`, `SITE_URL` | +| Gitea | `GITEA_URL`, `GITEA_TOKEN` | +| Model gateway | `AGENT_MODEL_*` | +| AgentOS/RivetKit | `RIVET_ENDPOINT` (optional) | +| Zopu agent | `FLUE_DB_TOKEN`, `PORT` | +| Daemon | `DAEMON_ID`, `DAEMON_NAME`, `DAEMON_VERSION`, `DAEMON_HEARTBEAT_MS`, `DAEMON_COMMAND_LEASE_MS` | +| Docker sandbox | group membership (no env vars) | +| Service auth | `AUTH_SECRET` (if needed) | + +## What is NOT deployed + +- **Web app**: the web frontend is not deployed in this lane. +- **Kubernetes**: no container orchestration. +- **PostgreSQL for Rivet**: the in-process RivetKit engine uses its own + storage; no external PostgreSQL is required. +- **Multi-node coordination**: single-node only. +- **Public administration endpoints**: no admin HTTP surface. +- **Secrets in source**: `.env` is never committed; `.env.template` contains + only placeholder values. +- **Production generated-app hosting**: only the execution plane runs here. +- **Docker-backed Orb sandboxes**: Docker is installed and access is + provisioned, but no Orb sandbox code is wired. This is a boundary + prepared for the Orb lane, not a working feature. + +## Reproducibility + +The deployment does not require the developer's MacBook to remain online. +Once bootstrap completes and `.env` is filled in: + +1. Services run under systemd with `Restart=always`. +2. Logs persist in journald. +3. Health checks run every 60 seconds via systemd timer. +4. Docker cleanup runs daily. +5. Disk usage is monitored daily. +6. Unattended-upgrades handles Debian security patches. diff --git a/deploy/zopu-runtime/bootstrap.sh b/deploy/zopu-runtime/bootstrap.sh new file mode 100755 index 0000000..1f42897 --- /dev/null +++ b/deploy/zopu-runtime/bootstrap.sh @@ -0,0 +1,246 @@ +#!/usr/bin/env bash +# +# bootstrap.sh — One-shot installer for the Zopu single-node execution plane. +# +# Run as root on a fresh Debian 12 host: +# +# sudo bash bootstrap.sh +# +# Environment overrides (set before running): +# ZOPU_REPO_URL — SSH clone URL (default: ssh://git@git.openputer.com:2222/puter/zopu-code.git) +# ZOPU_REPO_BRANCH — branch to deploy (default: dogfood/v0) +# ZOPU_INSTALL_DIR — install path (default: /opt/zopu) +# ZOPU_SERVICE_USER — system user (default: zopu) +# TAILSCALE_AUTHKEY — if set, configure Tailscale +# TAILSCALE_HOSTNAME — Tailscale hostname (default: zopu-runtime) +# +# Installs: Docker Engine, Bun, clones the repo, runs bun install, builds the +# daemon and agent, creates a non-root service user, installs systemd units, +# and configures firewall/Tailscale defaults. + +set -euo pipefail + +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- +REPO_URL="${ZOPU_REPO_URL:-ssh://git@git.openputer.com:2222/puter/zopu-code.git}" +REPO_BRANCH="${ZOPU_REPO_BRANCH:-dogfood/v0}" +INSTALL_DIR="${ZOPU_INSTALL_DIR:-/opt/zopu}" +SERVICE_USER="${ZOPU_SERVICE_USER:-zopu}" +DEPLOY_DIR="${INSTALL_DIR}/deploy/zopu-runtime" + +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +RED='\033[0;31m' +NC='\033[0m' + +log() { echo -e "${GREEN}[bootstrap]${NC} $*"; } +warn() { echo -e "${YELLOW}[bootstrap]${NC} $*"; } +err() { echo -e "${RED}[bootstrap]${NC} $*" >&2; } + +# --------------------------------------------------------------------------- +# Pre-flight +# --------------------------------------------------------------------------- +if [[ "$EUID" -ne 0 ]]; then + err "This script must be run as root." + exit 1 +fi + +if [[ -f /etc/debian_version ]]; then + log "Detected Debian $(cat /etc/debian_version)" +else + warn "This script targets Debian 12. Other distributions may need manual adjustments." +fi + +# --------------------------------------------------------------------------- +# 1. System packages +# --------------------------------------------------------------------------- +log "Updating apt and installing base packages..." +apt-get update -y +apt-get install -y \ + ca-certificates \ + curl \ + gnupg \ + ufw \ + git \ + jq \ + openssh-client \ + unattended-upgrades \ + netcat-openbsd \ + rsyslog + +# --------------------------------------------------------------------------- +# 2. Docker Engine +# --------------------------------------------------------------------------- +if ! command -v docker &>/dev/null; then + log "Installing Docker Engine..." + install -m 0755 -d /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/debian/gpg \ + -o /etc/apt/keyrings/docker.asc + chmod a+r /etc/apt/keyrings/docker.asc + + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] \ + https://download.docker.com/linux/debian \ + $(. /etc/os-release && echo "$VERSION_CODENAME") stable" \ + > /etc/apt/sources.list.d/docker.list + + apt-get update -y + apt-get install -y \ + docker-ce \ + docker-ce-cli \ + containerd.io \ + docker-buildx-plugin \ + docker-compose-plugin +else + log "Docker Engine already installed: $(docker --version)" +fi + +systemctl enable --now docker + +# --------------------------------------------------------------------------- +# 3. Bun +# --------------------------------------------------------------------------- +if ! command -v bun &>/dev/null; then + log "Installing Bun..." + curl -fsSL https://bun.sh/install | bash + ln -sf /root/.bun/bin/bun /usr/local/bin/bun +else + log "Bun already installed: $(bun --version)" +fi + +# --------------------------------------------------------------------------- +# 4. Service user +# --------------------------------------------------------------------------- +if ! id "$SERVICE_USER" &>/dev/null; then + log "Creating service user: $SERVICE_USER" + useradd -r -m -d "/home/$SERVICE_USER" -s /bin/bash "$SERVICE_USER" +fi + +if ! id -nG "$SERVICE_USER" | grep -qw docker; then + usermod -aG docker "$SERVICE_USER" + log "Added $SERVICE_USER to docker group" +fi + +# --------------------------------------------------------------------------- +# 5. Clone or update repository +# --------------------------------------------------------------------------- +if [[ -d "$INSTALL_DIR/.git" ]]; then + log "Repository exists at $INSTALL_DIR, fetching latest..." + cd "$INSTALL_DIR" + git fetch origin + git checkout "$REPO_BRANCH" + git reset --hard "origin/$REPO_BRANCH" +else + log "Cloning $REPO_URL (branch $REPO_BRANCH) into $INSTALL_DIR..." + git clone --branch "$REPO_BRANCH" "$REPO_URL" "$INSTALL_DIR" + cd "$INSTALL_DIR" +fi + +# --------------------------------------------------------------------------- +# 6. Install dependencies and build +# --------------------------------------------------------------------------- +log "Running bun install..." +sudo -u "$SERVICE_USER" bun install + +log "Building daemon binary..." +sudo -u "$SERVICE_USER" bun run build:daemon + +log "Building agent service..." +sudo -u "$SERVICE_USER" bun run build:agents + +# --------------------------------------------------------------------------- +# 7. Environment file +# --------------------------------------------------------------------------- +ENV_FILE="$INSTALL_DIR/.env" +if [[ ! -f "$ENV_FILE" ]]; then + log "Copying .env.template to .env — EDIT BEFORE STARTING SERVICES" + cp "$DEPLOY_DIR/.env.template" "$ENV_FILE" + chown "$SERVICE_USER":"$SERVICE_USER" "$ENV_FILE" + chmod 600 "$ENV_FILE" + warn "Edit $ENV_FILE with real values before starting services." +else + log ".env already exists at $ENV_FILE" +fi + +# --------------------------------------------------------------------------- +# 8. Persistent log directory +# --------------------------------------------------------------------------- +LOG_DIR="/var/log/zopu" +mkdir -p "$LOG_DIR" +chown "$SERVICE_USER":"$SERVICE_USER" "$LOG_DIR" + +# --------------------------------------------------------------------------- +# 9. Install systemd units (substitute placeholders) +# --------------------------------------------------------------------------- +log "Installing systemd units..." +for unit in zopu-daemon.service zopu-agent.service \ + zopu-health.timer zopu-health.service \ + zopu-docker-cleanup.timer zopu-docker-cleanup.service; do + SRC="$DEPLOY_DIR/systemd/$unit" + DST="/etc/systemd/system/$unit" + if [[ -f "$SRC" ]]; then + sed \ + -e "s|__INSTALL_DIR__|$INSTALL_DIR|g" \ + -e "s|__SERVICE_USER__|$SERVICE_USER|g" \ + "$SRC" > "$DST" + log " installed $unit" + fi +done +systemctl daemon-reload + +# --------------------------------------------------------------------------- +# 10. Firewall (deny-by-default posture) +# --------------------------------------------------------------------------- +log "Configuring firewall..." +if ! ufw status 2>/dev/null | grep -q "Status: active"; then + ufw allow 22/tcp + ufw default deny incoming + ufw default allow outgoing + ufw --force enable + log "Firewall enabled: SSH (22) allowed, all other incoming denied." + warn "Agent and RivetKit ports are NOT exposed publicly." + warn "Access is via Tailscale or direct private IP only." +else + log "Firewall already active." +fi + +# --------------------------------------------------------------------------- +# 11. Tailscale (optional) +# --------------------------------------------------------------------------- +if [[ -n "${TAILSCALE_AUTHKEY:-}" ]]; then + log "Installing and configuring Tailscale..." + if ! command -v tailscaled &>/dev/null; then + curl -fsSL https://tailscale.com/install.sh | sh + fi + tailscale up --authkey "$TAILSCALE_AUTHKEY" \ + --hostname "${TAILSCALE_HOSTNAME:-zopu-runtime}" \ + --accept-routes + log "Tailscale configured: $(tailscale ip -4 2>/dev/null || echo 'waiting for IP')" +else + warn "TAILSCALE_AUTHKEY not set — skipping Tailscale setup." +fi + +# --------------------------------------------------------------------------- +# 12. Disk-space monitoring cron +# --------------------------------------------------------------------------- +log "Installing disk-space monitor (daily at 06:00)..." +CRON_LINE="0 6 * * * $DEPLOY_DIR/scripts/disk-monitor.sh --warn-percent 80 >> /var/log/zopu/disk-monitor.log 2>&1" +echo "$CRON_LINE" > /etc/cron.d/zopu-disk-monitor +chmod 644 /etc/cron.d/zopu-disk-monitor + +# --------------------------------------------------------------------------- +# Done +# --------------------------------------------------------------------------- +log "Bootstrap complete." +echo "" +echo "Next steps:" +echo " 1. Edit $ENV_FILE with real values" +echo " 2. Start services:" +echo " systemctl start zopu-daemon zopu-agent" +echo " 3. Enable health monitoring:" +echo " systemctl enable --now zopu-health.timer zopu-docker-cleanup.timer" +echo " 4. Verify health:" +echo " $DEPLOY_DIR/scripts/health-check.sh" +echo "" +warn "Services are NOT started automatically. Edit .env first." diff --git a/deploy/zopu-runtime/caddy/Caddyfile b/deploy/zopu-runtime/caddy/Caddyfile new file mode 100644 index 0000000..7d5192b --- /dev/null +++ b/deploy/zopu-runtime/caddy/Caddyfile @@ -0,0 +1,26 @@ +# Caddyfile — Minimal reverse proxy for the Zopu agent service. +# +# This is OPTIONAL. The agent service does not need to be publicly exposed +# for the execution plane to function. Use Caddy only when an HTTP endpoint +# is genuinely needed (e.g., a webhook ingress from Convex site functions). +# +# In the default single-node deployment, the agent listens on localhost:3583 +# and is reachable over Tailscale. Install Caddy only if you need TLS +# termination or a public ingress point. +# +# --- Private/Tailscale deployment (recommended) --- +# +# Listen only on the Tailscale interface. Replace the hostname with your +# Tailscale machine name or IP. +# +# http://zopu-runtime.tail-xxxx.ts.net { +# reverse_proxy localhost:3583 +# } +# +# --- Public deployment with automatic TLS (only if needed) --- +# +# agent.example.invalid { +# reverse_proxy localhost:3583 +# } + +# Default: do not ship an active Caddyfile. The file is documentation. diff --git a/deploy/zopu-runtime/scripts/disk-monitor.sh b/deploy/zopu-runtime/scripts/disk-monitor.sh new file mode 100755 index 0000000..225adb3 --- /dev/null +++ b/deploy/zopu-runtime/scripts/disk-monitor.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# +# disk-monitor.sh — Check disk usage and alert when above threshold. +# +# Usage: +# disk-monitor.sh # print usage, warn at 80% +# disk-monitor.sh --warn-percent 90 # custom threshold +# +# Requires GNU coreutils df (standard on Debian). Uses --output for +# deterministic column ordering regardless of locale. +# +# Exit code 0 if under threshold, 1 if at or above. + +set -euo pipefail + +WARN_PERCENT=80 + +while [[ $# -gt 0 ]]; do + case "$1" in + --warn-percent) + WARN_PERCENT="$2" + shift 2 + ;; + *) + echo "Unknown argument: $1" >&2 + exit 2 + ;; + esac +done + +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +RED='\033[0;31m' +NC='\033[0m' + +EXIT_CODE=0 + +# Partitions to check: install root and /var (Docker data-root is often here) +PARTITIONS="${PARTITIONS:-/ /var}" + +for partition in $PARTITIONS; do + if [[ ! -d "$partition" ]]; then + continue + fi + # GNU df --output columns: pcent (Use%), size, avail, target (Mounted on) + read -r USAGE_PCT SIZE AVAIL MOUNT <<< "$(df -h --output=pcent,size,avail,target "$partition" | awk 'NR==2 {gsub(/%/,"",$1); print $1, $2, $3, $4}')" + + if [[ -z "${USAGE_PCT:-}" || ! "${USAGE_PCT:-}" =~ ^[0-9]+$ ]]; then + echo " [skip] Could not read usage for ${partition}" + continue + fi + + if [[ "$USAGE_PCT" -ge "$WARN_PERCENT" ]]; then + echo -e " ${RED}WARN${NC} ${MOUNT}: ${USAGE_PCT}% used (${AVAIL} avail of ${SIZE}) — threshold ${WARN_PERCENT}%" + EXIT_CODE=1 + elif [[ "$USAGE_PCT" -ge $((WARN_PERCENT - 10)) ]]; then + echo -e " ${YELLOW}NOTE${NC} ${MOUNT}: ${USAGE_PCT}% used (${AVAIL} avail of ${SIZE}) — approaching threshold" + else + echo -e " ${GREEN}OK${NC} ${MOUNT}: ${USAGE_PCT}% used (${AVAIL} avail of ${SIZE})" + fi +done + +exit "$EXIT_CODE" diff --git a/deploy/zopu-runtime/scripts/docker-cleanup.sh b/deploy/zopu-runtime/scripts/docker-cleanup.sh new file mode 100755 index 0000000..0c24207 --- /dev/null +++ b/deploy/zopu-runtime/scripts/docker-cleanup.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# +# docker-cleanup.sh — Remove stopped containers, dangling images, and unused networks. +# +# Safe to run via systemd timer (daily). Uses Docker's built-in pruning +# commands with conservative scope. +# +# Does NOT prune volumes. Named volumes may hold durable data and cannot be +# safely auto-pruned in a deployment that mixes stateful workloads. When the +# Orb lane lands with labeled resources, volume pruning can be scoped to +# Orb-managed labels (e.g. --filter label=org.openputer.orb). Until then, +# manage volumes manually. + +set -euo pipefail + +echo "[docker-cleanup] $(date -u +%Y-%m-%dT%H:%M:%SZ)" + +# Remove stopped containers older than 24 hours +echo "[docker-cleanup] Pruning stopped containers (>24h old)..." +docker container prune -f --filter "until=24h" + +# Remove dangling images (untagged intermediate layers only) +echo "[docker-cleanup] Pruning dangling images..." +docker image prune -f + +# Remove unused networks +echo "[docker-cleanup] Pruning unused networks..." +docker network prune -f + +# Volumes are intentionally NOT pruned. See header comment. + +# Show remaining disk usage +echo "[docker-cleanup] Docker disk usage:" +docker system df + +echo "[docker-cleanup] Done." diff --git a/deploy/zopu-runtime/scripts/health-check.sh b/deploy/zopu-runtime/scripts/health-check.sh new file mode 100755 index 0000000..e3bb0b8 --- /dev/null +++ b/deploy/zopu-runtime/scripts/health-check.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +# +# health-check.sh — Probe the Zopu execution plane. +# +# Checks (TCP/process-level; no assumed HTTP health routes): +# 1. zopu-daemon systemd unit is active +# 2. zopu-agent systemd unit is active +# 3. RivetKit engine TCP port (RIVET_ENDPOINT, default 6420) accepts connections +# 4. Flue agent TCP port (PORT, default 3583) accepts connections +# 5. Docker daemon is reachable +# +# Flue does not expose a health endpoint by design. RivetKit's health route +# is internal to the registry runtime. We use TCP connection checks only. +# +# Usage: +# health-check.sh # print results +# health-check.sh --quiet # suppress output, exit 0 only if all healthy +# +# Exit codes: 0 = all healthy, 1 = one or more unhealthy + +set -euo pipefail + +QUIET=false +[[ "${1:-}" == "--quiet" ]] && QUIET=true + +GREEN='\033[0;32m' +RED='\033[0;31m' +NC='\033[0m' + +pass() { $QUIET || echo -e " ${GREEN}PASS${NC} $*"; } +fail() { echo -e " ${RED}FAIL${NC} $*" >&2; FAILURES=$((FAILURES + 1)); } + +FAILURES=0 + +# --------------------------------------------------------------------------- +# Load environment +# --------------------------------------------------------------------------- +ENV_FILE="${ENV_FILE:-/opt/zopu/.env}" +if [[ -f "$ENV_FILE" ]]; then + # shellcheck source=/dev/null + set -a + . "$ENV_FILE" + set +a +fi + +RIVET_PORT="6420" +if [[ -n "${RIVET_ENDPOINT:-}" ]]; then + RIVET_PORT=$(echo "$RIVET_ENDPOINT" | sed -n 's|.*://[^:]*:\([0-9]*\).*|\1|p') + [[ -z "$RIVET_PORT" ]] && RIVET_PORT="6420" +fi + +AGENT_PORT="${PORT:-3583}" + +# --------------------------------------------------------------------------- +# TCP probe helper: works on Debian (nc from netcat-openbsd) and macOS. +# Falls back to bash /dev/tcp if nc is unavailable. +# --------------------------------------------------------------------------- +tcp_probe() { + local host="$1" port="$2" + if command -v nc &>/dev/null; then + nc -z -w 5 "$host" "$port" 2>/dev/null + else + timeout 5 bash -c "echo > /dev/tcp/${host}/${port}" 2>/dev/null + fi +} + +# --------------------------------------------------------------------------- +# 1. Daemon systemd unit +# --------------------------------------------------------------------------- +if systemctl is-active --quiet zopu-daemon 2>/dev/null; then + pass "zopu-daemon service is active" +else + fail "zopu-daemon service is not active" +fi + +# --------------------------------------------------------------------------- +# 2. Agent systemd unit +# --------------------------------------------------------------------------- +if systemctl is-active --quiet zopu-agent 2>/dev/null; then + pass "zopu-agent service is active" +else + fail "zopu-agent service is not active" +fi + +# --------------------------------------------------------------------------- +# 3. RivetKit engine TCP port +# --------------------------------------------------------------------------- +if tcp_probe localhost "$RIVET_PORT"; then + pass "RivetKit engine port ${RIVET_PORT} is accepting connections" +else + fail "RivetKit engine port ${RIVET_PORT} is not accepting connections" +fi + +# --------------------------------------------------------------------------- +# 4. Flue agent TCP port +# --------------------------------------------------------------------------- +if tcp_probe localhost "$AGENT_PORT"; then + pass "Flue agent port ${AGENT_PORT} is accepting connections" +else + fail "Flue agent port ${AGENT_PORT} is not accepting connections" +fi + +# --------------------------------------------------------------------------- +# 5. Docker daemon +# --------------------------------------------------------------------------- +if docker info &>/dev/null; then + pass "Docker daemon is reachable" +else + fail "Docker daemon is not reachable" +fi + +# --------------------------------------------------------------------------- +# Summary +# --------------------------------------------------------------------------- +$QUIET || echo "" +if [[ "$FAILURES" -eq 0 ]]; then + $QUIET || echo -e "${GREEN}All checks passed.${NC}" + exit 0 +else + echo -e "${RED}${FAILURES} check(s) failed.${NC}" >&2 + exit 1 +fi diff --git a/deploy/zopu-runtime/scripts/rollback.sh b/deploy/zopu-runtime/scripts/rollback.sh new file mode 100755 index 0000000..fe2f09d --- /dev/null +++ b/deploy/zopu-runtime/scripts/rollback.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# +# rollback.sh — Roll back to the previously deployed commit. +# +# Usage: +# rollback.sh # roll back to .last-deployed-sha +# rollback.sh # roll back to a specific commit +# +# .env is gitignored and is never touched by git operations. It survives +# updates and rollbacks unchanged. + +set -euo pipefail + +INSTALL_DIR="${ZOPU_INSTALL_DIR:-/opt/zopu}" +SERVICE_USER="${ZOPU_SERVICE_USER:-zopu}" + +GREEN='\033[0;32m' +RED='\033[0;31m' +NC='\033[0m' + +log() { echo -e "${GREEN}[rollback]${NC} $*"; } +err() { echo -e "${RED}[rollback]${NC} $*" >&2; } + +cd "$INSTALL_DIR" + +# Determine target +ROLLBACK_SHA="${1:-}" +if [[ -z "$ROLLBACK_SHA" ]]; then + LAST_SHA_FILE="${INSTALL_DIR}/.last-deployed-sha" + if [[ ! -f "$LAST_SHA_FILE" ]]; then + err "No previous deployment recorded in ${LAST_SHA_FILE}." + err "Pass a commit SHA explicitly: rollback.sh " + exit 1 + fi + ROLLBACK_SHA=$(cat "$LAST_SHA_FILE") +fi + +# Validate commit exists +if ! git rev-parse --verify "${ROLLBACK_SHA}^{commit}" &>/dev/null; then + err "Commit ${ROLLBACK_SHA} does not exist in the local repository." + exit 1 +fi + +CURRENT_SHA=$(git rev-parse HEAD) +log "Current: ${CURRENT_SHA:0:12}" +log "Rolling back to: ${ROLLBACK_SHA:0:12}" + +# Save current state before rolling back (enables re-rollback) +echo "$CURRENT_SHA" > "${INSTALL_DIR}/.pre-rollback-sha" + +# Checkout target +git checkout "$ROLLBACK_SHA" + +# Confirm .env is intact +if [[ -f "${INSTALL_DIR}/.env" ]]; then + log ".env preserved." +else + err ".env is missing! Restore it from backup before starting services." +fi + +log "Running bun install..." +sudo -u "$SERVICE_USER" bun install + +log "Building daemon..." +sudo -u "$SERVICE_USER" bun run build:daemon + +log "Building agent service..." +sudo -u "$SERVICE_USER" bun run build:agents + +log "Restarting services..." +systemctl restart zopu-daemon +sleep 3 +systemctl restart zopu-agent +sleep 5 + +# Health check +HEALTH_SCRIPT="${INSTALL_DIR}/deploy/zopu-runtime/scripts/health-check.sh" +if [[ -x "$HEALTH_SCRIPT" ]]; then + log "Running health check..." + if "$HEALTH_SCRIPT"; then + log "Rollback complete and healthy." + else + err "Health check failed after rollback!" + err " journalctl -u zopu-daemon -n 50" + err " journalctl -u zopu-agent -n 50" + exit 1 + fi +fi diff --git a/deploy/zopu-runtime/scripts/update.sh b/deploy/zopu-runtime/scripts/update.sh new file mode 100755 index 0000000..bffcd71 --- /dev/null +++ b/deploy/zopu-runtime/scripts/update.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# +# update.sh — Update the Zopu runtime to a specific branch or commit. +# +# Usage: +# update.sh # update to latest dogfood/v0 +# update.sh dogfood/v0 # update to latest of branch dogfood/v0 +# update.sh # checkout and build a specific commit +# +# The argument is treated as a branch name first; if no matching remote +# tracking branch exists it is treated as a commit SHA. +# +# .env is gitignored and is never touched by git operations. It survives +# updates and rollbacks unchanged. + +set -euo pipefail + +INSTALL_DIR="${ZOPU_INSTALL_DIR:-/opt/zopu}" +SERVICE_USER="${ZOPU_SERVICE_USER:-zopu}" +TARGET="${1:-dogfood/v0}" + +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +RED='\033[0;31m' +NC='\033[0m' + +log() { echo -e "${GREEN}[update]${NC} $*"; } +warn() { echo -e "${YELLOW}[update]${NC} $*"; } +err() { echo -e "${RED}[update]${NC} $*" >&2; } + +cd "$INSTALL_DIR" + +# Record current commit for rollback +CURRENT_SHA=$(git rev-parse HEAD) +log "Current HEAD: ${CURRENT_SHA:0:12}" +echo "$CURRENT_SHA" > "${INSTALL_DIR}/.last-deployed-sha" + +# Fetch all remotes +log "Fetching from origin..." +git fetch origin + +# Resolve target: branch first, then commit +if git show-ref --verify --quiet "refs/remotes/origin/${TARGET}"; then + log "Target is a branch: ${TARGET}" + git checkout -B "$TARGET" "origin/${TARGET}" +elif git rev-parse --verify "${TARGET}^{commit}" &>/dev/null; then + log "Target is a commit: ${TARGET:0:12}" + git checkout "$TARGET" +else + err "'${TARGET}' is not a known branch or valid commit." + err "Available branches: $(git branch -r | tr '\n' ' ')" + exit 1 +fi + +NEW_SHA=$(git rev-parse HEAD) +log "Now at: ${NEW_SHA:0:12}" + +# Confirm .env is intact +if [[ -f "${INSTALL_DIR}/.env" ]]; then + log ".env preserved." +else + err ".env is missing! Restore it from backup before starting services." +fi + +# Install and build +log "Running bun install..." +sudo -u "$SERVICE_USER" bun install + +log "Building daemon binary..." +sudo -u "$SERVICE_USER" bun run build:daemon + +log "Building agent service..." +sudo -u "$SERVICE_USER" bun run build:agents + +# Graceful restart: daemon first (owns RivetKit engine), then agent +log "Restarting services..." +systemctl restart zopu-daemon +sleep 3 +systemctl restart zopu-agent +sleep 5 + +# Health check +HEALTH_SCRIPT="${INSTALL_DIR}/deploy/zopu-runtime/scripts/health-check.sh" +if [[ -x "$HEALTH_SCRIPT" ]]; then + log "Running health check..." + if "$HEALTH_SCRIPT"; then + log "Update complete and healthy." + else + err "Health check failed after update!" + err " journalctl -u zopu-daemon -n 50" + err " journalctl -u zopu-agent -n 50" + err "To rollback: ${INSTALL_DIR}/deploy/zopu-runtime/scripts/rollback.sh" + exit 1 + fi +else + log "Update complete. Verify manually." +fi diff --git a/deploy/zopu-runtime/systemd/zopu-agent.service b/deploy/zopu-runtime/systemd/zopu-agent.service new file mode 100644 index 0000000..65a0837 --- /dev/null +++ b/deploy/zopu-runtime/systemd/zopu-agent.service @@ -0,0 +1,36 @@ +[Unit] +Description=Zopu Agent Service (Flue Node server) +After=network-online.target zopu-daemon.service +Wants=network-online.target + +[Service] +Type=simple +User=__SERVICE_USER__ +Group=__SERVICE_USER__ +WorkingDirectory=__INSTALL_DIR__/packages/agents + +EnvironmentFile=__INSTALL_DIR__/.env + +# Flue build output: packages/agents/dist/server.mjs +# Listens on PORT (default 3583 per .env.template). +ExecStart=/usr/local/bin/bun dist/server.mjs + +Restart=always +RestartSec=10 + +StandardOutput=journal +StandardError=journal +SyslogIdentifier=zopu-agent + +# Resource limits +MemoryMax=8G +CPUWeight=80 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=full +ProtectHome=true +PrivateTmp=true + +[Install] +WantedBy=multi-user.target diff --git a/deploy/zopu-runtime/systemd/zopu-daemon.service b/deploy/zopu-runtime/systemd/zopu-daemon.service new file mode 100644 index 0000000..12488db --- /dev/null +++ b/deploy/zopu-runtime/systemd/zopu-daemon.service @@ -0,0 +1,40 @@ +[Unit] +Description=Zopu Daemon (Bun/Effect + AgentOS/RivetKit) +After=network-online.target docker.service +Wants=network-online.target + +[Service] +Type=simple +User=__SERVICE_USER__ +Group=__SERVICE_USER__ +WorkingDirectory=__INSTALL_DIR__ + +EnvironmentFile=__INSTALL_DIR__/.env + +# RivetKit in-process engine: envoy mode (serverful). +# registry.start() boots the native Rust sidecar and actor envoy. +# RIVET_ENDPOINT defaults to http://localhost:6420 inside the daemon process. +ExecStart=__INSTALL_DIR__/apps/daemon/dist/code-daemon + +Restart=always +RestartSec=10 + +StandardOutput=journal +StandardError=journal +SyslogIdentifier=zopu-daemon + +# Resource limits (12-core, 40 GB host) +MemoryMax=8G +CPUWeight=100 + +# Security hardening +NoNewPrivileges=true +ProtectSystem=full +ProtectHome=true +PrivateTmp=true + +# Docker socket access (for future Orb sandboxes; group membership required) +SupplementaryGroups=docker + +[Install] +WantedBy=multi-user.target diff --git a/deploy/zopu-runtime/systemd/zopu-docker-cleanup.service b/deploy/zopu-runtime/systemd/zopu-docker-cleanup.service new file mode 100644 index 0000000..0c37668 --- /dev/null +++ b/deploy/zopu-runtime/systemd/zopu-docker-cleanup.service @@ -0,0 +1,7 @@ +[Unit] +Description=Zopu Docker Cleanup +After=docker.service + +[Service] +Type=oneshot +ExecStart=__INSTALL_DIR__/deploy/zopu-runtime/scripts/docker-cleanup.sh diff --git a/deploy/zopu-runtime/systemd/zopu-docker-cleanup.timer b/deploy/zopu-runtime/systemd/zopu-docker-cleanup.timer new file mode 100644 index 0000000..674a3ec --- /dev/null +++ b/deploy/zopu-runtime/systemd/zopu-docker-cleanup.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Zopu Docker Cleanup (daily) + +[Timer] +OnCalendar=daily +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/deploy/zopu-runtime/systemd/zopu-health.service b/deploy/zopu-runtime/systemd/zopu-health.service new file mode 100644 index 0000000..567494d --- /dev/null +++ b/deploy/zopu-runtime/systemd/zopu-health.service @@ -0,0 +1,8 @@ +[Unit] +Description=Zopu Health Check + +[Service] +Type=oneshot +User=__SERVICE_USER__ +EnvironmentFile=__INSTALL_DIR__/.env +ExecStart=__INSTALL_DIR__/deploy/zopu-runtime/scripts/health-check.sh --quiet diff --git a/deploy/zopu-runtime/systemd/zopu-health.timer b/deploy/zopu-runtime/systemd/zopu-health.timer new file mode 100644 index 0000000..a2092af --- /dev/null +++ b/deploy/zopu-runtime/systemd/zopu-health.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Zopu Health Check (every 60s) + +[Timer] +OnBootSec=30 +OnUnitActiveSec=60 +AccuracySec=5 + +[Install] +WantedBy=timers.target