From b4595cb117f723cd442d1d712db4c5b0f8276e65 Mon Sep 17 00:00:00 2001 From: Bas Nijholt Date: Sat, 20 Dec 2025 09:57:59 -0800 Subject: [PATCH] docs: add comprehensive Zensical-based documentation (#62) --- .gitattributes | 2 + .github/workflows/docs.yml | 56 ++++ docs/CNAME | 1 + docs/architecture.md | 352 +++++++++++++++++++++ docs/assets/apply.gif | 3 + docs/assets/apply.webm | 3 + docs/assets/install.gif | 3 + docs/assets/install.webm | 3 + docs/assets/logs.gif | 3 + docs/assets/logs.webm | 3 + docs/assets/migration.gif | 3 + docs/assets/migration.webm | 3 + docs/assets/quickstart.gif | 3 + docs/assets/quickstart.webm | 3 + docs/assets/update.gif | 3 + docs/assets/update.webm | 3 + docs/best-practices.md | 381 +++++++++++++++++++++++ docs/commands.md | 591 ++++++++++++++++++++++++++++++++++++ docs/configuration.md | 393 ++++++++++++++++++++++++ docs/demos/README.md | 26 ++ docs/demos/apply.tape | 39 +++ docs/demos/install.tape | 42 +++ docs/demos/logs.tape | 21 ++ docs/demos/migration.tape | 71 +++++ docs/demos/quickstart.tape | 90 ++++++ docs/demos/record.sh | 88 ++++++ docs/demos/update.tape | 32 ++ docs/getting-started.md | 284 +++++++++++++++++ docs/index.md | 127 ++++++++ docs/traefik.md | 385 +++++++++++++++++++++++ zensical.toml | 75 +++++ 31 files changed, 3092 insertions(+) create mode 100644 .gitattributes create mode 100644 .github/workflows/docs.yml create mode 100644 docs/CNAME create mode 100644 docs/architecture.md create mode 100644 docs/assets/apply.gif create mode 100644 docs/assets/apply.webm create mode 100644 docs/assets/install.gif create mode 100644 docs/assets/install.webm create mode 100644 docs/assets/logs.gif create mode 100644 docs/assets/logs.webm create mode 100644 docs/assets/migration.gif create mode 100644 docs/assets/migration.webm create mode 100644 docs/assets/quickstart.gif create mode 100644 docs/assets/quickstart.webm create mode 100644 docs/assets/update.gif create mode 100644 docs/assets/update.webm create mode 100644 docs/best-practices.md create mode 100644 docs/commands.md create mode 100644 docs/configuration.md create mode 100644 docs/demos/README.md create mode 100644 docs/demos/apply.tape create mode 100644 docs/demos/install.tape create mode 100644 docs/demos/logs.tape create mode 100644 docs/demos/migration.tape create mode 100644 docs/demos/quickstart.tape create mode 100755 docs/demos/record.sh create mode 100644 docs/demos/update.tape create mode 100644 docs/getting-started.md create mode 100644 docs/index.md create mode 100644 docs/traefik.md create mode 100644 zensical.toml diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..f894f00 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +*.gif filter=lfs diff=lfs merge=lfs -text +*.webm filter=lfs diff=lfs merge=lfs -text diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..9ec27db --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,56 @@ +name: Docs + +on: + push: + branches: [main] + paths: + - "docs/**" + - "zensical.toml" + - ".github/workflows/docs.yml" + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: "pages" + cancel-in-progress: false + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v4 + + - name: Set up Python + run: uv python install 3.12 + + - name: Install Zensical + run: uv tool install zensical + + - name: Build docs + run: zensical build + + - name: Setup Pages + uses: actions/configure-pages@v5 + + - name: Upload artifact + uses: actions/upload-pages-artifact@v3 + with: + path: "./site" + + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + needs: build + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/docs/CNAME b/docs/CNAME new file mode 100644 index 0000000..5454340 --- /dev/null +++ b/docs/CNAME @@ -0,0 +1 @@ +compose-farm.nijho.lt diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..0245854 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,352 @@ +--- +icon: lucide/layers +--- + +# Architecture + +This document explains how Compose Farm works under the hood. + +## Design Philosophy + +Compose Farm follows three core principles: + +1. **KISS** - Keep it simple. It's a thin wrapper around `docker compose` over SSH. +2. **YAGNI** - No orchestration, no service discovery, no health checks until needed. +3. **Zero changes** - Your existing compose files work unchanged. + +## High-Level Architecture + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ Compose Farm CLI │ +│ │ +│ ┌──────────┐ ┌──────────┐ ┌──────────┐ ┌──────────────────┐ │ +│ │ Config │ │ State │ │Operations│ │ Executor │ │ +│ │ Parser │ │ Tracker │ │ Logic │ │ (SSH/Local) │ │ +│ └────┬─────┘ └────┬─────┘ └────┬─────┘ └────────┬─────────┘ │ +└───────┼─────────────┼─────────────┼─────────────────┼───────────┘ + │ │ │ │ + ▼ ▼ ▼ ▼ +┌───────────────────────────────────────────────────────────────┐ +│ SSH / Local │ +└───────────────────────────────────────────────────────────────┘ + │ │ + ▼ ▼ +┌───────────────┐ ┌───────────────┐ +│ Host: nuc │ │ Host: hp │ +│ │ │ │ +│ docker compose│ │ docker compose│ +│ up -d │ │ up -d │ +└───────────────┘ └───────────────┘ +``` + +## Core Components + +### Configuration (`compose_farm/config.py`) + +Pydantic models for YAML configuration: + +- **Config** - Root configuration with compose_dir, hosts, services +- **HostConfig** - Host address and SSH user +- **ServiceConfig** - Service-to-host mappings + +Key features: +- Validation with Pydantic +- Multi-host service expansion (`all` → list of hosts) +- YAML loading with sensible defaults + +### State Tracking (`compose_farm/state.py`) + +Tracks deployment state in `~/.config/compose-farm/state.yaml`: + +```yaml +services: + plex: + host: nuc + running: true + sonarr: + host: nuc + running: true +``` + +Used for: +- Detecting migrations (service moved to different host) +- Identifying orphans (services removed from config) +- `cf ps` status display + +### Operations (`compose_farm/operations.py`) + +Business logic for service operations: + +- **up** - Start service, handle migration if needed +- **down** - Stop service +- **preflight checks** - Verify mounts, networks exist before operations +- **discover** - Find running services on hosts +- **migrate** - Down on old host, up on new host + +### Executor (`compose_farm/executor.py`) + +SSH and local command execution: + +- **Hybrid SSH approach**: asyncssh for parallel streaming, native `ssh -t` for raw mode +- **Parallel by default**: Multiple services via `asyncio.gather` +- **Streaming output**: Real-time stdout/stderr with `[service]` prefix +- **Local detection**: Skips SSH when target matches local machine IP + +### CLI (`compose_farm/cli/`) + +Typer-based CLI with subcommand modules: + +``` +cli/ +├── app.py # Shared Typer app, version callback +├── common.py # Shared helpers, options, progress utilities +├── config.py # config subcommand +├── lifecycle.py # up, down, pull, restart, update, apply +├── management.py # refresh, check, init-network, traefik-file +└── monitoring.py # logs, ps, stats +``` + +## Command Flow + +### cf up plex + +``` +1. Load configuration + └─► Parse compose-farm.yaml + └─► Validate service exists + +2. Check state + └─► Load state.yaml + └─► Is plex already running? + └─► Is it on a different host? (migration needed) + +3. Pre-flight checks + └─► SSH to target host + └─► Check compose file exists + └─► Check required mounts exist + └─► Check required networks exist + +4. Execute migration (if needed) + └─► SSH to old host + └─► Run: docker compose down + +5. Start service + └─► SSH to target host + └─► cd /opt/compose/plex + └─► Run: docker compose up -d + +6. Update state + └─► Write new state to state.yaml + +7. Generate Traefik config (if configured) + └─► Regenerate traefik file-provider +``` + +### cf apply + +``` +1. Load configuration and state + +2. Compute diff + ├─► Orphans: in state, not in config + ├─► Migrations: in both, different host + └─► Missing: in config, not in state + +3. Stop orphans + └─► For each orphan: cf down + +4. Migrate services + └─► For each migration: down old, up new + +5. Start missing + └─► For each missing: cf up + +6. Update state +``` + +## SSH Execution + +### Parallel Streaming (asyncssh) + +For most operations, Compose Farm uses asyncssh: + +```python +async def run_command(host, command): + async with asyncssh.connect(host) as conn: + result = await conn.run(command) + return result.stdout, result.stderr +``` + +Multiple services run concurrently via `asyncio.gather`. + +### Raw Mode (native ssh) + +For commands needing PTY (progress bars, interactive): + +```bash +ssh -t user@host "docker compose pull" +``` + +### Local Detection + +When target host IP matches local machine: + +```python +if is_local(host_address): + # Run locally, no SSH + subprocess.run(command) +else: + # SSH to remote + ssh.run(command) +``` + +## State Management + +### State File + +Location: `~/.config/compose-farm/state.yaml` + +```yaml +services: + plex: + host: nuc + running: true + digests: + plex: sha256:abc123... + sonarr: + host: nuc + running: true +``` + +### State Transitions + +``` +Config Change State Change Action +───────────────────────────────────────────────────── +Add service Missing cf up +Remove service Orphaned cf down +Change host Migration down old, up new +No change No change none (or refresh) +``` + +### cf refresh + +Syncs state with reality by querying Docker on each host: + +```bash +docker ps --format '{{.Names}}' +``` + +Updates state.yaml to match what's actually running. + +## Compose File Discovery + +For each service, Compose Farm looks for compose files in: + +``` +{compose_dir}/{service}/ +├── compose.yaml # preferred +├── compose.yml +├── docker-compose.yml +└── docker-compose.yaml +``` + +First match wins. + +## Traefik Integration + +### Label Extraction + +Compose Farm parses Traefik labels from compose files: + +```yaml +services: + plex: + labels: + - traefik.enable=true + - traefik.http.routers.plex.rule=Host(`plex.example.com`) + - traefik.http.services.plex.loadbalancer.server.port=32400 +``` + +### File Provider Generation + +Converts labels to Traefik file-provider YAML: + +```yaml +http: + routers: + plex: + rule: Host(`plex.example.com`) + service: plex + services: + plex: + loadBalancer: + servers: + - url: http://192.168.1.10:32400 +``` + +### Variable Resolution + +Supports `${VAR}` and `${VAR:-default}` from: +1. Service's `.env` file +2. Current environment + +## Error Handling + +### Pre-flight Failures + +Before any operation, Compose Farm checks: +- SSH connectivity +- Compose file existence +- Required mounts +- Required networks + +If checks fail, operation aborts with clear error. + +### Partial Failures + +When operating on multiple services: +- Each service is independent +- Failures are logged, but other services continue +- Exit code reflects overall success/failure + +## Performance Considerations + +### Parallel Execution + +Services are started/stopped in parallel: + +```python +await asyncio.gather(*[ + up_service(service) for service in services +]) +``` + +### SSH Multiplexing + +For repeated connections to the same host, SSH reuses connections. + +### Caching + +- Config is parsed once per command +- State is loaded once, written once +- Host discovery results are cached during command + +## Web UI Architecture + +``` +┌─────────────────────────────────────────────────────────────┐ +│ Web UI │ +│ │ +│ ┌─────────────┐ ┌─────────────┐ ┌─────────────────────┐ │ +│ │ FastAPI │ │ Jinja │ │ HTMX │ │ +│ │ Backend │ │ Templates │ │ Dynamic Updates │ │ +│ └─────────────┘ └─────────────┘ └─────────────────────┘ │ +│ │ +│ Pattern: Custom events, not hx-swap-oob │ +│ Elements trigger on: cf:refresh from:body │ +└─────────────────────────────────────────────────────────────┘ +``` + +Icons use [Lucide](https://lucide.dev/). Add new icons as macros in `web/templates/partials/icons.html`. diff --git a/docs/assets/apply.gif b/docs/assets/apply.gif new file mode 100644 index 0000000..ae172eb --- /dev/null +++ b/docs/assets/apply.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb1372a59a4ed1ac74d3864d7a84dd5311fce4cb6c6a00bf3a574bc2f98d5595 +size 895927 diff --git a/docs/assets/apply.webm b/docs/assets/apply.webm new file mode 100644 index 0000000..88fe8fb --- /dev/null +++ b/docs/assets/apply.webm @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f339a85f3d930db5a020c9f77e106edc5f44ea7dee6f68557106721493c24ef8 +size 205907 diff --git a/docs/assets/install.gif b/docs/assets/install.gif new file mode 100644 index 0000000..7c5f056 --- /dev/null +++ b/docs/assets/install.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:388aa49a1269145698f9763452aaf6b9c6232ea9229abe1dae304df558e29695 +size 403442 diff --git a/docs/assets/install.webm b/docs/assets/install.webm new file mode 100644 index 0000000..c41c3bf --- /dev/null +++ b/docs/assets/install.webm @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b8bf4dcb8ee67270d4a88124b4dd4abe0dab518e73812ee73f7c66d77f146e2 +size 228025 diff --git a/docs/assets/logs.gif b/docs/assets/logs.gif new file mode 100644 index 0000000..79b561b --- /dev/null +++ b/docs/assets/logs.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16b9a28137dfae25488e2094de85766a039457f5dca20c2d84ac72e3967c10b9 +size 164237 diff --git a/docs/assets/logs.webm b/docs/assets/logs.webm new file mode 100644 index 0000000..769bb7e --- /dev/null +++ b/docs/assets/logs.webm @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0fbe697a1f8256ce3b9a6a64c7019d42769134df9b5b964e5abe98a29e918fd +size 68242 diff --git a/docs/assets/migration.gif b/docs/assets/migration.gif new file mode 100644 index 0000000..2666fae --- /dev/null +++ b/docs/assets/migration.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:629b8c80b98eb996b75439745676fd99a83f391ca25f778a71bd59173f814c2f +size 1194931 diff --git a/docs/assets/migration.webm b/docs/assets/migration.webm new file mode 100644 index 0000000..40cdf9f --- /dev/null +++ b/docs/assets/migration.webm @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33fd46f2d8538cc43be4cb553b3af9d8b412f282ee354b6373e2793fe41c799b +size 405057 diff --git a/docs/assets/quickstart.gif b/docs/assets/quickstart.gif new file mode 100644 index 0000000..3dc0d07 --- /dev/null +++ b/docs/assets/quickstart.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e3a8cbefad1c0045d61b9c4bbba1074df550def1c1e39aa7d73e830355f821a +size 3298967 diff --git a/docs/assets/quickstart.webm b/docs/assets/quickstart.webm new file mode 100644 index 0000000..5baa98a --- /dev/null +++ b/docs/assets/quickstart.webm @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45959c423412e112c03dc5377783b7fe831699e0d4231ee809c7c2a7c30dda90 +size 979908 diff --git a/docs/assets/update.gif b/docs/assets/update.gif new file mode 100644 index 0000000..228d599 --- /dev/null +++ b/docs/assets/update.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2067f4967a93b7ee3a8db7750c435f41b1fccd2919f3443da4b848c20cc54f23 +size 124559 diff --git a/docs/assets/update.webm b/docs/assets/update.webm new file mode 100644 index 0000000..3355065 --- /dev/null +++ b/docs/assets/update.webm @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5471bd94e6d1b9d415547fa44de6021fdad2e1cc5b8b295680e217104aa749d6 +size 98149 diff --git a/docs/best-practices.md b/docs/best-practices.md new file mode 100644 index 0000000..90ee4e2 --- /dev/null +++ b/docs/best-practices.md @@ -0,0 +1,381 @@ +--- +icon: lucide/lightbulb +--- + +# Best Practices + +Tips, limitations, and recommendations for using Compose Farm effectively. + +## Limitations + +### No Cross-Host Networking + +Compose Farm moves containers between hosts but **does not provide cross-host networking**. Docker's internal DNS and networks don't span hosts. + +**What breaks when you move a service:** + +| Feature | Works? | Why | +|---------|--------|-----| +| `http://redis:6379` | No | Docker DNS doesn't cross hosts | +| Docker network names | No | Networks are per-host | +| `DATABASE_URL=postgres://db:5432` | No | Container name won't resolve | +| Host IP addresses | Yes | Use `192.168.1.10:5432` | + +### What Compose Farm Doesn't Do + +- No overlay networking (use Swarm/Kubernetes) +- No service discovery across hosts +- No automatic dependency tracking between compose files +- No health checks or restart policies beyond Docker's +- No secrets management beyond Docker's + +## Service Organization + +### Keep Dependencies Together + +If services talk to each other, keep them in the same compose file on the same host: + +```yaml +# /opt/compose/myapp/docker-compose.yml +services: + app: + image: myapp + depends_on: + - db + - redis + + db: + image: postgres + + redis: + image: redis +``` + +```yaml +# compose-farm.yaml +services: + myapp: nuc # All three containers stay together +``` + +### Separate Standalone Services + +Services that don't talk to other containers can be anywhere: + +```yaml +services: + # These can run on any host + plex: nuc + jellyfin: hp + homeassistant: nas + + # These should stay together + myapp: nuc # includes app + db + redis +``` + +### Cross-Host Communication + +If services MUST communicate across hosts, publish ports: + +```yaml +# Instead of +DATABASE_URL=postgres://db:5432 + +# Use +DATABASE_URL=postgres://192.168.1.10:5432 +``` + +```yaml +# And publish the port +services: + db: + ports: + - "5432:5432" +``` + +## Multi-Host Services + +### When to Use `all` + +Use `all` for services that need local access to each host: + +```yaml +services: + # Need Docker socket + dozzle: all # Log viewer + portainer-agent: all # Portainer agents + autokuma: all # Auto-creates monitors + + # Need host metrics + node-exporter: all # Prometheus metrics + promtail: all # Log shipping +``` + +### Host-Specific Lists + +For services on specific hosts only: + +```yaml +services: + # Only on compute nodes + gitlab-runner: [nuc, hp] + + # Only on storage nodes + minio: [nas-1, nas-2] +``` + +## Migration Safety + +### Pre-flight Checks + +Before migrating, Compose Farm verifies: +- Compose file is accessible on new host +- Required mounts exist on new host +- Required networks exist on new host + +### Data Considerations + +**Compose Farm doesn't move data.** Ensure: + +1. **Shared storage**: Data volumes on NFS/shared storage +2. **External databases**: Data in external DB, not container +3. **Backup first**: Always backup before migration + +### Safe Migration Pattern + +```bash +# 1. Preview changes +cf apply --dry-run + +# 2. Verify target host can run the service +cf check myservice + +# 3. Apply changes +cf apply +``` + +## State Management + +### When to Refresh + +Run `cf refresh` after: +- Manual `docker compose` commands +- Container restarts +- Host reboots +- Any changes outside Compose Farm + +```bash +cf refresh --dry-run # Preview +cf refresh # Sync +``` + +### State Conflicts + +If state doesn't match reality: + +```bash +# See what's actually running +cf refresh --dry-run + +# Sync state +cf refresh + +# Then apply config +cf apply +``` + +## Shared Storage + +### NFS Best Practices + +```bash +# Mount options for Docker compatibility +nas:/compose /opt/compose nfs rw,hard,intr,rsize=8192,wsize=8192 0 0 +``` + +### Directory Ownership + +Ensure consistent UID/GID across hosts: + +```yaml +services: + myapp: + environment: + - PUID=1000 + - PGID=1000 +``` + +### Config vs Data + +Keep config and data separate: + +``` +/opt/compose/ # Shared: compose files + config +├── plex/ +│ ├── docker-compose.yml +│ └── config/ # Small config files OK + +/mnt/data/ # Shared: large media files +├── movies/ +├── tv/ +└── music/ + +/opt/appdata/ # Local: per-host app data +├── plex/ +└── sonarr/ +``` + +## Performance + +### Parallel Operations + +Compose Farm runs operations in parallel. For large deployments: + +```bash +# Good: parallel by default +cf up --all + +# Avoid: sequential updates when possible +for svc in plex sonarr radarr; do + cf update $svc +done +``` + +### SSH Connection Reuse + +SSH connections are reused within a command. For many operations: + +```bash +# One command, one connection per host +cf update --all + +# Multiple commands, multiple connections (slower) +cf update plex && cf update sonarr && cf update radarr +``` + +## Traefik Setup + +### Service Placement + +Put Traefik on a reliable host: + +```yaml +services: + traefik: nuc # Primary host with good uptime +``` + +### Same-Host Services + +Services on the same host as Traefik use Docker provider: + +```yaml +traefik_service: traefik + +services: + traefik: nuc + portainer: nuc # Docker provider handles this + plex: hp # File provider handles this +``` + +### Middleware in Separate File + +Define middlewares outside Compose Farm's generated file: + +```yaml +# /opt/traefik/dynamic.d/middlewares.yml +http: + middlewares: + redirect-https: + redirectScheme: + scheme: https +``` + +## Backup Strategy + +### What to Backup + +| Item | Location | Method | +|------|----------|--------| +| Compose Farm config | `~/.config/compose-farm/` | Git or copy | +| Compose files | `/opt/compose/` | Git | +| State file | `~/.config/compose-farm/state.yaml` | Optional (can refresh) | +| App data | `/opt/appdata/` | Backup solution | + +### Disaster Recovery + +```bash +# Restore config +cp backup/compose-farm.yaml ~/.config/compose-farm/ + +# Refresh state from running containers +cf refresh + +# Or start fresh +cf apply +``` + +## Troubleshooting + +### Common Issues + +**Service won't start:** +```bash +cf check myservice # Verify mounts/networks +cf logs myservice # Check container logs +``` + +**Migration fails:** +```bash +cf check myservice # Verify new host is ready +cf init-network newhost # Create network if missing +``` + +**State out of sync:** +```bash +cf refresh --dry-run # See differences +cf refresh # Sync state +``` + +**SSH issues:** +```bash +cf ssh status # Check key status +cf ssh setup # Re-setup keys +``` + +### Debug Mode + +For more verbose output: + +```bash +# See exact commands being run +cf --verbose up myservice +``` + +## Security Considerations + +### SSH Keys + +- Use dedicated SSH key for Compose Farm +- Limit key to specific hosts if possible +- Don't store keys in Docker images + +### Network Exposure + +- Published ports are accessible from network +- Use firewalls for sensitive services +- Consider VPN for cross-host communication + +### Secrets + +- Don't commit `.env` files with secrets +- Use Docker secrets or external secret management +- Avoid secrets in compose file labels + +## Comparison: When to Use Alternatives + +| Scenario | Solution | +|----------|----------| +| 2-10 hosts, static services | **Compose Farm** | +| Cross-host container networking | Docker Swarm | +| Auto-scaling, self-healing | Kubernetes | +| Infrastructure as code | Ansible + Compose Farm | +| High availability requirements | Kubernetes or Swarm | diff --git a/docs/commands.md b/docs/commands.md new file mode 100644 index 0000000..ca95164 --- /dev/null +++ b/docs/commands.md @@ -0,0 +1,591 @@ +--- +icon: lucide/terminal +--- + +# Commands Reference + +The Compose Farm CLI is available as both `compose-farm` and the shorter alias `cf`. + +## Command Overview + +| Category | Command | Description | +|----------|---------|-------------| +| **Lifecycle** | `apply` | Make reality match config | +| | `up` | Start services | +| | `down` | Stop services | +| | `restart` | Restart services (down + up) | +| | `update` | Update services (pull + down + up) | +| | `pull` | Pull latest images | +| **Monitoring** | `ps` | Show service status | +| | `logs` | Show service logs | +| | `stats` | Show overview statistics | +| **Configuration** | `check` | Validate config and mounts | +| | `refresh` | Sync state from reality | +| | `init-network` | Create Docker network | +| | `traefik-file` | Generate Traefik config | +| | `config` | Manage config files | +| | `ssh` | Manage SSH keys | +| **Server** | `web` | Start web UI | + +## Global Options + +```bash +cf --version, -v # Show version +cf --help, -h # Show help +``` + +--- + +## Lifecycle Commands + +### cf apply + +Make reality match your configuration. The primary reconciliation command. + + + +```bash +cf apply [OPTIONS] +``` + +**Options:** + +| Option | Description | +|--------|-------------| +| `--dry-run, -n` | Preview changes without executing | +| `--no-orphans` | Skip stopping orphaned services | +| `--full, -f` | Also refresh running services | +| `--config, -c PATH` | Path to config file | + +**What it does:** + +1. Stops orphaned services (in state but removed from config) +2. Migrates services on wrong host +3. Starts missing services (in config but not running) + +**Examples:** + +```bash +# Preview what would change +cf apply --dry-run + +# Apply all changes +cf apply + +# Only start/migrate, don't stop orphans +cf apply --no-orphans + +# Also refresh all running services +cf apply --full +``` + +--- + +### cf up + +Start services. Auto-migrates if host assignment changed. + +```bash +cf up [OPTIONS] [SERVICES]... +``` + +**Options:** + +| Option | Description | +|--------|-------------| +| `--all, -a` | Start all services | +| `--host, -H TEXT` | Filter to services on this host | +| `--config, -c PATH` | Path to config file | + +**Examples:** + +```bash +# Start specific services +cf up plex sonarr + +# Start all services +cf up --all + +# Start all services on a specific host +cf up --all --host nuc +``` + +**Auto-migration:** + +If you change a service's host in config and run `cf up`: + +1. Verifies mounts/networks exist on new host +2. Runs `down` on old host +3. Runs `up -d` on new host +4. Updates state + +--- + +### cf down + +Stop services. + +```bash +cf down [OPTIONS] [SERVICES]... +``` + +**Options:** + +| Option | Description | +|--------|-------------| +| `--all, -a` | Stop all services | +| `--orphaned` | Stop orphaned services only | +| `--host, -H TEXT` | Filter to services on this host | +| `--config, -c PATH` | Path to config file | + +**Examples:** + +```bash +# Stop specific services +cf down plex + +# Stop all services +cf down --all + +# Stop services removed from config +cf down --orphaned + +# Stop all services on a host +cf down --all --host nuc +``` + +--- + +### cf restart + +Restart services (down + up). + +```bash +cf restart [OPTIONS] [SERVICES]... +``` + +**Options:** + +| Option | Description | +|--------|-------------| +| `--all, -a` | Restart all services | +| `--config, -c PATH` | Path to config file | + +**Examples:** + +```bash +cf restart plex +cf restart --all +``` + +--- + +### cf update + +Update services (pull + build + down + up). + + + +```bash +cf update [OPTIONS] [SERVICES]... +``` + +**Options:** + +| Option | Description | +|--------|-------------| +| `--all, -a` | Update all services | +| `--config, -c PATH` | Path to config file | + +**Examples:** + +```bash +# Update specific service +cf update plex + +# Update all services +cf update --all +``` + +--- + +### cf pull + +Pull latest images. + +```bash +cf pull [OPTIONS] [SERVICES]... +``` + +**Options:** + +| Option | Description | +|--------|-------------| +| `--all, -a` | Pull for all services | +| `--config, -c PATH` | Path to config file | + +**Examples:** + +```bash +cf pull plex +cf pull --all +``` + +--- + +## Monitoring Commands + +### cf ps + +Show status of services. + +```bash +cf ps [OPTIONS] [SERVICES]... +``` + +**Options:** + +| Option | Description | +|--------|-------------| +| `--all, -a` | Show all services (default) | +| `--host, -H TEXT` | Filter to services on this host | +| `--config, -c PATH` | Path to config file | + +**Examples:** + +```bash +# Show all services +cf ps + +# Show specific services +cf ps plex sonarr + +# Filter by host +cf ps --host nuc +``` + +--- + +### cf logs + +Show service logs. + + + +```bash +cf logs [OPTIONS] [SERVICES]... +``` + +**Options:** + +| Option | Description | +|--------|-------------| +| `--all, -a` | Show logs for all services | +| `--host, -H TEXT` | Filter to services on this host | +| `--follow, -f` | Follow logs (live stream) | +| `--tail, -n INTEGER` | Number of lines (default: 20 for --all, 100 otherwise) | +| `--config, -c PATH` | Path to config file | + +**Examples:** + +```bash +# Show last 100 lines +cf logs plex + +# Follow logs +cf logs -f plex + +# Show last 50 lines of multiple services +cf logs -n 50 plex sonarr + +# Show last 20 lines of all services +cf logs --all +``` + +--- + +### cf stats + +Show overview statistics. + +```bash +cf stats [OPTIONS] +``` + +**Options:** + +| Option | Description | +|--------|-------------| +| `--live, -l` | Query Docker for live container counts | +| `--config, -c PATH` | Path to config file | + +**Examples:** + +```bash +# Config/state overview +cf stats + +# Include live container counts +cf stats --live +``` + +--- + +## Configuration Commands + +### cf check + +Validate configuration, mounts, and networks. + +```bash +cf check [OPTIONS] [SERVICES]... +``` + +**Options:** + +| Option | Description | +|--------|-------------| +| `--local` | Skip SSH-based checks (faster) | +| `--config, -c PATH` | Path to config file | + +**Examples:** + +```bash +# Full validation with SSH +cf check + +# Fast local-only validation +cf check --local + +# Check specific service and show host compatibility +cf check jellyfin +``` + +--- + +### cf refresh + +Update local state from running services. + +```bash +cf refresh [OPTIONS] +``` + +**Options:** + +| Option | Description | +|--------|-------------| +| `--dry-run, -n` | Show what would change | +| `--log-path, -l PATH` | Path to Dockerfarm TOML log | +| `--config, -c PATH` | Path to config file | + +**Examples:** + +```bash +# Sync state with reality +cf refresh + +# Preview changes +cf refresh --dry-run +``` + +--- + +### cf init-network + +Create Docker network on hosts with consistent settings. + +```bash +cf init-network [OPTIONS] [HOSTS]... +``` + +**Options:** + +| Option | Description | +|--------|-------------| +| `--network, -n TEXT` | Network name (default: mynetwork) | +| `--subnet, -s TEXT` | Network subnet (default: 172.20.0.0/16) | +| `--gateway, -g TEXT` | Network gateway (default: 172.20.0.1) | +| `--config, -c PATH` | Path to config file | + +**Examples:** + +```bash +# Create on all hosts +cf init-network + +# Create on specific hosts +cf init-network nuc hp + +# Custom network settings +cf init-network -n production -s 10.0.0.0/16 -g 10.0.0.1 +``` + +--- + +### cf traefik-file + +Generate Traefik file-provider config from compose labels. + +```bash +cf traefik-file [OPTIONS] [SERVICES]... +``` + +**Options:** + +| Option | Description | +|--------|-------------| +| `--all, -a` | Generate for all services | +| `--output, -o PATH` | Output file (stdout if omitted) | +| `--config, -c PATH` | Path to config file | + +**Examples:** + +```bash +# Preview to stdout +cf traefik-file --all + +# Write to file +cf traefik-file --all -o /opt/traefik/dynamic.d/cf.yml + +# Specific services +cf traefik-file plex jellyfin -o /opt/traefik/cf.yml +``` + +--- + +### cf config + +Manage configuration files. + +```bash +cf config COMMAND +``` + +**Subcommands:** + +| Command | Description | +|---------|-------------| +| `init` | Create new config with examples | +| `show` | Display config with highlighting | +| `path` | Print config file path | +| `validate` | Validate syntax and schema | +| `edit` | Open in $EDITOR | +| `symlink PATH` | Symlink from default location | + +**Examples:** + +```bash +cf config init +cf config show +cf config validate +cf config edit +cf config path +cf config symlink /opt/compose-farm/config.yaml +``` + +--- + +### cf ssh + +Manage SSH keys for passwordless authentication. + +```bash +cf ssh COMMAND +``` + +**Subcommands:** + +| Command | Description | +|---------|-------------| +| `setup` | Generate key and copy to all hosts | +| `status` | Show SSH key status | + +**Examples:** + +```bash +# Set up SSH keys +cf ssh setup + +# Check status +cf ssh status +``` + +--- + +## Server Commands + +### cf web + +Start the web UI server. + +```bash +cf web [OPTIONS] +``` + +See `cf web --help` for available options. + +--- + +## Common Patterns + +### Daily Operations + +```bash +# Morning: check status +cf ps +cf stats --live + +# Update a specific service +cf update plex + +# View logs +cf logs -f plex +``` + +### Maintenance + +```bash +# Update all services +cf update --all + +# Refresh state after manual changes +cf refresh +``` + +### Migration + +```bash +# Preview what would change +cf apply --dry-run + +# Move a service: edit config, then +cf up plex # auto-migrates + +# Or reconcile everything +cf apply +``` + +### Troubleshooting + +```bash +# Validate config +cf check --local +cf check + +# Check specific service +cf check jellyfin + +# Sync state +cf refresh --dry-run +cf refresh +``` diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 0000000..3747d1d --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,393 @@ +--- +icon: lucide/settings +--- + +# Configuration Reference + +Compose Farm uses a YAML configuration file to define hosts and service assignments. + +## Config File Location + +Compose Farm looks for configuration in this order: + +1. `./compose-farm.yaml` (current directory) +2. `~/.config/compose-farm/compose-farm.yaml` + +Use `-c` / `--config` to specify a custom path: + +```bash +cf ps -c /path/to/config.yaml +``` + +## Full Example + +```yaml +# Required: directory containing compose files +compose_dir: /opt/compose + +# Optional: Docker network name (default: mynetwork) +network: mynetwork + +# Optional: auto-regenerate Traefik config +traefik_file: /opt/traefik/dynamic.d/compose-farm.yml +traefik_service: traefik + +# Define Docker hosts +hosts: + nuc: + address: 192.168.1.10 + user: docker + hp: + address: 192.168.1.11 + user: admin + local: localhost + +# Map services to hosts +services: + # Single-host services + plex: nuc + sonarr: nuc + radarr: hp + jellyfin: local + + # Multi-host services + dozzle: all # Run on ALL hosts + node-exporter: [nuc, hp] # Run on specific hosts +``` + +## Settings Reference + +### compose_dir (required) + +Directory containing your compose service folders. Must be the same path on all hosts. + +```yaml +compose_dir: /opt/compose +``` + +**Directory structure:** + +``` +/opt/compose/ +├── plex/ +│ ├── docker-compose.yml # or compose.yaml +│ └── .env # optional environment file +├── sonarr/ +│ └── docker-compose.yml +└── ... +``` + +Supported compose file names (checked in order): +- `compose.yaml` +- `compose.yml` +- `docker-compose.yml` +- `docker-compose.yaml` + +### network + +Docker network name for `cf init-network`. + +```yaml +network: mynetwork # default +``` + +### traefik_file + +Path to auto-generated Traefik file-provider config. When set, Compose Farm regenerates this file after `up`, `down`, `restart`, and `update` commands. + +```yaml +traefik_file: /opt/traefik/dynamic.d/compose-farm.yml +``` + +### traefik_service + +Service name running Traefik. Services on the same host are skipped in file-provider config (Traefik's docker provider handles them). + +```yaml +traefik_service: traefik +``` + +## Hosts Configuration + +### Basic Host + +```yaml +hosts: + myserver: + address: 192.168.1.10 +``` + +### With SSH User + +```yaml +hosts: + myserver: + address: 192.168.1.10 + user: docker +``` + +If `user` is omitted, the current user is used. + +### Localhost + +For services running on the same machine where you invoke Compose Farm: + +```yaml +hosts: + local: localhost +``` + +No SSH is used for localhost services. + +### Multiple Hosts + +```yaml +hosts: + nuc: + address: 192.168.1.10 + user: docker + hp: + address: 192.168.1.11 + user: admin + truenas: + address: 192.168.1.100 + local: localhost +``` + +## Services Configuration + +### Single-Host Service + +```yaml +services: + plex: nuc + sonarr: nuc + radarr: hp +``` + +### Multi-Host Service + +For services that need to run on every host (e.g., log shippers, monitoring agents): + +```yaml +services: + # Run on ALL configured hosts + dozzle: all + promtail: all + + # Run on specific hosts + node-exporter: [nuc, hp, truenas] +``` + +**Common multi-host services:** +- **Dozzle** - Docker log viewer (needs local socket) +- **Promtail/Alloy** - Log shipping (needs local socket) +- **node-exporter** - Host metrics (needs /proc, /sys) +- **AutoKuma** - Uptime Kuma monitors (needs local socket) + +### Service Names + +Service names must match directory names in `compose_dir`: + +```yaml +compose_dir: /opt/compose +services: + plex: nuc # expects /opt/compose/plex/docker-compose.yml + my-app: hp # expects /opt/compose/my-app/docker-compose.yml +``` + +## State File + +Compose Farm tracks deployment state in: + +``` +~/.config/compose-farm/state.yaml +``` + +This file records: +- Which services are running +- Which host each service runs on +- Last known state + +**Don't edit manually.** Use `cf refresh` to sync state with reality. + +## Environment Variables + +### In Compose Files + +Your compose files can use `.env` files as usual: + +``` +/opt/compose/plex/ +├── docker-compose.yml +└── .env +``` + +Compose Farm runs `docker compose` which handles `.env` automatically. + +### In Traefik Labels + +When generating Traefik config, Compose Farm resolves `${VAR}` and `${VAR:-default}` from: + +1. The service's `.env` file +2. Current environment + +## Config Commands + +### Initialize Config + +```bash +cf config init +``` + +Creates a new config file with documented examples. + +### Validate Config + +```bash +cf config validate +``` + +Checks syntax and schema. + +### Show Config + +```bash +cf config show +``` + +Displays current config with syntax highlighting. + +### Edit Config + +```bash +cf config edit +``` + +Opens config in `$EDITOR`. + +### Show Config Path + +```bash +cf config path +``` + +Prints the config file location (useful for scripting). + +### Create Symlink + +```bash +cf config symlink /path/to/my-config.yaml +``` + +Creates a symlink from the default location to your config file. + +## Validation + +### Local Validation + +Fast validation without SSH: + +```bash +cf check --local +``` + +Checks: +- Config syntax +- Service-to-host mappings +- Compose file existence + +### Full Validation + +```bash +cf check +``` + +Additional SSH-based checks: +- Host connectivity +- Mount point existence +- Docker network existence +- Traefik label validation + +### Service-Specific Check + +```bash +cf check jellyfin +``` + +Shows which hosts can run the service (have required mounts/networks). + +## Example Configurations + +### Minimal + +```yaml +compose_dir: /opt/compose + +hosts: + server: 192.168.1.10 + +services: + myapp: server +``` + +### Home Lab + +```yaml +compose_dir: /opt/compose + +hosts: + nuc: + address: 192.168.1.10 + user: docker + nas: + address: 192.168.1.100 + user: admin + +services: + # Media + plex: nuc + sonarr: nuc + radarr: nuc + + # Infrastructure + traefik: nuc + portainer: nuc + + # Monitoring (on all hosts) + dozzle: all +``` + +### Production + +```yaml +compose_dir: /opt/compose +network: production +traefik_file: /opt/traefik/dynamic.d/cf.yml +traefik_service: traefik + +hosts: + web-1: + address: 10.0.1.10 + user: deploy + web-2: + address: 10.0.1.11 + user: deploy + db: + address: 10.0.1.20 + user: deploy + +services: + # Load balanced + api: [web-1, web-2] + + # Single instance + postgres: db + redis: db + + # Infrastructure + traefik: web-1 + + # Monitoring + promtail: all +``` diff --git a/docs/demos/README.md b/docs/demos/README.md new file mode 100644 index 0000000..486c6b5 --- /dev/null +++ b/docs/demos/README.md @@ -0,0 +1,26 @@ +# Terminal Demos + +[VHS](https://github.com/charmbracelet/vhs) tape files for recording terminal demos. + +## Demos + +| File | Shows | +|------|-------| +| `install.tape` | Installing with `uv tool install` | +| `quickstart.tape` | `cf ps`, `cf up`, `cf logs` | +| `logs.tape` | Viewing logs | +| `update.tape` | `cf update` | +| `migration.tape` | Service migration | +| `apply.tape` | `cf apply` | + +## Recording + +```bash +# Record all demos (outputs to docs/assets/) +./docs/demos/record.sh + +# Single demo +cd /opt/stacks && vhs /path/to/docs/demos/quickstart.tape +``` + +Output files (GIF + WebM) are tracked with Git LFS. diff --git a/docs/demos/apply.tape b/docs/demos/apply.tape new file mode 100644 index 0000000..aeee139 --- /dev/null +++ b/docs/demos/apply.tape @@ -0,0 +1,39 @@ +# Apply Demo +# Shows cf apply previewing and reconciling state + +Output docs/assets/apply.gif +Output docs/assets/apply.webm + +Set Shell "bash" +Set FontSize 14 +Set Width 900 +Set Height 600 +Set Theme "Catppuccin Mocha" +Set TypingSpeed 50ms + +Type "# Preview what would change" +Enter +Sleep 500ms + +Type "cf apply --dry-run" +Enter +Wait + +Type "# Check current status" +Enter +Sleep 500ms + +Type "cf stats" +Enter +Wait+Screen /Summary/ +Sleep 2s + +Type "# Apply the changes" +Enter +Sleep 500ms + +Type "cf apply" +Enter +# Wait for shell prompt (command complete) +Wait +Sleep 4s diff --git a/docs/demos/install.tape b/docs/demos/install.tape new file mode 100644 index 0000000..2f3c29e --- /dev/null +++ b/docs/demos/install.tape @@ -0,0 +1,42 @@ +# Installation Demo +# Shows installing compose-farm with uv + +Output docs/assets/install.gif +Output docs/assets/install.webm + +Set Shell "bash" +Set FontSize 14 +Set Width 900 +Set Height 600 +Set Theme "Catppuccin Mocha" +Set TypingSpeed 50ms +Env FORCE_COLOR "1" + +Hide +Type "export PATH=$HOME/.local/bin:$PATH && uv tool uninstall compose-farm 2>/dev/null; clear" +Enter +Show +Type "# Install with uv (recommended)" +Enter +Sleep 500ms + +Type "uv tool install compose-farm" +Enter +Wait+Screen /Installed|already installed/ + +Type "# Verify installation" +Enter +Sleep 500ms + +Type "cf --version" +Enter +Wait+Screen /compose-farm/ +Sleep 1s + +Type "cf --help | less" +Enter +Sleep 2s +PageDown +Sleep 2s +Type "q" +Sleep 2s diff --git a/docs/demos/logs.tape b/docs/demos/logs.tape new file mode 100644 index 0000000..173f50e --- /dev/null +++ b/docs/demos/logs.tape @@ -0,0 +1,21 @@ +# Logs Demo +# Shows viewing service logs + +Output docs/assets/logs.gif +Output docs/assets/logs.webm + +Set Shell "bash" +Set FontSize 14 +Set Width 900 +Set Height 550 +Set Theme "Catppuccin Mocha" +Set TypingSpeed 50ms + +Type "# View recent logs" +Enter +Sleep 500ms + +Type "cf logs immich --tail 20" +Enter +Wait+Screen /immich/ +Sleep 2s diff --git a/docs/demos/migration.tape b/docs/demos/migration.tape new file mode 100644 index 0000000..03169d0 --- /dev/null +++ b/docs/demos/migration.tape @@ -0,0 +1,71 @@ +# Migration Demo +# Shows automatic service migration when host changes + +Output docs/assets/migration.gif +Output docs/assets/migration.webm + +Set Shell "bash" +Set FontSize 14 +Set Width 1000 +Set Height 600 +Set Theme "Catppuccin Mocha" +Set TypingSpeed 50ms + +Type "# Current status: audiobookshelf on 'nas'" +Enter +Sleep 500ms + +Type "cf ps audiobookshelf" +Enter +Wait+Screen /PORTS/ + +Type "# Edit config to move it to 'anton'" +Enter +Sleep 1s + +Type "nvim /opt/stacks/compose-farm.yaml" +Enter +Wait+Screen /services:/ + +# Search for audiobookshelf +Type "/audiobookshelf" +Enter +Sleep 1s + +# Move to the host value (nas) and change it +Type "f:" +Sleep 500ms +Type "w" +Sleep 500ms +Type "ciw" +Sleep 500ms +Type "anton" +Escape +Sleep 1s + +# Save and quit +Type ":wq" +Enter +Sleep 1s + +Type "# Run up - automatically migrates!" +Enter +Sleep 500ms + +Type "cf up audiobookshelf" +Enter +# Wait for migration phases: first the stop on old host +Wait+Screen /Migrating|down/ +# Then wait for start on new host +Wait+Screen /Starting|up/ +# Finally wait for completion +Wait + +Type "# Verify: audiobookshelf now on 'anton'" +Enter +Sleep 500ms + +Type "cf ps audiobookshelf" +Enter +Wait+Screen /PORTS/ +Sleep 3s diff --git a/docs/demos/quickstart.tape b/docs/demos/quickstart.tape new file mode 100644 index 0000000..9da3f55 --- /dev/null +++ b/docs/demos/quickstart.tape @@ -0,0 +1,90 @@ +# Quick Start Demo +# Shows basic cf commands + +Output docs/assets/quickstart.gif +Output docs/assets/quickstart.webm + +Set Shell "bash" +Set FontSize 14 +Set Width 900 +Set Height 600 +Set Theme "Catppuccin Mocha" +Set TypingSpeed 50ms +Env BAT_PAGING "always" + +Type "# Config is just: service -> host" +Enter +Sleep 500ms + +Type "# First, define your hosts..." +Enter +Sleep 500ms + +Type "bat -r 1:11 compose-farm.yaml" +Enter +Sleep 3s +Type "q" +Sleep 500ms + +Type "# Then map each service to a host" +Enter +Sleep 500ms + +Type "bat -r 13:30 compose-farm.yaml" +Enter +Sleep 3s +Type "q" +Sleep 500ms + +Type "# Check service status" +Enter +Sleep 500ms + +Type "cf ps immich" +Enter +Wait+Screen /PORTS/ + +Type "# Start a service" +Enter +Sleep 500ms + +Type "cf up immich" +Enter +Wait + +Type "# View logs" +Enter +Sleep 500ms + +Type "cf logs immich --tail 5" +Enter +Wait+Screen /immich/ +Sleep 2s + +Type "# ✨ The magic: move between hosts (nas → anton)" +Enter +Sleep 500ms + +Type "# Change host in config (using sed)" +Enter +Sleep 500ms + +Type "sed -i 's/audiobookshelf: nas/audiobookshelf: anton/' compose-farm.yaml" +Enter +Sleep 500ms + +Type "# Apply changes - auto-migrates!" +Enter +Sleep 500ms + +Type "cf apply" +Enter +Sleep 15s + +Type "# Verify: now on anton" +Enter +Sleep 500ms + +Type "cf ps audiobookshelf" +Enter +Sleep 5s diff --git a/docs/demos/record.sh b/docs/demos/record.sh new file mode 100755 index 0000000..671ed31 --- /dev/null +++ b/docs/demos/record.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# Record all VHS demos +# Run this on a Docker host with compose-farm configured + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DOCS_DIR="$(dirname "$SCRIPT_DIR")" +REPO_DIR="$(dirname "$DOCS_DIR")" +OUTPUT_DIR="$DOCS_DIR/assets" + +# Colors +GREEN='\033[0;32m' +BLUE='\033[0;34m' +YELLOW='\033[0;33m' +RED='\033[0;31m' +NC='\033[0m' # No Color + +# Check for VHS +if ! command -v vhs &> /dev/null; then + echo "VHS not found. Install with:" + echo " brew install vhs" + echo " # or" + echo " go install github.com/charmbracelet/vhs@latest" + exit 1 +fi + +# Ensure output directory exists +mkdir -p "$OUTPUT_DIR" + +# Temp output dir (VHS runs from /opt/stacks, so relative paths go here) +TEMP_OUTPUT="/opt/stacks/docs/assets" +mkdir -p "$TEMP_OUTPUT" + +# Change to /opt/stacks so cf commands use installed version (not editable install) +cd /opt/stacks + +# Ensure compose-farm.yaml has no uncommitted changes (safety check) +if ! git diff --quiet compose-farm.yaml; then + echo -e "${RED}Error: compose-farm.yaml has uncommitted changes${NC}" + echo "Commit or stash your changes before recording demos" + exit 1 +fi + +echo -e "${BLUE}Recording VHS demos...${NC}" +echo "Output directory: $OUTPUT_DIR" +echo "" + +# Function to record a tape +record_tape() { + local tape=$1 + local name=$(basename "$tape" .tape) + echo -e "${GREEN}Recording:${NC} $name" + if vhs "$tape"; then + echo -e "${GREEN} ✓ Done${NC}" + else + echo -e "${RED} ✗ Failed${NC}" + return 1 + fi +} + +# Record demos in logical order +echo -e "${YELLOW}=== Phase 1: Basic demos ===${NC}" +record_tape "$SCRIPT_DIR/install.tape" +record_tape "$SCRIPT_DIR/quickstart.tape" +record_tape "$SCRIPT_DIR/logs.tape" + +echo -e "${YELLOW}=== Phase 2: Update demo ===${NC}" +record_tape "$SCRIPT_DIR/update.tape" + +echo -e "${YELLOW}=== Phase 3: Migration demo ===${NC}" +record_tape "$SCRIPT_DIR/migration.tape" +git -C /opt/stacks checkout compose-farm.yaml # Reset after migration + +echo -e "${YELLOW}=== Phase 4: Apply demo ===${NC}" +record_tape "$SCRIPT_DIR/apply.tape" + +# Move GIFs and WebMs from temp location to repo +echo "" +echo -e "${BLUE}Moving recordings to repo...${NC}" +mv "$TEMP_OUTPUT"/*.gif "$OUTPUT_DIR/" 2>/dev/null || true +mv "$TEMP_OUTPUT"/*.webm "$OUTPUT_DIR/" 2>/dev/null || true +rmdir "$TEMP_OUTPUT" 2>/dev/null || true +rmdir "$(dirname "$TEMP_OUTPUT")" 2>/dev/null || true + +echo "" +echo -e "${GREEN}Done!${NC} Recordings saved to $OUTPUT_DIR/" +ls -la "$OUTPUT_DIR"/*.gif "$OUTPUT_DIR"/*.webm 2>/dev/null || echo "No recordings found (check for errors above)" diff --git a/docs/demos/update.tape b/docs/demos/update.tape new file mode 100644 index 0000000..f951532 --- /dev/null +++ b/docs/demos/update.tape @@ -0,0 +1,32 @@ +# Update Demo +# Shows updating services (pull + down + up) + +Output docs/assets/update.gif +Output docs/assets/update.webm + +Set Shell "bash" +Set FontSize 14 +Set Width 900 +Set Height 500 +Set Theme "Catppuccin Mocha" +Set TypingSpeed 50ms + +Type "# Update a single service" +Enter +Sleep 500ms + +Type "cf update grocy" +Enter +# Wait for command to complete (chain waits for longer timeout) +Wait+Screen /pull/ +Wait+Screen /grocy/ +Wait@60s + +Type "# Check current status" +Enter +Sleep 500ms + +Type "cf ps grocy" +Enter +Wait+Screen /PORTS/ +Sleep 1s diff --git a/docs/getting-started.md b/docs/getting-started.md new file mode 100644 index 0000000..8e9a520 --- /dev/null +++ b/docs/getting-started.md @@ -0,0 +1,284 @@ +--- +icon: lucide/rocket +--- + +# Getting Started + +This guide walks you through installing Compose Farm and setting up your first multi-host deployment. + +## Prerequisites + +Before you begin, ensure you have: + +- **[uv](https://docs.astral.sh/uv/)** (recommended) or Python 3.11+ +- **SSH key-based authentication** to your Docker hosts +- **Docker and Docker Compose** installed on all target hosts +- **Shared storage** for compose files (NFS, Syncthing, etc.) + +## Installation + + + +### Using uv (recommended) + +[uv](https://docs.astral.sh/uv/) is the recommended way to install Compose Farm. It handles Python installation automatically. + +```bash +# Install uv first (if not already installed) +curl -LsSf https://astral.sh/uv/install.sh | sh + +# Install compose-farm +uv tool install compose-farm +``` + +### Using pip + +If you already have Python 3.11+ installed: + +```bash +pip install compose-farm +``` + +### Using Docker + +```bash +docker run --rm \ + -v $SSH_AUTH_SOCK:/ssh-agent -e SSH_AUTH_SOCK=/ssh-agent \ + -v ./compose-farm.yaml:/root/.config/compose-farm/compose-farm.yaml:ro \ + ghcr.io/basnijholt/compose-farm up --all +``` + +### Verify Installation + +```bash +cf --version +cf --help +``` + +## SSH Setup + +Compose Farm uses SSH to run commands on remote hosts. You need passwordless SSH access. + +### Option 1: SSH Agent (default) + +If you already have SSH keys loaded in your agent: + +```bash +# Verify keys are loaded +ssh-add -l + +# Test connection +ssh user@192.168.1.10 "docker --version" +``` + +### Option 2: Dedicated Key (recommended for Docker) + +For persistent access when running in Docker: + +```bash +# Generate and distribute key to all hosts +cf ssh setup + +# Check status +cf ssh status +``` + +This creates `~/.ssh/compose-farm/id_ed25519` and copies the public key to each host. + +## Shared Storage Setup + +Compose files must be accessible at the **same path** on all hosts. Common approaches: + +### NFS Mount + +```bash +# On each Docker host +sudo mount nas:/volume1/compose /opt/compose + +# Or add to /etc/fstab +nas:/volume1/compose /opt/compose nfs defaults 0 0 +``` + +### Directory Structure + +``` +/opt/compose/ # compose_dir in config +├── plex/ +│ └── docker-compose.yml +├── sonarr/ +│ └── docker-compose.yml +├── radarr/ +│ └── docker-compose.yml +└── jellyfin/ + └── docker-compose.yml +``` + +## Configuration + +### Create Config File + +Create `~/.config/compose-farm/compose-farm.yaml`: + +```yaml +# Where compose files are located (same path on all hosts) +compose_dir: /opt/compose + +# Define your Docker hosts +hosts: + nuc: + address: 192.168.1.10 + user: docker # SSH user + hp: + address: 192.168.1.11 + # user defaults to current user + local: localhost # Run locally without SSH + +# Map services to hosts +services: + plex: nuc + sonarr: nuc + radarr: hp + jellyfin: local +``` + +### Validate Configuration + +```bash +cf check --local +``` + +This validates syntax without SSH connections. For full validation: + +```bash +cf check +``` + +## First Commands + +### Check Status + +```bash +cf ps +``` + +Shows all configured services and their status. + +### Start All Services + +```bash +cf up --all +``` + +Starts all services on their assigned hosts. + +### Start Specific Services + +```bash +cf up plex sonarr +``` + +### Apply Configuration + +The most powerful command - reconciles reality with your config: + +```bash +cf apply --dry-run # Preview changes +cf apply # Execute changes +``` + +This will: +1. Start services in config but not running +2. Migrate services on wrong host +3. Stop services removed from config + +## Docker Network Setup + +If your services use an external Docker network: + +```bash +# Create network on all hosts +cf init-network + +# Or specific hosts +cf init-network nuc hp +``` + +Default network: `mynetwork` with subnet `172.20.0.0/16` + +## Example Workflow + +### 1. Add a New Service + +Create the compose file: + +```bash +# On any host (shared storage) +mkdir -p /opt/compose/prowlarr +cat > /opt/compose/prowlarr/docker-compose.yml << 'EOF' +services: + prowlarr: + image: lscr.io/linuxserver/prowlarr:latest + container_name: prowlarr + environment: + - PUID=1000 + - PGID=1000 + volumes: + - /opt/config/prowlarr:/config + ports: + - "9696:9696" + restart: unless-stopped +EOF +``` + +Add to config: + +```yaml +services: + # ... existing services + prowlarr: nuc +``` + +Start the service: + +```bash +cf up prowlarr +``` + +### 2. Move a Service to Another Host + +Edit `compose-farm.yaml`: + +```yaml +services: + plex: hp # Changed from nuc +``` + +Apply the change: + +```bash +cf up plex +# Automatically: down on nuc, up on hp +``` + +Or use apply to reconcile everything: + +```bash +cf apply +``` + +### 3. Update All Services + +```bash +cf update --all +# Runs: pull + down + up for each service +``` + +## Next Steps + +- [Configuration Reference](configuration.md) - All config options +- [Commands Reference](commands.md) - Full CLI documentation +- [Traefik Integration](traefik.md) - Multi-host routing +- [Best Practices](best-practices.md) - Tips and limitations diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..219d95d --- /dev/null +++ b/docs/index.md @@ -0,0 +1,127 @@ +--- +icon: lucide/server +--- + +# Compose Farm + +A minimal CLI tool to run Docker Compose commands across multiple hosts via SSH. + +## What is Compose Farm? + +Compose Farm lets you manage Docker Compose services across multiple machines from a single command line. Define which services run where in one YAML file, then use `cf apply` to make reality match your configuration. + +```yaml +# compose-farm.yaml +compose_dir: /opt/compose + +hosts: + server-1: + address: 192.168.1.10 + server-2: + address: 192.168.1.11 + +services: + plex: server-1 + jellyfin: server-2 + sonarr: server-1 +``` + +```bash +cf apply # Services start, migrate, or stop as needed +``` + +## Why Compose Farm? + +| Problem | Compose Farm Solution | +|---------|----------------------| +| 100+ containers on one machine | Distribute across multiple hosts | +| Kubernetes too complex | Just SSH + docker compose | +| Swarm in maintenance mode | Zero infrastructure changes | +| Manual SSH for each host | Single command for all | + +**It's a convenience wrapper, not a new paradigm.** Your existing `docker-compose.yml` files work unchanged. + +## Quick Start + + + +### Installation + +```bash +uv tool install compose-farm +# or +pip install compose-farm +``` + +### Configuration + +Create `~/.config/compose-farm/compose-farm.yaml`: + +```yaml +compose_dir: /opt/compose + +hosts: + nuc: + address: 192.168.1.10 + user: docker + hp: + address: 192.168.1.11 + +services: + plex: nuc + sonarr: nuc + radarr: hp +``` + +### Usage + +```bash +# Make reality match config +cf apply + +# Start specific services +cf up plex sonarr + +# Check status +cf ps + +# View logs +cf logs -f plex +``` + +## Key Features + +- **Declarative configuration**: One YAML defines where everything runs +- **Auto-migration**: Change a host assignment, run `cf up`, service moves automatically + + +- **Parallel execution**: Multiple services start/stop concurrently +- **State tracking**: Knows which services are running where +- **Traefik integration**: Generate file-provider config for cross-host routing +- **Zero changes**: Your compose files work as-is + +## Requirements + +- [uv](https://docs.astral.sh/uv/) (recommended) or Python 3.11+ +- SSH key-based authentication to your Docker hosts +- Docker and Docker Compose on all target hosts +- Shared storage (compose files at same path on all hosts) + +## Documentation + +- [Getting Started](getting-started.md) - Installation and first steps +- [Configuration](configuration.md) - All configuration options +- [Commands](commands.md) - CLI reference +- [Architecture](architecture.md) - How it works under the hood +- [Traefik Integration](traefik.md) - Multi-host routing setup +- [Best Practices](best-practices.md) - Tips and limitations + +## License + +MIT diff --git a/docs/traefik.md b/docs/traefik.md new file mode 100644 index 0000000..f41818d --- /dev/null +++ b/docs/traefik.md @@ -0,0 +1,385 @@ +--- +icon: lucide/globe +--- + +# Traefik Integration + +Compose Farm can generate Traefik file-provider configuration for routing traffic across multiple hosts. + +## The Problem + +When you run Traefik on one host but services on others, Traefik's docker provider can't see remote containers. The file provider bridges this gap. + +``` + Internet + │ + ▼ +┌─────────────────────────────────────────────────────────────┐ +│ Host: nuc │ +│ │ +│ ┌─────────┐ │ +│ │ Traefik │◄─── Docker provider sees local containers │ +│ │ │ │ +│ │ │◄─── File provider sees remote services │ +│ └────┬────┘ (from compose-farm.yml) │ +│ │ │ +└───────┼─────────────────────────────────────────────────────┘ + │ + ├────────────────────┐ + │ │ + ▼ ▼ +┌───────────────┐ ┌───────────────┐ +│ Host: hp │ │ Host: nas │ +│ │ │ │ +│ plex:32400 │ │ jellyfin:8096 │ +└───────────────┘ └───────────────┘ +``` + +## How It Works + +1. Your compose files have standard Traefik labels +2. Compose Farm reads labels and generates file-provider config +3. Traefik watches the generated file +4. Traffic routes to remote services via host IP + published port + +## Setup + +### Step 1: Configure Traefik File Provider + +Add directory watching to your Traefik config: + +```yaml +# traefik.yml or docker-compose.yml command +providers: + file: + directory: /opt/traefik/dynamic.d + watch: true +``` + +Or via command line: + +```yaml +services: + traefik: + command: + - --providers.file.directory=/dynamic.d + - --providers.file.watch=true + volumes: + - /opt/traefik/dynamic.d:/dynamic.d:ro +``` + +### Step 2: Add Traefik Labels to Services + +Your compose files use standard Traefik labels: + +```yaml +# /opt/compose/plex/docker-compose.yml +services: + plex: + image: lscr.io/linuxserver/plex + ports: + - "32400:32400" # IMPORTANT: Must publish port! + labels: + - traefik.enable=true + - traefik.http.routers.plex.rule=Host(`plex.example.com`) + - traefik.http.routers.plex.entrypoints=websecure + - traefik.http.routers.plex.tls.certresolver=letsencrypt + - traefik.http.services.plex.loadbalancer.server.port=32400 +``` + +**Important:** Services must publish ports for cross-host routing. Traefik connects via `host_ip:published_port`. + +### Step 3: Generate File Provider Config + +```bash +cf traefik-file --all -o /opt/traefik/dynamic.d/compose-farm.yml +``` + +This generates: + +```yaml +# /opt/traefik/dynamic.d/compose-farm.yml +http: + routers: + plex: + rule: Host(`plex.example.com`) + entryPoints: + - websecure + tls: + certResolver: letsencrypt + service: plex + services: + plex: + loadBalancer: + servers: + - url: http://192.168.1.11:32400 +``` + +## Auto-Regeneration + +Configure automatic regeneration in `compose-farm.yaml`: + +```yaml +compose_dir: /opt/compose +traefik_file: /opt/traefik/dynamic.d/compose-farm.yml +traefik_service: traefik + +hosts: + nuc: + address: 192.168.1.10 + hp: + address: 192.168.1.11 + +services: + traefik: nuc # Traefik runs here + plex: hp # Routed via file-provider + sonarr: hp +``` + +With `traefik_file` set, these commands auto-regenerate the config: +- `cf up` +- `cf down` +- `cf restart` +- `cf update` +- `cf apply` + +### traefik_service Option + +When set, services on the **same host as Traefik** are skipped in file-provider output. Traefik's docker provider handles them directly. + +```yaml +traefik_service: traefik # traefik runs on nuc +services: + traefik: nuc # NOT in file-provider (docker provider) + portainer: nuc # NOT in file-provider (docker provider) + plex: hp # IN file-provider (cross-host) +``` + +## Label Syntax + +### Routers + +```yaml +labels: + # Basic router + - traefik.http.routers.myapp.rule=Host(`app.example.com`) + - traefik.http.routers.myapp.entrypoints=websecure + + # With TLS + - traefik.http.routers.myapp.tls=true + - traefik.http.routers.myapp.tls.certresolver=letsencrypt + + # With middleware + - traefik.http.routers.myapp.middlewares=auth@file +``` + +### Services + +```yaml +labels: + # Load balancer port + - traefik.http.services.myapp.loadbalancer.server.port=8080 + + # Health check + - traefik.http.services.myapp.loadbalancer.healthcheck.path=/health +``` + +### Middlewares + +Middlewares should be defined in a separate file (not generated by Compose Farm): + +```yaml +# /opt/traefik/dynamic.d/middlewares.yml +http: + middlewares: + auth: + basicAuth: + users: + - "user:$apr1$..." +``` + +Reference in labels: + +```yaml +labels: + - traefik.http.routers.myapp.middlewares=auth@file +``` + +## Variable Substitution + +Labels can use environment variables: + +```yaml +labels: + - traefik.http.routers.myapp.rule=Host(`${DOMAIN}`) +``` + +Compose Farm resolves variables from: +1. Service's `.env` file +2. Current environment + +```bash +# /opt/compose/myapp/.env +DOMAIN=app.example.com +``` + +## Port Resolution + +Compose Farm determines the target URL from published ports: + +```yaml +ports: + - "8080:80" # Uses 8080 + - "192.168.1.11:8080:80" # Uses 8080 on specific IP +``` + +If no suitable port is found, a warning is shown. + +## Complete Example + +### compose-farm.yaml + +```yaml +compose_dir: /opt/compose +traefik_file: /opt/traefik/dynamic.d/compose-farm.yml +traefik_service: traefik + +hosts: + nuc: + address: 192.168.1.10 + hp: + address: 192.168.1.11 + nas: + address: 192.168.1.100 + +services: + traefik: nuc + plex: hp + jellyfin: nas + sonarr: nuc + radarr: nuc +``` + +### /opt/compose/plex/docker-compose.yml + +```yaml +services: + plex: + image: lscr.io/linuxserver/plex + container_name: plex + ports: + - "32400:32400" + labels: + - traefik.enable=true + - traefik.http.routers.plex.rule=Host(`plex.example.com`) + - traefik.http.routers.plex.entrypoints=websecure + - traefik.http.routers.plex.tls.certresolver=letsencrypt + - traefik.http.services.plex.loadbalancer.server.port=32400 + # ... other config +``` + +### Generated compose-farm.yml + +```yaml +http: + routers: + plex: + rule: Host(`plex.example.com`) + entryPoints: + - websecure + tls: + certResolver: letsencrypt + service: plex + jellyfin: + rule: Host(`jellyfin.example.com`) + entryPoints: + - websecure + tls: + certResolver: letsencrypt + service: jellyfin + + services: + plex: + loadBalancer: + servers: + - url: http://192.168.1.11:32400 + jellyfin: + loadBalancer: + servers: + - url: http://192.168.1.100:8096 +``` + +Note: `sonarr` and `radarr` are NOT in the file because they're on the same host as Traefik (`nuc`). + +## Combining with Existing Config + +If you have existing Traefik dynamic config: + +```bash +# Move existing config to directory +mkdir -p /opt/traefik/dynamic.d +mv /opt/traefik/dynamic.yml /opt/traefik/dynamic.d/manual.yml + +# Generate Compose Farm config +cf traefik-file --all -o /opt/traefik/dynamic.d/compose-farm.yml + +# Update Traefik to watch directory +# --providers.file.directory=/dynamic.d +``` + +Traefik merges all YAML files in the directory. + +## Troubleshooting + +### Service Not Accessible + +1. **Check port is published:** + ```yaml + ports: + - "8080:80" # Must be published, not just exposed + ``` + +2. **Check label syntax:** + ```bash + cf check myservice + ``` + +3. **Verify generated config:** + ```bash + cf traefik-file myservice + ``` + +4. **Check Traefik logs:** + ```bash + docker logs traefik + ``` + +### Config Not Regenerating + +1. **Verify traefik_file is set:** + ```bash + cf config show | grep traefik + ``` + +2. **Check file permissions:** + ```bash + ls -la /opt/traefik/dynamic.d/ + ``` + +3. **Manually regenerate:** + ```bash + cf traefik-file --all -o /opt/traefik/dynamic.d/compose-farm.yml + ``` + +### Variable Not Resolved + +1. **Check .env file exists:** + ```bash + cat /opt/compose/myservice/.env + ``` + +2. **Test variable resolution:** + ```bash + cd /opt/compose/myservice + docker compose config + ``` diff --git a/zensical.toml b/zensical.toml new file mode 100644 index 0000000..d60703a --- /dev/null +++ b/zensical.toml @@ -0,0 +1,75 @@ +# Compose Farm Documentation +# Built with Zensical - https://zensical.org + +[project] +site_name = "Compose Farm" +site_description = "A minimal CLI tool to run Docker Compose commands across multiple hosts via SSH" +site_author = "Bas Nijholt" +site_url = "https://compose-farm.nijho.lt/" +copyright = "Copyright © 2025 Bas Nijholt" + +repo_url = "https://github.com/basnijholt/compose-farm" +repo_name = "GitHub" +edit_uri = "edit/main/docs" + +nav = [ + { "Home" = "index.md" }, + { "Getting Started" = "getting-started.md" }, + { "Configuration" = "configuration.md" }, + { "Commands" = "commands.md" }, + { "Architecture" = "architecture.md" }, + { "Traefik Integration" = "traefik.md" }, + { "Best Practices" = "best-practices.md" }, +] + +[project.theme] +language = "en" + +features = [ + "announce.dismiss", + "content.action.edit", + "content.action.view", + "content.code.annotate", + "content.code.copy", + "content.code.select", + "content.footnote.tooltips", + "content.tabs.link", + "content.tooltips", + "navigation.footer", + "navigation.indexes", + "navigation.instant", + "navigation.instant.prefetch", + "navigation.path", + "navigation.sections", + "navigation.top", + "navigation.tracking", + "search.highlight", +] + +[[project.theme.palette]] +scheme = "default" +primary = "teal" +toggle.icon = "lucide/sun" +toggle.name = "Switch to dark mode" + +[[project.theme.palette]] +scheme = "slate" +primary = "teal" +toggle.icon = "lucide/moon" +toggle.name = "Switch to light mode" + +[project.theme.font] +text = "Inter" +code = "JetBrains Mono" + +[project.theme.icon] +logo = "lucide/server" +repo = "lucide/github" + +[[project.extra.social]] +icon = "fontawesome/brands/github" +link = "https://github.com/basnijholt/compose-farm" + +[[project.extra.social]] +icon = "fontawesome/brands/python" +link = "https://pypi.org/project/compose-farm/"