fix(web): Run self-updates via SSH to survive container restart (#35)

* refactor: Store SSH keys in subdirectory for cleaner volume mounting

Change SSH key location from ~/.ssh/compose-farm (file) to
~/.ssh/compose-farm/id_ed25519 (file in directory).

This allows docker-compose to mount just the compose-farm directory
to /root/.ssh without exposing all host SSH keys to the container.

Also make host path the default option in docker-compose.yml with
clearer comments about the two options.

* docs: Update README for new SSH key directory structure

* docs: Clarify cf ssh setup must run inside container

CLAUDE.md

@@ -43,6 +43,10 @@ Icons use [Lucide](https://lucide.dev/). Add new icons as macros in `web/templat
 7. **State tracking**: Tracks where services are deployed for auto-migration
 8. **Pre-flight checks**: Verifies NFS mounts and Docker networks exist before starting/migrating
 
+## Code Style
+
+- **Imports at top level**: Never add imports inside functions unless they are explicitly marked with `# noqa: PLC0415` and a comment explaining it speeds up CLI startup. Heavy modules like `pydantic`, `yaml`, and `rich.table` are lazily imported to keep `cf --help` fast.
+
 ## Communication Notes
 
 - Clarify ambiguous wording (e.g., homophones like "right"/"write", "their"/"there").

README.md

@@ -190,25 +190,31 @@ cf ssh setup
 cf ssh status
 ```
 
-This creates `~/.ssh/compose-farm` (ED25519, no passphrase) and copies the public key to each host's `authorized_keys`. Compose Farm tries the SSH agent first, then falls back to this key.
+This creates `~/.ssh/compose-farm/id_ed25519` (ED25519, no passphrase) and copies the public key to each host's `authorized_keys`. Compose Farm tries the SSH agent first, then falls back to this key.
 
 <details><summary>🐳 Docker volume options for SSH keys</summary>
 
-When running in Docker, mount a volume to persist the SSH keys:
+When running in Docker, mount a volume to persist the SSH keys. Choose ONE option and use it for both `cf` and `web` services:
 
-**Option 1: Named volume (default)**
-```yaml
-volumes:
-  - cf-ssh:/root/.ssh
-```
-
-**Option 2: Host path (easier to backup/inspect)**
+**Option 1: Host path (default)** - keys at `~/.ssh/compose-farm/id_ed25519`
 ```yaml
 volumes:
   - ~/.ssh/compose-farm:/root/.ssh
 ```
 
-Run `cf ssh setup` once after starting the container (while the SSH agent still works), and the keys will persist across restarts.
+**Option 2: Named volume** - managed by Docker
+```yaml
+volumes:
+  - cf-ssh:/root/.ssh
+```
+
+Run setup once after starting the container (while the SSH agent still works):
+
+```bash
+docker compose exec web cf ssh setup
+```
+
+The keys will persist across restarts.
 
 </details>
 

docker-compose.yml

@@ -6,10 +6,11 @@ services:
       # Compose directory (contains compose files AND compose-farm.yaml config)
       - ${CF_COMPOSE_DIR:-/opt/stacks}:${CF_COMPOSE_DIR:-/opt/stacks}
       # SSH keys for passwordless auth (generated by `cf ssh setup`)
-      # Option 1: Named volume (default) - managed by Docker
-      - cf-ssh:/root/.ssh
-      # Option 2: Host path - easier to backup/inspect, uncomment to use:
-      # - ${CF_SSH_DIR:-~/.ssh/compose-farm}:/root/.ssh
+      # Choose ONE option below (use the same option for both cf and web services):
+      # Option 1: Host path (default) - keys at ~/.ssh/compose-farm/id_ed25519
+      - ${CF_SSH_DIR:-~/.ssh/compose-farm}:/root/.ssh
+      # Option 2: Named volume - managed by Docker, shared between services
+      # - cf-ssh:/root/.ssh
     environment:
       - SSH_AUTH_SOCK=/ssh-agent
       # Config file path (state stored alongside it)
@@ -22,12 +23,16 @@ services:
     volumes:
       - ${SSH_AUTH_SOCK}:/ssh-agent:ro
       - ${CF_COMPOSE_DIR:-/opt/stacks}:${CF_COMPOSE_DIR:-/opt/stacks}
-      # SSH keys - use same option as cf service above
-      - cf-ssh:/root/.ssh
-      # - ${CF_SSH_DIR:-~/.ssh/compose-farm}:/root/.ssh
+      # SSH keys - use the SAME option as cf service above
+      # Option 1: Host path (default)
+      - ${CF_SSH_DIR:-~/.ssh/compose-farm}:/root/.ssh
+      # Option 2: Named volume
+      # - cf-ssh:/root/.ssh
     environment:
       - SSH_AUTH_SOCK=/ssh-agent
       - CF_CONFIG=${CF_COMPOSE_DIR:-/opt/stacks}/compose-farm.yaml
+      # Used to detect self-updates and run via SSH to survive container restart
+      - CF_WEB_SERVICE=compose-farm
     labels:
       - traefik.enable=true
       - traefik.http.routers.compose-farm.rule=Host(`compose-farm.${DOMAIN}`)
@@ -44,4 +49,4 @@ networks:
 
 volumes:
   cf-ssh:
-    # Persists SSH keys across container restarts
+    # Only used if Option 2 is selected above

src/compose_farm/cli/management.py

@@ -149,8 +149,14 @@ def _check_ssh_connectivity(cfg: Config) -> list[str]:
 
     async def check_host(host_name: str) -> tuple[str, bool]:
         host = cfg.hosts[host_name]
-        result = await run_command(host, "echo ok", host_name, stream=False)
-        return host_name, result.success
+        try:
+            result = await asyncio.wait_for(
+                run_command(host, "echo ok", host_name, stream=False),
+                timeout=5.0,
+            )
+            return host_name, result.success
+        except TimeoutError:
+            return host_name, False
 
     results = run_parallel_with_progress(
         "Checking SSH connectivity",

src/compose_farm/cli/ssh.py

@@ -2,14 +2,20 @@
 
 from __future__ import annotations
 
+import asyncio
 import subprocess
-from typing import Annotated
+from typing import TYPE_CHECKING, Annotated
 
 import typer
 
 from compose_farm.cli.app import app
-from compose_farm.cli.common import ConfigOption, load_config_or_exit
+from compose_farm.cli.common import ConfigOption, load_config_or_exit, run_parallel_with_progress
 from compose_farm.console import console, err_console
+from compose_farm.executor import run_command
+
+if TYPE_CHECKING:
+    from compose_farm.config import Host
+
 from compose_farm.ssh_keys import (
     SSH_KEY_PATH,
     SSH_PUBKEY_PATH,
@@ -123,7 +129,7 @@ def ssh_keygen(
 ) -> None:
     """Generate SSH key (does not distribute to hosts).
 
-    Creates an ED25519 key at ~/.ssh/compose-farm with no passphrase.
+    Creates an ED25519 key at ~/.ssh/compose-farm/id_ed25519 with no passphrase.
     Use 'cf ssh setup' to also distribute the key to all configured hosts.
     """
     success = _generate_key(force=force)
@@ -138,8 +144,8 @@ def ssh_setup(
 ) -> None:
     """Generate SSH key and distribute to all configured hosts.
 
-    Creates an ED25519 key at ~/.ssh/compose-farm (no passphrase) and
-    copies the public key to authorized_keys on each host.
+    Creates an ED25519 key at ~/.ssh/compose-farm/id_ed25519 (no passphrase)
+    and copies the public key to authorized_keys on each host.
 
     For each host, tries SSH agent first. If agent is unavailable,
     prompts for password.
@@ -192,7 +198,7 @@ def ssh_setup(
 
 
 @ssh_app.command("status")
-def ssh_status(  # noqa: PLR0912 - branches are clear and readable
+def ssh_status(
     config: ConfigOption = None,
 ) -> None:
     """Show SSH key status and host connectivity."""
@@ -232,50 +238,42 @@ def ssh_status(  # noqa: PLR0912 - branches are clear and readable
         console.print("  [dim]No remote hosts configured[/]")
         return
 
+    async def check_host(item: tuple[str, Host]) -> tuple[str, str, str]:
+        """Check connectivity to a single host."""
+        host_name, host = item
+        target = f"{host.user}@{host.address}"
+        if host.port != _DEFAULT_SSH_PORT:
+            target += f":{host.port}"
+
+        try:
+            result = await asyncio.wait_for(
+                run_command(host, "echo ok", host_name, stream=False),
+                timeout=5.0,
+            )
+            status = "[green]OK[/]" if result.success else "[red]Auth failed[/]"
+        except TimeoutError:
+            status = "[red]Timeout (5s)[/]"
+        except Exception as e:
+            status = f"[red]Error: {e}[/]"
+
+        return host_name, target, status
+
+    # Check connectivity in parallel with progress bar
+    results = run_parallel_with_progress(
+        "Checking hosts",
+        list(remote_hosts.items()),
+        check_host,
+    )
+
+    # Build table from results
     table = Table(show_header=True, header_style="bold")
     table.add_column("Host")
     table.add_column("Address")
     table.add_column("Status")
 
-    for host_name, host in remote_hosts.items():
-        target = f"{host.user}@{host.address}"
-        if host.port != _DEFAULT_SSH_PORT:
-            target += f":{host.port}"
-
-        # Test connectivity with a simple command
-        cmd = [
-            "ssh",
-            "-o",
-            "StrictHostKeyChecking=no",
-            "-o",
-            "UserKnownHostsFile=/dev/null",
-            "-o",
-            "BatchMode=yes",  # Fail immediately if password required
-            "-o",
-            "ConnectTimeout=5",
-        ]
-
-        # Add key file if it exists
-        if key_exists():
-            cmd.extend(["-i", str(SSH_KEY_PATH)])
-
-        if host.port != _DEFAULT_SSH_PORT:
-            cmd.extend(["-p", str(host.port)])
-
-        cmd.extend([f"{host.user}@{host.address}", "echo ok"])
-
-        try:
-            result = subprocess.run(
-                cmd, check=False, capture_output=True, timeout=10, env=get_ssh_env()
-            )
-            if result.returncode == 0:
-                table.add_row(host_name, target, "[green]OK[/]")
-            else:
-                table.add_row(host_name, target, "[red]Auth failed[/]")
-        except subprocess.TimeoutExpired:
-            table.add_row(host_name, target, "[red]Timeout[/]")
-        except Exception as e:
-            table.add_row(host_name, target, f"[red]Error: {e}[/]")
+    # Sort by host name for consistent order
+    for host_name, target, status in sorted(results, key=lambda r: r[0]):
+        table.add_row(host_name, target, status)
 
     console.print(table)
 

src/compose_farm/executor.py

@@ -23,6 +23,43 @@ LOCAL_ADDRESSES = frozenset({"local", "localhost", "127.0.0.1", "::1"})
 _DEFAULT_SSH_PORT = 22
 
 
+def build_ssh_command(host: Host, command: str, *, tty: bool = False) -> list[str]:
+    """Build SSH command args for executing a command on a remote host.
+
+    Args:
+        host: Host configuration with address, port, user
+        command: Command to run on the remote host
+        tty: Whether to allocate a TTY (for interactive/progress bar commands)
+
+    Returns:
+        List of command args suitable for subprocess
+
+    """
+    ssh_args = [
+        "ssh",
+        "-o",
+        "StrictHostKeyChecking=no",
+        "-o",
+        "UserKnownHostsFile=/dev/null",
+        "-o",
+        "LogLevel=ERROR",
+    ]
+    if tty:
+        ssh_args.insert(1, "-tt")  # Force TTY allocation
+
+    key_path = get_key_path()
+    if key_path:
+        ssh_args.extend(["-i", str(key_path)])
+
+    if host.port != _DEFAULT_SSH_PORT:
+        ssh_args.extend(["-p", str(host.port)])
+
+    ssh_args.append(f"{host.user}@{host.address}")
+    ssh_args.append(command)
+
+    return ssh_args
+
+
 @lru_cache(maxsize=1)
 def _get_local_ips() -> frozenset[str]:
     """Get all IP addresses of the current machine."""
@@ -172,23 +209,7 @@ async def _run_ssh_command(
     """Run a command on a remote host via SSH with streaming output."""
     if raw:
         # Use native ssh with TTY for proper progress bar rendering
-        ssh_args = [
-            "ssh",
-            "-tt",  # Force TTY allocation even without stdin TTY
-            "-o",
-            "StrictHostKeyChecking=no",  # Match asyncssh known_hosts=None behavior
-            "-o",
-            "UserKnownHostsFile=/dev/null",
-            "-o",
-            "LogLevel=ERROR",  # Suppress warnings about known_hosts
-        ]
-        # Add key file if it exists (fallback for when agent is unavailable)
-        key_path = get_key_path()
-        if key_path:
-            ssh_args.extend(["-i", str(key_path)])
-        if host.port != _DEFAULT_SSH_PORT:
-            ssh_args.extend(["-p", str(host.port)])
-        ssh_args.extend([f"{host.user}@{host.address}", command])
+        ssh_args = build_ssh_command(host, command, tty=True)
         # Run in thread to avoid blocking the event loop
         # Use get_ssh_env() to auto-detect SSH agent socket
         result = await asyncio.to_thread(subprocess.run, ssh_args, check=False, env=get_ssh_env())

src/compose_farm/ssh_keys.py

@@ -6,7 +6,9 @@ import os
 from pathlib import Path
 
 # Default key paths for compose-farm SSH key
-SSH_KEY_PATH = Path.home() / ".ssh" / "compose-farm"
+# Keys are stored in a subdirectory for cleaner docker volume mounting
+SSH_KEY_DIR = Path.home() / ".ssh" / "compose-farm"
+SSH_KEY_PATH = SSH_KEY_DIR / "id_ed25519"
 SSH_PUBKEY_PATH = SSH_KEY_PATH.with_suffix(".pub")
 
 

src/compose_farm/web/static/app.js

@@ -411,6 +411,16 @@ function initPage() {
 document.addEventListener('DOMContentLoaded', function() {
     initPage();
     initKeyboardShortcuts();
+
+    // Handle ?action= parameter (from command palette navigation)
+    const params = new URLSearchParams(window.location.search);
+    const action = params.get('action');
+    if (action && window.location.pathname === '/') {
+        // Clear the URL parameter
+        history.replaceState({}, '', '/');
+        // Trigger the action
+        htmx.ajax('POST', `/api/${action}`, {swap: 'none'});
+    }
 });
 
 // Re-initialize after HTMX swaps main content
@@ -491,21 +501,29 @@ document.body.addEventListener('htmx:afterRequest', function(evt) {
         });
     }
 
-    const colors = { service: '#22c55e', action: '#eab308', nav: '#3b82f6' };
+    const colors = { service: '#22c55e', action: '#eab308', nav: '#3b82f6', app: '#a855f7' };
     let commands = [];
     let filtered = [];
     let selected = 0;
 
     const post = (url) => () => htmx.ajax('POST', url, {swap: 'none'});
     const nav = (url) => () => window.location.href = url;
+    // Navigate to dashboard and trigger action (or just POST if already on dashboard)
+    const dashboardAction = (endpoint) => () => {
+        if (window.location.pathname === '/') {
+            htmx.ajax('POST', `/api/${endpoint}`, {swap: 'none'});
+        } else {
+            window.location.href = `/?action=${endpoint}`;
+        }
+    };
     const cmd = (type, name, desc, action, icon = null) => ({ type, name, desc, action, icon });
 
     function buildCommands() {
         const actions = [
-            cmd('action', 'Apply', 'Make reality match config', post('/api/apply'), icons.check),
-            cmd('action', 'Refresh', 'Update state from reality', post('/api/refresh'), icons.refresh_cw),
-            cmd('nav', 'Dashboard', 'Go to dashboard', nav('/'), icons.home),
-            cmd('nav', 'Console', 'Go to console', nav('/console'), icons.terminal),
+            cmd('action', 'Apply', 'Make reality match config', dashboardAction('apply'), icons.check),
+            cmd('action', 'Refresh', 'Update state from reality', dashboardAction('refresh'), icons.refresh_cw),
+            cmd('app', 'Dashboard', 'Go to dashboard', nav('/'), icons.home),
+            cmd('app', 'Console', 'Go to console', nav('/console'), icons.terminal),
         ];
 
         // Add service-specific actions if on a service page
@@ -545,6 +563,9 @@ document.body.addEventListener('htmx:afterRequest', function(evt) {
                 <span class="opacity-40 text-xs">${c.desc}</span>
             </a>
         `).join('') || '<div class="opacity-50 p-2">No matches</div>';
+        // Scroll selected item into view
+        const sel = list.querySelector(`[data-idx="${selected}"]`);
+        if (sel) sel.scrollIntoView({ block: 'nearest' });
     }
 
     function open() {

src/compose_farm/web/streaming.py

@@ -6,11 +6,15 @@ import asyncio
 import os
 from typing import TYPE_CHECKING, Any
 
+from compose_farm.executor import build_ssh_command
 from compose_farm.ssh_keys import get_ssh_auth_sock
 
 if TYPE_CHECKING:
     from compose_farm.config import Config
 
+# Environment variable to identify the web service (for self-update detection)
+CF_WEB_SERVICE = os.environ.get("CF_WEB_SERVICE", "")
+
 # ANSI escape codes for terminal output
 RED = "\x1b[31m"
 GREEN = "\x1b[32m"
@@ -79,6 +83,76 @@ async def run_cli_streaming(
         tasks[task_id]["status"] = "failed"
 
 
+def _is_self_update(service: str, command: str) -> bool:
+    """Check if this is a self-update (updating the web service itself).
+
+    Self-updates need special handling because running 'down' on the container
+    we're running in would kill the process before 'up' can execute.
+    """
+    if not CF_WEB_SERVICE or service != CF_WEB_SERVICE:
+        return False
+    # Commands that involve 'down' need SSH: update, restart, down
+    return command in ("update", "restart", "down")
+
+
+async def _run_cli_via_ssh(
+    config: Config,
+    args: list[str],
+    task_id: str,
+) -> None:
+    """Run a cf CLI command via SSH to the host.
+
+    Used for self-updates to ensure the command survives container restart.
+    """
+    try:
+        # Get the host for the web service
+        host = config.get_host(CF_WEB_SERVICE)
+
+        # Build the remote command
+        remote_cmd = f"cf {' '.join(args)} --config={config.config_path}"
+
+        # Show what we're doing
+        await stream_to_task(
+            task_id,
+            f"{DIM}$ ssh {host.user}@{host.address} {remote_cmd}{RESET}{CRLF}",
+        )
+        await stream_to_task(
+            task_id,
+            f"{GREEN}Running via SSH (self-update protection){RESET}{CRLF}",
+        )
+
+        # Build SSH command using shared helper
+        ssh_args = build_ssh_command(host, remote_cmd)
+
+        # Set up environment with SSH agent
+        env = {**os.environ, "FORCE_COLOR": "1", "TERM": "xterm-256color"}
+        ssh_sock = get_ssh_auth_sock()
+        if ssh_sock:
+            env["SSH_AUTH_SOCK"] = ssh_sock
+
+        process = await asyncio.create_subprocess_exec(
+            *ssh_args,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.STDOUT,
+            env=env,
+        )
+
+        # Stream output
+        if process.stdout:
+            async for line in process.stdout:
+                text = line.decode("utf-8", errors="replace")
+                if text.endswith("\n") and not text.endswith("\r\n"):
+                    text = text[:-1] + "\r\n"
+                await stream_to_task(task_id, text)
+
+        exit_code = await process.wait()
+        tasks[task_id]["status"] = "completed" if exit_code == 0 else "failed"
+
+    except Exception as e:
+        await stream_to_task(task_id, f"{RED}Error: {e}{RESET}{CRLF}")
+        tasks[task_id]["status"] = "failed"
+
+
 async def run_compose_streaming(
     config: Config,
     service: str,
@@ -93,4 +167,9 @@ async def run_compose_streaming(
 
     # Build CLI args
     cli_args = [cli_cmd, service, *extra_args]
-    await run_cli_streaming(config, cli_args, task_id)
+
+    # Use SSH for self-updates to survive container restart
+    if _is_self_update(service, cli_cmd):
+        await _run_cli_via_ssh(config, cli_args, task_id)
+    else:
+        await run_cli_streaming(config, cli_args, task_id)