mirror of
https://github.com/basnijholt/compose-farm.git
synced 2026-02-03 14:13:26 +00:00
perf: Batch snapshot collection to 1 SSH call per host (#130)
## Summary Optimize `cf refresh` SSH calls from O(stacks) to O(hosts): - Discovery: 1 SSH call per host (unchanged) - Snapshots: 1 SSH call per host (was 1 per stack) For 50 stacks across 4 hosts: 54 → 8 SSH calls. ## Changes **Performance:** - Use `docker ps` + `docker image inspect` instead of `docker compose images` per stack - Batch snapshot collection by host in `collect_stacks_entries_on_host()` **Architecture:** - Add `build_discovery_results()` to `operations.py` (business logic) - Keep progress bar wrapper in `cli/management.py` (presentation) - Remove dead code: `discover_all_stacks_on_all_hosts()`, `collect_all_stacks_entries()`
This commit is contained in:
Bas Nijholt
@@ -10,8 +10,8 @@ import pytest
|
||||
from compose_farm.config import Config, Host
|
||||
from compose_farm.executor import CommandResult
|
||||
from compose_farm.logs import (
|
||||
_parse_images_output,
|
||||
collect_stack_entries,
|
||||
_SECTION_SEPARATOR,
|
||||
collect_stacks_entries_on_host,
|
||||
isoformat,
|
||||
load_existing_entries,
|
||||
merge_entries,
|
||||
@@ -19,74 +19,252 @@ from compose_farm.logs import (
|
||||
)
|
||||
|
||||
|
||||
def test_parse_images_output_handles_list_and_lines() -> None:
|
||||
data = [
|
||||
{"Service": "svc", "Image": "redis", "Digest": "sha256:abc"},
|
||||
{"Service": "svc", "Image": "db", "Digest": "sha256:def"},
|
||||
def _make_mock_output(
|
||||
project_images: dict[str, list[str]], image_info: list[dict[str, object]]
|
||||
) -> str:
|
||||
"""Build mock output matching the 2-docker-command format."""
|
||||
# Section 1: project|image pairs from docker ps
|
||||
ps_lines = [
|
||||
f"{project}|{image}" for project, images in project_images.items() for image in images
|
||||
]
|
||||
as_array = _parse_images_output(json.dumps(data))
|
||||
assert len(as_array) == 2
|
||||
|
||||
as_lines = _parse_images_output("\n".join(json.dumps(item) for item in data))
|
||||
assert len(as_lines) == 2
|
||||
# Section 2: JSON array from docker image inspect
|
||||
image_json = json.dumps(image_info)
|
||||
|
||||
return f"{chr(10).join(ps_lines)}\n{_SECTION_SEPARATOR}\n{image_json}"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_snapshot_preserves_first_seen(tmp_path: Path) -> None:
|
||||
compose_dir = tmp_path / "compose"
|
||||
compose_dir.mkdir()
|
||||
stack_dir = compose_dir / "svc"
|
||||
stack_dir.mkdir()
|
||||
(stack_dir / "docker-compose.yml").write_text("services: {}\n")
|
||||
class TestCollectStacksEntriesOnHost:
|
||||
"""Tests for collect_stacks_entries_on_host (2 docker commands per host)."""
|
||||
|
||||
config = Config(
|
||||
compose_dir=compose_dir,
|
||||
hosts={"local": Host(address="localhost")},
|
||||
stacks={"svc": "local"},
|
||||
)
|
||||
@pytest.fixture
|
||||
def config_with_stacks(self, tmp_path: Path) -> Config:
|
||||
"""Create a config with multiple stacks."""
|
||||
compose_dir = tmp_path / "compose"
|
||||
compose_dir.mkdir()
|
||||
for stack in ["plex", "jellyfin", "sonarr"]:
|
||||
stack_dir = compose_dir / stack
|
||||
stack_dir.mkdir()
|
||||
(stack_dir / "docker-compose.yml").write_text("services: {}\n")
|
||||
|
||||
sample_output = json.dumps([{"Service": "svc", "Image": "redis", "Digest": "sha256:abc"}])
|
||||
|
||||
async def fake_run_compose(
|
||||
_cfg: Config, stack: str, compose_cmd: str, *, stream: bool = True
|
||||
) -> CommandResult:
|
||||
assert compose_cmd == "images --format json"
|
||||
assert stream is False or stream is True
|
||||
return CommandResult(
|
||||
stack=stack,
|
||||
exit_code=0,
|
||||
success=True,
|
||||
stdout=sample_output,
|
||||
stderr="",
|
||||
return Config(
|
||||
compose_dir=compose_dir,
|
||||
hosts={"host1": Host(address="localhost"), "host2": Host(address="localhost")},
|
||||
stacks={"plex": "host1", "jellyfin": "host1", "sonarr": "host2"},
|
||||
)
|
||||
|
||||
log_path = tmp_path / "dockerfarm-log.toml"
|
||||
@pytest.mark.asyncio
|
||||
async def test_single_ssh_call(
|
||||
self, config_with_stacks: Config, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""Verify only 1 SSH call is made regardless of stack count."""
|
||||
call_count = {"count": 0}
|
||||
|
||||
# First snapshot
|
||||
first_time = datetime(2025, 1, 1, tzinfo=UTC)
|
||||
first_entries = await collect_stack_entries(
|
||||
config, "svc", now=first_time, run_compose_fn=fake_run_compose
|
||||
)
|
||||
first_iso = isoformat(first_time)
|
||||
merged = merge_entries([], first_entries, now_iso=first_iso)
|
||||
meta = {"generated_at": first_iso, "compose_dir": str(config.compose_dir)}
|
||||
write_toml(log_path, meta=meta, entries=merged)
|
||||
async def mock_run_command(
|
||||
host: Host, command: str, stack: str, *, stream: bool, prefix: str
|
||||
) -> CommandResult:
|
||||
call_count["count"] += 1
|
||||
output = _make_mock_output(
|
||||
{"plex": ["plex:latest"], "jellyfin": ["jellyfin:latest"]},
|
||||
[
|
||||
{
|
||||
"RepoTags": ["plex:latest"],
|
||||
"Id": "sha256:aaa",
|
||||
"RepoDigests": ["plex@sha256:aaa"],
|
||||
},
|
||||
{
|
||||
"RepoTags": ["jellyfin:latest"],
|
||||
"Id": "sha256:bbb",
|
||||
"RepoDigests": ["jellyfin@sha256:bbb"],
|
||||
},
|
||||
],
|
||||
)
|
||||
return CommandResult(stack=stack, exit_code=0, success=True, stdout=output)
|
||||
|
||||
after_first = tomllib.loads(log_path.read_text())
|
||||
first_seen = after_first["entries"][0]["first_seen"]
|
||||
monkeypatch.setattr("compose_farm.logs.run_command", mock_run_command)
|
||||
|
||||
# Second snapshot
|
||||
second_time = datetime(2025, 2, 1, tzinfo=UTC)
|
||||
second_entries = await collect_stack_entries(
|
||||
config, "svc", now=second_time, run_compose_fn=fake_run_compose
|
||||
)
|
||||
second_iso = isoformat(second_time)
|
||||
existing = load_existing_entries(log_path)
|
||||
merged = merge_entries(existing, second_entries, now_iso=second_iso)
|
||||
meta = {"generated_at": second_iso, "compose_dir": str(config.compose_dir)}
|
||||
write_toml(log_path, meta=meta, entries=merged)
|
||||
now = datetime(2025, 1, 1, tzinfo=UTC)
|
||||
entries = await collect_stacks_entries_on_host(
|
||||
config_with_stacks, "host1", {"plex", "jellyfin"}, now=now
|
||||
)
|
||||
|
||||
after_second = tomllib.loads(log_path.read_text())
|
||||
entry = after_second["entries"][0]
|
||||
assert entry["first_seen"] == first_seen
|
||||
assert entry["last_seen"].startswith("2025-02-01")
|
||||
assert call_count["count"] == 1
|
||||
assert len(entries) == 2
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_filters_to_requested_stacks(
|
||||
self, config_with_stacks: Config, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""Only return entries for stacks we asked for, even if others are running."""
|
||||
|
||||
async def mock_run_command(
|
||||
host: Host, command: str, stack: str, *, stream: bool, prefix: str
|
||||
) -> CommandResult:
|
||||
# Docker ps shows 3 stacks, but we only want plex
|
||||
output = _make_mock_output(
|
||||
{
|
||||
"plex": ["plex:latest"],
|
||||
"jellyfin": ["jellyfin:latest"],
|
||||
"other": ["other:latest"],
|
||||
},
|
||||
[
|
||||
{
|
||||
"RepoTags": ["plex:latest"],
|
||||
"Id": "sha256:aaa",
|
||||
"RepoDigests": ["plex@sha256:aaa"],
|
||||
},
|
||||
{
|
||||
"RepoTags": ["jellyfin:latest"],
|
||||
"Id": "sha256:bbb",
|
||||
"RepoDigests": ["j@sha256:bbb"],
|
||||
},
|
||||
{
|
||||
"RepoTags": ["other:latest"],
|
||||
"Id": "sha256:ccc",
|
||||
"RepoDigests": ["o@sha256:ccc"],
|
||||
},
|
||||
],
|
||||
)
|
||||
return CommandResult(stack=stack, exit_code=0, success=True, stdout=output)
|
||||
|
||||
monkeypatch.setattr("compose_farm.logs.run_command", mock_run_command)
|
||||
|
||||
now = datetime(2025, 1, 1, tzinfo=UTC)
|
||||
entries = await collect_stacks_entries_on_host(
|
||||
config_with_stacks, "host1", {"plex"}, now=now
|
||||
)
|
||||
|
||||
assert len(entries) == 1
|
||||
assert entries[0].stack == "plex"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multiple_images_per_stack(
|
||||
self, config_with_stacks: Config, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""Stack with multiple containers/images returns multiple entries."""
|
||||
|
||||
async def mock_run_command(
|
||||
host: Host, command: str, stack: str, *, stream: bool, prefix: str
|
||||
) -> CommandResult:
|
||||
output = _make_mock_output(
|
||||
{"plex": ["plex:latest", "redis:7"]},
|
||||
[
|
||||
{
|
||||
"RepoTags": ["plex:latest"],
|
||||
"Id": "sha256:aaa",
|
||||
"RepoDigests": ["p@sha256:aaa"],
|
||||
},
|
||||
{"RepoTags": ["redis:7"], "Id": "sha256:bbb", "RepoDigests": ["r@sha256:bbb"]},
|
||||
],
|
||||
)
|
||||
return CommandResult(stack=stack, exit_code=0, success=True, stdout=output)
|
||||
|
||||
monkeypatch.setattr("compose_farm.logs.run_command", mock_run_command)
|
||||
|
||||
now = datetime(2025, 1, 1, tzinfo=UTC)
|
||||
entries = await collect_stacks_entries_on_host(
|
||||
config_with_stacks, "host1", {"plex"}, now=now
|
||||
)
|
||||
|
||||
assert len(entries) == 2
|
||||
images = {e.image for e in entries}
|
||||
assert images == {"plex:latest", "redis:7"}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_stacks_returns_empty(self, config_with_stacks: Config) -> None:
|
||||
"""Empty stack set returns empty entries without making SSH call."""
|
||||
now = datetime(2025, 1, 1, tzinfo=UTC)
|
||||
entries = await collect_stacks_entries_on_host(config_with_stacks, "host1", set(), now=now)
|
||||
assert entries == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ssh_failure_returns_empty(
|
||||
self, config_with_stacks: Config, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""SSH failure returns empty list instead of raising."""
|
||||
|
||||
async def mock_run_command(
|
||||
host: Host, command: str, stack: str, *, stream: bool, prefix: str
|
||||
) -> CommandResult:
|
||||
return CommandResult(stack=stack, exit_code=1, success=False, stdout="", stderr="error")
|
||||
|
||||
monkeypatch.setattr("compose_farm.logs.run_command", mock_run_command)
|
||||
|
||||
now = datetime(2025, 1, 1, tzinfo=UTC)
|
||||
entries = await collect_stacks_entries_on_host(
|
||||
config_with_stacks, "host1", {"plex"}, now=now
|
||||
)
|
||||
|
||||
assert entries == []
|
||||
|
||||
|
||||
class TestSnapshotMerging:
|
||||
"""Tests for merge_entries preserving first_seen."""
|
||||
|
||||
@pytest.fixture
|
||||
def config(self, tmp_path: Path) -> Config:
|
||||
compose_dir = tmp_path / "compose"
|
||||
compose_dir.mkdir()
|
||||
stack_dir = compose_dir / "svc"
|
||||
stack_dir.mkdir()
|
||||
(stack_dir / "docker-compose.yml").write_text("services: {}\n")
|
||||
|
||||
return Config(
|
||||
compose_dir=compose_dir,
|
||||
hosts={"local": Host(address="localhost")},
|
||||
stacks={"svc": "local"},
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_preserves_first_seen(
|
||||
self, tmp_path: Path, config: Config, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""Repeated snapshots preserve first_seen timestamp."""
|
||||
|
||||
async def mock_run_command(
|
||||
host: Host, command: str, stack: str, *, stream: bool, prefix: str
|
||||
) -> CommandResult:
|
||||
output = _make_mock_output(
|
||||
{"svc": ["redis:latest"]},
|
||||
[
|
||||
{
|
||||
"RepoTags": ["redis:latest"],
|
||||
"Id": "sha256:abc",
|
||||
"RepoDigests": ["r@sha256:abc"],
|
||||
}
|
||||
],
|
||||
)
|
||||
return CommandResult(stack=stack, exit_code=0, success=True, stdout=output)
|
||||
|
||||
monkeypatch.setattr("compose_farm.logs.run_command", mock_run_command)
|
||||
|
||||
log_path = tmp_path / "dockerfarm-log.toml"
|
||||
|
||||
# First snapshot
|
||||
first_time = datetime(2025, 1, 1, tzinfo=UTC)
|
||||
first_entries = await collect_stacks_entries_on_host(
|
||||
config, "local", {"svc"}, now=first_time
|
||||
)
|
||||
first_iso = isoformat(first_time)
|
||||
merged = merge_entries([], first_entries, now_iso=first_iso)
|
||||
meta = {"generated_at": first_iso, "compose_dir": str(config.compose_dir)}
|
||||
write_toml(log_path, meta=meta, entries=merged)
|
||||
|
||||
after_first = tomllib.loads(log_path.read_text())
|
||||
first_seen = after_first["entries"][0]["first_seen"]
|
||||
|
||||
# Second snapshot
|
||||
second_time = datetime(2025, 2, 1, tzinfo=UTC)
|
||||
second_entries = await collect_stacks_entries_on_host(
|
||||
config, "local", {"svc"}, now=second_time
|
||||
)
|
||||
second_iso = isoformat(second_time)
|
||||
existing = load_existing_entries(log_path)
|
||||
merged = merge_entries(existing, second_entries, now_iso=second_iso)
|
||||
meta = {"generated_at": second_iso, "compose_dir": str(config.compose_dir)}
|
||||
write_toml(log_path, meta=meta, entries=merged)
|
||||
|
||||
after_second = tomllib.loads(log_path.read_text())
|
||||
entry = after_second["entries"][0]
|
||||
assert entry["first_seen"] == first_seen
|
||||
assert entry["last_seen"].startswith("2025-02-01")
|
||||
|
||||
@@ -12,9 +12,8 @@ from compose_farm.cli import lifecycle
|
||||
from compose_farm.config import Config, Host
|
||||
from compose_farm.executor import CommandResult
|
||||
from compose_farm.operations import (
|
||||
StackDiscoveryResult,
|
||||
_migrate_stack,
|
||||
discover_all_stacks_on_all_hosts,
|
||||
build_discovery_results,
|
||||
)
|
||||
|
||||
|
||||
@@ -115,63 +114,18 @@ class TestUpdateCommandSequence:
|
||||
assert "up -d" in source
|
||||
|
||||
|
||||
class TestDiscoverAllStacksOnAllHosts:
|
||||
"""Tests for discover_all_stacks_on_all_hosts function."""
|
||||
class TestBuildDiscoveryResults:
|
||||
"""Tests for build_discovery_results function."""
|
||||
|
||||
async def test_returns_discovery_results_for_all_stacks(self, basic_config: Config) -> None:
|
||||
"""Function returns StackDiscoveryResult for each stack."""
|
||||
with patch(
|
||||
"compose_farm.operations.get_running_stacks_on_host",
|
||||
return_value={"test-service"},
|
||||
):
|
||||
results = await discover_all_stacks_on_all_hosts(basic_config)
|
||||
|
||||
assert len(results) == 1
|
||||
assert isinstance(results[0], StackDiscoveryResult)
|
||||
assert results[0].stack == "test-service"
|
||||
|
||||
async def test_detects_stray_stacks(self, tmp_path: Path) -> None:
|
||||
"""Function detects stacks running on wrong hosts."""
|
||||
compose_dir = tmp_path / "compose"
|
||||
(compose_dir / "plex").mkdir(parents=True)
|
||||
(compose_dir / "plex" / "docker-compose.yml").write_text("services: {}")
|
||||
|
||||
config = Config(
|
||||
compose_dir=compose_dir,
|
||||
hosts={
|
||||
"host1": Host(address="localhost"),
|
||||
"host2": Host(address="localhost"),
|
||||
},
|
||||
stacks={"plex": "host1"}, # Should run on host1
|
||||
)
|
||||
|
||||
# Mock: plex is running on host2 (wrong host)
|
||||
async def mock_get_running(cfg: Config, host: str) -> set[str]:
|
||||
if host == "host2":
|
||||
return {"plex"}
|
||||
return set()
|
||||
|
||||
with patch(
|
||||
"compose_farm.operations.get_running_stacks_on_host",
|
||||
side_effect=mock_get_running,
|
||||
):
|
||||
results = await discover_all_stacks_on_all_hosts(config)
|
||||
|
||||
assert len(results) == 1
|
||||
assert results[0].stack == "plex"
|
||||
assert results[0].running_hosts == ["host2"]
|
||||
assert results[0].configured_hosts == ["host1"]
|
||||
assert results[0].is_stray is True
|
||||
assert results[0].stray_hosts == ["host2"]
|
||||
|
||||
async def test_queries_each_host_once(self, tmp_path: Path) -> None:
|
||||
"""Function makes exactly one call per host, not per stack."""
|
||||
@pytest.fixture
|
||||
def config(self, tmp_path: Path) -> Config:
|
||||
"""Create a test config with multiple stacks."""
|
||||
compose_dir = tmp_path / "compose"
|
||||
for stack in ["plex", "jellyfin", "sonarr"]:
|
||||
(compose_dir / stack).mkdir(parents=True)
|
||||
(compose_dir / stack / "docker-compose.yml").write_text("services: {}")
|
||||
|
||||
config = Config(
|
||||
return Config(
|
||||
compose_dir=compose_dir,
|
||||
hosts={
|
||||
"host1": Host(address="localhost"),
|
||||
@@ -180,17 +134,61 @@ class TestDiscoverAllStacksOnAllHosts:
|
||||
stacks={"plex": "host1", "jellyfin": "host1", "sonarr": "host2"},
|
||||
)
|
||||
|
||||
call_count = {"count": 0}
|
||||
def test_discovers_correctly_running_stacks(self, config: Config) -> None:
|
||||
"""Stacks running on correct hosts are discovered."""
|
||||
running_on_host = {
|
||||
"host1": {"plex", "jellyfin"},
|
||||
"host2": {"sonarr"},
|
||||
}
|
||||
|
||||
async def mock_get_running(cfg: Config, host: str) -> set[str]:
|
||||
call_count["count"] += 1
|
||||
return set()
|
||||
discovered, strays, duplicates = build_discovery_results(config, running_on_host)
|
||||
|
||||
with patch(
|
||||
"compose_farm.operations.get_running_stacks_on_host",
|
||||
side_effect=mock_get_running,
|
||||
):
|
||||
await discover_all_stacks_on_all_hosts(config)
|
||||
assert discovered == {"plex": "host1", "jellyfin": "host1", "sonarr": "host2"}
|
||||
assert strays == {}
|
||||
assert duplicates == {}
|
||||
|
||||
# Should call once per host (2), not once per stack (3)
|
||||
assert call_count["count"] == 2
|
||||
def test_detects_stray_stacks(self, config: Config) -> None:
|
||||
"""Stacks running on wrong hosts are marked as strays."""
|
||||
running_on_host = {
|
||||
"host1": set(),
|
||||
"host2": {"plex"}, # plex should be on host1
|
||||
}
|
||||
|
||||
discovered, strays, _duplicates = build_discovery_results(config, running_on_host)
|
||||
|
||||
assert "plex" not in discovered
|
||||
assert strays == {"plex": ["host2"]}
|
||||
|
||||
def test_detects_duplicates(self, config: Config) -> None:
|
||||
"""Single-host stacks running on multiple hosts are duplicates."""
|
||||
running_on_host = {
|
||||
"host1": {"plex"},
|
||||
"host2": {"plex"}, # plex running on both hosts
|
||||
}
|
||||
|
||||
discovered, strays, duplicates = build_discovery_results(
|
||||
config, running_on_host, stacks=["plex"]
|
||||
)
|
||||
|
||||
# plex is correctly running on host1
|
||||
assert discovered == {"plex": "host1"}
|
||||
# plex is also a stray on host2
|
||||
assert strays == {"plex": ["host2"]}
|
||||
# plex is a duplicate (single-host stack on multiple hosts)
|
||||
assert duplicates == {"plex": ["host1", "host2"]}
|
||||
|
||||
def test_filters_to_requested_stacks(self, config: Config) -> None:
|
||||
"""Only returns results for requested stacks."""
|
||||
running_on_host = {
|
||||
"host1": {"plex", "jellyfin"},
|
||||
"host2": {"sonarr"},
|
||||
}
|
||||
|
||||
discovered, _strays, _duplicates = build_discovery_results(
|
||||
config, running_on_host, stacks=["plex"]
|
||||
)
|
||||
|
||||
# Only plex should be in results
|
||||
assert discovered == {"plex": "host1"}
|
||||
assert "jellyfin" not in discovered
|
||||
assert "sonarr" not in discovered
|
||||
|
||||
Reference in New Issue
Block a user