diff --git a/.skills/mcc-integration-testing/scripts/prepare_offline_mcc_config.sh b/.skills/mcc-integration-testing/scripts/prepare_offline_mcc_config.sh index 1e30f673..fd713212 100644 --- a/.skills/mcc-integration-testing/scripts/prepare_offline_mcc_config.sh +++ b/.skills/mcc-integration-testing/scripts/prepare_offline_mcc_config.sh @@ -24,7 +24,7 @@ OUTPUT_INI="" MC_VERSION="" LOGIN_NAME="" -if [[ $# -ge 3 && -f "$1" ]]; then +if [[ $# -ge 3 && "$2" == *.ini ]]; then TEMPLATE_INI="$1" OUTPUT_INI="$2" MC_VERSION="$3" diff --git a/docs/superpowers/plans/2026-04-18-parkour-worker-pool.md b/docs/superpowers/plans/2026-04-18-parkour-worker-pool.md new file mode 100644 index 00000000..25ca2935 --- /dev/null +++ b/docs/superpowers/plans/2026-04-18-parkour-worker-pool.md @@ -0,0 +1,402 @@ +# Parkour Worker Pool Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Refactor `tools/test-parkour.py` into a long-lived worker-pool harness that handles server readiness cleanly and emits inspectable per-run artifacts for unattended parallel live testing. + +**Architecture:** Keep the existing matrix generation and live outcome rules, but replace per-case MCC relaunch with stable worker contexts that are reset between cases. Add a run-artifact layer so each case and worker has traceable logs, and classify harness failures separately from product failures. + +**Tech Stack:** Python 3, unittest/pytest, MCC `mcc-debug`, local `1.21.11-Vanilla` server, RCON, tmux-backed worker sessions. + +--- + +### Task 1: Lock The New Result Schema With Tests + +**Files:** +- Modify: `tools/tests/test_test_parkour_metrics.py` +- Test: `tools/tests/test_test_parkour_metrics.py` + +- [ ] **Step 1: Write failing tests for run artifacts and skip rows** + +Add tests that assert: + +```python +def test_result_to_record_includes_worker_session_and_paths(self) -> None: + case = module.TestCase( + case_id="linear-flat-gap1", + family="linear", + subfamily="flat", + gap_or_wall=1, + delta_y=0.0, + ceiling_height=None, + wall_offset=None, + expected="pass", + ) + result = module.TestResult( + case=case, + outcome="pass", + matched_expected=True, + replan_count=0, + turn_stall_count=0, + near_goal=True, + total_ticks=42, + final_position=(109.5, 80.0, 200.5), + session="parkour-run-1", + log_path="/tmp/parkour-runs/run-1/workers/1/worker.log", + event_log_path="/tmp/parkour-runs/run-1/events.jsonl", + duration_ms=4200, + error_kind=None, + skip_reason=None, + ) + + record = module.result_to_record(result, worker_id=1) + + self.assertEqual(record["session"], "parkour-run-1") + self.assertEqual(record["worker"], 1) + self.assertEqual(record["duration_ms"], 4200) + self.assertEqual(record["skip_reason"], None) + + +def test_make_skip_result_marks_case_as_skipped(self) -> None: + case = module.TestCase( + case_id="linear-flat-gap4", + family="linear", + subfamily="flat", + gap_or_wall=4, + delta_y=0.0, + ceiling_height=None, + wall_offset=None, + expected="pass", + ) + + result = module.make_skip_result(case, "group_failed_earlier") + + self.assertEqual(result.outcome, "skipped") + self.assertEqual(result.skip_reason, "group_failed_earlier") + self.assertFalse(result.matched_expected) +``` + +- [ ] **Step 2: Run the focused test file and confirm RED** + +Run: + +```bash +python3 -m pytest -q tools/tests/test_test_parkour_metrics.py +``` + +Expected: failures complaining that `TestResult` lacks the new fields and `make_skip_result` does not exist. + +- [ ] **Step 3: Implement the minimal production fields and helpers** + +Update `tools/test-parkour.py` so `TestResult` contains: + +```python +session: str | None = None +log_path: str | None = None +event_log_path: str | None = None +duration_ms: int | None = None +error_kind: str | None = None +skip_reason: str | None = None +``` + +Add: + +```python +def make_skip_result(case: TestCase, reason: str) -> TestResult: + return TestResult( + case=case, + outcome="skipped", + matched_expected=False, + skip_reason=reason, + error_kind=None, + ) +``` + +- [ ] **Step 4: Re-run the focused tests and confirm GREEN** + +Run: + +```bash +python3 -m pytest -q tools/tests/test_test_parkour_metrics.py +``` + +Expected: all tests in that file pass. + +### Task 2: Add Readiness And Harness Error Coverage + +**Files:** +- Modify: `tools/tests/test_test_parkour_metrics.py` +- Modify: `tools/test-parkour.py` + +- [ ] **Step 1: Write failing tests for readiness and harness classification** + +Add tests like: + +```python +def test_classify_outcome_prefers_harness_error_when_rcon_is_unavailable(self) -> None: + metrics = module.LiveMetrics() + result = module.classify_outcome(metrics, near_goal=None, error_kind="harness_rcon_unavailable") + self.assertEqual(result, "harness_rcon_unavailable") + + +def test_wait_for_rcon_ready_retries_until_command_succeeds(self) -> None: + attempts = {"count": 0} + + class FakeRcon: + def command(self, _cmd: str) -> str: + attempts["count"] += 1 + if attempts["count"] < 3: + raise ConnectionRefusedError("not ready") + return "There are 0 of a max of 20 players online" + + with mock.patch.object(module.time, "sleep", lambda _seconds: None): + self.assertTrue(module.wait_for_rcon_ready(FakeRcon(), timeout_seconds=3.0, poll_interval=0.1)) +``` + +- [ ] **Step 2: Run the focused tests and confirm RED** + +Run: + +```bash +python3 -m pytest -q tools/tests/test_test_parkour_metrics.py -k "harness_error or rcon_ready" +``` + +Expected: failures because the readiness helper and new classification signature do not exist. + +- [ ] **Step 3: Implement minimal readiness helpers** + +Add to `tools/test-parkour.py`: + +```python +def wait_for_rcon_ready( + rcon: RconClient, + timeout_seconds: float = 20.0, + poll_interval: float = 0.5, +) -> bool: + deadline = time.monotonic() + timeout_seconds + while time.monotonic() < deadline: + try: + rcon.command("list") + return True + except Exception: + time.sleep(poll_interval) + return False +``` + +Update `classify_outcome` to accept `error_kind: str | None = None` and return the harness error directly when present. + +- [ ] **Step 4: Re-run focused tests and confirm GREEN** + +Run: + +```bash +python3 -m pytest -q tools/tests/test_test_parkour_metrics.py -k "harness_error or rcon_ready" +``` + +Expected: passing tests. + +### Task 3: Convert Per-Case Relaunch Into A Long-Lived Worker Pool + +**Files:** +- Modify: `tools/tests/test_test_parkour_metrics.py` +- Modify: `tools/test-parkour.py` + +- [ ] **Step 1: Write failing tests for worker reuse bookkeeping** + +Add tests asserting that one worker can execute multiple cases without changing session naming: + +```python +def test_build_worker_session_name_is_stable_for_multiple_cases(self) -> None: + self.assertEqual(module.build_worker_session_name("run123", 2), "parkour-run123-2") + + +def test_result_rows_can_share_one_worker_session_across_cases(self) -> None: + case1 = "linear-flat-gap1" + case2 = "linear-flat-gap2" + session = module.build_worker_session_name("run123", 2) + self.assertEqual(session, "parkour-run123-2") + self.assertEqual(session, module.build_worker_session_name("run123", 2)) +``` + +- [ ] **Step 2: Run the focused tests and confirm RED if needed** + +Run: + +```bash +python3 -m pytest -q tools/tests/test_test_parkour_metrics.py -k "worker_session" +``` + +Expected: either existing coverage passes already or new assertions fail because the worker model still depends on per-case session allocation elsewhere. + +- [ ] **Step 3: Implement long-lived worker contexts** + +In `tools/test-parkour.py`: + +- keep `build_worker_session_name()` as the canonical stable session name +- stop calling `build_case_session_name()` from `worker_loop()` +- launch one `WorkerContext` per thread at worker start +- add `reset_worker_state(ctx, layout)` to reposition and resync between cases +- restart only that worker when reset or health checks fail + +The main shape should become: + +```python +ctx = ensure_worker_context(...) +for case, layout in items: + if case.group_key() in failed_groups: + ... + continue + + ctx = ensure_worker_context(...) + reset = reset_worker_state(ctx, layout) + if not reset.ok: + cleanup_workers([ctx]) + ctx = relaunch_worker_context(...) + reset = reset_worker_state(ctx, layout) +``` + +- [ ] **Step 4: Re-run focused metric tests** + +Run: + +```bash +python3 -m pytest -q tools/tests/test_test_parkour_metrics.py +``` + +Expected: file stays green after the worker-pool refactor. + +### Task 4: Emit Run Directories, Per-Case Records, And Summaries + +**Files:** +- Modify: `tools/tests/test_test_parkour_metrics.py` +- Modify: `tools/test-parkour.py` + +- [ ] **Step 1: Write failing tests for summary aggregation** + +Add tests like: + +```python +def test_summarize_results_groups_by_family_and_outcome(self) -> None: + summary = module.summarize_results( + [ + {"family": "linear", "outcome": "pass", "matched": True}, + {"family": "linear", "outcome": "reject", "matched": True}, + {"family": "neo", "outcome": "reject", "matched": False}, + {"family": "linear", "outcome": "skipped", "matched": False}, + ] + ) + + self.assertEqual(summary["families"]["linear"]["outcomes"]["pass"], 1) + self.assertEqual(summary["families"]["linear"]["outcomes"]["skipped"], 1) + self.assertEqual(summary["families"]["neo"]["mismatches"], 1) +``` + +- [ ] **Step 2: Run the focused tests and confirm RED** + +Run: + +```bash +python3 -m pytest -q tools/tests/test_test_parkour_metrics.py -k summarize_results +``` + +Expected: missing helper failures. + +- [ ] **Step 3: Implement run-artifact helpers** + +Add helpers in `tools/test-parkour.py`: + +```python +def create_run_dir(base_dir: Path | None = None) -> Path: ... +def write_case_artifact(run_dir: Path, result: TestResult) -> None: ... +def summarize_results(records: list[dict[str, object]]) -> dict[str, object]: ... +def write_summary_files(run_dir: Path, summary: dict[str, object]) -> None: ... +``` + +Update `_run_parallel()` and `_run_serial()` to: + +- create one run directory +- append JSONL rows there even when `--results` is omitted +- persist skip rows +- write `summary.json` and `summary.md` + +- [ ] **Step 4: Re-run targeted tests and then full tools suite** + +Run: + +```bash +python3 -m pytest -q tools/tests/test_test_parkour_metrics.py +python3 -m pytest -q tools/tests +``` + +Expected: full `tools/tests` suite passes. + +### Task 5: Verify The Real Harness Fixes Against 1.21.11 + +**Files:** +- Modify: `tools/test-parkour.py` +- Test: real-server execution only + +- [ ] **Step 1: Run a filtered live smoke on linear with long-lived workers** + +Run: + +```bash +source tools/mcc-env.sh && \ +python3 tools/test-parkour.py \ + --filter linear \ + --parallel 6 \ + --version 1.21.11-Vanilla \ + --results /tmp/parkour-linear-worker-pool.jsonl +``` + +Expected: + +- worker startup lines show six stable worker sessions +- multiple cases reuse the same worker/session ids +- summary artifacts are written under `/tmp/parkour-runs/...` + +- [ ] **Step 2: Inspect the emitted summary and worker logs** + +Run: + +```bash +latest_run=$(ls -td /tmp/parkour-runs/* | head -n 1) +printf '%s\n' "$latest_run" +sed -n '1,220p' "$latest_run/summary.md" +find "$latest_run/workers" -maxdepth 2 -type f | sort +``` + +Expected: + +- `summary.md` exists +- worker logs exist +- skipped cases are represented in the summary + +- [ ] **Step 3: Run one full live matrix to prove unattended output quality** + +Run: + +```bash +source tools/mcc-env.sh && \ +python3 tools/test-parkour.py \ + --parallel 6 \ + --version 1.21.11-Vanilla \ + --results /tmp/parkour-full-worker-pool.jsonl +``` + +Expected: + +- no raw `ConnectionRefusedError` at startup when the server is merely late +- a completed `summary.md` and `summary.json` +- case rows include `worker`, `session`, `log_path`, `duration_ms`, and skip metadata + +- [ ] **Step 4: Commit** + +```bash +git add tools/test-parkour.py \ + tools/tests/test_test_parkour_metrics.py \ + tools/tests/test_pathing_live_scripts.py \ + docs/superpowers/specs/2026-04-18-parkour-worker-pool-design.md \ + docs/superpowers/plans/2026-04-18-parkour-worker-pool.md +git commit -m "test-harness: add long-lived parkour worker pool" +``` diff --git a/docs/superpowers/specs/2026-04-18-parkour-worker-pool-design.md b/docs/superpowers/specs/2026-04-18-parkour-worker-pool-design.md new file mode 100644 index 00000000..3cac1bb2 --- /dev/null +++ b/docs/superpowers/specs/2026-04-18-parkour-worker-pool-design.md @@ -0,0 +1,194 @@ +# Parkour Worker Pool Design + +**Date:** 2026-04-18 + +**Goal** + +Make `tools/test-parkour.py` behave like a real parallel harness on `1.21.11-Vanilla`: keep `--parallel N` as `N` long-lived MCC workers, classify harness failures separately from product failures, and emit artifacts that make unattended runs easy to inspect. + +## Problem Statement + +The current script has two real operational problems. + +1. Server and RCON readiness are assumed too early. In prior use this caused an immediate `ConnectionRefusedError` when the server had been stopped by a previous harness. +2. The parallel path launches and tears down a fresh MCC session for every case. That produces large startup overhead, leaves many session directories behind, and makes logs hard to correlate with results. + +The current script also has observability gaps. + +- Skipped cases are only visible in stdout. +- JSONL rows do not carry enough metadata to jump directly to the relevant logs. +- There is no per-run summary artifact beyond terminal output. + +## Non-Goals + +- Changing parkour theory generation or expected pass/reject boundaries. +- Reworking the live outcome rules that treat any replan or turn-stall as a failure for pass cases. +- Generalizing the harness to multi-version shared-state parallelism. + +## Chosen Approach + +Use a long-lived worker pool. + +- Start `N` MCC workers once. +- Assign whole `group_key()` batches to workers to preserve stop-at-first-failure semantics inside a group. +- Reuse the same worker for multiple cases by resetting player state between cases. +- Restart only the individual worker that becomes unhealthy. + +This keeps isolation strong enough for unattended live testing while removing the main startup bottleneck. + +## Architecture + +### Run Controller + +The main process will build the course matrix and create a dedicated run directory under `/tmp/parkour-runs/-/`. + +It will own: + +- server/RCON readiness checks +- world build phase +- worker pool startup +- group scheduling +- summary generation + +### Worker Lifecycle + +Each worker will keep a stable: + +- `worker_id` +- `session` +- `username` +- MCC log path +- worker event log + +Worker lifecycle: + +1. launch MCC once +2. wait for join confirmation +3. enable debug mode +4. run many assigned cases with reset in between +5. if unhealthy, recycle just that worker +6. quit cleanly during harness shutdown + +### Case Execution + +Each case will still: + +- teleport to the start +- verify local sync via `debug state` +- run `goto` +- sample position/yaw during execution +- parse `[PathMetric]` telemetry +- classify into `pass`, `reject`, `fail`, or a harness-specific error + +The zero-replan and zero-turn-stall rule remains unchanged. + +### Artifacts + +Each run directory will contain: + +- `manifest.json` +- `results.jsonl` +- `summary.json` +- `summary.md` +- `events.jsonl` +- `workers//worker.log` +- `cases/.json` + +Every result row will include: + +- `case_id` +- `family` +- `subfamily` +- `expected` +- `outcome` +- `matched` +- `worker` +- `session` +- `log_path` +- `event_log_path` +- `replan_count` +- `turn_stall_count` +- `near_goal` +- `total_ticks` +- `final_position` +- `duration_ms` +- `skip_reason` +- `error_kind` + +Skipped cases will be recorded, not just printed. + +## Failure Taxonomy + +Observed product behavior and harness behavior must be separated. + +Product-side outcomes: + +- `pass` +- `reject` +- `fail` + +Harness-side outcomes: + +- `harness_rcon_unavailable` +- `harness_worker_launch_failed` +- `harness_join_timeout` +- `harness_start_sync_failed` +- `harness_worker_lost` + +`matched` remains the top-level boolean used by summaries. + +## Logging Model + +Terminal output becomes high-signal progress output: + +- worker launch / restart +- case start / case finish +- group skip decisions +- mismatch lines +- final summaries by family and outcome + +Detailed evidence moves into run artifacts. + +## Testing Strategy + +### Python unit coverage + +Extend `tools/tests/test_test_parkour_metrics.py` to cover: + +- worker session naming +- run directory naming +- result row schema +- skip row emission +- summary aggregation +- harness error classification + +Extend `tools/tests/test_pathing_live_scripts.py` to cover: + +- new CLI-compatible output expectations +- `--list-cases` stability + +### Real-server validation + +After refactor: + +- run targeted Python tests +- run the full `tools/tests` suite +- run a real `tools/test-parkour.py --filter linear --parallel 6 --version 1.21.11-Vanilla` +- confirm that workers are reused across multiple cases and that summary artifacts are emitted + +## Risks And Mitigations + +- Worker state leaks between cases. + - Mitigation: centralize `reset_worker_state()` and recycle unhealthy workers. +- Mixed threaded stdout becomes unreadable. + - Mitigation: keep stdout terse and persist details to per-worker logs. +- Shared server state still limits aggressive parallelism. + - Mitigation: keep one shared version/server per run and continue atomic group scheduling. + +## Acceptance Criteria + +- A stopped or not-yet-ready server is reported as a harness problem instead of a raw socket traceback. +- `--parallel 6` keeps approximately six long-lived MCC workers instead of one worker per case. +- Results include executed cases and skipped cases. +- A completed unattended run can be inspected from `summary.md` and `summary.json` without replaying terminal output. +- The zero-replan and zero-turn-stall requirement remains enforced for pass cases. diff --git a/tools/test-parkour.py b/tools/test-parkour.py index 74fe082e..a7e0ecd1 100644 --- a/tools/test-parkour.py +++ b/tools/test-parkour.py @@ -55,7 +55,7 @@ import time import uuid from dataclasses import dataclass, field from pathlib import Path -from typing import Optional +from typing import Callable, Optional REPO_ROOT = Path(__file__).resolve().parent.parent CAPABILITIES_PATH = REPO_ROOT / "tools" / "pathing_data" / "momentum-capabilities.json" @@ -153,6 +153,35 @@ class RconClient: return data +def connect_rcon_with_retry( + host: str = "localhost", + port: int = 25575, + password: str = "test123", + timeout_seconds: float = 20.0, + poll_interval: float = 0.5, + client_factory: Callable[..., RconClient] | None = None, +) -> RconClient: + deadline = time.monotonic() + timeout_seconds + last_error: Exception | None = None + + while time.monotonic() < deadline: + client = client_factory(host=host, port=port, password=password) if client_factory else RconClient(host=host, port=port, password=password) + try: + client.connect() + return client + except Exception as exc: + last_error = exc + try: + client.close() + except Exception: + pass + time.sleep(poll_interval) + + if last_error is not None: + raise RuntimeError(f"RCON unavailable on {host}:{port}") from last_error + raise RuntimeError(f"RCON unavailable on {host}:{port}") + + # --------------------------------------------------------------------------- # MCC command interface # --------------------------------------------------------------------------- @@ -702,6 +731,12 @@ class TestResult: final_position: tuple[float, float, float] | None = None total_ticks: int | None = None log_excerpt: str = "" + session: str | None = None + log_path: str | None = None + event_log_path: str | None = None + duration_ms: int | None = None + error_kind: str | None = None + skip_reason: str | None = None @dataclass(frozen=True) @@ -732,6 +767,10 @@ def build_case_session_name(run_token: str, worker_id: int, case_index: int) -> return f"parkour-{run_token}-{worker_id}-c{case_index}" +def build_worker_username(base_username: str, worker_id: int) -> str: + return f"{base_username}{worker_id}" + + def build_case_username(base_username: str, worker_id: int, case_index: int) -> str: return f"{base_username}{worker_id}c{case_index}" @@ -980,6 +1019,21 @@ def has_terminal_metrics(metrics: LiveMetrics) -> bool: ) +def wait_for_rcon_ready( + rcon: RconClient, + timeout_seconds: float = 20.0, + poll_interval: float = 0.5, +) -> bool: + deadline = time.monotonic() + timeout_seconds + while time.monotonic() < deadline: + try: + rcon.command("list") + return True + except Exception: + time.sleep(poll_interval) + return False + + def is_near_goal( position: tuple[float, float, float] | None, layout: CourseLayout, @@ -998,7 +1052,14 @@ def is_near_goal( ) -def classify_outcome(metrics: LiveMetrics, near_goal: bool | None) -> str: +def classify_outcome( + metrics: LiveMetrics, + near_goal: bool | None, + error_kind: str | None = None, +) -> str: + if error_kind is not None: + return error_kind + if metrics.segment_failed_count > 0 or metrics.replan_failed_count > 0 or metrics.generic_fail_count > 0: return "fail" @@ -1023,36 +1084,9 @@ def run_single_test( username: str, wait_seconds: int = 15, ) -> TestResult: - expected_start_position = ( - layout.start_x + 0.5, - float(layout.start_y), - layout.start_z + 0.5, - ) - start_synced = False - for _attempt in range(2): - rcon.command(f"gamemode creative {username}") - rcon.command(f"tp {username} {layout.start_x}.5 {layout.start_y} {layout.start_z}.5") - time.sleep(2) - rcon.command(f"gamemode survival {username}") - time.sleep(0.5) - if wait_for_local_start_sync(mcc, expected_start_position): - start_synced = True - break - time.sleep(0.5) - + start_time = time.monotonic() log_offset = mcc.log_length() - if not start_synced: - return TestResult( - case=case, - outcome="invalid_live_case", - matched_expected=False, - log_excerpt=( - " Harness: local MCC position did not stabilize at test start " - f"goal=({expected_start_position[0]:.1f},{expected_start_position[1]:.1f},{expected_start_position[2]:.1f})" - ), - ) - mcc.send(f"send ===== TEST: {case.case_id} (expect: {case.expected}) =====") time.sleep(0.2) mcc.send(f"goto {layout.end_x} {layout.end_y} {layout.end_z}") @@ -1131,6 +1165,9 @@ def run_single_test( final_position=final_position, total_ticks=metrics.total_ticks, log_excerpt="\n".join(excerpt_lines), + session=mcc.session, + log_path=str(mcc.log_file), + duration_ms=int((time.monotonic() - start_time) * 1000), ) @@ -1143,6 +1180,33 @@ def should_skip(case: TestCase, failed_groups: set[tuple]) -> bool: return case.group_key() in failed_groups +def make_skip_result(case: TestCase, reason: str) -> TestResult: + return TestResult( + case=case, + outcome="skipped", + matched_expected=False, + skip_reason=reason, + ) + + +def make_harness_result( + case: TestCase, + error_kind: str, + log_excerpt: str, + session: str | None = None, + log_path: str | None = None, +) -> TestResult: + return TestResult( + case=case, + outcome=error_kind, + matched_expected=False, + log_excerpt=log_excerpt, + session=session, + log_path=log_path, + error_kind=error_kind, + ) + + def result_to_record(result: TestResult, worker_id: int | None = None) -> dict[str, object]: record: dict[str, object] = { "case_id": result.case.case_id, @@ -1157,12 +1221,108 @@ def result_to_record(result: TestResult, worker_id: int | None = None) -> dict[s "near_goal": result.near_goal, "total_ticks": result.total_ticks, "final_position": list(result.final_position) if result.final_position is not None else None, + "session": result.session, + "log_path": result.log_path, + "event_log_path": result.event_log_path, + "duration_ms": result.duration_ms, + "error_kind": result.error_kind, + "skip_reason": result.skip_reason, } if worker_id is not None: record["worker"] = worker_id return record +def summarize_results(records: list[dict[str, object]]) -> dict[str, object]: + summary: dict[str, object] = { + "total": len(records), + "matched": sum(1 for r in records if bool(r.get("matched"))), + "mismatched": sum(1 for r in records if not bool(r.get("matched"))), + "families": {}, + } + + families: dict[str, dict[str, object]] = {} + for record in records: + family = str(record.get("family", "unknown")) + outcome = str(record.get("outcome", "unknown")) + matched = bool(record.get("matched")) + family_summary = families.setdefault( + family, + { + "total": 0, + "matched": 0, + "mismatches": 0, + "outcomes": {}, + }, + ) + family_summary["total"] = int(family_summary["total"]) + 1 + if matched: + family_summary["matched"] = int(family_summary["matched"]) + 1 + else: + family_summary["mismatches"] = int(family_summary["mismatches"]) + 1 + outcomes = family_summary["outcomes"] + assert isinstance(outcomes, dict) + outcomes[outcome] = int(outcomes.get(outcome, 0)) + 1 + + summary["families"] = families + return summary + + +def create_run_dir(base_dir: Path | None = None) -> Path: + root = base_dir or (Path(os.environ.get("TMPDIR", "/tmp")) / "parkour-runs") + timestamp = time.strftime("%Y%m%d-%H%M%S", time.gmtime()) + run_dir = root / f"{timestamp}-{make_parallel_run_token()}" + run_dir.mkdir(parents=True, exist_ok=False) + return run_dir + + +def write_summary_files(run_dir: Path, summary: dict[str, object]) -> None: + run_dir.mkdir(parents=True, exist_ok=True) + summary_json = run_dir / "summary.json" + summary_md = run_dir / "summary.md" + + summary_json.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + lines = [ + "# Parkour Summary", + "", + f"- Total: {summary.get('total', 0)}", + f"- Matched: {summary.get('matched', 0)}", + f"- Mismatched: {summary.get('mismatched', 0)}", + "", + "## Families", + "", + ] + + families = summary.get("families", {}) + if isinstance(families, dict): + for family, family_summary in sorted(families.items()): + lines.append(f"### {family}") + if isinstance(family_summary, dict): + lines.append(f"- Total: {family_summary.get('total', 0)}") + lines.append(f"- Matched: {family_summary.get('matched', 0)}") + lines.append(f"- Mismatches: {family_summary.get('mismatches', 0)}") + outcomes = family_summary.get("outcomes", {}) + if isinstance(outcomes, dict): + for outcome, count in sorted(outcomes.items()): + lines.append(f"- {outcome}: {count}") + lines.append("") + + summary_md.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8") + + +def append_jsonl_record(paths: list[Path], record: dict[str, object]) -> None: + payload = json.dumps(record) + "\n" + seen: set[Path] = set() + for path in paths: + if path in seen: + continue + seen.add(path) + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8") as f: + f.write(payload) + + # --------------------------------------------------------------------------- # Parallel worker infrastructure # --------------------------------------------------------------------------- @@ -1232,8 +1392,7 @@ def launch_worker_context( _tprint(f" [W{worker_id}] Failed to launch, exiting case.") return None - rcon = RconClient(port=rcon_port, password=rcon_password) - rcon.connect() + rcon = connect_rcon_with_retry(port=rcon_port, password=rcon_password, timeout_seconds=10.0) mcc = MccClient(session) if _wait_for_join(mcc): @@ -1262,6 +1421,37 @@ def launch_worker_context( ) +def register_worker_context( + ctx: WorkerContext, + workers_registry: list[WorkerContext], + registry_lock: threading.Lock, +) -> None: + with registry_lock: + workers_registry.append(ctx) + + +def reset_worker_state(ctx: WorkerContext, layout: CourseLayout) -> bool: + expected_start_position = ( + layout.start_x + 0.5, + float(layout.start_y), + layout.start_z + 0.5, + ) + + for _attempt in range(2): + ctx.rcon.command(f"gamemode creative {ctx.username}") + ctx.rcon.command( + f"tp {ctx.username} {layout.start_x}.5 {layout.start_y} {layout.start_z}.5" + ) + time.sleep(2) + ctx.rcon.command(f"gamemode survival {ctx.username}") + time.sleep(0.5) + if wait_for_local_start_sync(ctx.mcc, expected_start_position): + return True + time.sleep(0.5) + + return False + + def worker_loop( worker_id: int, base_username: str, @@ -1274,20 +1464,45 @@ def worker_loop( all_results: list[TestResult], results_lock: threading.Lock, wait_seconds: int, - results_path: Path | None, + results_paths: list[Path], workers_registry: list[WorkerContext], registry_lock: threading.Lock, skipped_counter: list[int], ) -> None: - """Run assigned groups while launching a fresh MCC session for each case.""" + """Run assigned groups while reusing one MCC session per worker.""" local_results: list[TestResult] = [] local_skipped = 0 failed_groups: set[tuple] = set() - case_counter = 0 + worker_session = build_worker_session_name(run_token, worker_id) + worker_username = build_worker_username(base_username, worker_id) + ctx: WorkerContext | None = None + + def write_result(result: TestResult) -> None: + with results_lock: + append_jsonl_record(results_paths, result_to_record(result, worker_id)) + + def ensure_worker() -> WorkerContext | None: + nonlocal ctx + if ctx is not None: + return ctx + + launched = launch_worker_context( + worker_id=worker_id, + username=worker_username, + session=worker_session, + version=version, + server_port=server_port, + rcon_port=rcon_port, + rcon_password=rcon_password, + ) + if launched is not None: + ctx = launched + register_worker_context(ctx, workers_registry, registry_lock) + return ctx while True: try: - group_key, items = group_queue.get_nowait() + _, items = group_queue.get_nowait() except queue.Empty: break @@ -1295,39 +1510,67 @@ def worker_loop( if case.group_key() in failed_groups: local_skipped += 1 _tprint(f" [W{worker_id}] {case.case_id} -- SKIPPED") + skipped = make_skip_result(case, "group_failed_earlier") + local_results.append(skipped) + write_result(skipped) continue - case_counter += 1 - session = build_case_session_name(run_token, worker_id, case_counter) - username = build_case_username(base_username, worker_id, case_counter) _tprint(f" [W{worker_id}] {case.case_id} (expect: {case.expected})" f" route=({layout.start_x},{layout.start_y},{layout.start_z})" f" -> ({layout.end_x},{layout.end_y},{layout.end_z})") - ctx = launch_worker_context( - worker_id=worker_id, - username=username, - session=session, - version=version, - server_port=server_port, - rcon_port=rcon_port, - rcon_password=rcon_password, - ) - - if ctx is None: - result = TestResult( - case=case, - outcome="invalid_live_case", - matched_expected=False, - log_excerpt=" Harness: failed to launch fresh MCC worker session", + current_ctx = ensure_worker() + if current_ctx is None: + result = make_harness_result( + case, + error_kind="harness_worker_launch_failed", + log_excerpt=" Harness: failed to launch worker session", + session=worker_session, ) else: + reset_ok = False try: - result = run_single_test( - case, layout, ctx.rcon, ctx.mcc, ctx.username, wait_seconds, + reset_ok = reset_worker_state(current_ctx, layout) + except Exception: + reset_ok = False + + if not reset_ok: + cleanup_workers([current_ctx]) + ctx = None + current_ctx = ensure_worker() + if current_ctx is not None: + try: + reset_ok = reset_worker_state(current_ctx, layout) + except Exception: + reset_ok = False + + if current_ctx is None: + result = make_harness_result( + case, + error_kind="harness_worker_launch_failed", + log_excerpt=" Harness: failed to relaunch worker session", + session=worker_session, + ) + elif not reset_ok: + result = make_harness_result( + case, + error_kind="harness_start_sync_failed", + log_excerpt=( + " Harness: local MCC position did not stabilize at test start " + f"goal=({layout.start_x + 0.5:.1f},{float(layout.start_y):.1f},{layout.start_z + 0.5:.1f})" + ), + session=current_ctx.session, + log_path=str(current_ctx.mcc.log_file), + ) + else: + result = run_single_test( + case, + layout, + current_ctx.rcon, + current_ctx.mcc, + current_ctx.username, + wait_seconds, ) - finally: - cleanup_workers([ctx]) local_results.append(result) @@ -1342,10 +1585,7 @@ def worker_loop( _tprint(f" [W{worker_id}] >> Group failed -- " f"skipping larger values") - if results_path: - with results_lock: - with results_path.open("a") as f: - f.write(json.dumps(result_to_record(result, worker_id)) + "\n") + write_result(result) group_queue.task_done() @@ -1447,8 +1687,15 @@ def main() -> None: print(f" {c.case_id:<50} {metric}={c.gap_or_wall} [{marker}]{q}") return - rcon = RconClient(port=args.rcon_port, password=args.rcon_password) - rcon.connect() + try: + rcon = connect_rcon_with_retry( + port=args.rcon_port, + password=args.rcon_password, + timeout_seconds=30.0, + ) + except Exception as exc: + print(f"Harness error: RCON unavailable on localhost:{args.rcon_port}: {exc}") + sys.exit(2) rcon.command("difficulty peaceful") rcon.command("gamerule doMobSpawning false") @@ -1477,14 +1724,22 @@ def main() -> None: print(f"\nBuilt {len(all_cases)} courses.") return - results_path = Path(args.results) if args.results else None - if results_path: - results_path.parent.mkdir(parents=True, exist_ok=True) + run_dir = create_run_dir() + canonical_results_path = run_dir / "results.jsonl" + results_paths = [canonical_results_path] + if args.results: + results_paths.append(Path(args.results)) + for path in results_paths: + path.parent.mkdir(parents=True, exist_ok=True) # Phase 1: Clear region and build all courses up front print("=" * 60) print(" Phase 1: Building all courses") print("=" * 60) + print(f" Run artifacts: {run_dir}") + print(f" Results JSONL: {canonical_results_path}") + if args.results: + print(f" External Results JSONL: {Path(args.results)}") rcon.command(f"gamemode creative {args.username}") @@ -1507,9 +1762,9 @@ def main() -> None: n_parallel = args.parallel try: if n_parallel > 1: - _run_parallel(layouts, rcon, args, results_path, n_parallel) + _run_parallel(layouts, rcon, args, results_paths, run_dir, n_parallel) else: - _run_serial(layouts, rcon, args, results_path) + _run_serial(layouts, rcon, args, results_paths, run_dir) finally: builder.forceload_remove() @@ -1518,7 +1773,8 @@ def _run_serial( layouts: list[tuple[TestCase, CourseLayout]], rcon: RconClient, args: argparse.Namespace, - results_path: Path | None, + results_paths: list[Path], + run_dir: Path, ) -> None: """Original serial test execution path.""" session = resolve_session() @@ -1537,20 +1793,42 @@ def _run_serial( results: list[TestResult] = [] failed_groups: set[tuple] = set() skipped = 0 + serial_ctx = WorkerContext( + worker_id=0, + username=args.username, + session=session, + rcon=rcon, + mcc=mcc, + ) for i, (case, layout) in enumerate(layouts, 1): if should_skip(case, failed_groups): skipped += 1 print(f" [{i}/{len(layouts)}] {case.case_id} -- SKIPPED (group already failed)") + skipped_result = make_skip_result(case, "group_failed_earlier") + results.append(skipped_result) + append_jsonl_record(results_paths, result_to_record(skipped_result)) continue print(f"\n--- [{i}/{len(layouts)}] {case.case_id} (expect: {case.expected}) ---") print(f" Route: ({layout.start_x},{layout.start_y},{layout.start_z}) -> " f"({layout.end_x},{layout.end_y},{layout.end_z})") - result = run_single_test( - case, layout, rcon, mcc, args.username, args.wait, - ) + if reset_worker_state(serial_ctx, layout): + result = run_single_test( + case, layout, rcon, mcc, args.username, args.wait, + ) + else: + result = make_harness_result( + case, + error_kind="harness_start_sync_failed", + log_excerpt=( + " Harness: local MCC position did not stabilize at test start " + f"goal=({layout.start_x + 0.5:.1f},{float(layout.start_y):.1f},{layout.start_z + 0.5:.1f})" + ), + session=session, + log_path=str(mcc.log_file), + ) results.append(result) status = "OK" if result.matched_expected else "MISMATCH" @@ -1563,19 +1841,18 @@ def _run_serial( print(f" >> Group failed at {case.family}/{case.subfamily} " f"gap/wall={case.gap_or_wall} -- skipping larger values") - if results_path: - with results_path.open("a") as f: - f.write(json.dumps(result_to_record(result)) + "\n") + append_jsonl_record(results_paths, result_to_record(result)) rcon.close() - _print_summary(results, skipped) + _print_summary(results, skipped, run_dir) def _run_parallel( layouts: list[tuple[TestCase, CourseLayout]], rcon: RconClient, args: argparse.Namespace, - results_path: Path | None, + results_paths: list[Path], + run_dir: Path, n_parallel: int, ) -> None: """Parallel test execution with streaming worker launch. @@ -1615,7 +1892,7 @@ def _run_parallel( args=(i, args.username, run_token, args.version, args.server_port, args.rcon_port, args.rcon_password, group_q, all_results, results_lock, - args.wait, results_path, + args.wait, results_paths, workers_registry, registry_lock, skipped_counter), daemon=True, ) @@ -1634,20 +1911,23 @@ def _run_parallel( cleanup_workers(workers_registry) print(" All workers stopped.") - _print_summary(all_results, skipped=skipped_counter[0]) + _print_summary(all_results, skipped=skipped_counter[0], run_dir=run_dir) -def _print_summary(results: list[TestResult], skipped: int) -> None: +def _print_summary(results: list[TestResult], skipped: int, run_dir: Path) -> None: print("\n" + "=" * 60) print(" SUMMARY") print("=" * 60) passed = [r for r in results if r.matched_expected] failed = [r for r in results if not r.matched_expected] + summary = summarize_results([result_to_record(r) for r in results]) + write_summary_files(run_dir, summary) print(f"\n {len(passed)}/{len(results)} matched expectations") if skipped: print(f" {skipped} cases skipped (stop-at-first-failure)") + print(f" Summary dir: {run_dir}") if failed: print(f"\n MISMATCHES ({len(failed)}):") diff --git a/tools/tests/test_pathing_live_scripts.py b/tools/tests/test_pathing_live_scripts.py index 4d51e5cf..1106e3c5 100644 --- a/tools/tests/test_pathing_live_scripts.py +++ b/tools/tests/test_pathing_live_scripts.py @@ -1,8 +1,57 @@ import subprocess +import tempfile import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] class PathingLiveScriptTests(unittest.TestCase): + def test_prepare_offline_config_treats_existing_output_ini_as_output_not_template(self) -> None: + with tempfile.TemporaryDirectory() as tempdir: + temp_path = Path(tempdir) + output_ini = temp_path / "MinecraftClient.debug.ini" + output_ini.write_text( + "\n".join( + [ + "[Main.General]", + 'Account = { Login = "OldBot", Password = "" }', + 'AccountType = "microsoft"', + "", + "[Main.Advanced]", + 'MinecraftVersion = "auto"', + "TerrainAndMovements = false", + "InventoryHandling = false", + "EntityHandling = false", + "AutoRespawn = false", + "", + ] + ), + encoding="utf-8", + ) + + result = subprocess.run( + [ + "bash", + str(REPO_ROOT / ".skills/mcc-integration-testing/scripts/prepare_offline_mcc_config.sh"), + str(output_ini), + "1.21.11", + "MCCBot1", + ], + check=False, + capture_output=True, + text=True, + cwd=temp_path, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse((temp_path / "1.21.11").exists()) + + content = output_ini.read_text(encoding="utf-8") + self.assertIn('Account = { Login = "MCCBot1", Password = "-" }', content) + self.assertIn('AccountType = "mojang"', content) + self.assertIn('MinecraftVersion = "1.21.11"', content) + def test_test_parkour_lists_all_families(self) -> None: result = subprocess.run( ["python3", "tools/test-parkour.py", "--list-cases"], @@ -23,7 +72,7 @@ class PathingLiveScriptTests(unittest.TestCase): def test_test_parkour_linear_has_reject_at_max_plus_one(self) -> None: result = subprocess.run( - ["python3", "tools/test-parkour.py", "--list-cases", "--family", "linear"], + ["python3", "tools/test-parkour.py", "--list-cases", "--filter", "linear"], check=False, capture_output=True, text=True, @@ -35,18 +84,37 @@ class PathingLiveScriptTests(unittest.TestCase): self.assertIn("[PASS]", result.stdout) self.assertIn("[REJECT]", result.stdout) - def test_test_parkour_neo_covers_wall_range(self) -> None: + def test_test_parkour_linear_marks_live_boundary_cases_as_pass(self) -> None: result = subprocess.run( - ["python3", "tools/test-parkour.py", "--list-cases", "--family", "neo"], + ["python3", "tools/test-parkour.py", "--list-cases", "--filter", "linear"], check=False, capture_output=True, text=True, ) self.assertEqual(result.returncode, 0, result.stderr) - for w in range(5): + self.assertIn("linear-flat-gap4", result.stdout) + self.assertIn("linear-ascend-gap1-dy+1", result.stdout) + self.assertIn("linear-descend-gap2-dy-2", result.stdout) + self.assertIn("linear-descend-gap3-dy-1", result.stdout) + self.assertIn("linear-flat-gap4 gap=4 [PASS]", result.stdout) + self.assertIn("linear-ascend-gap1-dy+1 gap=1 [PASS]", result.stdout) + self.assertIn("linear-descend-gap2-dy-2 gap=2 [PASS]", result.stdout) + self.assertIn("linear-descend-gap3-dy-1 gap=3 [PASS]", result.stdout) + + def test_test_parkour_neo_covers_wall_range(self) -> None: + result = subprocess.run( + ["python3", "tools/test-parkour.py", "--list-cases", "--filter", "neo"], + check=False, + capture_output=True, + text=True, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + for w in range(1, 5): self.assertIn(f"neo-neo-wall{w}", result.stdout) - self.assertIn("neo-neo-wall5", result.stdout) + self.assertNotIn("neo-neo-wall0", result.stdout) + self.assertNotIn("neo-neo-wall5", result.stdout) self.assertIn("[REJECT]", result.stdout) def test_test_pathing_theory_neo_ceiling_lists_theory_cases(self) -> None: diff --git a/tools/tests/test_test_parkour_metrics.py b/tools/tests/test_test_parkour_metrics.py index 87ef0094..4c57d407 100644 --- a/tools/tests/test_test_parkour_metrics.py +++ b/tools/tests/test_test_parkour_metrics.py @@ -1,5 +1,8 @@ import importlib.util +import queue import sys +import threading +import tempfile import unittest from unittest import mock from pathlib import Path @@ -16,6 +19,20 @@ spec.loader.exec_module(module) class ParkourMetricsTests(unittest.TestCase): + class FakeRcon: + def __init__(self, responses: list[str | Exception]) -> None: + self._responses = list(responses) + self.commands: list[str] = [] + + def command(self, cmd: str) -> str: + self.commands.append(cmd) + if not self._responses: + return "ok" + response = self._responses.pop(0) + if isinstance(response, Exception): + raise response + return response + class FakeMccClient: def __init__(self, logs: list[str]) -> None: self._logs = logs @@ -119,6 +136,14 @@ class ParkourMetricsTests(unittest.TestCase): self.assertEqual(module.classify_outcome(metrics, near_goal=True), "fail") + def test_classify_outcome_prefers_harness_error(self) -> None: + metrics = module.LiveMetrics(route_complete_count=1, navigation_complete_count=1) + + self.assertEqual( + module.classify_outcome(metrics, near_goal=True, error_kind="harness_rcon_unavailable"), + "harness_rcon_unavailable", + ) + def test_classify_outcome_planner_reject_stays_reject(self) -> None: metrics = module.LiveMetrics(planner_reject_count=1) @@ -240,6 +265,247 @@ class ParkourMetricsTests(unittest.TestCase): self.assertTrue(synced) self.assertEqual(client.sent_commands, ["debug state"] * 6) + def test_wait_for_rcon_ready_retries_until_list_succeeds(self) -> None: + rcon = self.FakeRcon( + [ + ConnectionRefusedError("not ready"), + TimeoutError("still not ready"), + "There are 0 of a max of 20 players online", + ] + ) + + clock_ticks = [0] + + def fake_monotonic() -> float: + clock_ticks[0] += 1 + return clock_ticks[0] * 0.1 + + with mock.patch.object(module.time, "sleep", lambda _seconds: None): + with mock.patch.object(module.time, "monotonic", side_effect=fake_monotonic): + ready = module.wait_for_rcon_ready( + rcon, + timeout_seconds=1.0, + poll_interval=0.1, + ) + + self.assertTrue(ready) + self.assertEqual(rcon.commands, ["list", "list", "list"]) + + def test_connect_rcon_with_retry_retries_initial_connect(self) -> None: + events: list[str] = [] + attempts = {"count": 0} + + class FakeClient: + def connect(self) -> None: + attempts["count"] += 1 + events.append(f"connect-{attempts['count']}") + if attempts["count"] < 3: + raise ConnectionRefusedError("server not ready") + + def fake_factory(*_args, **_kwargs) -> FakeClient: + return FakeClient() + + clock_ticks = [0] + + def fake_monotonic() -> float: + clock_ticks[0] += 1 + return clock_ticks[0] * 0.1 + + with mock.patch.object(module.time, "sleep", lambda _seconds: None): + with mock.patch.object(module.time, "monotonic", side_effect=fake_monotonic): + client = module.connect_rcon_with_retry( + host="localhost", + port=25575, + password="test123", + timeout_seconds=1.0, + poll_interval=0.1, + client_factory=fake_factory, + ) + + self.assertIsNotNone(client) + self.assertEqual(events, ["connect-1", "connect-2", "connect-3"]) + + def test_make_skip_result_records_skip_reason(self) -> None: + case = module.TestCase( + case_id="linear-flat-gap4", + family="linear", + subfamily="flat", + gap_or_wall=4, + delta_y=0.0, + ceiling_height=None, + wall_offset=None, + expected="pass", + ) + + result = module.make_skip_result(case, "group_failed_earlier") + + self.assertEqual(result.outcome, "skipped") + self.assertEqual(result.skip_reason, "group_failed_earlier") + self.assertEqual(result.error_kind, None) + self.assertFalse(result.matched_expected) + + def test_result_to_record_includes_session_paths_and_duration(self) -> None: + case = module.TestCase( + case_id="linear-flat-gap1", + family="linear", + subfamily="flat", + gap_or_wall=1, + delta_y=0.0, + ceiling_height=None, + wall_offset=None, + expected="pass", + ) + result = module.TestResult( + case=case, + outcome="pass", + matched_expected=True, + replan_count=0, + turn_stall_count=0, + near_goal=True, + final_position=(109.5, 80.0, 200.5), + total_ticks=42, + session="parkour-run123-2", + log_path="/tmp/parkour-runs/run123/workers/2/worker.log", + event_log_path="/tmp/parkour-runs/run123/events.jsonl", + duration_ms=4200, + error_kind=None, + skip_reason=None, + ) + + record = module.result_to_record(result, worker_id=2) + + self.assertEqual(record["worker"], 2) + self.assertEqual(record["session"], "parkour-run123-2") + self.assertEqual(record["log_path"], "/tmp/parkour-runs/run123/workers/2/worker.log") + self.assertEqual(record["event_log_path"], "/tmp/parkour-runs/run123/events.jsonl") + self.assertEqual(record["duration_ms"], 4200) + self.assertEqual(record["skip_reason"], None) + self.assertEqual(record["error_kind"], None) + + def test_summarize_results_groups_outcomes_by_family(self) -> None: + summary = module.summarize_results( + [ + {"family": "linear", "outcome": "pass", "matched": True}, + {"family": "linear", "outcome": "reject", "matched": True}, + {"family": "linear", "outcome": "skipped", "matched": False}, + {"family": "neo", "outcome": "reject", "matched": False}, + ] + ) + + self.assertEqual(summary["total"], 4) + self.assertEqual(summary["matched"], 2) + self.assertEqual(summary["families"]["linear"]["outcomes"]["pass"], 1) + self.assertEqual(summary["families"]["linear"]["outcomes"]["skipped"], 1) + self.assertEqual(summary["families"]["neo"]["mismatches"], 1) + + def test_write_summary_files_persists_json_and_markdown(self) -> None: + summary = { + "total": 4, + "matched": 2, + "mismatched": 2, + "families": { + "linear": { + "total": 3, + "matched": 2, + "mismatches": 1, + "outcomes": {"pass": 1, "reject": 1, "skipped": 1}, + } + }, + } + + with tempfile.TemporaryDirectory() as tempdir: + run_dir = Path(tempdir) + module.write_summary_files(run_dir, summary) + + summary_json = run_dir / "summary.json" + summary_md = run_dir / "summary.md" + + self.assertTrue(summary_json.exists()) + self.assertTrue(summary_md.exists()) + self.assertIn('"total": 4', summary_json.read_text(encoding="utf-8")) + self.assertIn("linear", summary_md.read_text(encoding="utf-8")) + + def test_append_jsonl_record_writes_to_all_requested_paths(self) -> None: + record = {"case_id": "linear-flat-gap1", "outcome": "pass"} + + with tempfile.TemporaryDirectory() as tempdir: + base = Path(tempdir) + path1 = base / "results-a.jsonl" + path2 = base / "results-b.jsonl" + + module.append_jsonl_record([path1, path2], record) + + self.assertEqual(path1.read_text(encoding="utf-8"), path2.read_text(encoding="utf-8")) + self.assertIn('"case_id": "linear-flat-gap1"', path1.read_text(encoding="utf-8")) + + def test_worker_loop_reuses_one_worker_context_for_multiple_cases(self) -> None: + case1 = module.TestCase( + case_id="linear-flat-gap1", + family="linear", + subfamily="flat", + gap_or_wall=1, + delta_y=0.0, + ceiling_height=None, + wall_offset=None, + expected="pass", + ) + case2 = module.TestCase( + case_id="linear-flat-gap2", + family="linear", + subfamily="flat", + gap_or_wall=2, + delta_y=0.0, + ceiling_height=None, + wall_offset=None, + expected="pass", + ) + layout1 = module.CourseLayout(100, 80, 200, 109, 80, 200, (0, 0, 0), (0, 0, 0)) + layout2 = module.CourseLayout(100, 80, 210, 112, 80, 210, (0, 0, 0), (0, 0, 0)) + group_q: queue.Queue = queue.Queue() + group_q.put((case1.group_key(), [(case1, layout1)])) + group_q.put((case2.group_key(), [(case2, layout2)])) + + fake_ctx = module.WorkerContext( + worker_id=1, + username="MCCBot1", + session="parkour-run123-1", + rcon=mock.Mock(), + mcc=mock.Mock(), + ) + all_results: list[module.TestResult] = [] + skipped_counter = [0] + + def make_result(case: module.TestCase, *_args, **_kwargs) -> module.TestResult: + return module.TestResult(case=case, outcome="pass", matched_expected=True) + + with mock.patch.object(module, "launch_worker_context", return_value=fake_ctx) as launch_mock: + with mock.patch.object(module, "reset_worker_state", create=True, return_value=True) as reset_mock: + with mock.patch.object(module, "run_single_test", side_effect=make_result) as run_mock: + with mock.patch.object(module, "cleanup_workers") as cleanup_mock: + module.worker_loop( + worker_id=1, + base_username="MCCBot", + run_token="run123", + version="1.21.11-Vanilla", + server_port=25565, + rcon_port=25575, + rcon_password="test123", + group_queue=group_q, + all_results=all_results, + results_lock=threading.Lock(), + wait_seconds=15, + results_paths=[], + workers_registry=[], + registry_lock=threading.Lock(), + skipped_counter=skipped_counter, + ) + + self.assertEqual(launch_mock.call_count, 1) + self.assertEqual(reset_mock.call_count, 2) + self.assertEqual(run_mock.call_count, 2) + cleanup_mock.assert_not_called() + self.assertEqual(len(all_results), 2) + if __name__ == "__main__": unittest.main()