tests: add tests summary

tests/__main__.py

@@ -2,7 +2,7 @@ import sys
 from typing import Type
 
 from midas.cli.ansi import Ansi
-from tests.base import Tester
+from tests.base import Tester, TestsSummary
 from tests.checker import CheckerTester
 from tests.generator import GeneratorTester
 from tests.midas import MidasTester
@@ -16,12 +16,12 @@ def print_banner(name: str):
     print(horizontal)
 
 
-def run_tests(tester_cls: Type[Tester]) -> bool:
+def run_tests(tester_cls: Type[Tester]) -> TestsSummary:
     print_banner(tester_cls.__name__)
     tester: Tester = tester_cls()
-    success: bool = tester.run_all_tests()
+    summary: TestsSummary = tester.run_all_tests()
     print()
-    return success
+    return summary
 
 
 def main():
@@ -32,12 +32,16 @@ def main():
         GeneratorTester,
     ]
 
-    success: bool = all(list(map(run_tests, testers)))  # list to avoid early stop
+    summaries: list[TestsSummary] = list(
+        map(run_tests, testers)
+    )  # list to avoid early stop
+    summary: TestsSummary = TestsSummary.concat(*summaries)
 
-    if success:
+    if summary.success:
         print(Ansi.FG(Ansi.BRIGHT_GREEN) + "All tests passed!" + Ansi.RESET)
     else:
         print(Ansi.FG(Ansi.BRIGHT_RED) + "Some tests failed!" + Ansi.RESET)
+        summary.print()
         sys.exit(1)
 
 

tests/base.py

@@ -4,6 +4,7 @@ import argparse
 import difflib
 import sys
 from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Iterator, Protocol
 
@@ -14,6 +15,43 @@ class CaseResult(Protocol):
     def dumps(self) -> str: ...
 
 
+@dataclass
+class TestsSummary:
+    tests: list[tuple[str, bool]] = field(default_factory=list)
+
+    @property
+    def success(self) -> bool:
+        return all(map(lambda t: t[1], self.tests))
+
+    @property
+    def successes(self) -> int:
+        return sum(map(lambda t: int(t[1]), self.tests))
+
+    @property
+    def failures(self) -> int:
+        return len(self.tests) - self.successes
+
+    def add(self, name: str, success: bool):
+        self.tests.append((name, success))
+
+    @staticmethod
+    def concat(*summaries: TestsSummary) -> TestsSummary:
+        return TestsSummary(
+            tests=sum(
+                map(lambda s: s.tests, summaries),
+                start=[],
+            ),
+        )
+
+    def print(self):
+        print("Tests summary:")
+        tests: list[tuple[str, bool]] = sorted(self.tests, key=lambda t: t[0])
+        for test, success in tests:
+            print(f" - [{'.' if success else 'X'}] {test}")
+        print("-" * 20)
+        print(f"passed: {self.successes}, failed: {self.failures}")
+
+
 class Tester(ABC):
     """A test runner to check for regressions in the lexer and parser"""
 
@@ -34,40 +72,37 @@ class Tester(ABC):
     @abstractmethod
     def _list_tests(self) -> list[Path]: ...
 
-    def run_all_tests(self) -> bool:
+    def run_all_tests(self) -> TestsSummary:
         paths: list[Path] = sorted(self._list_tests())
         return self.run_tests(paths)
 
-    def run_tests(self, tests: list[Path]) -> bool:
+    def run_tests(self, tests: list[Path]) -> TestsSummary:
         rule: str = "-" * 80
         n: int = len(tests)
-        successes: int = 0
-        failures: int = 0
+
+        summary: TestsSummary = TestsSummary()
 
         print(rule)
         for i, test in enumerate(tests):
             path: Path = test.resolve().relative_to(self.CASES_DIR)
             print(f"{Ansi.FG(Ansi.BRIGHT_CYAN)}Case {i+1}/{n}: {path}{Ansi.RESET}")
-            print(Ansi.DIM, end="")
             success: bool = self._run_test(test)
-            print(Ansi.RESET, end="")
-            if success:
-                successes += 1
-            else:
-                failures += 1
+            summary.add(str(path), success)
 
         print(rule)
-        print(f"Success: {successes}/{n}")
-        print(f"Failed: {failures}/{n}")
+        print(f"Success: {summary.successes}/{n}")
+        print(f"Failed: {summary.failures}/{n}")
         print(rule)
-        return failures == 0
+        return summary
 
     def _run_test(self, path: Path) -> bool:
         result_path: Path = self._result_path(path)
         if not result_path.exists():
             print("Missing snapshot. Please run the update command first")
             return False
+        print(Ansi.DIM, end="")
         result: CaseResult = self._exec_case(path)
+        print(Ansi.RESET, end="")
         expected: str = result_path.read_text()
         actual: str = result.dumps()
 
@@ -143,16 +178,18 @@ class Tester(ABC):
                 else:
                     tester.update_tests(args.FILE)
             case "run":
-                success: bool
+                summary: TestsSummary
                 if args.all:
-                    success = tester.run_all_tests()
+                    summary = tester.run_all_tests()
                 else:
-                    success = tester.run_tests(args.FILE)
-                if not success:
+                    summary = tester.run_tests(args.FILE)
+                if not summary.success:
+                    summary.print()
                     sys.exit(1)
             case None:
-                success: bool = tester.run_all_tests()
-                if not success:
+                summary: TestsSummary = tester.run_all_tests()
+                if not summary.success:
+                    summary.print()
                     sys.exit(1)
             case _:
                 print(f"Unknown subcommand '{args.subcommand}'")