From 2c1adf18a97e74202ded4c5c413ab20965a32a3c Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 4 Mar 2026 22:10:45 -0800 Subject: [PATCH 1/4] Add conformance results invariant validator --- .github/workflows/conformance.yml | 5 ++ .../results/mypy/generics_defaults.toml | 4 + .../mypy/generics_defaults_referential.toml | 3 + .../generics_defaults_specialization.toml | 3 + conformance/src/validate_results.py | 86 +++++++++++++++++++ 5 files changed, 101 insertions(+) create mode 100644 conformance/src/validate_results.py diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 8ad72d8d3..be45db465 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -32,6 +32,11 @@ jobs: uv sync --python 3.12 --frozen uv run --python 3.12 --frozen python src/main.py + - name: Validate conformance invariants + working-directory: conformance + run: | + uv run --python 3.12 --frozen python src/validate_results.py + - name: Assert conformance results are up to date run: | if [ -n "$(git status --porcelain -- conformance/results)" ]; then diff --git a/conformance/results/mypy/generics_defaults.toml b/conformance/results/mypy/generics_defaults.toml index efa709a2f..5b18af90b 100644 --- a/conformance/results/mypy/generics_defaults.toml +++ b/conformance/results/mypy/generics_defaults.toml @@ -1,4 +1,8 @@ conformant = "Partial" +notes = """ +Does not detect a TypeVar with a default used after a TypeVarTuple. +Does not fully support defaults on TypeVarTuple and ParamSpec. +""" output = """ generics_defaults.py:24: error: "T" cannot appear after "DefaultStrT" in type parameter list because it has no default type [misc] generics_defaults.py:66: error: "AllTheDefaults" expects between 2 and 5 type arguments, but 1 given [type-arg] diff --git a/conformance/results/mypy/generics_defaults_referential.toml b/conformance/results/mypy/generics_defaults_referential.toml index bad589849..7f28d7cc5 100644 --- a/conformance/results/mypy/generics_defaults_referential.toml +++ b/conformance/results/mypy/generics_defaults_referential.toml @@ -1,4 +1,7 @@ conformant = "Partial" +notes = """ +Does not correctly handle defaults referencing other TypeVars. +""" output = """ generics_defaults_referential.py:23: error: Expression is of type "type[slice[StartT, StopT, StepT]]", not "type[slice[int, int, int | None]]" [assert-type] generics_defaults_referential.py:37: error: Argument 1 to "Foo" has incompatible type "str"; expected "int" [arg-type] diff --git a/conformance/results/mypy/generics_defaults_specialization.toml b/conformance/results/mypy/generics_defaults_specialization.toml index b29ba389c..a40e0dc91 100644 --- a/conformance/results/mypy/generics_defaults_specialization.toml +++ b/conformance/results/mypy/generics_defaults_specialization.toml @@ -1,4 +1,7 @@ conformant = "Partial" +notes = """ +Does not correctly resolve defaults when classes are used directly. +""" output = """ generics_defaults_specialization.py:30: error: Bad number of arguments for type alias, expected between 0 and 1, given 2 [type-arg] generics_defaults_specialization.py:45: error: Expression is of type "type[Bar[DefaultStrT]]", not "type[Bar[str]]" [assert-type] diff --git a/conformance/src/validate_results.py b/conformance/src/validate_results.py new file mode 100644 index 000000000..8361dbc7a --- /dev/null +++ b/conformance/src/validate_results.py @@ -0,0 +1,86 @@ +""" +Validate invariants for conformance result files. +""" + +from pathlib import Path +import sys +import tomllib +from typing import Any + + +def main() -> int: + results_dir = Path(__file__).resolve().parent.parent / "results" + issues: list[str] = [] + checked = 0 + + for type_checker_dir in sorted(results_dir.iterdir()): + if not type_checker_dir.is_dir(): + continue + for file in sorted(type_checker_dir.iterdir()): + if file.name == "version.toml": + continue + checked += 1 + try: + with file.open("rb") as f: + info = tomllib.load(f) + except Exception as e: + issues.append(f"{file.relative_to(results_dir)}: failed to parse TOML ({e})") + continue + + issues.extend(_validate_result(file, results_dir, info)) + + if issues: + print(f"Found {len(issues)} invariant violation(s) across {checked} file(s):") + for issue in issues: + print(f"- {issue}") + return 1 + + print(f"Validated {checked} conformance result file(s); no invariant violations found.") + return 0 + + +def _validate_result(file: Path, results_dir: Path, info: dict[str, Any]) -> list[str]: + issues: list[str] = [] + rel_path = file.relative_to(results_dir) + + automated = info.get("conformance_automated") + if automated not in {"Pass", "Fail"}: + issues.append( + f"{rel_path}: conformance_automated must be 'Pass' or 'Fail' (got {automated!r})" + ) + return issues + automated_is_pass = automated == "Pass" + + conformant = info.get("conformant") + if conformant is None: + if automated_is_pass: + conformant_is_pass = True + else: + issues.append( + f"{rel_path}: conformant is required when conformance_automated is 'Fail'" + ) + return issues + elif isinstance(conformant, str): + conformant_is_pass = conformant == "Pass" + else: + issues.append(f"{rel_path}: conformant must be a string when present") + return issues + + if conformant_is_pass != automated_is_pass: + issues.append( + f"{rel_path}: conformant={conformant!r} does not match " + f"conformance_automated={automated!r}" + ) + + if not conformant_is_pass: + notes = info.get("notes", "") + if not isinstance(notes, str) or not notes.strip(): + issues.append( + f"{rel_path}: notes must be present when checker is not fully conformant" + ) + + return issues + + +if __name__ == "__main__": + raise SystemExit(main()) From 64990f86ce59556f9786fced710cb6fdc81c54fb Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 4 Mar 2026 22:13:06 -0800 Subject: [PATCH 2/4] another check --- conformance/results/mypy/constructors_call_metaclass.toml | 2 +- conformance/src/validate_results.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/conformance/results/mypy/constructors_call_metaclass.toml b/conformance/results/mypy/constructors_call_metaclass.toml index 8ee382dfc..933983b1f 100644 --- a/conformance/results/mypy/constructors_call_metaclass.toml +++ b/conformance/results/mypy/constructors_call_metaclass.toml @@ -1,4 +1,4 @@ -conformant = "Unupported" +conformant = "Unsupported" notes = """ Does not honor metaclass __call__ method when evaluating constructor call. Does not skip evaluation of __new__ and __init__ if custom metaclass call returns non-class. diff --git a/conformance/src/validate_results.py b/conformance/src/validate_results.py index 8361dbc7a..49a3fc163 100644 --- a/conformance/src/validate_results.py +++ b/conformance/src/validate_results.py @@ -61,6 +61,8 @@ def _validate_result(file: Path, results_dir: Path, info: dict[str, Any]) -> lis ) return issues elif isinstance(conformant, str): + if conformant not in ("Pass", "Partial", "Unsupported"): + issues.append(f"{rel_path}: invalid conformance status {conformant!r}") conformant_is_pass = conformant == "Pass" else: issues.append(f"{rel_path}: conformant must be a string when present") From 585c7ec26982778903b7aa88ec381608b996870e Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Wed, 4 Mar 2026 22:16:13 -0800 Subject: [PATCH 3/4] regen --- conformance/results/results.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/conformance/results/results.html b/conformance/results/results.html index 058d7e553..ec78463ae 100644 --- a/conformance/results/results.html +++ b/conformance/results/results.html @@ -263,19 +263,19 @@

Python Type System Conformance Test Results

Partial

Incorrect rejects + between two AnyStr

Constrained type var resolves to subtype instead of explcitly listed constraint

     generics_defaults -Partial +
Partial

Does not detect a TypeVar with a default used after a TypeVarTuple.

Does not fully support defaults on TypeVarTuple and ParamSpec.

Pass Pass Pass      generics_defaults_referential -Partial +
Partial

Does not correctly handle defaults referencing other TypeVars.

Pass Pass Pass      generics_defaults_specialization -Partial +
Partial

Does not correctly resolve defaults when classes are used directly.

Pass Pass Pass @@ -644,7 +644,7 @@

Python Type System Conformance Test Results

Pass      constructors_call_metaclass -
Unupported

Does not honor metaclass __call__ method when evaluating constructor call.

Does not skip evaluation of __new__ and __init__ if custom metaclass call returns non-class.

+
Unsupported

Does not honor metaclass __call__ method when evaluating constructor call.

Does not skip evaluation of __new__ and __init__ if custom metaclass call returns non-class.

Pass Pass Pass From 1eaa23489f834511c5a5cdfa4fbae2b58e065a79 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 5 Mar 2026 07:48:45 -0800 Subject: [PATCH 4/4] reject unrecognized keys --- conformance/src/validate_results.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/conformance/src/validate_results.py b/conformance/src/validate_results.py index 49a3fc163..061808336 100644 --- a/conformance/src/validate_results.py +++ b/conformance/src/validate_results.py @@ -7,6 +7,17 @@ import tomllib from typing import Any +ALLOWED_RESULT_KEYS = frozenset( + { + "conformance_automated", + "conformant", + "errors_diff", + "ignore_errors", + "notes", + "output", + } +) + def main() -> int: results_dir = Path(__file__).resolve().parent.parent / "results" @@ -43,6 +54,12 @@ def _validate_result(file: Path, results_dir: Path, info: dict[str, Any]) -> lis issues: list[str] = [] rel_path = file.relative_to(results_dir) + unknown_keys = sorted(set(info) - ALLOWED_RESULT_KEYS) + if unknown_keys: + issues.append( + f"{rel_path}: unrecognized key(s): {', '.join(repr(key) for key in unknown_keys)}" + ) + automated = info.get("conformance_automated") if automated not in {"Pass", "Fail"}: issues.append(