|
2 | 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this |
3 | 3 | # file, You can obtain one at https://mozilla.org/MPL/2.0/. |
4 | 4 |
|
| 5 | +import dataclasses |
| 6 | +import json |
5 | 7 | import pathlib |
6 | 8 | import re |
7 | 9 | import tarfile |
| 10 | +from typing import Optional |
8 | 11 |
|
9 | 12 | import jsonschema |
10 | 13 | import yaml |
|
137 | 140 | }, |
138 | 141 | } |
139 | 142 |
|
| 143 | +STDLIB_TEST_ANNOTATION_COMMON_PROPERTIES = { |
| 144 | + "reason": {"type": "string"}, |
| 145 | + "targets": { |
| 146 | + "type": "array", |
| 147 | + "items": {"type": "string"}, |
| 148 | + }, |
| 149 | + "ignore-targets": { |
| 150 | + "type": "array", |
| 151 | + "items": {"type": "string"}, |
| 152 | + }, |
| 153 | + "minimum-python-version": {"type": "string"}, |
| 154 | + "maximum-python-version": {"type": "string"}, |
| 155 | + "build-option": {"type": "string"}, |
| 156 | + "no-build-option": {"type": "string"}, |
| 157 | +} |
| 158 | + |
| 159 | +STDLIB_TEST_ANNOTATIONS_SCHEMA = { |
| 160 | + "type": "object", |
| 161 | + "properties": { |
| 162 | + "harness-skips": { |
| 163 | + "type": "array", |
| 164 | + "items": { |
| 165 | + "type": "object", |
| 166 | + "properties": { |
| 167 | + "name": {"type": "string"}, |
| 168 | + **STDLIB_TEST_ANNOTATION_COMMON_PROPERTIES, |
| 169 | + }, |
| 170 | + "additionalProperties": False, |
| 171 | + "required": ["name", "reason"], |
| 172 | + }, |
| 173 | + }, |
| 174 | + "module-excludes": { |
| 175 | + "type": "array", |
| 176 | + "items": { |
| 177 | + "type": "object", |
| 178 | + "properties": { |
| 179 | + "module": {"type": "string"}, |
| 180 | + **STDLIB_TEST_ANNOTATION_COMMON_PROPERTIES, |
| 181 | + }, |
| 182 | + "additionalProperties": False, |
| 183 | + "required": ["module", "reason"], |
| 184 | + }, |
| 185 | + }, |
| 186 | + "expected-failures": { |
| 187 | + "type": "array", |
| 188 | + "items": { |
| 189 | + "type": "object", |
| 190 | + "properties": { |
| 191 | + "name": {"type": "string"}, |
| 192 | + "dont-verify": {"type": "boolean"}, |
| 193 | + "intermittent": {"type": "boolean"}, |
| 194 | + **STDLIB_TEST_ANNOTATION_COMMON_PROPERTIES, |
| 195 | + }, |
| 196 | + "additionalProperties": False, |
| 197 | + "required": ["name", "reason"], |
| 198 | + }, |
| 199 | + }, |
| 200 | + }, |
| 201 | +} |
140 | 202 |
|
141 | 203 | # Packages that define tests. |
142 | 204 | STDLIB_TEST_PACKAGES = { |
@@ -750,3 +812,139 @@ def extension_modules_config(yaml_path: pathlib.Path): |
750 | 812 | jsonschema.validate(data, EXTENSION_MODULES_SCHEMA) |
751 | 813 |
|
752 | 814 | return data |
| 815 | + |
| 816 | + |
| 817 | +TEST_ANNOTATION_HARNESS_SKIP = "harness-skip" |
| 818 | +TEST_ANNOTATION_MODULE_EXCLUDE = "module-exclude" |
| 819 | +TEST_ANNOTATION_TEST_FAILURE = "test-failure" |
| 820 | + |
| 821 | + |
| 822 | +@dataclasses.dataclass |
| 823 | +class TestAnnotation: |
| 824 | + # Describes the type of annotation. |
| 825 | + flavor: str |
| 826 | + # Name/pattern of test. |
| 827 | + name: str |
| 828 | + # Describes why the annotation exists. |
| 829 | + reason: str |
| 830 | + # Whether the test is expected to fail. |
| 831 | + expect_test_failure: bool |
| 832 | + # Whether to skip verification of failures. |
| 833 | + dont_verify: bool |
| 834 | + # Whether test failure is intermittent. Should only be true if |
| 835 | + # expect_test_failure also true. |
| 836 | + intermittent_test_failure: bool |
| 837 | + # Whether to exclude loading the test module when running tests. |
| 838 | + exclude_testing: bool |
| 839 | + |
| 840 | + |
| 841 | +@dataclasses.dataclass |
| 842 | +class TestAnnotations: |
| 843 | + annotations: list[TestAnnotation] |
| 844 | + |
| 845 | + def json_dump(self, of): |
| 846 | + data = [dataclasses.asdict(a) for a in self.annotations] |
| 847 | + json.dump(data, of, indent=2) |
| 848 | + |
| 849 | + |
| 850 | +def filter_stdlib_test_entry( |
| 851 | + flavor: str, |
| 852 | + test, |
| 853 | + python_version: str, |
| 854 | + target_triple: str, |
| 855 | + build_options: set[str], |
| 856 | +) -> Optional[TestAnnotation]: |
| 857 | + name = test["name"] |
| 858 | + |
| 859 | + if targets := test.get("targets"): |
| 860 | + matches_target = any(re.match(p, target_triple) for p in targets) |
| 861 | + else: |
| 862 | + matches_target = True |
| 863 | + |
| 864 | + for m in test.get("ignore-targets", []): |
| 865 | + if re.match(m, target_triple): |
| 866 | + matches_target = False |
| 867 | + |
| 868 | + if not matches_target: |
| 869 | + log(f"ignoring {flavor} rule (target doesn't match): {name}") |
| 870 | + return None |
| 871 | + |
| 872 | + python_minimum_version = test.get("minimum-python-version", "1.0") |
| 873 | + python_maximum_version = test.get("maximum-python-version", "100.0") |
| 874 | + |
| 875 | + if not meets_python_minimum_version(python_version, python_minimum_version): |
| 876 | + log( |
| 877 | + f"ignoring {flavor} rule ({python_version} < {python_minimum_version} (min)): {name}" |
| 878 | + ) |
| 879 | + return None |
| 880 | + |
| 881 | + if not meets_python_maximum_version(python_version, python_maximum_version): |
| 882 | + log( |
| 883 | + f"ignoring {flavor} rule ({python_version} > {python_maximum_version} (max)): {name}" |
| 884 | + ) |
| 885 | + return None |
| 886 | + |
| 887 | + if option := test.get("build-option"): |
| 888 | + if option not in build_options: |
| 889 | + log(f"ignoring {flavor} rule (build option {option} not present): {name}") |
| 890 | + return None |
| 891 | + |
| 892 | + if option := test.get("no-build-option"): |
| 893 | + if option in build_options: |
| 894 | + log(f"ignoring {flavor} rule (build option {option} is present): {name}") |
| 895 | + return None |
| 896 | + |
| 897 | + # Filtering complete. This rule applies to the current build. |
| 898 | + |
| 899 | + log(f"relevant {flavor} test rule: {name}: {test['reason']}") |
| 900 | + |
| 901 | + return TestAnnotation( |
| 902 | + flavor=flavor, |
| 903 | + name=name, |
| 904 | + reason=test["reason"], |
| 905 | + expect_test_failure=True, |
| 906 | + dont_verify=test.get("dont-verify", False), |
| 907 | + intermittent_test_failure=test.get("intermittent", False), |
| 908 | + exclude_testing=test.get("exclude", False), |
| 909 | + ) |
| 910 | + |
| 911 | + |
| 912 | +def stdlib_test_annotations( |
| 913 | + yaml_path: pathlib.Path, |
| 914 | + python_version: str, |
| 915 | + target_triple: str, |
| 916 | + build_options: set[str], |
| 917 | +) -> TestAnnotations: |
| 918 | + """Processes the test-annotations.yml file for a given build configuration.""" |
| 919 | + with yaml_path.open("r", encoding="utf-8") as fh: |
| 920 | + data = yaml.load(fh, Loader=yaml.SafeLoader) |
| 921 | + |
| 922 | + jsonschema.validate(data, STDLIB_TEST_ANNOTATIONS_SCHEMA) |
| 923 | + |
| 924 | + annotations = [] |
| 925 | + |
| 926 | + log(f"processing {len(data['expected-failures'])} stdlib test annotations") |
| 927 | + |
| 928 | + raw_entries = [] |
| 929 | + |
| 930 | + for entry in data["harness-skips"]: |
| 931 | + raw_entries.append((TEST_ANNOTATION_HARNESS_SKIP, entry)) |
| 932 | + |
| 933 | + for entry in data["module-excludes"]: |
| 934 | + entry["name"] = entry["module"] |
| 935 | + raw_entries.append((TEST_ANNOTATION_MODULE_EXCLUDE, entry)) |
| 936 | + |
| 937 | + for entry in data["expected-failures"]: |
| 938 | + raw_entries.append((TEST_ANNOTATION_TEST_FAILURE, entry)) |
| 939 | + |
| 940 | + for flavor, entry in raw_entries: |
| 941 | + if a := filter_stdlib_test_entry( |
| 942 | + flavor, |
| 943 | + entry, |
| 944 | + python_version, |
| 945 | + target_triple, |
| 946 | + build_options, |
| 947 | + ): |
| 948 | + annotations.append(a) |
| 949 | + |
| 950 | + return TestAnnotations(annotations) |
0 commit comments