Update python to 3.12.4 (#3880)

* update python to 3.12

* drop down to 3.12.4 for compatibility

* setup rust toolchain for deps

* fix typo

* fix linter

* fixing linter

* adding rust dep

* trying to fix circleci

* trying to fix circleci

* trying to fix circleci

* trying to fix circleci

* trying to fix circleci

* trying to fix circleci

* trying to fix circleci

* trying to fix circleci

* trying to fix circleci

* trying to fix circleci

* trying to fix circleci

* trying to fix circleci

* trying to fix circleci

* trying to fix circleci

* trying to fix circleci

* trying to fix circleci

* trying new image

* trying new image

* trying new image

* trying new image

* invalidate cache

* bump reqs

* remove cache invalidation

* fix cache name

* update to use latest image

Author: parithosh

.circleci/config.yml

@@ -60,7 +60,7 @@ commands:
 jobs:
   checkout_specs:
     docker:
-      - image: circleci/python:3.9
+      - image: cimg/python:3.12.4
     working_directory: ~/specs-repo
     steps:
       # Restore git repo at point close to target branch/revision, to speed up checkout
@@ -80,19 +80,19 @@ jobs:
             - ~/specs-repo
   install_pyspec_test:
     docker:
-      - image: circleci/python:3.9
+      - image: ethpandaops/circleci-python-rust:latest
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
           key: v3-specs-repo-{{ .Branch }}-{{ .Revision }}
       - restore_pyspec_cached_venv
       - run:
           name: Install pyspec requirements
-          command: make install_test
+          command: make install_test && make preinstallation
       - save_pyspec_cached_venv
   test-phase0:
     docker:
-      - image: circleci/python:3.9
+      - image: cimg/python:3.12.4
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
@@ -105,7 +105,7 @@ jobs:
           path: tests/core/pyspec/test-reports
   test-altair:
     docker:
-      - image: circleci/python:3.9
+      - image: cimg/python:3.12.4
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
@@ -118,7 +118,7 @@ jobs:
           path: tests/core/pyspec/test-reports
   test-bellatrix:
     docker:
-      - image: circleci/python:3.9
+      - image: cimg/python:3.12.4
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
@@ -131,7 +131,7 @@ jobs:
           path: tests/core/pyspec/test-reports
   test-capella:
     docker:
-      - image: circleci/python:3.9
+      - image: cimg/python:3.12.4
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
@@ -144,7 +144,7 @@ jobs:
           path: tests/core/pyspec/test-reports
   test-deneb:
     docker:
-      - image: circleci/python:3.9
+      - image: cimg/python:3.12.4
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
@@ -157,7 +157,7 @@ jobs:
           path: tests/core/pyspec/test-reports
   test-electra:
     docker:
-      - image: circleci/python:3.9
+      - image: cimg/python:3.12.4
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
@@ -170,7 +170,7 @@ jobs:
           path: tests/core/pyspec/test-reports
   test-whisk:
     docker:
-      - image: circleci/python:3.9
+      - image: cimg/python:3.12.4
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
@@ -183,7 +183,7 @@ jobs:
           path: tests/core/pyspec/test-reports
   test-eip7594:
     docker:
-      - image: circleci/python:3.9
+      - image: cimg/python:3.12.4
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
@@ -205,7 +205,7 @@ jobs:
           command: sudo npm install -g doctoc@2.2.0 && make check_toc
   codespell:
     docker:
-      - image: circleci/python:3.9
+      - image: cimg/python:3.12.4
     working_directory: ~/specs-repo
     steps:
       - checkout
@@ -214,7 +214,7 @@ jobs:
           command: pip install 'codespell<3.0.0,>=2.0.0' --user &&  make codespell
   lint:
     docker:
-      - image: circleci/python:3.9
+      - image: cimg/python:3.12.4
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
@@ -270,7 +270,7 @@ jobs:
             - /nix
   install_deposit_contract_web3_tester:
     docker:
-      - image: circleci/python:3.9
+      - image: cimg/python:3.12.4
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
@@ -282,7 +282,7 @@ jobs:
       - save_deposit_contract_tester_cached_venv
   test_deposit_contract_web3_tests:
     docker:
-      - image: circleci/python:3.9
+      - image: cimg/python:3.12.4
     working_directory: ~/specs-repo
     steps:
       - restore_cache:

.github/workflows/generate_vectors.yml

@@ -32,7 +32,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
+          python-version: '3.12.4'
           cache: ''
       - name: Clean up Spec Repository
         run: |

.github/workflows/run-tests.yml

@@ -45,7 +45,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
+          python-version: '3.12.4'
           cache: ''
       - name: Check codespell
         run: make codespell
@@ -55,10 +55,12 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
+      - name: Setup Rust for dependencies
+        uses: actions-rust-lang/setup-rust-toolchain@v1
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
+          python-version: '3.12.4'
           cache: ''
       - name: Install pyspec requirements
         run: make install_test
@@ -76,10 +78,12 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
+      - name: Setup Rust for dependencies
+        uses: actions-rust-lang/setup-rust-toolchain@v1
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
+          python-version: '3.12.4'
           cache: ''
       - name: set TEST_PRESET_TYPE
         if: github.event.inputs.test_preset_type != ''

docker/Dockerfile

@@ -1,5 +1,5 @@
 # Rename the build stage from 'base' to 'builder' for clarification and code readability
-FROM python:3.11.0-slim-bullseye as builder
+FROM python:3.12.4-slim-bullseye as builder
 
 ENV DEBIAN_FRONTEND=noninteractive \
     WORKDIR=/consensus-specs  \

requirements_preinstallation.txt

@@ -1,4 +1,4 @@
-pip>=23.1.2
-wheel>=0.40.0
-setuptools>=68.0.0
-pylint>=3.0.0
+pip>=24.0.0
+wheel>=0.44.0
+setuptools>=72.0.0
+pylint>=3.2.0

tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py

@@ -132,7 +132,7 @@ def should_skip_case_dir(case_dir, is_force, diagnostics_obj):
             print(f'Skipping already existing test: {case_dir}')
             is_skip = True
         else:
-            print(f'Warning, output directory {case_dir} already exist,'
+            print(f'Warning, output directory {case_dir} already exist, '
                   ' old files will be deleted and it will generate test vector files with the latest version')
             # Clear the existing case_dir folder
             shutil.rmtree(case_dir)
@@ -356,7 +356,7 @@ def generate_test_vector(test_case, case_dir, log_file, file_mode):
     test_end = time.time()
     span = round(test_end - test_start, 2)
     if span > TIME_THRESHOLD_TO_PRINT:
-        print(f'    - generated in {span} seconds')
+        print(f'- generated in {span} seconds')
 
     return result
 

tests/generators/ssz_generic/ssz_basic_vector.py

@@ -33,7 +33,7 @@ def valid_cases():
         for length in [1, 2, 3, 4, 5, 8, 16, 31, 512, 513]:
             for mode in random_modes:
                 yield f'vec_{name}_{length}_{mode.to_name()}', \
-                      valid_test_case(lambda: basic_vector_case_fn(rng, mode, typ, length))
+                    valid_test_case(lambda: basic_vector_case_fn(rng, mode, typ, length))
 
 
 def invalid_cases():
@@ -52,14 +52,14 @@ def invalid_cases():
                 if length == 1:
                     # empty bytes, no elements. It may seem valid, but empty fixed-size elements are not valid SSZ.
                     yield f'vec_{name}_{length}_{mode.to_name()}_one_less', \
-                          invalid_test_case(lambda: b"")
+                        invalid_test_case(lambda: b"")
                 else:
                     yield f'vec_{name}_{length}_{mode.to_name()}_one_less', \
-                          invalid_test_case(lambda: serialize(basic_vector_case_fn(rng, mode, typ, length - 1)))
+                        invalid_test_case(lambda: serialize(basic_vector_case_fn(rng, mode, typ, length - 1)))
                 yield f'vec_{name}_{length}_{mode.to_name()}_one_more', \
-                      invalid_test_case(lambda: serialize(basic_vector_case_fn(rng, mode, typ, length + 1)))
+                    invalid_test_case(lambda: serialize(basic_vector_case_fn(rng, mode, typ, length + 1)))
                 yield f'vec_{name}_{length}_{mode.to_name()}_one_byte_less', \
-                      invalid_test_case(lambda: serialize(basic_vector_case_fn(rng, mode, typ, length))[:-1])
+                    invalid_test_case(lambda: serialize(basic_vector_case_fn(rng, mode, typ, length))[:-1])
                 yield f'vec_{name}_{length}_{mode.to_name()}_one_byte_more', \
-                      invalid_test_case(lambda: serialize(basic_vector_case_fn(rng, mode, typ, length))
-                                        + serialize(basic_vector_case_fn(rng, mode, uint8, 1)))
+                    invalid_test_case(lambda: serialize(basic_vector_case_fn(rng, mode, typ, length))
+                                      + serialize(basic_vector_case_fn(rng, mode, uint8, 1)))

tests/generators/ssz_generic/ssz_bitlist.py

@@ -22,7 +22,7 @@ def valid_cases():
                          RandomizationMode.mode_zero,
                          RandomizationMode.mode_max]:
                 yield f'bitlist_{size}_{mode.to_name()}_{variation}', \
-                      valid_test_case(lambda: bitlist_case_fn(rng, mode, size))
+                    valid_test_case(lambda: bitlist_case_fn(rng, mode, size))
 
 
 def invalid_cases():
@@ -33,5 +33,5 @@ def invalid_cases():
     for (typ_limit, test_limit) in [(1, 2), (1, 8), (1, 9), (2, 3), (3, 4), (4, 5),
                                     (5, 6), (8, 9), (32, 64), (32, 33), (512, 513)]:
         yield f'bitlist_{typ_limit}_but_{test_limit}', \
-              invalid_test_case(lambda: serialize(
-                  bitlist_case_fn(rng, RandomizationMode.mode_max_count, test_limit)))
+            invalid_test_case(lambda: serialize(
+                bitlist_case_fn(rng, RandomizationMode.mode_max_count, test_limit)))

tests/generators/ssz_generic/ssz_bitvector.py

@@ -38,5 +38,5 @@ def invalid_cases():
                                   (5, 6), (8, 9), (9, 8), (16, 8), (32, 33), (512, 513)]:
         for mode in [RandomizationMode.mode_random, RandomizationMode.mode_zero, RandomizationMode.mode_max]:
             yield f'bitvec_{typ_size}_{mode.to_name()}_{test_size}', \
-                  invalid_test_case(lambda: serialize(bitvector_case_fn(rng, mode, test_size,
-                                                                        invalid_making_pos=typ_size)))
+                invalid_test_case(lambda: serialize(bitvector_case_fn(rng, mode, test_size,
+                                                    invalid_making_pos=typ_size)))

tests/generators/ssz_generic/ssz_container.py

@@ -46,7 +46,7 @@ class BitsStruct(Container):
     E: Bitvector[8]
 
 
-def container_case_fn(rng: Random, mode: RandomizationMode, typ: Type[View], chaos: bool=False):
+def container_case_fn(rng: Random, mode: RandomizationMode, typ: Type[View], chaos: bool = False):
     return get_random_ssz_object(rng, typ,
                                  max_bytes_length=2000,
                                  max_list_length=2000,
@@ -77,15 +77,15 @@ def valid_cases():
         for mode in modes:
             for variation in range(3):
                 yield f'{name}_{mode.to_name()}_chaos_{variation}', \
-                      valid_test_case(lambda: container_case_fn(rng, mode, typ, chaos=True))
+                    valid_test_case(lambda: container_case_fn(rng, mode, typ, chaos=True))
         # Notes: Below is the second wave of iteration, and only the random mode is selected
         # for container without offset since ``RandomizationMode.mode_zero`` and ``RandomizationMode.mode_max``
         # are deterministic.
         modes = [RandomizationMode.mode_random] if len(offsets) == 0 else list(RandomizationMode)
         for mode in modes:
             for variation in range(10):
                 yield f'{name}_{mode.to_name()}_{variation}', \
-                      valid_test_case(lambda: container_case_fn(rng, mode, typ))
+                    valid_test_case(lambda: container_case_fn(rng, mode, typ))
 
 
 def mod_offset(b: bytes, offset_index: int, change: Callable[[int], int]):
@@ -100,8 +100,8 @@ def invalid_cases():
     for (name, (typ, offsets)) in PRESET_CONTAINERS.items():
         # using mode_max_count, so that the extra byte cannot be picked up as normal list content
         yield f'{name}_extra_byte', \
-              invalid_test_case(lambda: serialize(
-                  container_case_fn(rng, RandomizationMode.mode_max_count, typ)) + b'\xff')
+            invalid_test_case(lambda: serialize(
+                container_case_fn(rng, RandomizationMode.mode_max_count, typ)) + b'\xff')
 
         if len(offsets) != 0:
             # Note: there are many more ways to have invalid offsets,
@@ -112,23 +112,22 @@ def invalid_cases():
                          RandomizationMode.mode_max_count]:
                 for index, offset_index in enumerate(offsets):
                     yield f'{name}_{mode.to_name()}_offset_{offset_index}_plus_one', \
-                          invalid_test_case(lambda: mod_offset(
-                              b=serialize(container_case_fn(rng, mode, typ)),
-                              offset_index=offset_index,
-                              change=lambda x: x + 1
-                          ))
+                        invalid_test_case(lambda: mod_offset(
+                            b=serialize(container_case_fn(rng, mode, typ)),
+                            offset_index=offset_index,
+                            change=lambda x: x + 1))
                     yield f'{name}_{mode.to_name()}_offset_{offset_index}_zeroed', \
-                          invalid_test_case(lambda: mod_offset(
-                              b=serialize(container_case_fn(rng, mode, typ)),
-                              offset_index=offset_index,
-                              change=lambda x: 0
-                          ))
+                        invalid_test_case(lambda: mod_offset(
+                            b=serialize(container_case_fn(rng, mode, typ)),
+                            offset_index=offset_index,
+                            change=lambda x: 0
+                        ))
                     if index == 0:
                         yield f'{name}_{mode.to_name()}_offset_{offset_index}_minus_one', \
                             invalid_test_case(lambda: mod_offset(
-                              b=serialize(container_case_fn(rng, mode, typ)),
-                              offset_index=offset_index,
-                              change=lambda x: x - 1
+                                b=serialize(container_case_fn(rng, mode, typ)),
+                                offset_index=offset_index,
+                                change=lambda x: x - 1
                             ))
                     if mode == RandomizationMode.mode_max_count:
                         serialized = serialize(container_case_fn(rng, mode, typ))