refactor: port tidy3d to pydantic v2

fix: rebase changes for v2

chore(tidy3d): FXC-4559-add-type-checking-import-moving-and-checking-to-pre-commit-and-ci

fix: minor fixes for v2

fix: stabilize model hashing in tests

fix: restore copy(True) compatibility and accept doping lists

fix(tidy3d): FXC-4824-fix-corrupted-data-in-vtk-arrays

fix: more rebase fixes for v2

fix(tidy3d): FXC-5094-fix-deprecation-warnings-in-tests

fix: even more rebase changes

final rebase changes

fix: repair pydantic v2 validators

chore: regenerate schemas

Co-authored-by: mahlau-flex <yannik.mahlau@flexcompute.com>

chore(ci): fix move-type-imports and pin actions

chore(ci): fix misleading python step name

perf(pydantic): defer model build

Fix missing Field reference in TCAD grid

chore: remove stale pd references

test: restore default pytest coverage

docs: remove stale adjoint plugin warning

Author: yaugenst-flex

.github/workflows/tidy3d-python-client-tests.yml

@@ -101,6 +101,9 @@ on:
 permissions:
   contents: read
 
+env:
+  VERIFICATIONS_PY_VERSION: '3.11'
+
 jobs:
   determine-test-scope:
     runs-on: ubuntu-latest
@@ -223,6 +226,39 @@ jobs:
           echo "version_match_tests=$version_match_tests"
           echo "extras_integration_tests=$extras_integration_tests"
           echo "test_type=$test_type"
+      
+
+  move-type-imports:
+    name: move-type-imports
+    needs: determine-test-scope
+    if: needs.determine-test-scope.outputs.code_quality_tests == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout-branch
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+        with:
+          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: setup-python
+        uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4.9.1
+        with:
+          python-version: ${{ env.VERIFICATIONS_PY_VERSION }}
+
+      - name: install-dependencies
+        run: |
+          set -euo pipefail
+          python -m venv .venv
+          source .venv/bin/activate
+          pip install libcst
+
+      - name: verify-type-import-guards
+        run: |
+          set -euo pipefail
+          source .venv/bin/activate
+          python scripts/move_type_imports.py --mode check_on_change
 
   lint:
     needs: determine-test-scope
@@ -256,10 +292,10 @@ jobs:
           submodules: false
           persist-credentials: false
 
-      - name: set-python-3.10
+      - name: setup-python-${{ env.VERIFICATIONS_PY_VERSION }}
         uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4.9.1
         with:
-          python-version: '3.10'
+          python-version: ${{ env.VERIFICATIONS_PY_VERSION }}
 
       - name: Install mypy
         run: |
@@ -442,7 +478,7 @@ jobs:
       
       - name: install-depedencies
         run: |
-          uv venv $GITHUB_WORKSPACE/.venv -p 3.11
+          uv venv $GITHUB_WORKSPACE/.venv -p "$VERIFICATIONS_PY_VERSION"
           source $GITHUB_WORKSPACE/.venv/bin/activate
           uv pip install -e "$GITHUB_WORKSPACE"
       
@@ -1025,6 +1061,7 @@ jobs:
       - determine-test-scope
       - local-tests
       - remote-tests
+      - move-type-imports
       - lint
       - mypy
       - verify-schema-change
@@ -1037,6 +1074,12 @@ jobs:
       - extras-integration-tests
     runs-on: ubuntu-latest
     steps:
+      - name: move-type-imports
+        if: ${{ needs.determine-test-scope.outputs.code_quality_tests == 'true' && needs.move-type-imports.result != 'success' && needs.move-type-imports.result != 'skipped' }}
+        run: |
+          echo "❌ Found imports used only for typing that are not guarded by if TYPE_CHECKING."
+          exit 1
+
       - name: check-linting-result
         if: ${{ needs.determine-test-scope.outputs.code_quality_tests == 'true' && needs.lint.result != 'success' && needs.lint.result != 'skipped' }}
         run: |

.gitignore

@@ -1,6 +1,7 @@
 # Batch JSON files
 batch*.json
 *.vtu
+simulation.json
 
 # Byte-compiled / optimized / DLL files
 *$py.class

.pre-commit-config.yaml

@@ -3,6 +3,14 @@ default_install_hook_types:
   - pre-commit
   - commit-msg
 repos:
+  - repo: local
+    hooks:
+      - id: move-type-imports
+        name: move type-only imports under TYPE_CHECKING
+        entry: poetry run python scripts/move_type_imports.py --mode fix --only-changed
+        language: system
+        pass_filenames: false
+        stages: [pre-commit]
   - repo: https://github.com/astral-sh/ruff-pre-commit
     rev: "v0.11.11"
     hooks:

pyproject.toml

@@ -33,7 +33,8 @@ numpy = ">=2.2.6,<2.5.0"
 matplotlib = "^3.10.0"
 shapely = "^2.0"
 pandas = "^2.2"
-pydantic = "^2.0"
+pydantic = ">=2.9,<3"
+pydantic-settings = "^2.10.0"
 PyYAML = "^6.0.3"
 dask = "^2025.12.0"
 toml = "^0.10.2"
@@ -175,6 +176,7 @@ dev = [
   'diff-cover',
   'openpyxl',
   'zizmor',
+  'libcst',
 ]
 tests = [
   'psutil',
@@ -309,7 +311,9 @@ banned-module-level-imports = ["scipy", "matplotlib"]
 
 [tool.pytest.ini_options]
 # TODO: remove --assert=plain when https://github.com/scipy/scipy/issues/22236 is resolved
-addopts = "--cov=tidy3d --doctest-modules -n auto --dist worksteal --assert=plain -m 'not numerical and not perf and not generate_profile and not performance_profile'"
+addopts = """
+--cov=tidy3d --doctest-modules -n auto --dist worksteal --assert=plain -m 'not numerical and not perf and not generate_profile and not performance_profile'
+"""
 markers = [
   "numerical: marks numerical tests for adjoint gradients that require running simulations (deselect with '-m \"not numerical\"')",
   "generate_profile: regenerate fixture data required by the performance tests (select with '-m \"generate_profile\"')",
@@ -336,82 +340,24 @@ python_files = "*.py"
 [tool.mypy]
 python_version = "3.10"
 files = [
-  "tidy3d/web",
-  "tidy3d/config",
-  "tidy3d/material_library",
-  "tidy3d/components/geometry"
+  "tidy3d",
 ]
 ignore_missing_imports = true
 follow_imports = "skip"
 disallow_untyped_defs = true
 disable_error_code = [
-  "abstract",
-  "annotation-unchecked",
   "arg-type",
-  "assert-type",
   "assignment",
   "attr-defined",
-  "await-not-async",
   "call-arg",
   "call-overload",
-  "comparison-overlap",
-  "dict-item",
-  "empty-body",
-  "exit-return",
-  "explicit-override",
-  "func-returns-value",
-  "has-type",
-  "ignore-without-code",
-  "import",
-  "import-not-found",
   "import-untyped",
   "index",
-  "list-item",
-  "literal-required",
-  "method-assign",
   "misc",
-  "mutable-override",
-  "name-defined",
-  "name-match",
-  "narrowed-type-not-subtype",
-  "no-any-return",
-  "no-any-unimported",
-  "no-overload-impl",
-  "no-redef",
-  "no-untyped-call",
   "operator",
-  "overload-cannot-match",
-  "overload-overlap",
   "override",
-  "possibly-undefined",
-  "prop-decorator",
-  "redundant-cast",
-  "redundant-expr",
-  "redundant-self",
-  "return",
   "return-value",
-  "safe-super",
-  "str-bytes-safe",
-  "str-format",
-  "syntax",
-  "top-level-await",
-  "truthy-bool",
-  "truthy-function",
-  "truthy-iterable",
-  "type-abstract",
-  "type-arg",
-  "type-var",
-  "typeddict-item",
-  "typeddict-readonly-mutated",
-  "typeddict-unknown-key",
-  "unimported-reveal",
   "union-attr",
-  "unreachable",
-  "unused-awaitable",
-  "unused-coroutine",
-  "unused-ignore",
-  "used-before-def",
-  "valid-newtype",
   "valid-type",
   "var-annotated",
 ]