Merge remote-tracking branch 'upstream/main' into feat/pep770-sbom

Author: fangchenli

.gitattributes

@@ -13,7 +13,6 @@
 *.dta binary
 *.xls binary
 *.xlsx binary
-pandas/_version.py export-subst
 
 
 *.bz2 export-ignore
@@ -72,7 +71,6 @@ web export-ignore
 CITATION.cff export-ignore
 codecov.yml export-ignore
 environment.yml export-ignore
-setup.py export-ignore
 
 
 # GH 39321

.github/PULL_REQUEST_TEMPLATE.md

@@ -3,4 +3,5 @@
 - [ ] All [code checks passed](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#pre-commit).
 - [ ] Added [type annotations](https://pandas.pydata.org/pandas-docs/dev/development/contributing_codebase.html#type-hints) to new arguments/methods/functions.
 - [ ] Added an entry in the latest `doc/source/whatsnew/vX.X.X.rst` file if fixing a bug or adding a new feature.
+- [ ] I have reviewed and followed all the [contribution guidelines](https://pandas.pydata.org/docs/dev/development/contributing.html)
 - [ ] If I used AI to develop this pull request, I prompted it to follow `AGENTS.md`.

.github/workflows/code-checks.yml

@@ -43,15 +43,8 @@ jobs:
     - name: Build Pandas
       id: build
       uses: ./.github/actions/build_pandas
-      with:
-        editable: false
 
     # The following checks are independent of each other and should still be run if one fails
-
-    # TODO: The doctests have to be run first right now, since the Cython doctests only work
-    # with pandas installed in non-editable mode
-    # This can be removed once pytest-cython doesn't require C extensions to be installed inplace
-
     - name: Extra installs
       # https://pytest-qt.readthedocs.io/en/latest/troubleshooting.html#github-actions-azure-pipelines-travis-ci-and-gitlab-ci-cd
       run: sudo apt-get update && sudo apt-get install -y libegl1 libopengl0
@@ -60,13 +53,6 @@ jobs:
       run: cd ci && ./code_checks.sh doctests
       if: ${{ steps.build.outcome == 'success' && always() }}
 
-    - name: Install pandas in editable mode
-      id: build-editable
-      if: ${{ steps.build.outcome == 'success' && always() }}
-      uses: ./.github/actions/build_pandas
-      with:
-        editable: true
-
     - name: Check for no warnings when building single-page docs
       run: ci/code_checks.sh single-docs
       if: ${{ steps.build.outcome == 'success' && always() }}

.github/workflows/docbuild-and-upload.yml

@@ -17,13 +17,15 @@ on:
         description: 'The pandas version to override'
         required: false
         type: string
+      publish_prod:
+        description: 'Publish to production'
+        type: boolean
+        default: false
 
 env:
   ENV_FILE: environment.yml
   PANDAS_CI: 1
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-  PANDAS_VERSION: "${{ github.event_name == 'workflow_dispatch' && github.event.inputs.version || '' }}"
-  PANDAS_VERSION_OVERRIDE: "${{ github.event_name == 'workflow_dispatch' && github.event.inputs.version || '' }}"
 
 permissions:
   contents: read
@@ -45,13 +47,34 @@ jobs:
     steps:
     - name: Set pandas version
       run: |
-        echo "PANDAS_VERSION=${GITHUB_REF_NAME:1}" >> "$GITHUB_ENV"
-      if: github.event_name != 'workflow_dispatch'
-
-    - name: Show environment variables
-      run: |
-        echo "PANDAS_VERSION=${{ env.PANDAS_VERSION }}"
-        echo "PANDAS_VERSION_OVERRIDE=${{ env.PANDAS_VERSION_OVERRIDE }}"
+        # tags include a `v` prefix.
+        tag_version_pat="^v\d+\.\d+\.\d+(rc\d+)?$"
+        version_pat="^\d+\.\d+\.\d+(rc\d+)?$"
+
+        if [[ -n "${{ github.event.inputs.version }}" ]]; then
+          PANDAS_VERSION="${{ github.event.inputs.version }}"
+        elif [[ "${{ github.event_name }}" == "push" && "$GITHUB_REF_NAME" =~ $tag_version_pat ]]; then
+          PANDAS_VERSION="${GITHUB_REF_NAME:1}"
+        else
+          PANDAS_VERSION=""
+        fi
+        echo "PANDAS_VERSION=$PANDAS_VERSION" >> "$GITHUB_ENV"
+
+        if [[ "${{ github.event_name }}" == "push" && -n "$PANDAS_VERSION" ]]; then
+          PUBLISH_PROD="true"
+        elif [[ "${{ github.event.inputs.publish_prod }}" == "true" ]]; then
+          PUBLISH_PROD="true"
+        else
+          PUBLISH_PROD="false"
+        fi
+        echo "PUBLISH_PROD=$PUBLISH_PROD" >> "$GITHUB_ENV"
+
+        if [[ "$PUBLISH_PROD" == "true" ]] &&
+           [[ ! "$PANDAS_VERSION" =~ $version_pat ]]
+        then
+          echo "Invalid version $PANDAS_VERSION for publishing to prod."
+          exit 1
+        fi
 
     - name: Checkout
       uses: actions/checkout@v6
@@ -105,7 +128,7 @@ jobs:
 
     - name: Upload prod docs
       run: rsync -az --delete doc/build/html/ web@${{ secrets.server_ip }}:/var/www/html/pandas-docs/version/${{ env.PANDAS_VERSION }}
-      if: (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')) || (github.event_name == 'workflow_dispatch' && env.PANDAS_VERSION_OVERRIDE != '')
+      if: ${{ env.PUBLISH_PROD == 'true' }}
 
     - name: Move docs into site directory
       run: mv doc/build/html web/build/docs

.github/workflows/unit-tests.yml

@@ -27,7 +27,7 @@ jobs:
     strategy:
       matrix:
         platform: [ubuntu-24.04, ubuntu-24.04-arm]
-        env_file: [actions-311.yaml, actions-312.yaml, actions-313.yaml]
+        env_file: [actions-311.yaml, actions-312.yaml, actions-313.yaml, actions-314.yaml]
         # Prevent the include jobs from overriding other jobs
         pattern: [""]
         pandas_future_infer_string: ["1"]
@@ -190,7 +190,7 @@ jobs:
     strategy:
       matrix:
         os: [macos-15-intel, macos-15, windows-2025]
-        env_file: [actions-311.yaml, actions-312.yaml, actions-313.yaml]
+        env_file: [actions-311.yaml, actions-312.yaml, actions-313.yaml, actions-314.yaml]
       fail-fast: false
     runs-on: ${{ matrix.os }}
     name: ${{ format('{0} {1}', matrix.os, matrix.env_file) }}
@@ -246,7 +246,7 @@ jobs:
         run: |
           /opt/python/cp313-cp313/bin/python -m venv ~/virtualenvs/pandas-dev
           . ~/virtualenvs/pandas-dev/bin/activate
-          python -m pip install --no-cache-dir -U pip wheel setuptools meson[ninja]==1.2.1 meson-python==0.13.1
+          python -m pip install --no-cache-dir -U pip wheel meson[ninja]==1.2.1 meson-python==0.13.1
           python -m pip install numpy -Csetup-args="-Dallow-noblas=true"
           python -m pip install --no-cache-dir versioneer[toml] cython python-dateutil pytest>=8.3.4 pytest-xdist>=3.6.1 hypothesis>=6.116.0
           python -m pip install --no-cache-dir --no-build-isolation -e . -Csetup-args="--werror"
@@ -284,7 +284,7 @@ jobs:
         run: |
           /opt/python/cp313-cp313/bin/python -m venv ~/virtualenvs/pandas-dev
           . ~/virtualenvs/pandas-dev/bin/activate
-          python -m pip install --no-cache-dir -U pip wheel setuptools meson-python==0.13.1 meson[ninja]==1.2.1
+          python -m pip install --no-cache-dir -U pip wheel meson-python==0.13.1 meson[ninja]==1.2.1
           python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytest>=8.3.4 pytest-xdist>=3.6.1 hypothesis>=6.116.0
           python -m pip install --no-cache-dir --no-build-isolation -e . -Csetup-args="--werror"
           python -m pip list --no-cache-dir
@@ -318,7 +318,7 @@ jobs:
     #    To freeze this file, uncomment out the ``if: false`` condition, and migrate the jobs
     #    to the corresponding posix/windows-macos/sdist etc. workflows.
     # Feel free to modify this comment as necessary.
-    # if: false
+    if: false
     defaults:
       run:
         shell: bash -eou pipefail {0}
@@ -349,12 +349,12 @@ jobs:
       - name: Set up Python Dev Version
         uses: actions/setup-python@v6
         with:
-          python-version: '3.14-dev'
+          python-version: '3.15-dev'
 
       - name: Build Environment
         run: |
           python --version
-          python -m pip install --upgrade pip setuptools wheel meson[ninja]==1.2.1 meson-python==0.13.1
+          python -m pip install --upgrade pip wheel meson[ninja]==1.2.1 meson-python==0.13.1
           python -m pip install --pre --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy
           python -m pip install versioneer[toml] python-dateutil tzdata cython hypothesis>=6.116.0 pytest>=8.3.4 pytest-xdist>=3.6.1 pytest-cov
           python -m pip install -ve . --no-build-isolation --no-index --no-deps -Csetup-args="--werror"

.github/workflows/wheels.yml

@@ -116,6 +116,10 @@ jobs:
         # BackendUnavailable: Cannot import 'mesonpy'
           - buildplat: [windows-11-arm, win_arm64]
             python: ["cp313t", "3.13"]
+    concurrency:
+      # https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.buildplat[0] }}-${{ matrix.buildplat[1] }}-${{ matrix.python[0]}}-build-wheels
+      cancel-in-progress: true
 
     env:
       IS_PUSH: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }}
@@ -192,7 +196,8 @@ jobs:
         # installing wheel here because micromamba step was skipped
         if: matrix.buildplat[1] == 'win_arm64'
         shell: bash -el {0}
-        run: python -m pip install wheel anaconda-client
+        # cryptography pin due to https://github.com/pyca/cryptography/pull/14216
+        run: python -m pip install wheel anaconda-client cryptography==46.0.3
 
       - name: Validate wheel RECORD
         shell: bash -el {0}

.gitignore

@@ -45,11 +45,11 @@ debug
 
 # Python files #
 ################
-# setup.py working directory
+# meson working directory
 build
 # sphinx build directory
 doc/_build
-# setup.py dist directory
+# dist directory
 dist
 # Egg metadata
 *.egg-info

.pre-commit-config.yaml

@@ -19,7 +19,7 @@ ci:
     skip: [pyright, mypy]
 repos:
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.14.10
+    rev: v0.14.14
     hooks:
     -   id: ruff
         args: [--exit-non-zero-on-fix]
@@ -46,7 +46,7 @@ repos:
     -   id: codespell
         types_or: [python, rst, markdown, cython, c]
 -   repo: https://github.com/MarcoGorelli/cython-lint
-    rev: v0.18.1
+    rev: v0.19.0
     hooks:
     -   id: cython-lint
     -   id: double-quote-cython-strings
@@ -99,7 +99,7 @@ repos:
       args: ['-i', '--style', '{IndentPPDirectives: AfterHash}']
       types_or: [c, c++]
 -   repo: https://github.com/trim21/pre-commit-mirror-meson
-    rev: v1.10.0
+    rev: v1.10.1
     hooks:
     - id: meson-fmt
       args: ['--inplace']
@@ -300,7 +300,7 @@ repos:
         entry: python scripts/validate_exception_location.py
         language: python
         files: ^pandas/
-        exclude: ^(pandas/_libs/|pandas/tests/|pandas/errors/__init__.py$|pandas/_version.py)
+        exclude: ^pandas/_libs/|pandas/tests/|pandas/errors/__init__.py$
         types: [python]
     -   id: check-test-naming
         name: check that test names start with 'test'

asv_bench/benchmarks/indexing.py

@@ -438,6 +438,28 @@ def time_frame_getitem_single_column_int(self):
         self.df_int_col[0]
 
 
+class DataFrameGetitemDuplicateColumns:
+    """
+    Benchmark df[key] when columns have duplicate names but key is unique.
+
+    Previously each access called columns.drop_duplicates(keep=False), which
+    built a new Index (O(n)). Now we use get_loc(key), so this path is O(1)
+    for hash-based indexes.
+    """
+
+    params = [1_000, 10_000, 100_000, 1_000_000]
+    param_names = ["ncols"]
+
+    def setup(self, ncols):
+        # ncols-1 duplicate names + one unique column we access
+        cols = ["a"] * (ncols - 1) + ["key"]
+        self.df = DataFrame(0, index=range(100), columns=cols)
+
+    def time_getitem_single_column_with_duplicate_columns(self, ncols):
+        for _ in range(100):
+            self.df["key"]
+
+
 class IndexSingleRow:
     params = [True, False]
     param_names = ["unique_cols"]