Merge main

Author: shuoweil

.github/workflows/librarian_tidy.yml

@@ -2,8 +2,6 @@ name: librarian tidy
 
 on:
   pull_request:
-    paths:
-      - 'librarian.yaml'
 
 permissions:
   contents: read
@@ -14,19 +12,21 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v6
-
-      - name: Install Go
-        uses: actions/setup-go@v6
+      - uses: dorny/paths-filter@v4
+        id: changes
         with:
-          go-version: '1.26.x'
+          filters: |
+            librarian:
+              - 'librarian.yaml'
+
+      -  uses: googleapis/librarian@main
 
       - name: Run librarian tidy
-        run: |
-          V=$(go run github.com/googleapis/librarian/cmd/librarian@latest config get version)
-          go run github.com/googleapis/librarian/cmd/librarian@${V} tidy
-          git diff
+        if: steps.changes.outputs.librarian == 'true'
+        run: librarian tidy
 
       - name: Check for diff
+        if: steps.changes.outputs.librarian == 'true'
         run: |
           if ! git diff --exit-code; then
             V=$(go run github.com/googleapis/librarian/cmd/librarian@latest config get version)

.github/workflows/regenerate-all.yml

@@ -20,15 +20,10 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v6
 
-      - name: Install Go
-        uses: actions/setup-go@v6
+      - uses: googleapis/librarian@main
         with:
-          go-version: '1.26.x'
-
-      - name: Install protoc
-        uses: arduino/setup-protoc@v3
-        with:
-          version: "25.3"
+          protoc-version: "25.3"
+          protoc-checksum: "5ec3474ca09df0511bb2ca66b5ca091fa8943c30aa26285f225d0b1ba60b5665b3419be4cd2322decbb55464039ca0a0405a47e86bcc11491589405d615d280e"
 
       - name: Install pandoc
         run: |
@@ -38,21 +33,20 @@ jobs:
           tar -xvf /tmp/pandoc.tar.gz -C /tmp/pandoc --strip-components=1
 
       - name: Install Python packages for Librarian
-        run: |
-          version=$(sed -n 's/^version: *//p' librarian.yaml)
-          go run "github.com/googleapis/librarian/cmd/librarian@${version}" install
+        run: librarian install
 
       - name: Clone Synthtool Templates
         run: |
           git clone --recurse-submodules --single-branch https://github.com/googleapis/synthtool.git /home/runner/synthtool
 
       - name: Regenerate
         run: |
-          version=$(sed -n 's/^version: *//p' librarian.yaml)
           PATH=$PATH:/tmp/pandoc/bin
-          go run "github.com/googleapis/librarian/cmd/librarian@${version}" generate -all -v
+          librarian generate -all -v
+          git diff --exit-code
 
       - name: Create issue on diff
+        if: failure()
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |

.github/workflows/unittest.yml

@@ -48,71 +48,6 @@ jobs:
       with:
         name: coverage-artifact-${{ '{{' }} matrix.python {{ '}}' }}
         path: .coverage-${{ matrix.python }}
-  unit-prerelease:
-    name: ${{ matrix.option }}
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python: ["3.14"]
-        option: ["prerelease"]
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v4
-      # Use a fetch-depth of 2 to avoid error `fatal: origin/main...HEAD: no merge base`
-      # See https://github.com/googleapis/google-cloud-python/issues/12013
-      # and https://github.com/actions/checkout#checkout-head.
-      with:
-        fetch-depth: 2
-    - name: Setup Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: ${{ matrix.python }}
-        allow-prereleases: true
-    - name: Install nox
-      run: |
-        python -m pip install --upgrade setuptools pip wheel
-        python -m pip install nox
-    - name: Run ${{ matrix.option }} tests
-      env:
-        BUILD_TYPE: presubmit
-        TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref }}
-        TEST_TYPE: ${{ matrix.option }}
-        # TODO(https://github.com/googleapis/google-cloud-python/issues/13775): Specify `PY_VERSION` rather than relying on the default python version of the nox session.
-        PY_VERSION: "unused"
-      run: |
-        ci/run_conditional_tests.sh
-  unit-core-deps-from-source:
-    name: ${{ matrix.option }}
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python: ["3.14"]
-        option: ["core_deps_from_source-3.14"]
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v4
-      # Use a fetch-depth of 2 to avoid error `fatal: origin/main...HEAD: no merge base`
-      # See https://github.com/googleapis/google-cloud-python/issues/12013
-      # and https://github.com/actions/checkout#checkout-head.
-      with:
-        fetch-depth: 2
-    - name: Setup Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: ${{ matrix.python }}
-    - name: Install nox
-      run: |
-        python -m pip install --upgrade setuptools pip wheel
-        python -m pip install nox
-    - name: Run ${{ matrix.option }} tests
-      env:
-        BUILD_TYPE: presubmit
-        TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref }}
-        TEST_TYPE: ${{ matrix.option }}
-        # TODO(https://github.com/googleapis/google-cloud-python/issues/13775): Specify `PY_VERSION` rather than relying on the default python version of the nox session.
-        PY_VERSION: "unused"
-      run: |
-        ci/run_conditional_tests.sh
 
   cover:
     runs-on: ubuntu-latest

.kokoro/continuous/benchmark-bigframes.cfg

@@ -0,0 +1,22 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Only run this nox session.
+env_vars: {
+    key: "NOX_SESSION"
+    value: "benchmark"
+}
+
+env_vars: {
+    key: "BENCHMARK_AND_PUBLISH"
+    value: "true"
+}
+
+env_vars: {
+    key: "GOOGLE_CLOUD_PROJECT"
+    value: "bigframes-benchmarking"
+}
+
+env_vars: {
+    key: "BIGFRAMES_TEST_MODEL_VERTEX_ENDPOINT"
+    value: "https://us-central1-aiplatform.googleapis.com/v1/projects/272725758477/locations/us-central1/endpoints/590545496255234048"
+}

.kokoro/continuous/continuous-bigframes.cfg

@@ -3,7 +3,7 @@
 # Only run these nox sessions.
 env_vars: {
     key: "NOX_SESSION"
-    value: "e2e load system_prerelease notebook system_noextras"
+    value: "e2e system_prerelease notebook system_noextras"
 }
 
 env_vars: {

.kokoro/continuous/head.cfg .kokoro/continuous/core_deps.cfg.kokoro/continuous/head.cfg renamed to .kokoro/continuous/core_deps.cfg

@@ -3,10 +3,10 @@
 # Only run this nox session.
 env_vars: {
     key: "NOX_SESSION"
-    value: "prerelease_deps"
+    value: "notebook"
 }
 
 env_vars: {
-  key: "BIGFRAMES_TEST_PROJECT"
-  value: "bigframes-testing"
-}
+    key: "GOOGLE_CLOUD_PROJECT"
+    value: "bigframes-testing"
+}

.kokoro/presubmit/prerelease.cfg .kokoro/continuous/notebook-bigframes.cfg.kokoro/presubmit/prerelease.cfg renamed to .kokoro/continuous/notebook-bigframes.cfg

@@ -90,35 +90,9 @@ run_package_test() {
   return $res
 }
 
-packages_with_system_tests=(
-  "bigframes"
-  "google-auth"
-  "google-cloud-bigquery-storage"
-  "google-cloud-bigtable"
-  "google-cloud-compute"
-  "google-cloud-compute-v1beta"
-  "google-cloud-datastore"
-  "google-cloud-dns"
-  "google-cloud-error-reporting"
-  "google-cloud-firestore"
-  "google-cloud-logging"
-  "google-cloud-ndb"
-  "google-cloud-pubsub"
-  "google-cloud-spanner"
-  "google-cloud-storage"
-  "google-cloud-testutils"
-  "pandas-gbq"
-  "sqlalchemy-bigquery"
-  "sqlalchemy-spanner"
-)
-
 # A file for running system tests
 system_test_script="${PROJECT_ROOT}/.kokoro/system-single.sh"
 
-# Join array elements with | for the pattern match
-packages_with_system_tests_pattern=$(printf "|*%s*" "${packages_with_system_tests[@]}")
-packages_with_system_tests_pattern="${packages_with_system_tests_pattern:1}" # Remove the leading pipe
-
 # Run system tests for each package with directory packages/*/tests/system
 for path in `find 'packages' \
   \( -type d -wholename 'packages/*/tests/system' \) -o \
@@ -142,8 +116,20 @@ for path in `find 'packages' \
     "${package_path}/**/version.py"
   )
 
-  # If the package is in our "always run full system tests" list, check the whole directory
-  if [[ $package_name == @($packages_with_system_tests_pattern) ]]; then
+  # Hand-written (non-GAPIC_AUTO) packages or google-cloud-compute should check
+  # the whole directory for changes.
+  metadata="${package_path}/.repo-metadata.json"
+  library_type="UNKNOWN"
+  if [ -f "$metadata" ]; then
+    library_type=$(sed -n 's/.*"library_type":[[:space:]]*"\([^"]*\)".*/\1/p' "$metadata")
+    library_type="${library_type:-UNKNOWN}"
+  fi
+
+  # System tests always run in release PRs, regardless of library type.
+  # Automated GAPIC libraries bypass system tests in non-release PRs because their generation is deterministic.
+  # However, google-cloud-compute is included because its Discovery-based nature requires additional
+  # verification.
+  if [[ "${library_type}" != "GAPIC_AUTO" || "${package_name}" == "google-cloud-compute"* ]]; then
     files_to_check=("${package_path}")
   fi
 

.kokoro/presubmit/head.cfg

@@ -23,9 +23,6 @@ libraries:
   # Allow release for google-crc32c once this bug is fixed.
   - id: "google-crc32c"
     release_blocked: true
-  # TODO(b/501132869): Disabling automatic releases until resolved.
-  - id: "google-cloud-bigtable"
-    release_blocked: true
   # TODO(https://github.com/googleapis/google-cloud-python/issues/16962):
   # Disable automatic releases until tests stabilize.
   - id: "pandas-gbq"
@@ -34,4 +31,18 @@ libraries:
   # Disable automatic releases until system tests are sped up or reorganized.
   - id: "google-cloud-firestore"
     release_blocked: true
+  # TODO(https://github.com/googleapis/google-cloud-python/issues/17287):
+  # Allow releases for sqlalchemy-bigquery once the bug above is fixed.
+  - id: "sqlalchemy-bigquery"
+    release_blocked: true
+  # TODO(https://github.com/googleapis/google-cloud-python/issues/17327)
+  - id: "google-cloud-bigquery"
+    release_blocked: true
+  # TODO(https://github.com/googleapis/google-cloud-python/issues/17327)
+  - id: "google-cloud-bigtable"
+    release_blocked: true
+  # TODO(https://github.com/googleapis/google-cloud-python/issues/17334)
+  - id: "google-auth"
+    release_blocked: true
+