diff --git a/.github/trivyignore/kafka-connect-runtime.trivyignore b/.github/trivyignore/kafka-connect-runtime.trivyignore new file mode 100644 index 000000000000..2da8c6d2f484 --- /dev/null +++ b/.github/trivyignore/kafka-connect-runtime.trivyignore @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Trivy ignore file for the Kafka Connect runtime distribution. +# +# The Kafka Connect runtime's own jackson-databind is already aligned to the +# project version (2.22.x). The remaining finding comes from parquet-jackson, +# which ships its own shaded copy of jackson-databind (relocated under +# shaded.parquet.*). That copy cannot be upgraded independently of Apache +# Parquet, and 1.17.1 still bundles a vulnerable version. +# +# jackson-databind CVEs shaded into parquet-jackson, only fixed in jackson +# >= 2.21.4 / 2.18.8. +CVE-2026-54512 +CVE-2026-54513 diff --git a/.github/trivyignore/spark-runtime-3.5.trivyignore b/.github/trivyignore/spark-runtime-3.5.trivyignore new file mode 100644 index 000000000000..a8f4671634c5 --- /dev/null +++ b/.github/trivyignore/spark-runtime-3.5.trivyignore @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Trivy ignore file for the Spark 3.5 runtime bundle. +# +# The Spark 3.5 runtime bundles the Jackson version provided by Spark 3.5 +# (jackson 2.15.x), which cannot be safely upgraded without breaking +# compatibility with Spark 3.5. Newer Spark lines (4.0, 4.1) align Jackson to a +# fixed version instead of ignoring the finding. +# +# jackson-databind CVEs that are only fixed in jackson >= 2.18.8. +CVE-2026-54512 +CVE-2026-54513 diff --git a/.github/workflows/api-binary-compatibility.yml b/.github/workflows/api-binary-compatibility.yml index 4a4954bad3de..e8bd23c9b475 100644 --- a/.github/workflows/api-binary-compatibility.yml +++ b/.github/workflows/api-binary-compatibility.yml @@ -46,7 +46,7 @@ jobs: revapi: runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: # fetch-depth of zero ensures that the tags are pulled in and we're not in a detached HEAD state # revapi depends on the tags, specifically the tag from git describe, to find the relevant override @@ -55,11 +55,11 @@ jobs: # See https://github.com/actions/checkout/issues/124 fetch-depth: 0 persist-credentials: false - - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + - uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 with: distribution: zulu java-version: 17 - - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2 + - uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0 with: # Read-only: small job; restore opportunistically from other jobs' caches but never write. cache-read-only: true diff --git a/.github/workflows/asf-allowlist-check.yml b/.github/workflows/asf-allowlist-check.yml index 71311bca7235..c5690c113228 100644 --- a/.github/workflows/asf-allowlist-check.yml +++ b/.github/workflows/asf-allowlist-check.yml @@ -25,13 +25,9 @@ name: "ASF Allowlist Check" on: pull_request: - paths: - - ".github/**" push: branches: - main - paths: - - ".github/**" permissions: contents: read @@ -40,7 +36,7 @@ jobs: asf-allowlist-check: runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - uses: apache/infrastructure-actions/allowlist-check@4e9c961f587f72b170874b6f5cd4ac15f7f26eb8 # main diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index c22c8b5438aa..9b7b204e9272 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -41,7 +41,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false diff --git a/.github/workflows/cve-scan.yml b/.github/workflows/cve-scan.yml index 78bfb9937d61..2365f86837fb 100644 --- a/.github/workflows/cve-scan.yml +++ b/.github/workflows/cve-scan.yml @@ -31,6 +31,7 @@ on: - '**' - '!.github/**' - '.github/workflows/cve-scan.yml' + - '.github/trivyignore/**' - '!.baseline/**' - '!.palantir/**' - '!.gitignore' @@ -54,6 +55,7 @@ on: - '**' - '!.github/**' - '.github/workflows/cve-scan.yml' + - '.github/trivyignore/**' - '!.baseline/**' - '!.palantir/**' - '!.gitignore' @@ -86,12 +88,13 @@ jobs: # Trivy CVE scan — scans bundled jars for known vulnerabilities. # # Behaviour: - # - On PRs: the scan blocks CI if CVEs are found (exit-code 1). - # SARIF upload is skipped because GitHub's Security tab only - # accepts results from default/protected branches. - # - On push to main/release branches: the scan is informational - # (exit-code 0) and results are uploaded as SARIF to the GitHub - # Security tab for ongoing tracking. + # - Trivy always writes SARIF with exit-code 0. The reporting step owns + # finding-based failures so GitHub opens the step with actionable details. + # - On PRs: the reporting step blocks CI if HIGH/CRITICAL CVEs are found. + # - On push to main/release branches and release tags: findings are + # informational, then SARIF is uploaded to the GitHub Security tab. + # - Missing or unparseable SARIF is still a failure on every event because + # it means the scan did not produce usable results. # ------------------------------------------------------------------ cve-scan: runs-on: ubuntu-24.04 @@ -113,6 +116,7 @@ jobs: :iceberg-kafka-connect:iceberg-kafka-connect-runtime:distZip scan-path: kafka-connect/kafka-connect-runtime/build/distributions unpack: true + trivyignores: .github/trivyignore/kafka-connect-runtime.trivyignore - distribution: aws-bundle build-task: :iceberg-aws-bundle:shadowJar scan-path: aws-bundle/build/libs @@ -131,6 +135,7 @@ jobs: :iceberg-spark:iceberg-spark-runtime-3.5_2.12:shadowJar scan-path: spark/v3.5/spark-runtime/build/libs unpack: false + trivyignores: .github/trivyignore/spark-runtime-3.5.trivyignore - distribution: spark-runtime-4.0_2.13 build-task: >- -DsparkVersions=4.0 @@ -167,14 +172,14 @@ jobs: unpack: false name: ${{ matrix.distribution }} steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + - uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 with: distribution: zulu java-version: 21 - - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2 + - uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0 with: # Read-only: small job; restore opportunistically from other jobs' caches but never write. cache-read-only: true @@ -214,20 +219,109 @@ jobs: scan-ref: '/tmp/cve-scan' scanners: 'vuln' severity: 'HIGH,CRITICAL' + trivyignores: ${{ matrix.trivyignores }} limit-severities-for-sarif: true - # Block PRs on CVE findings; on main/release branches report without failing - exit-code: ${{ github.event_name == 'pull_request' && '1' || '0' }} + # Let Trivy generate SARIF without failing on findings. GitHub opens the + # failed step by default, so PRs fail later in the reporting step that + # prints the actionable CVE details. + exit-code: '0' format: 'sarif' output: 'trivy-results.sarif' trivy-image: ${{ env.TRIVY_IMAGE }} - - name: Print Trivy scan results + - name: Report Trivy scan results if: always() + env: + # PRs block on findings; push runs report findings without failing so + # SARIF can be uploaded to GitHub Security for tracking. + FAIL_ON_FINDINGS: ${{ github.event_name == 'pull_request' && 'true' || 'false' }} run: | - if [ -f trivy-results.sarif ]; then - echo "## Trivy CVE Scan Results — ${{ matrix.distribution }}" - jq -r '.runs[].results[] | "- \(.ruleId): \(.message.text)"' trivy-results.sarif 2>/dev/null || echo "No findings or unable to parse SARIF." - else - echo "No SARIF file found — scan may have failed to install." + results_file="trivy-results.sarif" + summary="${GITHUB_STEP_SUMMARY:-/dev/null}" + + log() { + printf '%s\n' "$*" | tee -a "${summary}" + } + + markdown_escape() { + value="$1" + value="${value//|/\\|}" + printf '%s' "${value}" + } + + escape_annotation() { + value="$1" + value="${value//'%'/%25}" + value="${value//$'\r'/%0D}" + value="${value//$'\n'/%0A}" + printf '%s' "${value}" + } + + extract_findings() { + jq -r ' + def field($prefix): + (.message.text | split("\n") | map(select(startswith($prefix))) | first // "") + | ltrimstr($prefix); + + .runs[].results[]? + | [ + .ruleId, + field("Severity: "), + field("Package: "), + field("Installed Version: "), + field("Fixed Version: "), + field("Link: ") + ] + | @tsv + ' "${results_file}" + } + + report_findings() { + log "Found ${finding_count} HIGH/CRITICAL vulnerabilities." + log "" + log "| CVE | Severity | Package | Installed | Fixed | Link |" + log "| --- | --- | --- | --- | --- | --- |" + + while IFS=$'\t' read -r cve severity package installed fixed link; do + cve="$(markdown_escape "${cve}")" + severity="$(markdown_escape "${severity}")" + package="$(markdown_escape "${package}")" + installed="$(markdown_escape "${installed}")" + fixed="$(markdown_escape "${fixed}")" + link="$(markdown_escape "${link}")" + log "| ${cve} | ${severity} | \`${package}\` | \`${installed}\` | ${fixed} | ${link} |" + done <<< "${findings}" + } + + if [ ! -f "${results_file}" ]; then + log "No SARIF file found — scan may have failed to run." + exit 1 + fi + + if ! findings="$(extract_findings)"; then + log "Unable to parse Trivy SARIF results." + exit 1 + fi + + log "## Trivy CVE Scan Results — ${{ matrix.distribution }}" + + if [ -z "${findings}" ]; then + log "No HIGH or CRITICAL vulnerabilities found." + exit 0 + fi + + finding_count="$(printf '%s\n' "${findings}" | awk 'END { print NR }')" + finding_ids="$(printf '%s\n' "${findings}" | cut -f1 | awk 'BEGIN { sep="" } { printf "%s%s", sep, $0; sep=", " } END { print "" }')" + + report_findings + + if [ "${FAIL_ON_FINDINGS}" = "true" ]; then + # Surface findings in the PR checks UI, not just in the workflow logs. + annotation_message="Trivy found ${finding_count} HIGH/CRITICAL vulnerabilities in ${{ matrix.distribution }}: ${finding_ids}. See the 'Report Trivy scan results' step for details." + annotation="$(escape_annotation "${annotation_message}")" + echo "::error title=Trivy CVE scan failed::${annotation}" + log "" + log "Failing because ${finding_count} HIGH/CRITICAL vulnerabilities were found." + exit 1 fi - name: Upload Trivy results to GitHub Security tab if: always() && github.event_name == 'push' diff --git a/.github/workflows/delta-conversion-ci.yml b/.github/workflows/delta-conversion-ci.yml index a35b21dcb4a6..a1a5e952f095 100644 --- a/.github/workflows/delta-conversion-ci.yml +++ b/.github/workflows/delta-conversion-ci.yml @@ -82,14 +82,14 @@ jobs: env: SPARK_LOCAL_IP: localhost steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + - uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 with: distribution: zulu java-version: ${{ matrix.jvm }} - - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2 + - uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0 with: # Read-only: java-ci's build-checks (17) is the global canonical writer. cache-read-only: true @@ -111,14 +111,14 @@ jobs: env: SPARK_LOCAL_IP: localhost steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + - uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 with: distribution: zulu java-version: ${{ matrix.jvm }} - - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2 + - uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0 with: # Read-only: java-ci's build-checks (17) is the global canonical writer. cache-read-only: true diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml index df49e215e370..615b290a3119 100644 --- a/.github/workflows/docs-ci.yml +++ b/.github/workflows/docs-ci.yml @@ -36,7 +36,7 @@ jobs: matrix: os: [ubuntu-latest, macos-latest] steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 diff --git a/.github/workflows/flink-ci.yml b/.github/workflows/flink-ci.yml index 9fec6df124d6..f7964315e174 100644 --- a/.github/workflows/flink-ci.yml +++ b/.github/workflows/flink-ci.yml @@ -86,14 +86,14 @@ jobs: env: SPARK_LOCAL_IP: localhost steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + - uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 with: distribution: zulu java-version: ${{ matrix.jvm }} - - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2 + - uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0 with: # Read-only: java-ci's build-checks (17) is the global canonical writer. cache-read-only: true diff --git a/.github/workflows/hive-ci.yml b/.github/workflows/hive-ci.yml index b4fd3cc8027c..59b5c832c853 100644 --- a/.github/workflows/hive-ci.yml +++ b/.github/workflows/hive-ci.yml @@ -83,14 +83,14 @@ jobs: env: SPARK_LOCAL_IP: localhost steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + - uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 with: distribution: zulu java-version: ${{ matrix.jvm }} - - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2 + - uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0 with: # Read-only: java-ci's build-checks (17) is the global canonical writer. cache-read-only: true diff --git a/.github/workflows/java-ci.yml b/.github/workflows/java-ci.yml index bf4120d2022c..ec6ea01c5096 100644 --- a/.github/workflows/java-ci.yml +++ b/.github/workflows/java-ci.yml @@ -78,14 +78,14 @@ jobs: env: SPARK_LOCAL_IP: localhost steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + - uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 with: distribution: zulu java-version: ${{ matrix.jvm }} - - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2 + - uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0 with: # Read-only: java-ci's build-checks (17) is the global canonical writer. cache-read-only: true @@ -105,14 +105,14 @@ jobs: matrix: jvm: [17, 21] steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + - uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 with: distribution: zulu java-version: ${{ matrix.jvm }} - - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2 + - uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0 with: # Writes cache on main; read-only otherwise. cache-read-only: ${{ !(github.ref == 'refs/heads/main' && matrix.jvm == 17) }} @@ -125,14 +125,14 @@ jobs: matrix: jvm: [17, 21] steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + - uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 with: distribution: zulu java-version: ${{ matrix.jvm }} - - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2 + - uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0 with: # Read-only: java-ci's build-checks (17) is the global canonical writer. cache-read-only: true @@ -141,14 +141,14 @@ jobs: check-runtime-deps: runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + - uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 with: distribution: zulu java-version: 17 - - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2 + - uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0 with: # Read-only: java-ci's build-checks (17) is the global canonical writer. cache-read-only: true diff --git a/.github/workflows/jmh-benchmarks.yml b/.github/workflows/jmh-benchmarks.yml index f6e561a7eae8..77856f9bec20 100644 --- a/.github/workflows/jmh-benchmarks.yml +++ b/.github/workflows/jmh-benchmarks.yml @@ -49,7 +49,7 @@ jobs: matrix: ${{ steps.set-matrix.outputs.matrix }} foundlabel: ${{ steps.set-matrix.outputs.foundlabel }} steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: repository: ${{ github.event.inputs.repo }} ref: ${{ github.event.inputs.ref }} @@ -94,16 +94,16 @@ jobs: env: SPARK_LOCAL_IP: localhost steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: repository: ${{ github.event.inputs.repo }} ref: ${{ github.event.inputs.ref }} persist-credentials: false - - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + - uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 with: distribution: zulu java-version: 17 - - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2 + - uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0 with: # Disabled: dispatched against arbitrary repo/ref inputs; never restore or write to avoid cache poisoning. cache-disabled: true diff --git a/.github/workflows/kafka-connect-ci.yml b/.github/workflows/kafka-connect-ci.yml index 562fd970a111..799bf9a39ae8 100644 --- a/.github/workflows/kafka-connect-ci.yml +++ b/.github/workflows/kafka-connect-ci.yml @@ -83,14 +83,14 @@ jobs: env: SPARK_LOCAL_IP: localhost steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + - uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 with: distribution: zulu java-version: ${{ matrix.jvm }} - - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2 + - uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0 with: # Read-only: java-ci's build-checks (17) is the global canonical writer. cache-read-only: true diff --git a/.github/workflows/license-check.yml b/.github/workflows/license-check.yml index 997f698b1d52..6fdeda86f4e6 100644 --- a/.github/workflows/license-check.yml +++ b/.github/workflows/license-check.yml @@ -27,7 +27,7 @@ jobs: rat: runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - run: | diff --git a/.github/workflows/open-api.yml b/.github/workflows/open-api.yml index 8a6fb17c9863..92732cb118ff 100644 --- a/.github/workflows/open-api.yml +++ b/.github/workflows/open-api.yml @@ -44,7 +44,7 @@ jobs: runs-on: ubuntu-slim steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Install uv diff --git a/.github/workflows/publish-iceberg-rest-fixture-docker.yml b/.github/workflows/publish-iceberg-rest-fixture-docker.yml index 3275aab4c9d1..8927f187e998 100644 --- a/.github/workflows/publish-iceberg-rest-fixture-docker.yml +++ b/.github/workflows/publish-iceberg-rest-fixture-docker.yml @@ -41,14 +41,14 @@ jobs: runs-on: ubuntu-latest environment: docker-publish steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + - uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 with: distribution: zulu java-version: 21 - - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2 + - uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0 with: # Read-only: small job; restore opportunistically from other jobs' caches but never write. cache-read-only: true diff --git a/.github/workflows/publish-snapshot.yml b/.github/workflows/publish-snapshot.yml index cade0d93e0fa..5e34aab78998 100644 --- a/.github/workflows/publish-snapshot.yml +++ b/.github/workflows/publish-snapshot.yml @@ -34,16 +34,16 @@ jobs: runs-on: ubuntu-24.04 environment: maven-publish steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: # we need to fetch all tags so that getProjectVersion() in build.gradle correctly determines the next SNAPSHOT version from the newest tag fetch-depth: 0 persist-credentials: false - - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + - uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 with: distribution: zulu java-version: 17 - - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2 + - uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0 with: # Read-only: small job; restore opportunistically from other jobs' caches but never write. cache-read-only: true diff --git a/.github/workflows/recurring-jmh-benchmarks.yml b/.github/workflows/recurring-jmh-benchmarks.yml index 50b2231df841..e91ea55be8d3 100644 --- a/.github/workflows/recurring-jmh-benchmarks.yml +++ b/.github/workflows/recurring-jmh-benchmarks.yml @@ -51,14 +51,14 @@ jobs: env: SPARK_LOCAL_IP: localhost steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + - uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 with: distribution: zulu java-version: 17 - - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2 + - uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0 with: # Read-only: small job; restore opportunistically from other jobs' caches but never write. cache-read-only: true diff --git a/.github/workflows/site-ci.yml b/.github/workflows/site-ci.yml index 43e61ff1feb2..c762fd421f3c 100644 --- a/.github/workflows/site-ci.yml +++ b/.github/workflows/site-ci.yml @@ -37,7 +37,7 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 diff --git a/.github/workflows/spark-ci.yml b/.github/workflows/spark-ci.yml index 06230ddc78d6..32fd7c3a1a58 100644 --- a/.github/workflows/spark-ci.yml +++ b/.github/workflows/spark-ci.yml @@ -97,14 +97,14 @@ jobs: env: SPARK_LOCAL_IP: localhost steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + - uses: actions/setup-java@ad2b38190b15e4d6bdf0c97fb4fca8412226d287 # v5.3.0 with: distribution: zulu java-version: ${{ matrix.jvm }} - - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2 + - uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6.2.0 with: # Read-only: java-ci's build-checks (17) is the global canonical writer. cache-read-only: true diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index 7bf94e1943aa..4f231f62155d 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -34,7 +34,7 @@ jobs: permissions: {} steps: - name: Checkout repository - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false diff --git a/AGENTS.md b/AGENTS.md index e6c771d4b482..ada467efa4ce 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -139,7 +139,8 @@ The `api/` module has the strongest stability guarantees — breaking changes ar - Test classes and methods should be package private unless required by inheritance. - Compute expected values, don't hardcode. Tests belong in the module that owns the code. - Write the most direct test for the bug. Parameterized tests for type variations. -- JUnit 5 + AssertJ: `@Test` (no `test` prefix), `assertThat`, `assertThatThrownBy`. +- JUnit 5 + AssertJ: `@Test`, `assertThat`, `assertThatThrownBy`. +- Avoid using `test` prefixes for newly added tests. - `waitUntilAfter` for time-dependent tests. Separate tests over combined. ### REST / OpenAPI Spec diff --git a/aliyun/src/main/java/org/apache/iceberg/aliyun/oss/OSSFileIO.java b/aliyun/src/main/java/org/apache/iceberg/aliyun/oss/OSSFileIO.java index be85b93a75f5..3096cfe4e986 100644 --- a/aliyun/src/main/java/org/apache/iceberg/aliyun/oss/OSSFileIO.java +++ b/aliyun/src/main/java/org/apache/iceberg/aliyun/oss/OSSFileIO.java @@ -76,6 +76,11 @@ public InputFile newInputFile(String path) { return new OSSInputFile(client(), new OSSURI(path), aliyunProperties, metrics); } + @Override + public InputFile newInputFile(String path, long length) { + return new OSSInputFile(client(), new OSSURI(path), aliyunProperties, length, metrics); + } + @Override public OutputFile newOutputFile(String path) { return new OSSOutputFile(client(), new OSSURI(path), aliyunProperties, metrics); diff --git a/aliyun/src/test/java/org/apache/iceberg/aliyun/oss/TestOSSFileIO.java b/aliyun/src/test/java/org/apache/iceberg/aliyun/oss/TestOSSFileIO.java index dda4e75c2d00..eff7daf8b369 100644 --- a/aliyun/src/test/java/org/apache/iceberg/aliyun/oss/TestOSSFileIO.java +++ b/aliyun/src/test/java/org/apache/iceberg/aliyun/oss/TestOSSFileIO.java @@ -19,6 +19,10 @@ package org.apache.iceberg.aliyun.oss; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.AdditionalAnswers.delegatesTo; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import com.aliyun.oss.OSS; import com.aliyun.oss.OSSClient; @@ -99,6 +103,29 @@ public void testInputFile() throws IOException { assertThat(inFileContent(in, dataSize)).as("Should have expected content").isEqualTo(data); } + @Test + public void testNewInputFileWithLength() throws IOException { + String location = randomLocation(); + int dataSize = 1024 * 10; + byte[] data = randomData(dataSize); + OutputFile out = fileIO().newOutputFile(location); + writeOSSData(out, data); + + OSSURI uri = new OSSURI(location); + OSS ossMock = mock(OSS.class, delegatesTo(ossClient().get())); + try (FileIO io = new OSSFileIO(() -> ossMock)) { + InputFile in = io.newInputFile(location, dataSize); + assertThat(in.getLength()).as("Should return the known length").isEqualTo(dataSize); + verify(ossMock, times(0)).getSimplifiedObjectMeta(uri.bucket(), uri.key()); + + InputFile inWithoutLength = io.newInputFile(location); + assertThat(inWithoutLength.getLength()) + .as("Should return the actual length") + .isEqualTo(dataSize); + verify(ossMock, times(1)).getSimplifiedObjectMeta(uri.bucket(), uri.key()); + } + } + @Test public void testDeleteFile() throws IOException { String location = randomLocation(); diff --git a/api/src/main/java/org/apache/iceberg/types/IndexByName.java b/api/src/main/java/org/apache/iceberg/types/IndexByName.java index 60258f5c5c3e..9ca2a1d3396c 100644 --- a/api/src/main/java/org/apache/iceberg/types/IndexByName.java +++ b/api/src/main/java/org/apache/iceberg/types/IndexByName.java @@ -32,20 +32,26 @@ import org.apache.iceberg.relocated.com.google.common.collect.Maps; public class IndexByName extends TypeUtil.SchemaVisitor> { - private static final Joiner DOT = Joiner.on("."); - private final Deque fieldNames = Lists.newLinkedList(); private final Deque shortFieldNames = Lists.newLinkedList(); private final Map nameToId = Maps.newHashMap(); private final Map shortNameToId = Maps.newHashMap(); + private final Joiner joiner; private final Function quotingFunc; + private final boolean useShortNames; public IndexByName() { this(Function.identity()); } public IndexByName(Function quotingFunc) { + this(".", quotingFunc, false); + } + + IndexByName(String separator, Function quotingFunc, boolean useShortNames) { + this.joiner = Joiner.on(separator); this.quotingFunc = quotingFunc; + this.useShortNames = useShortNames; } /** @@ -76,8 +82,15 @@ public Map byName() { * @return a map from field ID to name */ public Map byId() { + // a builder is used to swap key and value ImmutableMap.Builder builder = ImmutableMap.builder(); nameToId.forEach((key, value) -> builder.put(value, key)); + + if (useShortNames) { + shortNameToId.forEach((key, value) -> builder.put(value, key)); + return builder.buildKeepingLast(); + } + return builder.build(); } @@ -193,7 +206,7 @@ private void addField(String name, int fieldId) { if (!fieldNames.isEmpty()) { Iterator quotedFieldNames = Iterators.transform(fieldNames.descendingIterator(), quotingFunc::apply); - fullName = DOT.join(DOT.join(quotedFieldNames), quotedName); + fullName = joiner.join(joiner.join(quotedFieldNames), quotedName); } Integer existingFieldId = nameToId.put(fullName, fieldId); @@ -208,7 +221,7 @@ private void addField(String name, int fieldId) { if (!shortFieldNames.isEmpty()) { Iterator quotedShortFieldNames = Iterators.transform(shortFieldNames.descendingIterator(), quotingFunc::apply); - String shortName = DOT.join(DOT.join(quotedShortFieldNames), quotedName); + String shortName = joiner.join(joiner.join(quotedShortFieldNames), quotedName); if (!shortNameToId.containsKey(shortName)) { shortNameToId.put(shortName, fieldId); } diff --git a/api/src/main/java/org/apache/iceberg/types/TypeUtil.java b/api/src/main/java/org/apache/iceberg/types/TypeUtil.java index 22e461a39ef9..a3bee3e3d860 100644 --- a/api/src/main/java/org/apache/iceberg/types/TypeUtil.java +++ b/api/src/main/java/org/apache/iceberg/types/TypeUtil.java @@ -195,6 +195,19 @@ public static Map indexQuotedNameById( return indexer.byId(); } + /** + * Indexes the fields of a struct from ID to a flattened name that can be used for stats struct + * field names. + * + * @param struct a struct type + * @return an index from field ID to short names, joined by _ + */ + public static Map indexStatsNames(Types.StructType struct) { + IndexByName indexer = new IndexByName("_", Function.identity(), true /* use short names */); + visit(struct, indexer); + return indexer.byId(); + } + /** * Creates a mapping from lower-case field names to their corresponding field IDs. * diff --git a/api/src/main/java/org/apache/iceberg/types/Types.java b/api/src/main/java/org/apache/iceberg/types/Types.java index ec6076b04fa0..f082915920ea 100644 --- a/api/src/main/java/org/apache/iceberg/types/Types.java +++ b/api/src/main/java/org/apache/iceberg/types/Types.java @@ -59,8 +59,8 @@ private Types() {} .put(BinaryType.get().toString(), BinaryType.get()) .put(UnknownType.get().toString(), UnknownType.get()) .put(VariantType.get().toString(), VariantType.get()) - .put(GeometryType.crs84().toString(), GeometryType.crs84()) - .put(GeographyType.crs84().toString(), GeographyType.crs84()) + .put(GeometryType.NAME, GeometryType.crs84()) + .put(GeographyType.NAME, GeographyType.crs84()) .buildOrThrow(); private static final Pattern FIXED = Pattern.compile("fixed\\[\\s*(\\d+)\\s*\\]"); @@ -570,8 +570,10 @@ public int hashCode() { } } + /** A geometry type, optionally parameterized by a CRS. The default CRS is {@code OGC:CRS84}. */ public static class GeometryType extends PrimitiveType { public static final String DEFAULT_CRS = "OGC:CRS84"; + private static final String NAME = "geometry"; public static GeometryType crs84() { return new GeometryType(); @@ -584,12 +586,14 @@ public static GeometryType of(String crs) { private final String crs; private GeometryType() { - crs = null; + this(null); } private GeometryType(String crs) { Preconditions.checkArgument(crs == null || !crs.isEmpty(), "Invalid CRS: (empty string)"); - this.crs = DEFAULT_CRS.equalsIgnoreCase(crs) ? null : crs; + // an omitted CRS canonicalizes to the default; a provided value is kept as-is (case + // preserved) + this.crs = crs == null ? DEFAULT_CRS : crs; } @Override @@ -598,9 +602,13 @@ public TypeID typeId() { } public String crs() { - return crs != null ? crs : DEFAULT_CRS; + return crs; } + /** + * Two geometry types are equal when their CRS match case-insensitively, so {@code OGC:CRS84} + * and {@code ogc:crs84} are equal. + */ @Override public boolean equals(Object o) { if (this == o) { @@ -610,27 +618,29 @@ public boolean equals(Object o) { } GeometryType that = (GeometryType) o; - return Objects.equals(crs, that.crs); + return crs.equalsIgnoreCase(that.crs); } @Override public int hashCode() { - return Objects.hash(GeometryType.class, crs); + // hash the upper-cased CRS so it stays consistent with the case-insensitive equals + return Objects.hash(GeometryType.class, crs.toUpperCase(Locale.ROOT)); } @Override public String toString() { - if (crs == null) { - return "geometry"; - } - - return String.format("geometry(%s)", crs); + return String.format("%s(%s)", NAME, crs()); } } + /** + * A geography type, optionally parameterized by a CRS and an edge-interpolation algorithm. The + * default CRS is {@code OGC:CRS84} and the default algorithm is {@code spherical}. + */ public static class GeographyType extends PrimitiveType { public static final String DEFAULT_CRS = "OGC:CRS84"; public static final EdgeAlgorithm DEFAULT_ALGORITHM = EdgeAlgorithm.SPHERICAL; + private static final String NAME = "geography"; public static GeographyType crs84() { return new GeographyType(); @@ -648,14 +658,15 @@ public static GeographyType of(String crs, EdgeAlgorithm algorithm) { private final EdgeAlgorithm algorithm; private GeographyType() { - this.crs = null; - this.algorithm = null; + this(null, null); } private GeographyType(String crs, EdgeAlgorithm algorithm) { Preconditions.checkArgument(crs == null || !crs.isEmpty(), "Invalid CRS: (empty string)"); - this.crs = DEFAULT_CRS.equalsIgnoreCase(crs) ? null : crs; - this.algorithm = algorithm; + // an omitted CRS/algorithm canonicalizes to the default; a provided CRS is kept as-is (case + // preserved) + this.crs = crs == null ? DEFAULT_CRS : crs; + this.algorithm = algorithm == null ? DEFAULT_ALGORITHM : algorithm; } @Override @@ -664,13 +675,17 @@ public TypeID typeId() { } public String crs() { - return crs != null ? crs : DEFAULT_CRS; + return crs; } public EdgeAlgorithm algorithm() { - return algorithm != null ? algorithm : DEFAULT_ALGORITHM; + return algorithm; } + /** + * Two geography types are equal when their edge algorithms are equal and their CRS match + * case-insensitively, so {@code OGC:CRS84} and {@code ogc:crs84} are equal. + */ @Override public boolean equals(Object o) { if (this == o) { @@ -680,23 +695,18 @@ public boolean equals(Object o) { } GeographyType that = (GeographyType) o; - return Objects.equals(crs, that.crs) && Objects.equals(algorithm, that.algorithm); + return crs.equalsIgnoreCase(that.crs) && Objects.equals(algorithm, that.algorithm); } @Override public int hashCode() { - return Objects.hash(GeographyType.class, crs, algorithm); + // hash the upper-cased CRS so it stays consistent with the case-insensitive equals + return Objects.hash(GeographyType.class, crs.toUpperCase(Locale.ROOT), algorithm); } @Override public String toString() { - if (algorithm != null) { - return String.format("geography(%s, %s)", crs != null ? crs : DEFAULT_CRS, algorithm); - } else if (crs != null) { - return String.format("geography(%s)", crs); - } else { - return "geography"; - } + return String.format("%s(%s, %s)", NAME, crs(), algorithm()); } } diff --git a/api/src/test/java/org/apache/iceberg/TestPartitionSpecValidation.java b/api/src/test/java/org/apache/iceberg/TestPartitionSpecValidation.java index a1709d2a2e06..5c37ad527291 100644 --- a/api/src/test/java/org/apache/iceberg/TestPartitionSpecValidation.java +++ b/api/src/test/java/org/apache/iceberg/TestPartitionSpecValidation.java @@ -351,8 +351,12 @@ public void testUnsupported(int fieldId, String partitionName, String expectedEr private static Object[][] unsupportedFieldsProvider() { return new Object[][] { {7, "variant_partition1", "Cannot partition by non-primitive source field: variant"}, - {8, "geom_partition1", "Invalid source type geometry for transform: bucket[5]"}, - {9, "geog_partition1", "Invalid source type geography for transform: bucket[5]"}, + {8, "geom_partition1", "Invalid source type geometry(OGC:CRS84) for transform: bucket[5]"}, + { + 9, + "geog_partition1", + "Invalid source type geography(OGC:CRS84, spherical) for transform: bucket[5]" + }, {10, "unknown_partition1", "Invalid source type unknown for transform: bucket[5]"} }; } diff --git a/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java b/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java index dd8afebab84d..b7da4b3108e6 100644 --- a/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java +++ b/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java @@ -571,7 +571,7 @@ public void testReassignIdsIllegalArgumentException() { } @Test - public void testValidateSchemaViaIndexByName() { + public void testValidateSchemaViaByName() { Types.NestedField nestedType = Types.NestedField.required( 1, @@ -970,4 +970,71 @@ public void ancestorFieldsInNestedSchema() { assertThat(TypeUtil.ancestorFields(schema, 16)).containsExactly(pointsElement, points); assertThat(TypeUtil.ancestorFields(schema, 17)).containsExactly(pointsElement, points); } + + @Test + public void testIndexStatsNames() { + Schema schema = + new Schema( + required(1, "id", Types.LongType.get()), + optional(2, "data", Types.StringType.get()), + optional( + 3, + "location", + Types.StructType.of( + required(10, "lat", Types.FloatType.get()), + required(11, "long", Types.FloatType.get()))), + optional(4, "values", Types.ListType.ofOptional(12, IntegerType.get())), + optional( + 5, + "points", + Types.ListType.ofRequired( + 13, + Types.StructType.of( + required(14, "x", IntegerType.get()), + required(15, "y", IntegerType.get())))), + optional( + 6, + "properties", + Types.MapType.ofOptional(16, 17, Types.StringType.get(), Types.StringType.get())), + optional( + 7, + "addresses", + Types.MapType.ofRequired( + 18, + 19, + Types.StringType.get(), + Types.StructType.of( + required(20, "street", Types.StringType.get()), + optional(21, "city", Types.StringType.get()), + optional(22, "state", Types.StringType.get()), + required(23, "zip", Types.IntegerType.get()), + required(24, "value", Types.IntegerType.get()))))); + + Map statsNameIndex = TypeUtil.indexStatsNames(schema.asStruct()); + + assertThat(statsNameIndex) + .containsEntry(1, "id") + .containsEntry(2, "data") + .containsEntry(3, "location") + .containsEntry(10, "location_lat") + .containsEntry(11, "location_long") + .containsEntry(4, "values") + .containsEntry(12, "values_element") + .containsEntry(5, "points") + .containsEntry(13, "points_element") + .containsEntry(14, "points_x") + .containsEntry(15, "points_y") + .containsEntry(6, "properties") + .containsEntry(16, "properties_key") + .containsEntry(17, "properties_value") + .containsEntry(7, "addresses") + .containsEntry(18, "addresses_key") + .containsEntry(19, "addresses_value") + .containsEntry(20, "addresses_street") + .containsEntry(21, "addresses_city") + .containsEntry(22, "addresses_state") + .containsEntry(23, "addresses_zip") + .containsEntry(24, "addresses_value") // the leaf takes precedence + .hasSize(22); + } } diff --git a/api/src/test/java/org/apache/iceberg/types/TestTypes.java b/api/src/test/java/org/apache/iceberg/types/TestTypes.java index fa5ed4304d3c..2fb224aefb15 100644 --- a/api/src/test/java/org/apache/iceberg/types/TestTypes.java +++ b/api/src/test/java/org/apache/iceberg/types/TestTypes.java @@ -165,13 +165,54 @@ public void fromPrimitiveString() { @Test public void testGeospatialTypeToString() { - assertThat(Types.GeometryType.crs84().toString()).isEqualTo("geometry"); + assertThat(Types.GeometryType.crs84().toString()).isEqualTo("geometry(OGC:CRS84)"); + assertThat(Types.GeometryType.of(Types.GeometryType.DEFAULT_CRS).toString()) + .isEqualTo("geometry(OGC:CRS84)"); assertThat(Types.GeometryType.of("srid:4326").toString()).isEqualTo("geometry(srid:4326)"); - assertThat(Types.GeographyType.crs84().toString()).isEqualTo("geography"); + assertThat(Types.GeographyType.crs84().toString()).isEqualTo("geography(OGC:CRS84, spherical)"); assertThat(Types.GeographyType.of("srid:4326", EdgeAlgorithm.KARNEY).toString()) .isEqualTo("geography(srid:4326, karney)"); assertThat(Types.GeographyType.of(null, EdgeAlgorithm.KARNEY).toString()) .isEqualTo("geography(OGC:CRS84, karney)"); + assertThat( + Types.GeographyType.of(Types.GeographyType.DEFAULT_CRS, EdgeAlgorithm.SPHERICAL) + .toString()) + .isEqualTo("geography(OGC:CRS84, spherical)"); + assertThat(Types.GeographyType.of("srid:4326", EdgeAlgorithm.SPHERICAL).toString()) + .isEqualTo("geography(srid:4326, spherical)"); + + // the CRS keeps its original casing in toString even though comparison is case-insensitive + assertThat(Types.GeometryType.of("ogc:crs84").toString()).isEqualTo("geometry(ogc:crs84)"); + assertThat(Types.GeographyType.of("ogc:crs84").toString()) + .isEqualTo("geography(ogc:crs84, spherical)"); + } + + @Test + public void testGeospatialTypeDefaultNormalization() { + // an omitted default and an explicit default (exact CRS / spherical algorithm) are equal + assertThat(Types.GeometryType.of(Types.GeometryType.DEFAULT_CRS)) + .isEqualTo(Types.GeometryType.crs84()); + assertThat(Types.GeographyType.of(Types.GeographyType.DEFAULT_CRS)) + .isEqualTo(Types.GeographyType.crs84()); + assertThat(Types.GeographyType.of(Types.GeographyType.DEFAULT_CRS, EdgeAlgorithm.SPHERICAL)) + .isEqualTo(Types.GeographyType.crs84()) + .hasSameHashCodeAs(Types.GeographyType.crs84()); + assertThat(Types.GeographyType.of("srid:4326", EdgeAlgorithm.SPHERICAL)) + .isEqualTo(Types.GeographyType.of("srid:4326")) + .hasSameHashCodeAs(Types.GeographyType.of("srid:4326")); + assertThat(Types.GeographyType.of("srid:4326", EdgeAlgorithm.SPHERICAL).algorithm()) + .isEqualTo(EdgeAlgorithm.SPHERICAL); + assertThat(Types.GeographyType.of("srid:4326", EdgeAlgorithm.KARNEY)) + .isNotEqualTo(Types.GeographyType.of("srid:4326")); + assertThat(Types.GeographyType.of("srid:4326")).isNotEqualTo(Types.GeographyType.crs84()); + + // CRS comparison is case-insensitive: any casing of a CRS is equal and hashes the same + assertThat(Types.GeometryType.of("ogc:crs84")) + .isEqualTo(Types.GeometryType.crs84()) + .hasSameHashCodeAs(Types.GeometryType.crs84()); + assertThat(Types.GeographyType.of("ogc:crs84")) + .isEqualTo(Types.GeographyType.crs84()) + .hasSameHashCodeAs(Types.GeographyType.crs84()); } @Test diff --git a/api/src/test/java/org/apache/iceberg/util/RandomUtil.java b/api/src/test/java/org/apache/iceberg/util/RandomUtil.java index b437b0bbf51c..4dd3ac887044 100644 --- a/api/src/test/java/org/apache/iceberg/util/RandomUtil.java +++ b/api/src/test/java/org/apache/iceberg/util/RandomUtil.java @@ -20,6 +20,8 @@ import java.math.BigDecimal; import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -157,6 +159,12 @@ public static Object generatePrimitive(Type.PrimitiveType primitive, Random rand BigDecimal bigDecimal = new BigDecimal(unscaled, type.scale()); return negate(choice) ? bigDecimal.negate() : bigDecimal; + case GEOMETRY: + case GEOGRAPHY: + // geometry and geography values are stored as WKB + return wkbPoint( + (random.nextDouble() * 360.0) - 180.0, (random.nextDouble() * 180.0) - 90.0); + default: throw new IllegalArgumentException( "Cannot generate random value for unknown type: " + primitive); @@ -202,12 +210,27 @@ public static Object generateDictionaryEncodablePrimitive( byte[] uuidBytes = new byte[16]; random.nextBytes(uuidBytes); return uuidBytes; + case GEOMETRY: + case GEOGRAPHY: + // a small set of distinct points so the WKB column stays dictionary encodable + return wkbPoint(value, value); default: throw new IllegalArgumentException( "Cannot generate random value for unknown type: " + primitive); } } + /** Encodes a point as little-endian WKB, the on-disk representation for geo values. */ + public static byte[] wkbPoint(double xCoord, double yCoord) { + return ByteBuffer.allocate(21) + .order(ByteOrder.LITTLE_ENDIAN) + .put((byte) 1) // byte order: little endian + .putInt(1) // WKB geometry type: Point + .putDouble(xCoord) + .putDouble(yCoord) + .array(); + } + private static final long FIFTY_YEARS_IN_MICROS = (50L * (365 * 3 + 366) * 24 * 60 * 60 * 1_000_000) / 4; private static final long ABOUT_TEN_YEARS_IN_NANOS = 10L * 365 * 24 * 60 * 60 * 1_000_000_000; diff --git a/aws/src/main/java/org/apache/iceberg/aws/glue/IcebergToGlueConverter.java b/aws/src/main/java/org/apache/iceberg/aws/glue/IcebergToGlueConverter.java index 56b38a47e968..2363dc4e2a87 100644 --- a/aws/src/main/java/org/apache/iceberg/aws/glue/IcebergToGlueConverter.java +++ b/aws/src/main/java/org/apache/iceberg/aws/glue/IcebergToGlueConverter.java @@ -263,18 +263,22 @@ static void setTableInputInformation( Optional.ofNullable(existingTable.description()).ifPresent(tableInputBuilder::description); } - Map existingColumnMap = null; + Map existingColumnNameToComment = null; if (existingTable != null) { List existingColumns = existingTable.storageDescriptor().columns(); - existingColumnMap = + // First-seen-wins on duplicate names. toColumns writes current-schema columns before + // historical-schema columns. Preserve the current-schema comment. + existingColumnNameToComment = existingColumns.stream() .filter(column -> column.comment() != null) - .collect(Collectors.toMap(Column::name, Column::comment)); + .collect( + Collectors.toMap( + Column::name, Column::comment, (existing, duplicate) -> existing)); } else { - existingColumnMap = Collections.emptyMap(); + existingColumnNameToComment = Collections.emptyMap(); } - List columns = toColumns(metadata, existingColumnMap); + List columns = toColumns(metadata, existingColumnNameToComment); tableInputBuilder.storageDescriptor( storageDescriptor.location(metadata.location()).columns(columns).build()); @@ -340,19 +344,20 @@ private static String toTypeString(Type type) { } private static List toColumns( - TableMetadata metadata, Map existingColumnMap) { + TableMetadata metadata, Map existingColumnNameToComment) { List columns = Lists.newArrayList(); Set addedNames = Sets.newHashSet(); for (NestedField field : metadata.schema().columns()) { - addColumnWithDedupe(columns, addedNames, field, true /* is current */, existingColumnMap); + addColumnWithDedupe( + columns, addedNames, field, true /* is current */, existingColumnNameToComment); } for (Schema schema : metadata.schemas()) { if (schema.schemaId() != metadata.currentSchemaId()) { for (NestedField field : schema.columns()) { addColumnWithDedupe( - columns, addedNames, field, false /* is not current */, existingColumnMap); + columns, addedNames, field, false /* is not current */, existingColumnNameToComment); } } } @@ -365,7 +370,7 @@ private static void addColumnWithDedupe( Set dedupe, NestedField field, boolean isCurrent, - Map existingColumnMap) { + Map existingColumnNameToComment) { if (!dedupe.contains(field.name())) { Column.Builder builder = Column.builder() @@ -379,8 +384,9 @@ private static void addColumnWithDedupe( if (field.doc() != null && !field.doc().isEmpty()) { builder.comment(field.doc()); - } else if (existingColumnMap != null && existingColumnMap.containsKey(field.name())) { - builder.comment(existingColumnMap.get(field.name())); + } else if (existingColumnNameToComment != null + && existingColumnNameToComment.containsKey(field.name())) { + builder.comment(existingColumnNameToComment.get(field.name())); } columns.add(builder.build()); diff --git a/aws/src/test/java/org/apache/iceberg/aws/glue/TestIcebergToGlueConverter.java b/aws/src/test/java/org/apache/iceberg/aws/glue/TestIcebergToGlueConverter.java index edebfd3420e2..a797ac7b2ffa 100644 --- a/aws/src/test/java/org/apache/iceberg/aws/glue/TestIcebergToGlueConverter.java +++ b/aws/src/test/java/org/apache/iceberg/aws/glue/TestIcebergToGlueConverter.java @@ -384,4 +384,57 @@ public void testSetTableInputInformationWithExistingTable() { .as("Columns should match") .isEqualTo(expectedTableInput.storageDescriptor().columns()); } + + @Test + public void testDuplicateExistingColumnsKeepFirstComment() { + TableInput.Builder actualTableInputBuilder = TableInput.builder(); + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.StringType.get())); + PartitionSpec partitionSpec = + PartitionSpec.builderFor(schema).identity("id").withSpecId(1000).build(); + TableMetadata tableMetadata = + TableMetadata.newTableMetadata(schema, partitionSpec, "s3://test", tableLocationProperties); + + Table existingGlueTable = + Table.builder() + .storageDescriptor( + StorageDescriptor.builder() + .columns( + ImmutableList.of( + Column.builder().name("id").comment("current comment").build(), + Column.builder().name("id").comment("historical comment").build())) + .build()) + .build(); + + IcebergToGlueConverter.setTableInputInformation( + actualTableInputBuilder, tableMetadata, existingGlueTable); + TableInput actualTableInput = actualTableInputBuilder.build(); + + TableInput expectedTableInput = + TableInput.builder() + .storageDescriptor( + StorageDescriptor.builder() + .location("s3://test") + .additionalLocations(Sets.newHashSet(tableLocationProperties.values())) + .columns( + ImmutableList.of( + Column.builder() + .name("id") + .type("string") + .comment("current comment") + .parameters( + ImmutableMap.of( + IcebergToGlueConverter.ICEBERG_FIELD_ID, "1", + IcebergToGlueConverter.ICEBERG_FIELD_OPTIONAL, "false", + IcebergToGlueConverter.ICEBERG_FIELD_CURRENT, "true")) + .build())) + .build()) + .build(); + + assertThat(actualTableInput.storageDescriptor()) + .as("Storage descriptor should be set") + .isNotNull(); + assertThat(actualTableInput.storageDescriptor().columns()) + .as("Columns should match") + .isEqualTo(expectedTableInput.storageDescriptor().columns()); + } } diff --git a/azure-bundle/build.gradle b/azure-bundle/build.gradle index fde8adbfc539..8d0fe41a5789 100644 --- a/azure-bundle/build.gradle +++ b/azure-bundle/build.gradle @@ -31,6 +31,9 @@ project(":iceberg-azure-bundle") { dependencies { implementation platform(libs.azuresdk.bom) + // Align the shaded Jackson to the project version; the Azure SDK BOM otherwise + // pulls in an older jackson-databind with known CVEs. + implementation platform(libs.jackson.bom) implementation "com.azure:azure-storage-file-datalake" implementation "com.azure:azure-security-keyvault-keys" implementation "com.azure:azure-identity" diff --git a/azure-bundle/runtime-deps.txt b/azure-bundle/runtime-deps.txt index 2caed26153e8..adb6044657ee 100644 --- a/azure-bundle/runtime-deps.txt +++ b/azure-bundle/runtime-deps.txt @@ -8,10 +8,10 @@ com.azure:azure-storage-common:12.33 com.azure:azure-storage-file-datalake:12.27 com.azure:azure-storage-internal-avro:12.19 com.azure:azure-xml:1.2 -com.fasterxml.jackson.core:jackson-annotations:2.18 -com.fasterxml.jackson.core:jackson-core:2.18 -com.fasterxml.jackson.core:jackson-databind:2.18 -com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.18 +com.fasterxml.jackson.core:jackson-annotations:2.22 +com.fasterxml.jackson.core:jackson-core:2.22 +com.fasterxml.jackson.core:jackson-databind:2.22 +com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.22 com.microsoft.azure:msal4j-persistence-extension:1.3 com.microsoft.azure:msal4j:1.23 io.netty:netty-buffer:4.2 diff --git a/build.gradle b/build.gradle index 90a38f633fa3..7a8f9a91128f 100644 --- a/build.gradle +++ b/build.gradle @@ -28,7 +28,7 @@ buildscript { dependencies { classpath 'com.gradleup.shadow:shadow-gradle-plugin:8.3.11' classpath 'com.palantir.baseline:gradle-baseline-java:6.90.0' - classpath 'com.diffplug.spotless:spotless-plugin-gradle:8.6.0' + classpath 'com.diffplug.spotless:spotless-plugin-gradle:8.7.0' classpath 'gradle.plugin.org.inferred:gradle-processors:3.7.0' classpath 'me.champeau.jmh:jmh-gradle-plugin:0.7.3' classpath 'gradle.plugin.io.morethan.jmhreport:gradle-jmh-report:0.9.6' @@ -36,7 +36,7 @@ buildscript { classpath 'org.revapi:gradle-revapi:1.8.0' classpath 'com.gorylenko.gradle-git-properties:gradle-git-properties:2.5.7' classpath 'com.palantir.gradle.gitversion:gradle-git-version:4.3.0' - classpath 'org.openapitools:openapi-generator-gradle-plugin:7.22.0' + classpath 'org.openapitools:openapi-generator-gradle-plugin:7.23.0' } } diff --git a/core/src/main/java/org/apache/iceberg/BaseSnapshot.java b/core/src/main/java/org/apache/iceberg/BaseSnapshot.java index b8ea6db22938..826b9624c0e6 100644 --- a/core/src/main/java/org/apache/iceberg/BaseSnapshot.java +++ b/core/src/main/java/org/apache/iceberg/BaseSnapshot.java @@ -340,7 +340,10 @@ public boolean equals(Object o) { && Objects.equal(this.parentId, other.parentId()) && this.sequenceNumber == other.sequenceNumber() && this.timestampMillis == other.timestampMillis() - && Objects.equal(this.schemaId, other.schemaId()); + && Objects.equal(this.schemaId, other.schemaId()) + && Objects.equal(this.firstRowId, other.firstRowId()) + && Objects.equal(this.addedRows, other.addedRows()) + && Objects.equal(this.keyId, other.keyId()); } return false; @@ -349,7 +352,14 @@ public boolean equals(Object o) { @Override public int hashCode() { return Objects.hashCode( - this.snapshotId, this.parentId, this.sequenceNumber, this.timestampMillis, this.schemaId); + this.snapshotId, + this.parentId, + this.sequenceNumber, + this.timestampMillis, + this.schemaId, + this.firstRowId, + this.addedRows, + this.keyId); } @Override @@ -361,6 +371,9 @@ public String toString() { .add("summary", summary) .add("manifest-list", manifestListLocation) .add("schema-id", schemaId) + .add("first-row-id", firstRowId) + .add("added-rows", addedRows) + .add("key-id", keyId) .toString(); } } diff --git a/core/src/main/java/org/apache/iceberg/DeletionVectorStruct.java b/core/src/main/java/org/apache/iceberg/DeletionVectorStruct.java index 04d23fa33abe..3f5be0756fad 100644 --- a/core/src/main/java/org/apache/iceberg/DeletionVectorStruct.java +++ b/core/src/main/java/org/apache/iceberg/DeletionVectorStruct.java @@ -19,6 +19,7 @@ package org.apache.iceberg; import java.io.Serializable; +import java.util.Objects; import org.apache.iceberg.avro.SupportsIndexProjection; import org.apache.iceberg.relocated.com.google.common.base.MoreObjects; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; @@ -128,6 +129,26 @@ static Builder builder() { return new Builder(); } + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } else if (!(other instanceof DeletionVectorStruct)) { + return false; + } + + DeletionVectorStruct that = (DeletionVectorStruct) other; + return Objects.equals(location, that.location) + && offset == that.offset + && sizeInBytes == that.sizeInBytes + && cardinality == that.cardinality; + } + + @Override + public int hashCode() { + return Objects.hash(location, offset, sizeInBytes, cardinality); + } + @Override public String toString() { return MoreObjects.toStringHelper(this) diff --git a/core/src/main/java/org/apache/iceberg/RewriteTablePathUtil.java b/core/src/main/java/org/apache/iceberg/RewriteTablePathUtil.java index 69f82931833e..e72dbef6dbcc 100644 --- a/core/src/main/java/org/apache/iceberg/RewriteTablePathUtil.java +++ b/core/src/main/java/org/apache/iceberg/RewriteTablePathUtil.java @@ -47,6 +47,7 @@ import org.apache.iceberg.puffin.PuffinReader; import org.apache.iceberg.puffin.PuffinWriter; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.relocated.com.google.common.collect.Sets; @@ -377,7 +378,12 @@ public static RewriteResult rewriteDataManifest( * @param stagingLocation staging location for rewritten files (referred delete file will be * rewritten here) * @return a copy plan of content files in the manifest that was rewritten + * @deprecated since 1.12.0, will be removed in 1.13.0; use the overload that accepts the map of + * rewritten position delete file sizes. This overload records the original {@code + * file_size_in_bytes}, which can be inconsistent with the rewritten file size on disk once + * embedded data file paths change length. */ + @Deprecated public static RewriteResult rewriteDeleteManifest( ManifestFile manifestFile, Set snapshotIds, @@ -389,6 +395,52 @@ public static RewriteResult rewriteDeleteManifest( String targetPrefix, String stagingLocation) throws IOException { + return rewriteDeleteManifest( + manifestFile, + snapshotIds, + outputFile, + io, + format, + specsById, + sourcePrefix, + targetPrefix, + stagingLocation, + ImmutableMap.of()); + } + + /** + * Rewrite a delete manifest, replacing path references. + * + *

This is a metadata-only operation: position delete file content is rewritten separately (see + * {@link #rewritePositionDelete}). The actual sizes of those rewritten files are supplied via + * {@code rewrittenDeleteFileSizes} and recorded in the manifest so that {@code + * file_size_in_bytes} stays consistent with the rewritten file on disk. + * + * @param manifestFile source delete manifest to rewrite + * @param snapshotIds snapshot ids for filtering returned delete manifest entries + * @param outputFile output file to rewrite manifest file to + * @param io file io + * @param format format of the manifest file + * @param specsById map of partition specs by id + * @param sourcePrefix source prefix that will be replaced + * @param targetPrefix target prefix that will replace it + * @param stagingLocation staging location for rewritten position delete files + * @param rewrittenDeleteFileSizes map from source position delete file path to the actual size of + * the rewritten file; entries absent from the map keep their original size + * @return a copy plan of content files in the manifest that was rewritten + */ + public static RewriteResult rewriteDeleteManifest( + ManifestFile manifestFile, + Set snapshotIds, + OutputFile outputFile, + FileIO io, + int format, + Map specsById, + String sourcePrefix, + String targetPrefix, + String stagingLocation, + Map rewrittenDeleteFileSizes) + throws IOException { PartitionSpec spec = specsById.get(manifestFile.partitionSpecId()); try (ManifestWriter writer = ManifestFiles.writeDeleteManifest(format, spec, outputFile, manifestFile.snapshotId()); @@ -405,7 +457,8 @@ public static RewriteResult rewriteDeleteManifest( sourcePrefix, targetPrefix, stagingLocation, - writer)) + writer, + rewrittenDeleteFileSizes)) .reduce(new RewriteResult<>(), RewriteResult::append); } } @@ -445,14 +498,20 @@ private static RewriteResult writeDeleteFileEntry( String sourcePrefix, String targetPrefix, String stagingLocation, - ManifestWriter writer) { + ManifestWriter writer, + Map rewrittenDeleteFileSizes) { DeleteFile file = entry.file(); RewriteResult result = new RewriteResult<>(); switch (file.content()) { case POSITION_DELETES: - DeleteFile posDeleteFile = newPositionDeleteEntry(file, spec, sourcePrefix, targetPrefix); + // Path rewrites change the file size; use the measured size, falling back to the original + // for entries that were not rewritten (e.g. deleted entries not copied to the target). + long fileSizeInBytes = + rewrittenDeleteFileSizes.getOrDefault(file.location(), file.fileSizeInBytes()); + DeleteFile posDeleteFile = + newPositionDeleteEntry(file, spec, sourcePrefix, targetPrefix, fileSizeInBytes); appendEntryWithFile(entry, writer, posDeleteFile); // keep the following entries in metadata but exclude them from copyPlan // 1) deleted position delete files @@ -523,7 +582,11 @@ private static DeleteFile newEqualityDeleteEntry( } private static DeleteFile newPositionDeleteEntry( - DeleteFile file, PartitionSpec spec, String sourcePrefix, String targetPrefix) { + DeleteFile file, + PartitionSpec spec, + String sourcePrefix, + String targetPrefix, + long fileSizeInBytes) { String path = file.location(); Preconditions.checkArgument( path.startsWith(sourcePrefix), @@ -535,6 +598,7 @@ private static DeleteFile newPositionDeleteEntry( FileMetadata.deleteFileBuilder(spec) .copy(file) .withPath(newPath(path, sourcePrefix, targetPrefix)) + .withFileSizeInBytes(fileSizeInBytes) .withMetrics(ContentFileUtil.replacePathBounds(file, sourcePrefix, targetPrefix)); // Update referencedDataFile for DV files @@ -607,7 +671,11 @@ PositionDeleteWriter writer( * @param sourcePrefix source prefix that will be replaced * @param targetPrefix target prefix to replace it * @param posDeleteReaderWriter class to read and write position delete files + * @deprecated since 1.12.0, will be removed in 1.13.0; use {@link #rewritePositionDelete} which + * returns the size of the rewritten file so callers can record an accurate {@code + * file_size_in_bytes}. */ + @Deprecated public static void rewritePositionDeleteFile( DeleteFile deleteFile, OutputFile outputFile, @@ -617,6 +685,37 @@ public static void rewritePositionDeleteFile( String targetPrefix, PositionDeleteReaderWriter posDeleteReaderWriter) throws IOException { + rewritePositionDelete( + deleteFile, outputFile, io, spec, sourcePrefix, targetPrefix, posDeleteReaderWriter); + } + + /** + * Rewrite a position delete file, replacing path references, and return the size of the rewritten + * file. + * + *

The size is measured from the writer after it is closed (rather than via a separate {@code + * getLength()}/HEAD call), so it is accurate even on file systems where the length of an + * in-progress write underreports. Callers record this size as {@code file_size_in_bytes} in the + * rewritten manifest. + * + * @param deleteFile source position delete file to rewrite + * @param outputFile output file to write the rewritten delete file to + * @param io file io + * @param spec spec of delete file + * @param sourcePrefix source prefix that will be replaced + * @param targetPrefix target prefix to replace it + * @param posDeleteReaderWriter class to read and write position delete files + * @return the size in bytes of the rewritten file + */ + public static long rewritePositionDelete( + DeleteFile deleteFile, + OutputFile outputFile, + FileIO io, + PartitionSpec spec, + String sourcePrefix, + String targetPrefix, + PositionDeleteReaderWriter posDeleteReaderWriter) + throws IOException { String path = deleteFile.location(); if (!path.startsWith(sourcePrefix)) { throw new UnsupportedOperationException( @@ -625,8 +724,7 @@ public static void rewritePositionDeleteFile( // DV files (Puffin format for v3+) need special handling to rewrite internal blob metadata if (ContentFileUtil.isDV(deleteFile)) { - rewriteDVFile(deleteFile, outputFile, io, sourcePrefix, targetPrefix); - return; + return rewriteDVFile(deleteFile, outputFile, io, sourcePrefix, targetPrefix); } // For non-DV position delete files (v2), rewrite using the reader/writer @@ -655,9 +753,14 @@ record = recordIt.next(); writer.write(newPositionDeleteRecord(record, sourcePrefix, targetPrefix)); } } + + writer.close(); + return writer.length(); } } } + + return 0; } /** @@ -668,8 +771,9 @@ record = recordIt.next(); * @param io file io * @param sourcePrefix source prefix that will be replaced * @param targetPrefix target prefix to replace it + * @return the size in bytes of the rewritten DV file */ - private static void rewriteDVFile( + private static long rewriteDVFile( DeleteFile deleteFile, OutputFile outputFile, FileIO io, @@ -708,6 +812,8 @@ private static void rewriteDVFile( try (PuffinWriter writer = Puffin.write(outputFile).createdBy(IcebergBuild.fullVersion()).build()) { rewrittenBlobs.forEach(writer::write); + writer.close(); + return writer.length(); } } diff --git a/core/src/main/java/org/apache/iceberg/TableMetadata.java b/core/src/main/java/org/apache/iceberg/TableMetadata.java index b811d8c42f15..07f6d3342727 100644 --- a/core/src/main/java/org/apache/iceberg/TableMetadata.java +++ b/core/src/main/java/org/apache/iceberg/TableMetadata.java @@ -1519,10 +1519,10 @@ public Builder addEncryptionKey(EncryptedKey key) { } public Builder removeEncryptionKey(String keyId) { - boolean removed = encryptionKeys.removeIf(key -> key.keyId().equals(keyId)); - keysById.remove(keyId); + EncryptedKey removedKey = keysById.remove(keyId); - if (removed) { + if (removedKey != null) { + encryptionKeys.remove(removedKey); changes.add(new MetadataUpdate.RemoveEncryptionKey(keyId)); } diff --git a/core/src/main/java/org/apache/iceberg/TableProperties.java b/core/src/main/java/org/apache/iceberg/TableProperties.java index 881c00b71263..56d381632607 100644 --- a/core/src/main/java/org/apache/iceberg/TableProperties.java +++ b/core/src/main/java/org/apache/iceberg/TableProperties.java @@ -357,16 +357,51 @@ private TableProperties() {} public static final String DELETE_TARGET_FILE_SIZE_BYTES = "write.delete.target-file-size-bytes"; public static final long DELETE_TARGET_FILE_SIZE_BYTES_DEFAULT = 64 * 1024 * 1024; // 64 MB + /** + * @deprecated will be removed in 1.14.0, use + * SparkTableProperties.WRITE_PARTITIONED_FANOUT_ENABLED in iceberg-spark instead. + */ + @Deprecated public static final String SPARK_WRITE_PARTITIONED_FANOUT_ENABLED = "write.spark.fanout.enabled"; - public static final boolean SPARK_WRITE_PARTITIONED_FANOUT_ENABLED_DEFAULT = false; + /** + * @deprecated will be removed in 1.14.0, use + * SparkTableProperties.WRITE_PARTITIONED_FANOUT_ENABLED_DEFAULT in iceberg-spark instead. + */ + @Deprecated public static final boolean SPARK_WRITE_PARTITIONED_FANOUT_ENABLED_DEFAULT = false; + + /** + * @deprecated will be removed in 1.14.0, use SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA in + * iceberg-spark instead. + */ + @Deprecated public static final String SPARK_WRITE_ACCEPT_ANY_SCHEMA = "write.spark.accept-any-schema"; - public static final boolean SPARK_WRITE_ACCEPT_ANY_SCHEMA_DEFAULT = false; + /** + * @deprecated will be removed in 1.14.0, use SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA_DEFAULT + * in iceberg-spark instead. + */ + @Deprecated public static final boolean SPARK_WRITE_ACCEPT_ANY_SCHEMA_DEFAULT = false; + + /** + * @deprecated will be removed in 1.14.0, use SparkTableProperties.WRITE_AUTO_SCHEMA_EVOLUTION in + * iceberg-spark instead. + */ + @Deprecated public static final String SPARK_WRITE_AUTO_SCHEMA_EVOLUTION = "write.spark.auto-schema-evolution.enabled"; - public static final boolean SPARK_WRITE_AUTO_SCHEMA_EVOLUTION_DEFAULT = true; + /** + * @deprecated will be removed in 1.14.0, use + * SparkTableProperties.WRITE_AUTO_SCHEMA_EVOLUTION_DEFAULT in iceberg-spark instead. + */ + @Deprecated public static final boolean SPARK_WRITE_AUTO_SCHEMA_EVOLUTION_DEFAULT = true; + + /** + * @deprecated will be removed in 1.14.0, use + * SparkTableProperties.WRITE_ADVISORY_PARTITION_SIZE_BYTES in iceberg-spark instead. + */ + @Deprecated public static final String SPARK_WRITE_ADVISORY_PARTITION_SIZE_BYTES = "write.spark.advisory-partition-size-bytes"; diff --git a/core/src/main/java/org/apache/iceberg/TrackedFile.java b/core/src/main/java/org/apache/iceberg/TrackedFile.java index 7dd7e1b3b48d..5b9cdd9ab46c 100644 --- a/core/src/main/java/org/apache/iceberg/TrackedFile.java +++ b/core/src/main/java/org/apache/iceberg/TrackedFile.java @@ -35,9 +35,9 @@ interface TrackedFile { "content_type", Types.IntegerType.get(), "Type of content: 0=DATA, 2=EQUALITY_DELETES, 3=DATA_MANIFEST, 4=DELETE_MANIFEST"); - Types.NestedField WRITER_FORMAT_VERSION = + Types.NestedField FORMAT_VERSION = Types.NestedField.required( - 157, "writer_format_version", Types.IntegerType.get(), "Writer format version"); + 157, "format_version", Types.IntegerType.get(), "Format version of this file"); Types.NestedField LOCATION = Types.NestedField.required(100, "location", Types.StringType.get(), "Location of the file"); Types.NestedField FILE_FORMAT = @@ -100,7 +100,7 @@ static Types.StructType schemaWithContentStats( return Types.StructType.of( TRACKING, CONTENT_TYPE, - WRITER_FORMAT_VERSION, + FORMAT_VERSION, LOCATION, FILE_FORMAT, RECORD_COUNT, @@ -123,8 +123,8 @@ static Types.StructType schemaWithContentStats( /** Returns the type of content stored by this entry. */ FileContent contentType(); - /** Returns the version of the writer that wrote this file */ - int writerFormatVersion(); + /** Returns the format version of this file. */ + int formatVersion(); /** Returns the location of the file. */ String location(); diff --git a/core/src/main/java/org/apache/iceberg/TrackedFileBuilder.java b/core/src/main/java/org/apache/iceberg/TrackedFileBuilder.java new file mode 100644 index 000000000000..d7aa28c3290f --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/TrackedFileBuilder.java @@ -0,0 +1,361 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import java.nio.ByteBuffer; +import java.util.List; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; + +class TrackedFileBuilder { + private final long snapshotId; + private final FileContent contentType; + + // Required fields + private Integer formatVersion = null; + private String location = null; + private FileFormat fileFormat = null; + private Long recordCount = null; + private Long fileSizeInBytes = null; + private PartitionData partitionData = null; + + // optional fields + private Integer specId = null; + private ContentStats contentStats = null; + private Integer sortOrderId = null; + private DeletionVector deletionVector = null; + private ManifestInfo manifestInfo = null; + private ByteBuffer keyMetadata = null; + private List splitOffsets = null; + private List equalityIds = null; + + // tracking-related fields + private Tracking sourceTracking = null; + private boolean dvUpdated = false; + private ByteBuffer deletedPositions = null; + private ByteBuffer replacedPositions = null; + + /** + * Creates a builder for a newly added data file entry. + * + * @param newSnapshotId the snapshot ID in which the new tracked file will be committed + */ + static TrackedFileBuilder data(long newSnapshotId) { + return new TrackedFileBuilder(FileContent.DATA, newSnapshotId); + } + + /** + * Creates a builder for a newly added equality delete file entry. + * + * @param newSnapshotId the snapshot ID in which the new tracked file will be committed + */ + static TrackedFileBuilder equalityDelete(long newSnapshotId) { + return new TrackedFileBuilder(FileContent.EQUALITY_DELETES, newSnapshotId); + } + + /** + * Creates a builder for a newly added data manifest entry. + * + * @param newSnapshotId the snapshot ID in which the new tracked file will be committed + */ + static TrackedFileBuilder dataManifest(long newSnapshotId) { + return new TrackedFileBuilder(FileContent.DATA_MANIFEST, newSnapshotId); + } + + /** + * Creates a builder for a newly added delete manifest entry. + * + * @param newSnapshotId the snapshot ID in which the new tracked file will be committed + */ + static TrackedFileBuilder deleteManifest(long newSnapshotId) { + return new TrackedFileBuilder(FileContent.DELETE_MANIFEST, newSnapshotId); + } + + /** + * Creates a builder for a tracked file derived from {@code source}. + * + * @param source source tracked file to copy fields from + * @param newSnapshotId the snapshot ID in which the new tracked file will be committed + */ + static TrackedFileBuilder from(TrackedFile source, long newSnapshotId) { + Preconditions.checkArgument(source != null, "Invalid source: null"); + return new TrackedFileBuilder(source, newSnapshotId); + } + + /** + * Returns a DELETED tracked file derived from {@code source}. + * + * @param source source tracked file + * @param newSnapshotId the snapshot ID in which the new tracked file will be committed + */ + static TrackedFile deleted(TrackedFile source, long newSnapshotId) { + Preconditions.checkArgument(source != null, "Invalid source: null"); + return terminal(source, TrackingBuilder.deleted(source.tracking(), newSnapshotId)); + } + + /** + * Returns a REPLACED tracked file derived from {@code source}. + * + *

Manifest entries cannot transition to REPLACED. + * + * @param source source tracked file + * @param newSnapshotId the snapshot ID in which the new tracked file will be committed + */ + static TrackedFile replaced(TrackedFile source, long newSnapshotId) { + Preconditions.checkArgument(source != null, "Invalid source: null"); + Preconditions.checkArgument( + !isLeafManifest(source.contentType()), + "Manifest entries cannot transition to REPLACED, but entry type is: %s", + source.contentType()); + return terminal(source, TrackingBuilder.replaced(source.tracking(), newSnapshotId)); + } + + private static TrackedFile terminal(TrackedFile source, Tracking tracking) { + return new TrackedFileStruct( + tracking, + source.contentType(), + source.formatVersion(), + source.location(), + source.fileFormat(), + (PartitionData) source.partition(), + source.recordCount(), + source.fileSizeInBytes(), + source.specId(), + source.contentStats(), + source.sortOrderId(), + source.deletionVector(), + source.manifestInfo(), + source.keyMetadata(), + source.splitOffsets(), + source.equalityIds()); + } + + private TrackedFileBuilder(FileContent contentType, long snapshotId) { + this.contentType = contentType; + this.snapshotId = snapshotId; + } + + private TrackedFileBuilder(TrackedFile source, long snapshotId) { + this.contentType = source.contentType(); + this.snapshotId = snapshotId; + this.formatVersion = source.formatVersion(); + this.location = source.location(); + this.fileFormat = source.fileFormat(); + this.recordCount = source.recordCount(); + this.fileSizeInBytes = source.fileSizeInBytes(); + this.partitionData = (PartitionData) source.partition(); + this.specId = source.specId(); + this.contentStats = source.contentStats(); + this.sortOrderId = source.sortOrderId(); + this.deletionVector = source.deletionVector(); + this.manifestInfo = source.manifestInfo(); + this.keyMetadata = source.keyMetadata(); + this.splitOffsets = source.splitOffsets(); + this.equalityIds = source.equalityIds(); + this.sourceTracking = source.tracking(); + } + + TrackedFileBuilder formatVersion(int newFormatVersion) { + Preconditions.checkArgument( + newFormatVersion >= 0, "Invalid format version: %s (must be >= 0)", newFormatVersion); + this.formatVersion = newFormatVersion; + return this; + } + + TrackedFileBuilder location(String newLocation) { + Preconditions.checkArgument(newLocation != null, "Invalid location: null"); + this.location = newLocation; + return this; + } + + TrackedFileBuilder fileFormat(FileFormat newFileFormat) { + Preconditions.checkArgument(newFileFormat != null, "Invalid file format: null"); + this.fileFormat = newFileFormat; + return this; + } + + TrackedFileBuilder recordCount(long newRecordCount) { + Preconditions.checkArgument( + newRecordCount >= 0, "Invalid record count: %s (must be >= 0)", newRecordCount); + this.recordCount = newRecordCount; + return this; + } + + TrackedFileBuilder fileSizeInBytes(long newFileSizeInBytes) { + Preconditions.checkArgument( + newFileSizeInBytes >= 0, + "Invalid file size in bytes: %s (must be >= 0)", + newFileSizeInBytes); + this.fileSizeInBytes = newFileSizeInBytes; + return this; + } + + TrackedFileBuilder specId(int newSpecId) { + Preconditions.checkArgument(newSpecId >= 0, "Invalid spec ID: %s (must be >= 0)", newSpecId); + this.specId = newSpecId; + return this; + } + + TrackedFileBuilder partition(PartitionData newPartitionData) { + Preconditions.checkArgument(newPartitionData != null, "Invalid partition: null"); + this.partitionData = newPartitionData; + return this; + } + + TrackedFileBuilder contentStats(ContentStats newContentStats) { + Preconditions.checkArgument(newContentStats != null, "Invalid content stats: null"); + this.contentStats = newContentStats; + return this; + } + + TrackedFileBuilder sortOrderId(int newSortOrderId) { + Preconditions.checkArgument( + !isLeafManifest(contentType), + "Sort order ID cannot be added to manifest entries, but entry type is: %s", + contentType); + Preconditions.checkArgument( + newSortOrderId >= 0, "Invalid sort order ID: %s (must be >= 0)", newSortOrderId); + this.sortOrderId = newSortOrderId; + return this; + } + + TrackedFileBuilder deletionVector(DeletionVector newDeletionVector) { + Preconditions.checkArgument(newDeletionVector != null, "Invalid deletion vector: null"); + Preconditions.checkArgument( + contentType == FileContent.DATA, + "Deletion vector can only be added to DATA entries, but entry type is: %s", + contentType); + Preconditions.checkArgument( + this.deletionVector == null || !this.deletionVector.equals(newDeletionVector), + "The same deletion vector already added"); + this.deletionVector = newDeletionVector; + this.dvUpdated = true; + return this; + } + + TrackedFileBuilder manifestInfo(ManifestInfo newManifestInfo) { + Preconditions.checkArgument(newManifestInfo != null, "Invalid manifest info: null"); + Preconditions.checkArgument( + isLeafManifest(contentType), + "Manifest info can only be added to manifests, but entry type is: %s", + contentType); + this.manifestInfo = newManifestInfo; + return this; + } + + TrackedFileBuilder keyMetadata(ByteBuffer newKeyMetadata) { + Preconditions.checkArgument(newKeyMetadata != null, "Invalid key metadata: null"); + this.keyMetadata = newKeyMetadata; + return this; + } + + TrackedFileBuilder splitOffsets(List newSplitOffsets) { + Preconditions.checkArgument(newSplitOffsets != null, "Invalid split offsets: null"); + Preconditions.checkArgument( + !isLeafManifest(contentType), + "Split offsets cannot be added to manifest entries, but entry type is: %s", + contentType); + this.splitOffsets = newSplitOffsets; + return this; + } + + TrackedFileBuilder equalityIds(List newEqualityIds) { + Preconditions.checkArgument(newEqualityIds != null, "Invalid equality IDs: null"); + Preconditions.checkArgument( + contentType == FileContent.EQUALITY_DELETES, + "Equality IDs can only be added to EQUALITY_DELETES entries, but entry type is: %s", + contentType); + this.equalityIds = newEqualityIds; + return this; + } + + TrackedFileBuilder deletedPositions(ByteBuffer newDeletedPositions) { + Preconditions.checkArgument(newDeletedPositions != null, "Invalid deleted positions: null"); + Preconditions.checkArgument( + isLeafManifest(contentType), + "Deleted positions can only be added to manifest entries, but entry type is: %s", + contentType); + this.deletedPositions = newDeletedPositions; + return this; + } + + TrackedFileBuilder replacedPositions(ByteBuffer newReplacedPositions) { + Preconditions.checkArgument(newReplacedPositions != null, "Invalid replaced positions: null"); + Preconditions.checkArgument( + isLeafManifest(contentType), + "Replaced positions can only be added to manifest entries, but entry type is: %s", + contentType); + this.replacedPositions = newReplacedPositions; + return this; + } + + private static boolean isLeafManifest(FileContent contentType) { + return contentType == FileContent.DATA_MANIFEST || contentType == FileContent.DELETE_MANIFEST; + } + + TrackedFile build() { + Preconditions.checkArgument(formatVersion != null, "Missing required field: format version"); + Preconditions.checkArgument(location != null, "Missing required field: location"); + Preconditions.checkArgument(fileFormat != null, "Missing required field: file format"); + Preconditions.checkArgument(recordCount != null, "Missing required field: record count"); + Preconditions.checkArgument( + fileSizeInBytes != null, "Missing required field: file size in bytes"); + Preconditions.checkArgument(partitionData != null, "Missing required field: partition data"); + Preconditions.checkArgument( + !isLeafManifest(contentType) || manifestInfo != null, + "Missing required field: manifest info"); + Preconditions.checkArgument( + contentType != FileContent.EQUALITY_DELETES || equalityIds != null, + "Missing required field: equality IDs"); + + TrackingBuilder trackingBuilder = + sourceTracking == null + ? TrackingBuilder.added(snapshotId) + : TrackingBuilder.from(sourceTracking, snapshotId); + + if (dvUpdated) { + trackingBuilder.dvUpdated(); + } + + if (deletedPositions != null) { + trackingBuilder.deletedPositions(deletedPositions); + } + + if (replacedPositions != null) { + trackingBuilder.replacedPositions(replacedPositions); + } + + return new TrackedFileStruct( + trackingBuilder.build(), + contentType, + formatVersion, + location, + fileFormat, + partitionData, + recordCount, + fileSizeInBytes, + specId, + contentStats, + sortOrderId, + deletionVector, + manifestInfo, + keyMetadata, + splitOffsets, + equalityIds); + } +} diff --git a/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java b/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java index 3c350b89373d..958ddfbbc4cf 100644 --- a/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java +++ b/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java @@ -45,7 +45,7 @@ public PartitionData copy() { Types.StructType.of( TrackedFile.TRACKING, TrackedFile.CONTENT_TYPE, - TrackedFile.WRITER_FORMAT_VERSION, + TrackedFile.FORMAT_VERSION, TrackedFile.LOCATION, TrackedFile.FILE_FORMAT, TrackedFile.RECORD_COUNT, @@ -69,16 +69,16 @@ public PartitionData copy() { TrackedFile.EQUALITY_IDS); private FileContent contentType = null; - private int writerFormatVersion = -1; + private int formatVersion = -1; private String location = null; private FileFormat fileFormat = null; + private Tracking tracking = null; private long recordCount = -1L; private long fileSizeInBytes = -1L; - private Integer specId = null; private PartitionData partitionData = EMPTY_PARTITION_DATA; // optional fields - private Tracking tracking = null; + private Integer specId = null; private ContentStats contentStats = null; private Integer sortOrderId = null; private DeletionVector deletionVector = null; @@ -102,20 +102,27 @@ public PartitionData copy() { super(BASE_TYPE.fields().size()); } - /** Constructor that accepts required fields. */ TrackedFileStruct( Tracking tracking, FileContent contentType, - int writerFormatVersion, + int formatVersion, String location, FileFormat fileFormat, PartitionData partition, long recordCount, - long fileSizeInBytes) { + long fileSizeInBytes, + Integer specId, + ContentStats contentStats, + Integer sortOrderId, + DeletionVector deletionVector, + ManifestInfo manifestInfo, + ByteBuffer keyMetadata, + List splitOffsets, + List equalityIds) { super(BASE_TYPE.fields().size()); this.tracking = tracking; this.contentType = contentType; - this.writerFormatVersion = writerFormatVersion; + this.formatVersion = formatVersion; this.location = location; this.fileFormat = fileFormat; this.recordCount = recordCount; @@ -123,13 +130,22 @@ public PartitionData copy() { if (partition != null) { this.partitionData = partition; } + + this.specId = specId; + this.contentStats = contentStats; + this.sortOrderId = sortOrderId; + this.deletionVector = deletionVector; + this.manifestInfo = manifestInfo; + this.keyMetadata = ByteBuffers.toByteArray(keyMetadata); + this.splitOffsets = ArrayUtil.toLongArray(splitOffsets); + this.equalityIds = ArrayUtil.toIntArray(equalityIds); } /** Copy constructor. */ private TrackedFileStruct(TrackedFileStruct toCopy, boolean withStats, Set statsIds) { super(toCopy); this.contentType = toCopy.contentType; - this.writerFormatVersion = toCopy.writerFormatVersion; + this.formatVersion = toCopy.formatVersion; this.location = toCopy.location; this.fileFormat = toCopy.fileFormat; this.recordCount = toCopy.recordCount; @@ -173,8 +189,8 @@ public FileContent contentType() { } @Override - public int writerFormatVersion() { - return writerFormatVersion; + public int formatVersion() { + return formatVersion; } @Override @@ -258,98 +274,51 @@ protected T internalGet(int pos, Class javaClass) { } private Object getByPos(int pos) { - switch (pos) { - case 0: - return tracking; - case 1: - return contentType != null ? contentType.id() : null; - case 2: - return writerFormatVersion; - case 3: - return location; - case 4: - return fileFormat != null ? fileFormat.toString() : null; - case 5: - return recordCount; - case 6: - return fileSizeInBytes; - case 7: - return specId; - case 8: - return partitionData; - case 9: - return contentStats; - case 10: - return sortOrderId; - case 11: - return deletionVector; - case 12: - return manifestInfo; - case 13: - return keyMetadata(); - case 14: - return splitOffsets(); - case 15: - return equalityIds(); - default: - throw new UnsupportedOperationException("Unknown field ordinal: " + pos); - } + return switch (pos) { + case 0 -> tracking; + case 1 -> contentType != null ? contentType.id() : null; + case 2 -> formatVersion; + case 3 -> location; + case 4 -> fileFormat != null ? fileFormat.toString() : null; + case 5 -> recordCount; + case 6 -> fileSizeInBytes; + case 7 -> specId; + case 8 -> partitionData; + case 9 -> contentStats; + case 10 -> sortOrderId; + case 11 -> deletionVector; + case 12 -> manifestInfo; + case 13 -> keyMetadata(); + case 14 -> splitOffsets(); + case 15 -> equalityIds(); + default -> throw new UnsupportedOperationException("Unknown field ordinal: " + pos); + }; } @Override protected void internalSet(int pos, T value) { switch (pos) { - case 0: - this.tracking = (Tracking) value; - break; - case 1: - this.contentType = FileContent.fromId((Integer) value); - break; - case 2: - this.writerFormatVersion = (int) value; - break; - case 3: - // always coerce to String for Serializable - this.location = value.toString(); - break; - case 4: - this.fileFormat = FileFormat.fromString(value.toString()); - break; - case 5: - this.recordCount = (long) value; - break; - case 6: - this.fileSizeInBytes = (long) value; - break; - case 7: - this.specId = (Integer) value; - break; - case 8: - this.partitionData = (PartitionData) value; - break; - case 9: - this.contentStats = (ContentStats) value; - break; - case 10: - this.sortOrderId = (Integer) value; - break; - case 11: - this.deletionVector = (DeletionVector) value; - break; - case 12: - this.manifestInfo = (ManifestInfo) value; - break; - case 13: - this.keyMetadata = ByteBuffers.toByteArray((ByteBuffer) value); - break; - case 14: - this.splitOffsets = ArrayUtil.toLongArray((List) value); - break; - case 15: - this.equalityIds = ArrayUtil.toIntArray((List) value); - break; - default: + case 0 -> this.tracking = (Tracking) value; + case 1 -> this.contentType = FileContent.fromId((Integer) value); + case 2 -> this.formatVersion = (int) value; + case 3 -> + // always coerce to String for Serializable + this.location = value.toString(); + case 4 -> this.fileFormat = FileFormat.fromString(value.toString()); + case 5 -> this.recordCount = (long) value; + case 6 -> this.fileSizeInBytes = (long) value; + case 7 -> this.specId = (Integer) value; + case 8 -> this.partitionData = (PartitionData) value; + case 9 -> this.contentStats = (ContentStats) value; + case 10 -> this.sortOrderId = (Integer) value; + case 11 -> this.deletionVector = (DeletionVector) value; + case 12 -> this.manifestInfo = (ManifestInfo) value; + case 13 -> this.keyMetadata = ByteBuffers.toByteArray((ByteBuffer) value); + case 14 -> this.splitOffsets = ArrayUtil.toLongArray((List) value); + case 15 -> this.equalityIds = ArrayUtil.toIntArray((List) value); + default -> { // ignore the object, it must be from a newer version of the format + } } } @@ -357,7 +326,7 @@ protected void internalSet(int pos, T value) { public String toString() { return MoreObjects.toStringHelper(this) .add("content", contentType != null ? contentType.lowerCaseName() : null) - .add("writer_format_version", writerFormatVersion) + .add("format_version", formatVersion) .add("location", location) .add("file_format", fileFormat) .add("record_count", recordCount) diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java b/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java index dbba1becade1..1d8e9da6d43c 100644 --- a/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java +++ b/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java @@ -72,6 +72,7 @@ import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.rest.RESTCatalogProperties.ScanPlanningMode; import org.apache.iceberg.rest.RESTCatalogProperties.SnapshotMode; import org.apache.iceberg.rest.RESTTableCache.TableWithETag; import org.apache.iceberg.rest.auth.AuthManager; @@ -174,6 +175,7 @@ public class RESTSessionCatalog extends BaseViewSessionCatalog private Set endpoints; private Supplier> mutationHeaders = Map::of; private String namespaceSeparator = null; + private ScanPlanningMode clientScanPlanningMode = null; private RESTTableCache tableCache; @@ -291,6 +293,10 @@ public void initialize(String name, Map unresolved) { RESTCatalogProperties.NAMESPACE_SEPARATOR, RESTCatalogProperties.NAMESPACE_SEPARATOR_DEFAULT); + String scanPlanningModeConfig = mergedProps.get(RESTCatalogProperties.SCAN_PLANNING_MODE); + this.clientScanPlanningMode = + scanPlanningModeConfig == null ? null : ScanPlanningMode.fromString(scanPlanningModeConfig); + this.tableCache = createTableCache(mergedProps); this.closeables.addCloseable(this.tableCache); @@ -603,31 +609,34 @@ private RESTTable restTableForScanPlanning( TableIdentifier finalIdentifier, RESTClient restClient, Map tableConf) { - // Get client-side and server-side scan planning modes - String planningModeClientConfig = properties().get(RESTCatalogProperties.SCAN_PLANNING_MODE); String planningModeServerConfig = tableConf.get(RESTCatalogProperties.SCAN_PLANNING_MODE); - - // Warn if client and server configs conflict; server config takes precedence - if (planningModeClientConfig != null - && planningModeServerConfig != null - && !planningModeClientConfig.equalsIgnoreCase(planningModeServerConfig)) { + ScanPlanningMode serverScanPlanningMode = + planningModeServerConfig == null + ? null + : ScanPlanningMode.fromString(planningModeServerConfig); + + // Warn if client and server configs conflict + if (clientScanPlanningMode != null + && serverScanPlanningMode != null + && clientScanPlanningMode != serverScanPlanningMode) { LOG.warn( "Scan planning mode mismatch for table {}: client config={}, server config={}. " + "Server config will take precedence.", finalIdentifier, - planningModeClientConfig, + clientScanPlanningMode.modeName(), planningModeServerConfig); } - // Determine effective mode: prefer server config if present, otherwise use client config - String effectiveModeConfig = - planningModeServerConfig != null ? planningModeServerConfig : planningModeClientConfig; - RESTCatalogProperties.ScanPlanningMode effectiveMode = - effectiveModeConfig != null - ? RESTCatalogProperties.ScanPlanningMode.fromString(effectiveModeConfig) - : RESTCatalogProperties.SCAN_PLANNING_MODE_DEFAULT; + // Determine effective mode: prefer server config if present, otherwise use client config, + // fall back to default if both are null + ScanPlanningMode effectiveMode = RESTCatalogProperties.SCAN_PLANNING_MODE_DEFAULT; + if (serverScanPlanningMode != null) { + effectiveMode = serverScanPlanningMode; + } else if (clientScanPlanningMode != null) { + effectiveMode = clientScanPlanningMode; + } - if (effectiveMode == RESTCatalogProperties.ScanPlanningMode.SERVER) { + if (effectiveMode == ScanPlanningMode.SERVER) { Preconditions.checkState( endpoints.contains(Endpoint.V1_SUBMIT_TABLE_SCAN_PLAN), "Server requires server-side scan planning for table %s but does not support endpoint %s", diff --git a/core/src/test/java/org/apache/iceberg/InternalTestHelpers.java b/core/src/test/java/org/apache/iceberg/InternalTestHelpers.java index 781051f11d7b..226f105f5310 100644 --- a/core/src/test/java/org/apache/iceberg/InternalTestHelpers.java +++ b/core/src/test/java/org/apache/iceberg/InternalTestHelpers.java @@ -91,6 +91,8 @@ private static void assertEquals(Type type, Object expected, Object actual) { case FIXED: case BINARY: case DECIMAL: + case GEOMETRY: + case GEOGRAPHY: assertThat(actual).as("Primitive value should be equal to expected").isEqualTo(expected); break; case STRUCT: diff --git a/core/src/test/java/org/apache/iceberg/RandomInternalData.java b/core/src/test/java/org/apache/iceberg/RandomInternalData.java index e458ce054cc0..a0638ad010a9 100644 --- a/core/src/test/java/org/apache/iceberg/RandomInternalData.java +++ b/core/src/test/java/org/apache/iceberg/RandomInternalData.java @@ -99,6 +99,8 @@ public Object primitive(Type.PrimitiveType primitive) { switch (primitive.typeId()) { case FIXED: case BINARY: + case GEOMETRY: + case GEOGRAPHY: return ByteBuffer.wrap((byte[]) result); case UUID: return UUID.nameUUIDFromBytes((byte[]) result); diff --git a/core/src/test/java/org/apache/iceberg/TestDeletionVectorStruct.java b/core/src/test/java/org/apache/iceberg/TestDeletionVectorStruct.java index 8242be38e94a..0f08b59e150d 100644 --- a/core/src/test/java/org/apache/iceberg/TestDeletionVectorStruct.java +++ b/core/src/test/java/org/apache/iceberg/TestDeletionVectorStruct.java @@ -163,6 +163,64 @@ void testBuilderMissingRequiredFields() { .hasMessage("Missing required value: cardinality"); } + @Test + void testDvEquality() { + DeletionVectorStruct dv = + DeletionVectorStruct.builder() + .location("s3://bucket/data/dv.puffin") + .offset(256L) + .sizeInBytes(128L) + .cardinality(42L) + .build(); + + DeletionVectorStruct sameDv = + DeletionVectorStruct.builder() + .location("s3://bucket/data/dv.puffin") + .offset(256L) + .sizeInBytes(128L) + .cardinality(42L) + .build(); + + DeletionVectorStruct dvWithDifferentLocation = + DeletionVectorStruct.builder() + .location("s3://bucket/data/dv2.puffin") + .offset(256L) + .sizeInBytes(128L) + .cardinality(42L) + .build(); + + DeletionVectorStruct dvWithDifferentOffset = + DeletionVectorStruct.builder() + .location("s3://bucket/data/dv.puffin") + .offset(1L) + .sizeInBytes(128L) + .cardinality(42L) + .build(); + + DeletionVectorStruct dvWithDifferentSize = + DeletionVectorStruct.builder() + .location("s3://bucket/data/dv.puffin") + .offset(256L) + .sizeInBytes(8L) + .cardinality(42L) + .build(); + + DeletionVectorStruct dvWithDifferentCardinality = + DeletionVectorStruct.builder() + .location("s3://bucket/data/dv.puffin") + .offset(256L) + .sizeInBytes(128L) + .cardinality(2L) + .build(); + + assertThat(dv).isEqualTo(dv); + assertThat(dv).isEqualTo(sameDv); + assertThat(dv).isNotEqualTo(dvWithDifferentLocation); + assertThat(dv).isNotEqualTo(dvWithDifferentOffset); + assertThat(dv).isNotEqualTo(dvWithDifferentSize); + assertThat(dv).isNotEqualTo(dvWithDifferentCardinality); + } + @Test void testBuilderRejectsInvalidValuesAtSetter() { assertThatThrownBy(() -> DeletionVectorStruct.builder().location(null)) diff --git a/core/src/test/java/org/apache/iceberg/TestRewriteTablePathUtil.java b/core/src/test/java/org/apache/iceberg/TestRewriteTablePathUtil.java index bedd8dd66d71..1b0f5f6b1c70 100644 --- a/core/src/test/java/org/apache/iceberg/TestRewriteTablePathUtil.java +++ b/core/src/test/java/org/apache/iceberg/TestRewriteTablePathUtil.java @@ -24,6 +24,9 @@ import java.io.IOException; import java.util.Set; +import org.apache.iceberg.io.InputFile; +import org.apache.iceberg.io.OutputFile; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestTemplate; import org.junit.jupiter.api.extension.ExtendWith; @@ -277,8 +280,100 @@ public void testRewritingMultiplePositionDeleteEntriesWithinManifestFile() throw table.specs(), sourcePrefix, targetPrefix, - stagingDir); + stagingDir, + ImmutableMap.of()); assertThat(deleteFileRewriteResult.toRewrite()).hasSize(2); } + + // A position delete entry that is not rewritten (e.g. a DELETED entry not copied to the target) + // is absent from the measured-size map and must keep its original file_size_in_bytes. + @TestTemplate + public void testRewriteDeleteManifestFallsBackToOriginalSizeForDeletedEntries() + throws IOException { + assumeThat(formatVersion) + .as("Delete files only work for format version 2+") + .isGreaterThanOrEqualTo(2); + + String sourcePrefix = "/path/to/"; + String targetPrefix = "/path/new/"; + String stagingDir = "/staging/"; + + // FILE_A_DELETES is live, so its rewritten size is measured and supplied; FILE_B_DELETES is a + // DELETED entry, absent from the size map. + ManifestFile manifest = deleteManifestWithLiveAndDeletedEntry(FILE_A_DELETES, FILE_B_DELETES); + + long measuredSizeForA = 9999L; + OutputFile output = + Files.localOutput( + FileFormat.AVRO.addExtension( + temp.resolve("junit" + System.nanoTime()).toFile().toString())); + RewriteTablePathUtil.rewriteDeleteManifest( + manifest, + Set.of(1000L), + output, + table.io(), + formatVersion, + table.specs(), + sourcePrefix, + targetPrefix, + stagingDir, + ImmutableMap.of(FILE_A_DELETES.location(), measuredSizeForA)); + + InputFile rewrittenInput = output.toInputFile(); + ManifestFile rewritten = + new GenericManifestFile( + rewrittenInput.location(), + rewrittenInput.getLength(), + SPEC.specId(), + ManifestContent.DELETES, + 0L, + 0L, + 1000L, + null, + null, + null, + null, + null, + null, + null, + null, + null); + int seen = 0; + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(rewritten, table.io(), table.specs())) { + for (ManifestEntry entry : reader.entries()) { + seen++; + if (entry.status() == ManifestEntry.Status.DELETED) { + assertThat(entry.file().fileSizeInBytes()) + .as("DELETED entry should fall back to its original size") + .isEqualTo(FILE_B_DELETES.fileSizeInBytes()); + } else { + assertThat(entry.file().fileSizeInBytes()) + .as("Live entry should use the measured rewritten size") + .isEqualTo(measuredSizeForA); + } + } + } + + assertThat(seen).as("Both the live and deleted entries should be present").isEqualTo(2); + } + + private ManifestFile deleteManifestWithLiveAndDeletedEntry(DeleteFile live, DeleteFile deleted) + throws IOException { + OutputFile manifestFile = + Files.localOutput( + FileFormat.AVRO.addExtension( + temp.resolve("junit" + System.nanoTime()).toFile().toString())); + ManifestWriter writer = + ManifestFiles.writeDeleteManifest(formatVersion, SPEC, manifestFile, 1000L); + try { + writer.add(live); + writer.delete(deleted, 1, null); + } finally { + writer.close(); + } + + return writer.toManifestFile(); + } } diff --git a/core/src/test/java/org/apache/iceberg/TestSnapshot.java b/core/src/test/java/org/apache/iceberg/TestSnapshot.java index ff9aebffc3a6..398bce3e94d9 100644 --- a/core/src/test/java/org/apache/iceberg/TestSnapshot.java +++ b/core/src/test/java/org/apache/iceberg/TestSnapshot.java @@ -30,6 +30,36 @@ @ExtendWith(ParameterizedTestExtension.class) public class TestSnapshot extends TestBase { + @TestTemplate + public void snapshotValueMethodsIncludeMetadataFields() { + Snapshot snapshot = + new BaseSnapshot( + 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", 10L, 5L, "key-1"); + Snapshot same = + new BaseSnapshot( + 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", 10L, 5L, "key-1"); + Snapshot differentFirstRowId = + new BaseSnapshot( + 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", 11L, 5L, "key-1"); + Snapshot differentAddedRows = + new BaseSnapshot( + 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", 10L, 6L, "key-1"); + Snapshot differentKeyId = + new BaseSnapshot( + 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", 10L, 5L, "key-2"); + Snapshot withoutMetadataFields = + new BaseSnapshot( + 0, 1, null, 0, DataOperations.APPEND, null, 1, "ml.avro", null, null, null); + + assertThat(snapshot).isEqualTo(same); + assertThat(snapshot.hashCode()).isEqualTo(same.hashCode()); + assertThat(snapshot).isNotEqualTo(differentFirstRowId); + assertThat(snapshot).isNotEqualTo(differentAddedRows); + assertThat(snapshot).isNotEqualTo(differentKeyId); + assertThat(snapshot).isNotEqualTo(withoutMetadataFields); + assertThat(snapshot.toString()).contains("first-row-id=10", "added-rows=5", "key-id=key-1"); + } + @TestTemplate public void testAppendFilesFromTable() { table.newFastAppend().appendFile(FILE_A).appendFile(FILE_B).commit(); diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFile.java b/core/src/test/java/org/apache/iceberg/TestTrackedFile.java index 2c4bf93e4945..170c01ef7dc4 100644 --- a/core/src/test/java/org/apache/iceberg/TestTrackedFile.java +++ b/core/src/test/java/org/apache/iceberg/TestTrackedFile.java @@ -47,7 +47,7 @@ public void schemaWithContentStatsFieldOrder() { .containsExactly( "tracking", "content_type", - "writer_format_version", + "format_version", "location", "file_format", "record_count", diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java index e13a342b8d5a..f4d5675ee94a 100644 --- a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java +++ b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java @@ -34,7 +34,7 @@ class TestTrackedFileAdapters { - private static final int WRITER_FORMAT_VERSION = 4; + private static final int FORMAT_VERSION_V4 = 4; private static final String MANIFEST_LOCATION = "s3://bucket/table/manifest.parquet"; private static final String DATA_FILE_LOCATION = "s3://bucket/data/file.parquet"; private static final String DV_LOCATION = "s3://bucket/puffin/dv-file.bin"; @@ -57,13 +57,9 @@ class TestTrackedFileAdapters { .identity("category") .withSpecId(PARTITIONED_SPEC_ID) .build(); - - // Passed for unpartitioned test files, where there is no partition tuple. - private static final PartitionData NO_PARTITION = null; + private static final PartitionData PARTITION = partition("books"); // Tracking field ordinals, looked up from the schema so the tests do not hard-code offsets. - private static final int STATUS_ORDINAL = ordinalOf(Tracking.schema(), "status"); - private static final int SNAPSHOT_ID_ORDINAL = ordinalOf(Tracking.schema(), "snapshot_id"); private static final int DATA_SEQUENCE_NUMBER_ORDINAL = ordinalOf(Tracking.schema(), "sequence_number"); private static final int FILE_SEQUENCE_NUMBER_ORDINAL = @@ -75,40 +71,34 @@ class TestTrackedFileAdapters { // TrackedFile optional field ordinals, looked up from the schema. private static final Types.StructType TRACKED_FILE_SCHEMA = TrackedFile.schemaWithContentStats(Types.StructType.of(), Types.StructType.of()); + private static final int CONTENT_TYPE_ORDINAL = ordinalOf(TRACKED_FILE_SCHEMA, "content_type"); private static final int SPEC_ID_ORDINAL = ordinalOf(TRACKED_FILE_SCHEMA, "spec_id"); - private static final int CONTENT_STATS_ORDINAL = ordinalOf(TRACKED_FILE_SCHEMA, "content_stats"); - private static final int SORT_ORDER_ID_ORDINAL = ordinalOf(TRACKED_FILE_SCHEMA, "sort_order_id"); private static final int DELETION_VECTOR_ORDINAL = ordinalOf(TRACKED_FILE_SCHEMA, "deletion_vector"); - private static final int KEY_METADATA_ORDINAL = ordinalOf(TRACKED_FILE_SCHEMA, "key_metadata"); - private static final int SPLIT_OFFSETS_ORDINAL = ordinalOf(TRACKED_FILE_SCHEMA, "split_offsets"); - private static final int EQUALITY_IDS_ORDINAL = ordinalOf(TRACKED_FILE_SCHEMA, "equality_ids"); @Test void testDataFileAdapterDelegation() { - PartitionData partition = partition("books"); - - TrackedFileStruct file = - new TrackedFileStruct( - createTracking(), - FileContent.DATA, - WRITER_FORMAT_VERSION, - DATA_FILE_LOCATION, - FileFormat.PARQUET, - partition, - 100L, - 1024L); - file.set(SPEC_ID_ORDINAL, PARTITIONED_SPEC_ID); - file.set(CONTENT_STATS_ORDINAL, createContentStats()); - file.set(SORT_ORDER_ID_ORDINAL, 3); - file.set(KEY_METADATA_ORDINAL, ByteBuffer.wrap(new byte[] {1, 2, 3})); - file.set(SPLIT_OFFSETS_ORDINAL, ImmutableList.of(50L, 100L)); + TrackedFile file = + TrackedFileBuilder.data(42L) + .formatVersion(FORMAT_VERSION_V4) + .location(DATA_FILE_LOCATION) + .fileFormat(FileFormat.PARQUET) + .partition(PARTITION) + .recordCount(100L) + .fileSizeInBytes(1024L) + .specId(PARTITIONED_SPEC_ID) + .contentStats(createContentStats()) + .sortOrderId(3) + .keyMetadata(ByteBuffer.wrap(new byte[] {1, 2, 3})) + .splitOffsets(ImmutableList.of(50L, 100L)) + .build(); + populateTrackingFields(file); DataFile dataFile = TrackedFileAdapters.asDataFile(file, specsById(PARTITIONED_SPEC)); assertThat(dataFile.pos()).isEqualTo(MANIFEST_POS); assertThat(dataFile.specId()).isEqualTo(PARTITIONED_SPEC_ID); - assertThat(dataFile.partition()).isSameAs(partition); + assertThat(dataFile.partition()).isSameAs(PARTITION); assertThat(dataFile.content()).isEqualTo(FileContent.DATA); assertThat(dataFile.location()).isEqualTo(DATA_FILE_LOCATION); assertThat(dataFile.format()).isEqualTo(FileFormat.PARQUET); @@ -139,7 +129,7 @@ void testDataFileAdapterDelegation() { @ParameterizedTest @EnumSource(value = FileContent.class, mode = EnumSource.Mode.EXCLUDE, names = "DATA") void testDataFileAdapterRejectsNonDataContent(FileContent contentType) { - TrackedFileStruct file = trackedFile(contentType); + TrackedFileStruct file = dummyTrackedFile(contentType); assertThatThrownBy(() -> TrackedFileAdapters.asDataFile(file, UNPARTITIONED)) .isInstanceOf(IllegalArgumentException.class) @@ -148,31 +138,29 @@ void testDataFileAdapterRejectsNonDataContent(FileContent contentType) { @Test void testEqualityDeleteFileAdapterDelegation() { - PartitionData partition = partition("books"); - - TrackedFileStruct file = - new TrackedFileStruct( - createTracking(), - FileContent.EQUALITY_DELETES, - WRITER_FORMAT_VERSION, - "s3://bucket/eq-delete.avro", - FileFormat.AVRO, - partition, - 50L, - 512L); - file.set(SPEC_ID_ORDINAL, PARTITIONED_SPEC_ID); - file.set(CONTENT_STATS_ORDINAL, createContentStats()); - file.set(SORT_ORDER_ID_ORDINAL, 5); - file.set(KEY_METADATA_ORDINAL, ByteBuffer.wrap(new byte[] {4, 5})); - file.set(SPLIT_OFFSETS_ORDINAL, ImmutableList.of(200L)); - file.set(EQUALITY_IDS_ORDINAL, ImmutableList.of(1, 2, 3)); + TrackedFile file = + TrackedFileBuilder.equalityDelete(42L) + .formatVersion(FORMAT_VERSION_V4) + .location("s3://bucket/eq-delete.avro") + .fileFormat(FileFormat.AVRO) + .partition(PARTITION) + .recordCount(50L) + .fileSizeInBytes(512L) + .specId(PARTITIONED_SPEC_ID) + .contentStats(createContentStats()) + .sortOrderId(5) + .keyMetadata(ByteBuffer.wrap(new byte[] {4, 5})) + .splitOffsets(ImmutableList.of(200L)) + .equalityIds(ImmutableList.of(1, 2, 3)) + .build(); + populateTrackingFields(file); DeleteFile deleteFile = TrackedFileAdapters.asEqualityDeleteFile(file, specsById(PARTITIONED_SPEC)); assertThat(deleteFile.pos()).isEqualTo(MANIFEST_POS); assertThat(deleteFile.specId()).isEqualTo(PARTITIONED_SPEC_ID); - assertThat(deleteFile.partition()).isSameAs(partition); + assertThat(deleteFile.partition()).isSameAs(PARTITION); assertThat(deleteFile.content()).isEqualTo(FileContent.EQUALITY_DELETES); assertThat(deleteFile.location()).isEqualTo("s3://bucket/eq-delete.avro"); assertThat(deleteFile.format()).isEqualTo(FileFormat.AVRO); @@ -203,7 +191,7 @@ void testEqualityDeleteFileAdapterDelegation() { @ParameterizedTest @EnumSource(value = FileContent.class, mode = EnumSource.Mode.EXCLUDE, names = "EQUALITY_DELETES") void testEqualityDeleteFileAdapterRejectsNonEqualityContent(FileContent contentType) { - TrackedFileStruct file = trackedFile(contentType); + TrackedFileStruct file = dummyTrackedFile(contentType); assertThatThrownBy(() -> TrackedFileAdapters.asEqualityDeleteFile(file, UNPARTITIONED)) .isInstanceOf(IllegalArgumentException.class) @@ -220,19 +208,18 @@ void testDVDeleteFileAdapterDelegation() { .cardinality(10L) .build(); - PartitionData partition = partition("books"); - TrackedFileStruct file = - new TrackedFileStruct( - createTracking(), - FileContent.DATA, - WRITER_FORMAT_VERSION, - DATA_FILE_LOCATION, - FileFormat.PARQUET, - partition, - 100L, - 1024L); - file.set(SPEC_ID_ORDINAL, PARTITIONED_SPEC_ID); - file.set(DELETION_VECTOR_ORDINAL, dv); + TrackedFile file = + TrackedFileBuilder.data(42L) + .formatVersion(FORMAT_VERSION_V4) + .location(DATA_FILE_LOCATION) + .fileFormat(FileFormat.PARQUET) + .partition(PARTITION) + .recordCount(100L) + .fileSizeInBytes(1024L) + .specId(PARTITIONED_SPEC_ID) + .deletionVector(dv) + .build(); + populateTrackingFields(file); DeleteFile dvFile = TrackedFileAdapters.asDVDeleteFile(file, specsById(PARTITIONED_SPEC)); @@ -251,7 +238,7 @@ void testDVDeleteFileAdapterDelegation() { // fields delegated from TrackedFile / Tracking assertThat(dvFile.pos()).isEqualTo(MANIFEST_POS); assertThat(dvFile.specId()).isEqualTo(PARTITIONED_SPEC_ID); - assertThat(dvFile.partition()).isSameAs(partition); + assertThat(dvFile.partition()).isSameAs(PARTITION); assertThat(dvFile.dataSequenceNumber()).isEqualTo(DATA_SEQUENCE_NUMBER); assertThat(dvFile.fileSequenceNumber()).isEqualTo(FILE_SEQUENCE_NUMBER); assertThat(dvFile.manifestLocation()).isEqualTo(MANIFEST_LOCATION); @@ -273,7 +260,7 @@ void testDVDeleteFileAdapterDelegation() { @ParameterizedTest @EnumSource(value = FileContent.class, mode = EnumSource.Mode.EXCLUDE, names = "DATA") void testDVDeleteFileAdapterRejectsNonDataContent(FileContent contentType) { - TrackedFileStruct file = trackedFile(contentType); + TrackedFileStruct file = dummyTrackedFile(contentType); assertThatThrownBy(() -> TrackedFileAdapters.asDVDeleteFile(file, UNPARTITIONED)) .isInstanceOf(IllegalArgumentException.class) @@ -282,7 +269,7 @@ void testDVDeleteFileAdapterRejectsNonDataContent(FileContent contentType) { @Test void testDVDeleteFileAdapterRejectsNullDeletionVector() { - TrackedFileStruct file = trackedFile(FileContent.DATA); + TrackedFileStruct file = dummyTrackedFile(FileContent.DATA); assertThatThrownBy(() -> TrackedFileAdapters.asDVDeleteFile(file, UNPARTITIONED)) .isInstanceOf(IllegalArgumentException.class) @@ -291,7 +278,7 @@ void testDVDeleteFileAdapterRejectsNullDeletionVector() { @Test void testNullContentStatsReturnsNullStats() { - TrackedFileStruct file = trackedFile(FileContent.DATA); + TrackedFileStruct file = dummyTrackedFile(FileContent.DATA); DataFile dataFile = TrackedFileAdapters.asDataFile(file, UNPARTITIONED); @@ -307,19 +294,19 @@ void testNullTrackingReturnsNullTrackingFields() { // Files read before manifest inheritance have no tracking; tracking-derived fields must be // null rather than throwing. assertNullTrackingFields( - TrackedFileAdapters.asDataFile(trackedFile(FileContent.DATA), UNPARTITIONED)); + TrackedFileAdapters.asDataFile(dummyTrackedFile(FileContent.DATA), UNPARTITIONED)); assertNullTrackingFields( TrackedFileAdapters.asEqualityDeleteFile( - trackedFile(FileContent.EQUALITY_DELETES), UNPARTITIONED)); + dummyTrackedFile(FileContent.EQUALITY_DELETES), UNPARTITIONED)); - TrackedFileStruct dvFile = trackedFile(FileContent.DATA); + TrackedFileStruct dvFile = dummyTrackedFile(FileContent.DATA); dvFile.set(DELETION_VECTOR_ORDINAL, deletionVector()); assertNullTrackingFields(TrackedFileAdapters.asDVDeleteFile(dvFile, UNPARTITIONED)); } @Test void testUnpartitionedFilePartitionIsEmpty() { - TrackedFileStruct file = trackedFile(FileContent.DATA); + TrackedFileStruct file = dummyTrackedFile(FileContent.DATA); DataFile dataFile = TrackedFileAdapters.asDataFile(file, UNPARTITIONED); @@ -330,7 +317,7 @@ void testUnpartitionedFilePartitionIsEmpty() { @Test void testNullSpecIdResolvesToUnpartitionedSpec() { PartitionSpec unpartitioned = PartitionSpec.builderFor(new Schema()).withSpecId(5).build(); - TrackedFileStruct file = trackedFile(FileContent.DATA); + TrackedFileStruct file = dummyTrackedFile(FileContent.DATA); DataFile dataFile = TrackedFileAdapters.asDataFile(file, specsById(unpartitioned)); @@ -341,7 +328,7 @@ void testNullSpecIdResolvesToUnpartitionedSpec() { void testNullSpecIdThrowsWhenNoUnpartitionedSpec() { Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); PartitionSpec partitioned = PartitionSpec.builderFor(schema).identity("id").build(); - TrackedFileStruct file = trackedFile(FileContent.DATA); + TrackedFileStruct file = dummyTrackedFile(FileContent.DATA); assertThatThrownBy(() -> TrackedFileAdapters.asDataFile(file, specsById(partitioned))) .isInstanceOf(IllegalArgumentException.class) @@ -350,7 +337,7 @@ void testNullSpecIdThrowsWhenNoUnpartitionedSpec() { @Test void testUnknownSpecIdThrows() { - TrackedFileStruct file = trackedFile(FileContent.DATA); + TrackedFileStruct file = dummyTrackedFile(FileContent.DATA); file.set(SPEC_ID_ORDINAL, 99); assertThatThrownBy(() -> TrackedFileAdapters.asDataFile(file, ImmutableMap.of())) @@ -361,7 +348,7 @@ void testUnknownSpecIdThrows() { @Test void testSpecIdMismatchThrows() { int mismatchedSpecId = PARTITIONED_SPEC_ID + 1; - TrackedFileStruct file = trackedFile(FileContent.DATA); + TrackedFileStruct file = dummyTrackedFile(FileContent.DATA); file.set(SPEC_ID_ORDINAL, PARTITIONED_SPEC_ID); PartitionSpec mismatched = PartitionSpec.builderFor(PARTITION_SCHEMA) @@ -399,28 +386,19 @@ private static PartitionData partition(String category) { } /** Minimal file with no tracking, used by the rejection and null-tracking tests. */ - private static TrackedFileStruct trackedFile(FileContent contentType) { - return new TrackedFileStruct( - null, - contentType, - WRITER_FORMAT_VERSION, - "s3://bucket/file", - FileFormat.PARQUET, - NO_PARTITION, - 1L, - 1L); + private static TrackedFileStruct dummyTrackedFile(FileContent contentType) { + TrackedFileStruct file = new TrackedFileStruct(); + file.set(CONTENT_TYPE_ORDINAL, contentType.id()); + return file; } - private static TrackingStruct createTracking() { - TrackingStruct tracking = new TrackingStruct(); - tracking.set(STATUS_ORDINAL, EntryStatus.ADDED.id()); - tracking.set(SNAPSHOT_ID_ORDINAL, 42L); + private static void populateTrackingFields(TrackedFile file) { + TrackingStruct tracking = (TrackingStruct) file.tracking(); tracking.set(DATA_SEQUENCE_NUMBER_ORDINAL, DATA_SEQUENCE_NUMBER); tracking.set(FILE_SEQUENCE_NUMBER_ORDINAL, FILE_SEQUENCE_NUMBER); tracking.set(FIRST_ROW_ID_ORDINAL, FIRST_ROW_ID); tracking.setManifestLocation(MANIFEST_LOCATION); tracking.set(MANIFEST_POS_ORDINAL, MANIFEST_POS); - return tracking; } private static DeletionVector deletionVector() { diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFileBuilder.java b/core/src/test/java/org/apache/iceberg/TestTrackedFileBuilder.java new file mode 100644 index 000000000000..d6b0701fdfd7 --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/TestTrackedFileBuilder.java @@ -0,0 +1,863 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import static org.apache.iceberg.types.Types.NestedField.optional; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.nio.ByteBuffer; +import java.util.stream.Stream; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.types.Types; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +public class TestTrackedFileBuilder { + private static final int FORMAT_VERSION_V4 = 4; + private static final Schema TABLE_SCHEMA = + new Schema( + optional(1, "id", Types.IntegerType.get()), optional(2, "data", Types.StringType.get())); + private static final Types.StructType PARTITION_TYPE = + PartitionSpec.builderFor(TABLE_SCHEMA).identity("id").build().partitionType(); + private static final PartitionData PARTITION_DATA = new PartitionData(PARTITION_TYPE); + private static final ManifestInfo MANIFEST_INFO = + ManifestInfoStruct.builder() + .addedFilesCount(10) + .existingFilesCount(20) + .deletedFilesCount(3) + .replacedFilesCount(2) + .addedRowsCount(1000L) + .existingRowsCount(2000L) + .deletedRowsCount(300L) + .replacedRowsCount(200L) + .minSequenceNumber(5L) + .build(); + private static final DeletionVector DELETION_VECTOR = + DeletionVectorStruct.builder() + .location("s3://bucket/data/dv.puffin") + .offset(0L) + .sizeInBytes(128L) + .cardinality(10L) + .build(); + private static final ContentStats CONTENT_STATS = + BaseContentStats.builder() + .withTableSchema(TABLE_SCHEMA) + .withFieldStats( + BaseFieldStats.builder() + .fieldId(1) + .type(Types.IntegerType.get()) + .valueCount(2000L) + .nullValueCount(0L) + .lowerBound(1) + .upperBound(1000) + .build()) + .withFieldStats( + BaseFieldStats.builder() + .fieldId(2) + .type(Types.StringType.get()) + .valueCount(2000L) + .nullValueCount(5L) + .lowerBound("a") + .upperBound("z") + .build()) + .build(); + private static final ByteBuffer KEY_METADATA = ByteBuffer.wrap(new byte[] {1, 2, 3}); + private static final ImmutableList SPLIT_OFFSETS = ImmutableList.of(0L, 4096L, 8192L); + private static final ByteBuffer DELETED_POSITIONS = ByteBuffer.wrap(new byte[] {10, 11, 12}); + private static final ByteBuffer REPLACED_POSITIONS = ByteBuffer.wrap(new byte[] {20, 21, 22}); + + private static Stream missingRequiredFieldCases() { + return Stream.of( + Arguments.of("formatVersion", "Missing required field: format version"), + Arguments.of("location", "Missing required field: location"), + Arguments.of("fileFormat", "Missing required field: file format"), + Arguments.of("recordCount", "Missing required field: record count"), + Arguments.of("fileSizeInBytes", "Missing required field: file size in bytes"), + Arguments.of("partition", "Missing required field: partition data")); + } + + @ParameterizedTest + @MethodSource("missingRequiredFieldCases") + public void missingRequiredFields(String missingField, String expectedMessage) { + TrackedFileBuilder dataBuilder = + builderWithMissingRequiredField(TrackedFileBuilder.data(50L), missingField); + TrackedFileBuilder equalityDeleteBuilder = + builderWithMissingRequiredField(TrackedFileBuilder.equalityDelete(50L), missingField); + TrackedFileBuilder dataManifestBuilder = + builderWithMissingRequiredField(TrackedFileBuilder.dataManifest(50L), missingField); + TrackedFileBuilder deleteManifestBuilder = + builderWithMissingRequiredField(TrackedFileBuilder.deleteManifest(50L), missingField); + + assertThatThrownBy(dataBuilder::build) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage(expectedMessage); + assertThatThrownBy(equalityDeleteBuilder::build) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage(expectedMessage); + assertThatThrownBy(dataManifestBuilder::build) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage(expectedMessage); + assertThatThrownBy(deleteManifestBuilder::build) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage(expectedMessage); + } + + private TrackedFileBuilder builderWithMissingRequiredField( + TrackedFileBuilder builder, String missingField) { + if (!"formatVersion".equals(missingField)) { + builder.formatVersion(FORMAT_VERSION_V4); + } + if (!"location".equals(missingField)) { + builder.location("s3://bucket/data/file"); + } + if (!"fileFormat".equals(missingField)) { + builder.fileFormat(FileFormat.PARQUET); + } + if (!"recordCount".equals(missingField)) { + builder.recordCount(2000L); + } + if (!"fileSizeInBytes".equals(missingField)) { + builder.fileSizeInBytes(12345L); + } + if (!"partition".equals(missingField)) { + builder.partition(PARTITION_DATA); + } + return builder; + } + + @ParameterizedTest + @MethodSource("manifestBuilders") + public void missingFieldsForManifests(TrackedFileBuilder builder, FileContent contentType) { + assertThatThrownBy( + () -> + builder + .formatVersion(FORMAT_VERSION_V4) + .location("s3://bucket/data/manifest.avro") + .fileFormat(FileFormat.AVRO) + .recordCount(420L) + .fileSizeInBytes(556L) + .partition(PARTITION_DATA) + .build()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Missing required field: manifest info"); + } + + @Test + public void missingEqualityIdsForEqualityDeletes() { + assertThatThrownBy( + () -> + TrackedFileBuilder.equalityDelete(50L) + .formatVersion(FORMAT_VERSION_V4) + .location("s3://bucket/data/eq_delete.parquet") + .fileFormat(FileFormat.PARQUET) + .recordCount(2000L) + .fileSizeInBytes(12345L) + .partition(PARTITION_DATA) + .build()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Missing required field: equality IDs"); + } + + private static Stream nonEqualityDeleteBuilders() { + return Stream.of( + Arguments.of(TrackedFileBuilder.data(10L), FileContent.DATA), + Arguments.of(TrackedFileBuilder.dataManifest(10L), FileContent.DATA_MANIFEST), + Arguments.of(TrackedFileBuilder.deleteManifest(10L), FileContent.DELETE_MANIFEST), + Arguments.of(TrackedFileBuilder.from(sourceData(12L), 20L), FileContent.DATA), + Arguments.of( + TrackedFileBuilder.from(sourceDataManifest(21L), 25L), FileContent.DATA_MANIFEST), + Arguments.of( + TrackedFileBuilder.from(sourceDeleteManifest(12L), 20L), FileContent.DELETE_MANIFEST)); + } + + @ParameterizedTest + @MethodSource("nonEqualityDeleteBuilders") + public void invalidEqualityIdsForContentType( + TrackedFileBuilder builder, FileContent contentType) { + assertThatThrownBy(() -> builder.equalityIds(ImmutableList.of(1))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage( + "Equality IDs can only be added to EQUALITY_DELETES entries, but entry type is: " + + contentType); + } + + private static Stream nonDataBuilders() { + return Stream.of( + Arguments.of(TrackedFileBuilder.equalityDelete(10L), FileContent.EQUALITY_DELETES), + Arguments.of(TrackedFileBuilder.dataManifest(10L), FileContent.DATA_MANIFEST), + Arguments.of(TrackedFileBuilder.deleteManifest(10L), FileContent.DELETE_MANIFEST), + Arguments.of( + TrackedFileBuilder.from(sourceEqualityDelete(12L), 20L), FileContent.EQUALITY_DELETES), + Arguments.of( + TrackedFileBuilder.from(sourceDataManifest(21L), 25L), FileContent.DATA_MANIFEST), + Arguments.of( + TrackedFileBuilder.from(sourceDeleteManifest(12L), 20L), FileContent.DELETE_MANIFEST)); + } + + @ParameterizedTest + @MethodSource("nonDataBuilders") + public void invalidDeletionVectorForContentType( + TrackedFileBuilder builder, FileContent contentType) { + assertThatThrownBy(() -> builder.deletionVector(DELETION_VECTOR)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage( + "Deletion vector can only be added to DATA entries, but entry type is: " + contentType); + } + + @ParameterizedTest + @MethodSource("manifestBuilders") + public void invalidSortOrderIdForContentType( + TrackedFileBuilder builder, FileContent contentType) { + assertThatThrownBy(() -> builder.sortOrderId(1)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage( + "Sort order ID cannot be added to manifest entries, but entry type is: " + contentType); + } + + @ParameterizedTest + @MethodSource("manifestBuilders") + public void invalidSplitOffsetsForContentType( + TrackedFileBuilder builder, FileContent contentType) { + assertThatThrownBy(() -> builder.splitOffsets(SPLIT_OFFSETS)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage( + "Split offsets cannot be added to manifest entries, but entry type is: " + contentType); + } + + private static Stream nonManifestBuilders() { + return Stream.of( + Arguments.of(TrackedFileBuilder.data(10L), FileContent.DATA), + Arguments.of(TrackedFileBuilder.equalityDelete(10L), FileContent.EQUALITY_DELETES), + Arguments.of(TrackedFileBuilder.from(sourceData(12L), 20L), FileContent.DATA), + Arguments.of( + TrackedFileBuilder.from(sourceEqualityDelete(12L), 20L), FileContent.EQUALITY_DELETES)); + } + + @ParameterizedTest + @MethodSource("nonManifestBuilders") + public void invalidManifestInfoForContentType( + TrackedFileBuilder builder, FileContent contentType) { + assertThatThrownBy(() -> builder.manifestInfo(MANIFEST_INFO)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage( + "Manifest info can only be added to manifests, but entry type is: " + contentType); + } + + @ParameterizedTest + @MethodSource("nonManifestBuilders") + public void invalidDeletedPositionsForContentType( + TrackedFileBuilder builder, FileContent contentType) { + assertThatThrownBy(() -> builder.deletedPositions(DELETED_POSITIONS)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage( + "Deleted positions can only be added to manifest entries, but entry type is: " + + contentType); + } + + @ParameterizedTest + @MethodSource("nonManifestBuilders") + public void invalidReplacedPositionsForContentType( + TrackedFileBuilder builder, FileContent contentType) { + assertThatThrownBy(() -> builder.replacedPositions(REPLACED_POSITIONS)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage( + "Replaced positions can only be added to manifest entries, but entry type is: " + + contentType); + } + + @Test + public void invalidNullInputs() { + assertThatThrownBy(() -> TrackedFileBuilder.data(30L).location(null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid location: null"); + + assertThatThrownBy(() -> TrackedFileBuilder.data(30L).fileFormat(null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid file format: null"); + + assertThatThrownBy(() -> TrackedFileBuilder.data(30L).partition(null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid partition: null"); + + assertThatThrownBy(() -> TrackedFileBuilder.data(30L).contentStats(null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid content stats: null"); + + assertThatThrownBy(() -> TrackedFileBuilder.data(30L).deletionVector(null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid deletion vector: null"); + + assertThatThrownBy(() -> TrackedFileBuilder.dataManifest(30L).manifestInfo(null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid manifest info: null"); + + assertThatThrownBy(() -> TrackedFileBuilder.data(30L).keyMetadata(null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid key metadata: null"); + + assertThatThrownBy(() -> TrackedFileBuilder.data(30L).splitOffsets(null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid split offsets: null"); + + assertThatThrownBy(() -> TrackedFileBuilder.equalityDelete(30L).equalityIds(null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid equality IDs: null"); + + assertThatThrownBy(() -> TrackedFileBuilder.from(null, 20L)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid source: null"); + + assertThatThrownBy( + () -> + TrackedFileBuilder.from( + entryWithInheritedSeqNums(sourceDataManifest(10L), 15L), 20L) + .deletedPositions(null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid deleted positions: null"); + + assertThatThrownBy( + () -> + TrackedFileBuilder.from( + entryWithInheritedSeqNums(sourceDeleteManifest(100L), 150L), 200L) + .replacedPositions(null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid replaced positions: null"); + + assertThatThrownBy(() -> TrackedFileBuilder.deleted(null, 20L)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid source: null"); + + assertThatThrownBy(() -> TrackedFileBuilder.replaced(null, 20L)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid source: null"); + } + + @Test + public void invalidNegativeInputs() { + assertThatThrownBy(() -> TrackedFileBuilder.dataManifest(40L).formatVersion(-1)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid format version: -1 (must be >= 0)"); + + assertThatThrownBy(() -> TrackedFileBuilder.dataManifest(40L).recordCount(-1)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid record count: -1 (must be >= 0)"); + + assertThatThrownBy(() -> TrackedFileBuilder.dataManifest(40L).fileSizeInBytes(-1)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid file size in bytes: -1 (must be >= 0)"); + + assertThatThrownBy(() -> TrackedFileBuilder.dataManifest(40L).specId(-1)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid spec ID: -1 (must be >= 0)"); + + assertThatThrownBy(() -> TrackedFileBuilder.data(40L).sortOrderId(-1)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid sort order ID: -1 (must be >= 0)"); + } + + @Test + public void buildDataFileWithRequiredFieldsOnly() { + TrackedFile trackedFile = + TrackedFileBuilder.data(50L) + .formatVersion(FORMAT_VERSION_V4) + .location("s3://bucket/data/file.parquet") + .fileFormat(FileFormat.PARQUET) + .recordCount(2000L) + .fileSizeInBytes(12345L) + .partition(PARTITION_DATA) + .build(); + + assertThat(trackedFile.formatVersion()).isEqualTo(FORMAT_VERSION_V4); + assertThat(trackedFile.contentType()).isEqualTo(FileContent.DATA); + assertThat(trackedFile.location()).isEqualTo("s3://bucket/data/file.parquet"); + assertThat(trackedFile.fileFormat()).isEqualTo(FileFormat.PARQUET); + assertThat(trackedFile.recordCount()).isEqualTo(2000L); + assertThat(trackedFile.fileSizeInBytes()).isEqualTo(12345L); + assertThat(trackedFile.partition()).isSameAs(PARTITION_DATA); + + assertThat(trackedFile.tracking().status()).isEqualTo(EntryStatus.ADDED); + assertThat(trackedFile.tracking().snapshotId()).isEqualTo(50L); + assertThat(trackedFile.tracking().dvSnapshotId()).isNull(); + + assertThat(trackedFile.specId()).isNull(); + assertThat(trackedFile.contentStats()).isNull(); + assertThat(trackedFile.sortOrderId()).isNull(); + assertThat(trackedFile.deletionVector()).isNull(); + assertThat(trackedFile.manifestInfo()).isNull(); + assertThat(trackedFile.keyMetadata()).isNull(); + assertThat(trackedFile.splitOffsets()).isNull(); + assertThat(trackedFile.equalityIds()).isNull(); + } + + @Test + public void buildDataFileWithAllFields() { + TrackedFile trackedFile = + TrackedFileBuilder.data(50L) + .formatVersion(FORMAT_VERSION_V4) + .location("s3://bucket/data/file.parquet") + .fileFormat(FileFormat.PARQUET) + .recordCount(2000L) + .fileSizeInBytes(12345L) + .specId(7) + .partition(PARTITION_DATA) + .contentStats(CONTENT_STATS) + .sortOrderId(3) + .deletionVector(DELETION_VECTOR) + .keyMetadata(KEY_METADATA) + .splitOffsets(SPLIT_OFFSETS) + .build(); + + assertThat(trackedFile.formatVersion()).isEqualTo(FORMAT_VERSION_V4); + assertThat(trackedFile.contentType()).isEqualTo(FileContent.DATA); + assertThat(trackedFile.location()).isEqualTo("s3://bucket/data/file.parquet"); + assertThat(trackedFile.fileFormat()).isEqualTo(FileFormat.PARQUET); + assertThat(trackedFile.recordCount()).isEqualTo(2000L); + assertThat(trackedFile.fileSizeInBytes()).isEqualTo(12345L); + assertThat(trackedFile.specId()).isEqualTo(7); + assertThat(trackedFile.partition()).isSameAs(PARTITION_DATA); + assertThat(trackedFile.contentStats()).isSameAs(CONTENT_STATS); + assertThat(trackedFile.sortOrderId()).isEqualTo(3); + assertThat(trackedFile.deletionVector()).isSameAs(DELETION_VECTOR); + assertThat(trackedFile.keyMetadata()).isEqualTo(KEY_METADATA); + assertThat(trackedFile.splitOffsets()).isEqualTo(SPLIT_OFFSETS); + + assertThat(trackedFile.tracking().status()).isEqualTo(EntryStatus.ADDED); + assertThat(trackedFile.tracking().snapshotId()).isEqualTo(50L); + assertThat(trackedFile.tracking().dvSnapshotId()).isEqualTo(50L); + + // Unsupported fields for data files + assertThat(trackedFile.manifestInfo()).isNull(); + assertThat(trackedFile.equalityIds()).isNull(); + } + + @Test + public void buildEqualityDeleteFileWithRequiredFieldsOnly() { + TrackedFile trackedFile = + TrackedFileBuilder.equalityDelete(50L) + .formatVersion(FORMAT_VERSION_V4) + .location("s3://bucket/data/eq_delete.parquet") + .fileFormat(FileFormat.PARQUET) + .recordCount(2000L) + .fileSizeInBytes(12345L) + .partition(PARTITION_DATA) + .equalityIds(ImmutableList.of(1)) + .build(); + + assertThat(trackedFile.formatVersion()).isEqualTo(FORMAT_VERSION_V4); + assertThat(trackedFile.contentType()).isEqualTo(FileContent.EQUALITY_DELETES); + assertThat(trackedFile.location()).isEqualTo("s3://bucket/data/eq_delete.parquet"); + assertThat(trackedFile.fileFormat()).isEqualTo(FileFormat.PARQUET); + assertThat(trackedFile.recordCount()).isEqualTo(2000L); + assertThat(trackedFile.fileSizeInBytes()).isEqualTo(12345L); + assertThat(trackedFile.partition()).isSameAs(PARTITION_DATA); + assertThat(trackedFile.equalityIds()).containsExactly(1); + + assertThat(trackedFile.tracking().status()).isEqualTo(EntryStatus.ADDED); + assertThat(trackedFile.tracking().snapshotId()).isEqualTo(50L); + + assertThat(trackedFile.specId()).isNull(); + assertThat(trackedFile.contentStats()).isNull(); + assertThat(trackedFile.sortOrderId()).isNull(); + assertThat(trackedFile.deletionVector()).isNull(); + assertThat(trackedFile.manifestInfo()).isNull(); + assertThat(trackedFile.keyMetadata()).isNull(); + assertThat(trackedFile.splitOffsets()).isNull(); + } + + @Test + public void buildEqualityDeleteFileWithAllFields() { + TrackedFile trackedFile = + TrackedFileBuilder.equalityDelete(50L) + .formatVersion(FORMAT_VERSION_V4) + .location("s3://bucket/data/eq_delete.parquet") + .fileFormat(FileFormat.PARQUET) + .recordCount(2000L) + .fileSizeInBytes(12345L) + .specId(7) + .partition(PARTITION_DATA) + .contentStats(CONTENT_STATS) + .keyMetadata(KEY_METADATA) + .splitOffsets(SPLIT_OFFSETS) + .sortOrderId(3) + .equalityIds(ImmutableList.of(1, 2)) + .build(); + + assertThat(trackedFile.formatVersion()).isEqualTo(FORMAT_VERSION_V4); + assertThat(trackedFile.contentType()).isEqualTo(FileContent.EQUALITY_DELETES); + assertThat(trackedFile.location()).isEqualTo("s3://bucket/data/eq_delete.parquet"); + assertThat(trackedFile.fileFormat()).isEqualTo(FileFormat.PARQUET); + assertThat(trackedFile.recordCount()).isEqualTo(2000L); + assertThat(trackedFile.fileSizeInBytes()).isEqualTo(12345L); + assertThat(trackedFile.specId()).isEqualTo(7); + assertThat(trackedFile.partition()).isSameAs(PARTITION_DATA); + assertThat(trackedFile.contentStats()).isSameAs(CONTENT_STATS); + assertThat(trackedFile.keyMetadata()).isEqualTo(KEY_METADATA); + assertThat(trackedFile.splitOffsets()).isEqualTo(SPLIT_OFFSETS); + assertThat(trackedFile.sortOrderId()).isEqualTo(3); + assertThat(trackedFile.equalityIds()).containsExactly(1, 2); + + assertThat(trackedFile.tracking().status()).isEqualTo(EntryStatus.ADDED); + assertThat(trackedFile.tracking().snapshotId()).isEqualTo(50L); + + // Unsupported fields for equality delete files + assertThat(trackedFile.deletionVector()).isNull(); + assertThat(trackedFile.manifestInfo()).isNull(); + } + + private static Stream manifestBuilders() { + return Stream.of( + Arguments.of(TrackedFileBuilder.dataManifest(50L), FileContent.DATA_MANIFEST), + Arguments.of(TrackedFileBuilder.deleteManifest(50L), FileContent.DELETE_MANIFEST)); + } + + @ParameterizedTest + @MethodSource("manifestBuilders") + public void buildManifestWithRequiredFieldsOnly( + TrackedFileBuilder builder, FileContent contentType) { + TrackedFile trackedFile = + builder + .formatVersion(FORMAT_VERSION_V4) + .location("s3://bucket/data/manifest.avro") + .fileFormat(FileFormat.AVRO) + .recordCount(420L) + .fileSizeInBytes(556L) + .partition(PARTITION_DATA) + .manifestInfo(MANIFEST_INFO) + .build(); + + assertThat(trackedFile.formatVersion()).isEqualTo(FORMAT_VERSION_V4); + assertThat(trackedFile.contentType()).isEqualTo(contentType); + assertThat(trackedFile.location()).isEqualTo("s3://bucket/data/manifest.avro"); + assertThat(trackedFile.fileFormat()).isEqualTo(FileFormat.AVRO); + assertThat(trackedFile.recordCount()).isEqualTo(420L); + assertThat(trackedFile.fileSizeInBytes()).isEqualTo(556L); + assertThat(trackedFile.partition()).isSameAs(PARTITION_DATA); + assertThat(trackedFile.manifestInfo()).isSameAs(MANIFEST_INFO); + + assertThat(trackedFile.tracking().status()).isEqualTo(EntryStatus.ADDED); + assertThat(trackedFile.tracking().snapshotId()).isEqualTo(50L); + + assertThat(trackedFile.specId()).isNull(); + assertThat(trackedFile.contentStats()).isNull(); + assertThat(trackedFile.sortOrderId()).isNull(); + assertThat(trackedFile.deletionVector()).isNull(); + assertThat(trackedFile.keyMetadata()).isNull(); + assertThat(trackedFile.splitOffsets()).isNull(); + assertThat(trackedFile.equalityIds()).isNull(); + } + + @ParameterizedTest + @MethodSource("manifestBuilders") + public void buildManifestWithAllFields(TrackedFileBuilder builder, FileContent contentType) { + TrackedFile trackedFile = + builder + .formatVersion(FORMAT_VERSION_V4) + .location("s3://bucket/data/manifest.avro") + .fileFormat(FileFormat.AVRO) + .recordCount(420L) + .fileSizeInBytes(556L) + .specId(7) + .partition(PARTITION_DATA) + .contentStats(CONTENT_STATS) + .keyMetadata(KEY_METADATA) + .manifestInfo(MANIFEST_INFO) + .build(); + + assertThat(trackedFile.formatVersion()).isEqualTo(FORMAT_VERSION_V4); + assertThat(trackedFile.contentType()).isEqualTo(contentType); + assertThat(trackedFile.location()).isEqualTo("s3://bucket/data/manifest.avro"); + assertThat(trackedFile.fileFormat()).isEqualTo(FileFormat.AVRO); + assertThat(trackedFile.recordCount()).isEqualTo(420L); + assertThat(trackedFile.fileSizeInBytes()).isEqualTo(556L); + assertThat(trackedFile.specId()).isEqualTo(7); + assertThat(trackedFile.partition()).isSameAs(PARTITION_DATA); + assertThat(trackedFile.contentStats()).isSameAs(CONTENT_STATS); + assertThat(trackedFile.keyMetadata()).isEqualTo(KEY_METADATA); + assertThat(trackedFile.manifestInfo()).isSameAs(MANIFEST_INFO); + + assertThat(trackedFile.tracking().status()).isEqualTo(EntryStatus.ADDED); + assertThat(trackedFile.tracking().snapshotId()).isEqualTo(50L); + + // Unsupported fields for manifests + assertThat(trackedFile.sortOrderId()).isNull(); + assertThat(trackedFile.deletionVector()).isNull(); + assertThat(trackedFile.splitOffsets()).isNull(); + assertThat(trackedFile.equalityIds()).isNull(); + } + + private static Stream manifestSources() { + return Stream.of( + Arguments.of(sourceDataManifest(10L), FileContent.DATA_MANIFEST), + Arguments.of(sourceDeleteManifest(10L), FileContent.DELETE_MANIFEST)); + } + + @ParameterizedTest + @MethodSource("manifestSources") + public void buildManifestFromSourceWithDeletedPositions( + TrackedFile source, FileContent contentType) { + entryWithInheritedSeqNums(source, 7L); + + TrackedFile trackedFile = + TrackedFileBuilder.from(source, 20L).deletedPositions(DELETED_POSITIONS).build(); + + assertThat(trackedFile.contentType()).isEqualTo(contentType); + assertThat(trackedFile.tracking().status()).isEqualTo(EntryStatus.MODIFIED); + assertThat(trackedFile.tracking().dvSnapshotId()).isEqualTo(20L); + assertThat(trackedFile.tracking().deletedPositions()).isEqualTo(DELETED_POSITIONS); + assertThat(trackedFile.tracking().replacedPositions()).isNull(); + } + + @ParameterizedTest + @MethodSource("manifestSources") + public void buildManifestFromSourceWithReplacedPositions( + TrackedFile source, FileContent contentType) { + entryWithInheritedSeqNums(source, 7L); + + TrackedFile trackedFile = + TrackedFileBuilder.from(source, 20L).replacedPositions(REPLACED_POSITIONS).build(); + + assertThat(trackedFile.contentType()).isEqualTo(contentType); + assertThat(trackedFile.tracking().status()).isEqualTo(EntryStatus.MODIFIED); + assertThat(trackedFile.tracking().dvSnapshotId()).isEqualTo(20L); + assertThat(trackedFile.tracking().deletedPositions()).isNull(); + assertThat(trackedFile.tracking().replacedPositions()).isEqualTo(REPLACED_POSITIONS); + } + + @ParameterizedTest + @MethodSource("manifestSources") + public void buildManifestFromSourceClearsPositions(TrackedFile source, FileContent contentType) { + entryWithInheritedSeqNums(source, 7L); + + TrackedFile sourceWithPositions = + TrackedFileBuilder.from(source, 15L) + .deletedPositions(DELETED_POSITIONS) + .replacedPositions(REPLACED_POSITIONS) + .build(); + + TrackedFile newEntry = TrackedFileBuilder.from(sourceWithPositions, 20L).build(); + + // Building a new entry from this source should not carry the positions over + assertThat(newEntry.contentType()).isEqualTo(contentType); + assertThat(newEntry.tracking().status()).isEqualTo(EntryStatus.EXISTING); + assertThat(newEntry.tracking().deletedPositions()).isNull(); + assertThat(newEntry.tracking().replacedPositions()).isNull(); + assertThat(newEntry.tracking().dvSnapshotId()).isEqualTo(15L); + } + + @Test + public void buildDataFileFromSource() { + TrackedFile source = entryWithInheritedSeqNums(sourceData(10L), 45L); + + TrackedFile trackedFile = TrackedFileBuilder.from(source, 20L).build(); + + assertThat(trackedFile.contentType()).isEqualTo(FileContent.DATA); + assertThat(trackedFile.tracking().status()).isEqualTo(EntryStatus.EXISTING); + assertThat(trackedFile.tracking().snapshotId()).isEqualTo(source.tracking().snapshotId()); + verifyFieldsAreFromSource(trackedFile, source); + } + + @Test + public void updateDVWhenBuildingDataFileFromSource() { + TrackedFile source = entryWithInheritedSeqNums(sourceData(10L), 45L); + + DeletionVector dv = + DeletionVectorStruct.builder() + .location("s3://bucket/data/new_dv.puffin") + .offset(5L) + .sizeInBytes(256L) + .cardinality(40L) + .build(); + + TrackedFile trackedFile = TrackedFileBuilder.from(source, 20L).deletionVector(dv).build(); + + assertThat(trackedFile.deletionVector()).isNotSameAs(source.deletionVector()).isSameAs(dv); + assertThat(trackedFile.tracking().status()).isEqualTo(EntryStatus.MODIFIED); + assertThat(trackedFile.tracking().snapshotId()).isEqualTo(10L); + assertThat(trackedFile.tracking().dataSequenceNumber()).isEqualTo(45L); + assertThat(trackedFile.tracking().fileSequenceNumber()).isEqualTo(45L); + assertThat(trackedFile.tracking().dvSnapshotId()).isEqualTo(20L); + } + + @Test + public void addingSameDeletionVectorFails() { + TrackedFile source = entryWithInheritedSeqNums(sourceData(10L), 45L); + + DeletionVector dv = + DeletionVectorStruct.builder() + .location("s3://bucket/data/new_dv.puffin") + .offset(5L) + .sizeInBytes(256L) + .cardinality(40L) + .build(); + + DeletionVector dvCopy = dv.copy(); + + TrackedFile trackedFile = TrackedFileBuilder.from(source, 20L).deletionVector(dv).build(); + + assertThatThrownBy(() -> TrackedFileBuilder.from(trackedFile, 30L).deletionVector(dv)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("The same deletion vector already added"); + assertThatThrownBy(() -> TrackedFileBuilder.from(trackedFile, 30L).deletionVector(dvCopy)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("The same deletion vector already added"); + } + + private static Stream nonManifestSources() { + return Stream.of( + Arguments.of(sourceData(10L), FileContent.DATA), + Arguments.of(sourceEqualityDelete(10L), FileContent.EQUALITY_DELETES)); + } + + private static Stream allSources() { + return Stream.concat(nonManifestSources(), manifestSources()); + } + + @ParameterizedTest + @MethodSource("allSources") + public void deletedFromSource(TrackedFile source, FileContent contentType) { + entryWithInheritedSeqNums(source, 15L); + + TrackedFile deleted = TrackedFileBuilder.deleted(source, 20L); + + assertThat(deleted.contentType()).isEqualTo(contentType); + assertThat(deleted.tracking().status()).isEqualTo(EntryStatus.DELETED); + assertThat(deleted.tracking().snapshotId()).isEqualTo(20L); + verifyFieldsAreFromSource(deleted, source); + } + + @ParameterizedTest + @MethodSource("nonManifestSources") + public void replacedFromNonManifestSource(TrackedFile source, FileContent contentType) { + entryWithInheritedSeqNums(source, 15L); + + TrackedFile replaced = TrackedFileBuilder.replaced(source, 20L); + + assertThat(replaced.contentType()).isEqualTo(contentType); + assertThat(replaced.tracking().status()).isEqualTo(EntryStatus.REPLACED); + assertThat(replaced.tracking().snapshotId()).isEqualTo(20L); + verifyFieldsAreFromSource(replaced, source); + } + + @ParameterizedTest + @MethodSource("manifestSources") + public void replacedFromManifestSourceFails(TrackedFile source, FileContent contentType) { + entryWithInheritedSeqNums(source, 15L); + + assertThatThrownBy(() -> TrackedFileBuilder.replaced(source, 20L)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage( + "Manifest entries cannot transition to REPLACED, but entry type is: " + contentType); + } + + private static TrackedFile sourceData(long snapshotId) { + return TrackedFileBuilder.data(snapshotId) + .formatVersion(FORMAT_VERSION_V4) + .location("s3://bucket/data/file.parquet") + .fileFormat(FileFormat.PARQUET) + .recordCount(2000L) + .fileSizeInBytes(12345L) + .partition(PARTITION_DATA) + .specId(7) + .contentStats(CONTENT_STATS) + .sortOrderId(3) + .deletionVector(DELETION_VECTOR) + .keyMetadata(KEY_METADATA) + .splitOffsets(SPLIT_OFFSETS) + .build(); + } + + private static TrackedFile sourceEqualityDelete(long snapshotId) { + return TrackedFileBuilder.equalityDelete(snapshotId) + .formatVersion(FORMAT_VERSION_V4) + .location("s3://bucket/data/eq_delete.parquet") + .fileFormat(FileFormat.PARQUET) + .recordCount(2000L) + .fileSizeInBytes(12345L) + .partition(PARTITION_DATA) + .equalityIds(ImmutableList.of(1)) + .build(); + } + + private static TrackedFile sourceDataManifest(long snapshotId) { + return TrackedFileBuilder.dataManifest(snapshotId) + .formatVersion(FORMAT_VERSION_V4) + .location("s3://bucket/data/data_manifest.parquet") + .fileFormat(FileFormat.PARQUET) + .recordCount(420L) + .fileSizeInBytes(556L) + .partition(PARTITION_DATA) + .manifestInfo(MANIFEST_INFO) + .build(); + } + + private static TrackedFile sourceDeleteManifest(long snapshotId) { + return TrackedFileBuilder.deleteManifest(snapshotId) + .formatVersion(FORMAT_VERSION_V4) + .location("s3://bucket/data/delete_manifest.parquet") + .fileFormat(FileFormat.PARQUET) + .recordCount(100L) + .fileSizeInBytes(543L) + .partition(PARTITION_DATA) + .manifestInfo(MANIFEST_INFO) + .build(); + } + + private static TrackedFile entryWithInheritedSeqNums(TrackedFile entry, long sequenceNumber) { + Tracking manifestTrackingToInheritFrom = + new TrackingStruct( + EntryStatus.EXISTING, 123L, sequenceNumber, sequenceNumber, null, null, null, null); + + ((TrackingStruct) entry.tracking()).inheritFrom(manifestTrackingToInheritFrom); + return entry; + } + + /** + * Verifies that fields in entry are the same as in source. Note, snapshot ID can't be verified + * here, because based on the entry's status it is either carried over or not. + */ + private static void verifyFieldsAreFromSource(TrackedFile entry, TrackedFile source) { + assertThat(entry.formatVersion()).isEqualTo(source.formatVersion()); + assertThat(entry.location()).isEqualTo(source.location()); + assertThat(entry.fileFormat()).isEqualTo(source.fileFormat()); + assertThat(entry.recordCount()).isEqualTo(source.recordCount()); + assertThat(entry.fileSizeInBytes()).isEqualTo(source.fileSizeInBytes()); + assertThat(entry.specId()).isEqualTo(source.specId()); + assertThat(entry.partition()).isSameAs(source.partition()); + assertThat(entry.contentStats()).isSameAs(source.contentStats()); + assertThat(entry.sortOrderId()).isEqualTo(source.sortOrderId()); + assertThat(entry.deletionVector()).isSameAs(source.deletionVector()); + assertThat(entry.keyMetadata()).isEqualTo(source.keyMetadata()); + assertThat(entry.splitOffsets()).isEqualTo(source.splitOffsets()); + assertThat(entry.manifestInfo()).isSameAs(source.manifestInfo()); + assertThat(entry.equalityIds()).isEqualTo(source.equalityIds()); + + assertThat(entry.tracking().dataSequenceNumber()) + .isEqualTo(source.tracking().dataSequenceNumber()); + assertThat(entry.tracking().fileSequenceNumber()) + .isEqualTo(source.tracking().fileSequenceNumber()); + assertThat(entry.tracking().dvSnapshotId()).isEqualTo(source.tracking().dvSnapshotId()); + assertThat(entry.tracking().firstRowId()).isEqualTo(source.tracking().firstRowId()); + assertThat(entry.tracking().deletedPositions()).isEqualTo(source.tracking().deletedPositions()); + assertThat(entry.tracking().replacedPositions()) + .isEqualTo(source.tracking().replacedPositions()); + } +} diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java b/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java index 0b725a39fb6b..c3391bcbbe44 100644 --- a/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java +++ b/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java @@ -30,7 +30,7 @@ import org.junit.jupiter.api.Test; class TestTrackedFileStruct { - private static final int WRITER_FORMAT_VERSION_V4 = 4; + private static final int FORMAT_VERSION_V4 = 4; private static final Types.StructType PARTITION_TYPE = Types.StructType.of( Types.NestedField.optional(1000, "id_bucket", Types.IntegerType.get()), @@ -41,18 +41,15 @@ class TestTrackedFileStruct { TrackedFile.schemaWithContentStats(Types.StructType.of(), Types.StructType.of()).fields(); private static final int TRACKING_ORDINAL = SCHEMA_FIELDS.indexOf(TrackedFile.TRACKING); private static final int CONTENT_TYPE_ORDINAL = SCHEMA_FIELDS.indexOf(TrackedFile.CONTENT_TYPE); - private static final int WRITER_FORMAT_VERSION_ORDINAL = - SCHEMA_FIELDS.indexOf(TrackedFile.WRITER_FORMAT_VERSION); + private static final int FORMAT_VERSION_ORDINAL = + SCHEMA_FIELDS.indexOf(TrackedFile.FORMAT_VERSION); private static final int LOCATION_ORDINAL = SCHEMA_FIELDS.indexOf(TrackedFile.LOCATION); private static final int FILE_FORMAT_ORDINAL = SCHEMA_FIELDS.indexOf(TrackedFile.FILE_FORMAT); private static final int RECORD_COUNT_ORDINAL = SCHEMA_FIELDS.indexOf(TrackedFile.RECORD_COUNT); private static final int FILE_SIZE_IN_BYTES_ORDINAL = SCHEMA_FIELDS.indexOf(TrackedFile.FILE_SIZE_IN_BYTES); private static final int SPEC_ID_ORDINAL = SCHEMA_FIELDS.indexOf(TrackedFile.SPEC_ID); - // partition and content_stats are rebuilt with the supplied struct types inside - // schemaWithContentStats, so their ordinals are looked up by field ID. private static final int PARTITION_ORDINAL = ordinalOf(TrackedFile.PARTITION_ID); - private static final int CONTENT_STATS_ORDINAL = ordinalOf(TrackedFile.CONTENT_STATS_ID); private static final int SORT_ORDER_ID_ORDINAL = SCHEMA_FIELDS.indexOf(TrackedFile.SORT_ORDER_ID); private static final int DELETION_VECTOR_ORDINAL = SCHEMA_FIELDS.indexOf(TrackedFile.DELETION_VECTOR); @@ -100,7 +97,7 @@ void testFieldAccess() { file.set(TRACKING_ORDINAL, tracking); file.set(CONTENT_TYPE_ORDINAL, FileContent.EQUALITY_DELETES.id()); - file.set(WRITER_FORMAT_VERSION_ORDINAL, WRITER_FORMAT_VERSION_V4); + file.set(FORMAT_VERSION_ORDINAL, FORMAT_VERSION_V4); file.set(LOCATION_ORDINAL, "s3://bucket/data/eq-delete.avro"); file.set(FILE_FORMAT_ORDINAL, "avro"); file.set(RECORD_COUNT_ORDINAL, 50L); @@ -117,7 +114,7 @@ void testFieldAccess() { assertThat(file.tracking().status()).isEqualTo(EntryStatus.ADDED); assertThat(file.tracking().snapshotId()).isEqualTo(42L); assertThat(file.contentType()).isEqualTo(FileContent.EQUALITY_DELETES); - assertThat(file.writerFormatVersion()).isEqualTo(WRITER_FORMAT_VERSION_V4); + assertThat(file.formatVersion()).isEqualTo(FORMAT_VERSION_V4); assertThat(file.location()).isEqualTo("s3://bucket/data/eq-delete.avro"); assertThat(file.fileFormat()).isEqualTo(FileFormat.AVRO); assertThat(file.recordCount()).isEqualTo(50L); @@ -209,7 +206,7 @@ void testCopy() { assertThat(copy.sortOrderId()).isEqualTo(1); assertThat(copy.recordCount()).isEqualTo(100L); assertThat(copy.fileSizeInBytes()).isEqualTo(1024L); - assertThat(copy.writerFormatVersion()).isEqualTo(WRITER_FORMAT_VERSION_V4); + assertThat(copy.formatVersion()).isEqualTo(FORMAT_VERSION_V4); assertThat(copy.keyMetadata()).isNotNull(); assertThat(copy.splitOffsets()).containsExactly(50L); assertThat(copy.equalityIds()).isNull(); @@ -332,7 +329,7 @@ void testJavaSerializationRoundTrip() throws IOException, ClassNotFoundException TrackedFileStruct deserialized = TestHelpers.roundTripSerialize(file); assertThat(deserialized.contentType()).isEqualTo(FileContent.DATA); - assertThat(deserialized.writerFormatVersion()).isEqualTo(WRITER_FORMAT_VERSION_V4); + assertThat(deserialized.formatVersion()).isEqualTo(FORMAT_VERSION_V4); assertThat(deserialized.location()).isEqualTo("s3://bucket/data/file.parquet"); assertThat(deserialized.fileFormat()).isEqualTo(FileFormat.PARQUET); assertThat(deserialized.recordCount()).isEqualTo(100L); @@ -356,7 +353,7 @@ void testKryoSerializationRoundTrip() throws IOException { TrackedFileStruct deserialized = TestHelpers.KryoHelpers.roundTripSerialize(file); assertThat(deserialized.contentType()).isEqualTo(FileContent.DATA); - assertThat(deserialized.writerFormatVersion()).isEqualTo(WRITER_FORMAT_VERSION_V4); + assertThat(deserialized.formatVersion()).isEqualTo(FORMAT_VERSION_V4); assertThat(deserialized.location()).isEqualTo("s3://bucket/data/file.parquet"); assertThat(deserialized.fileFormat()).isEqualTo(FileFormat.PARQUET); assertThat(deserialized.recordCount()).isEqualTo(100L); @@ -374,10 +371,6 @@ void testKryoSerializationRoundTrip() throws IOException { } static TrackedFileStruct createFullTrackedFile() { - TrackingStruct tracking = (TrackingStruct) TrackingBuilder.added(42L).build(); - tracking.setManifestLocation("s3://bucket/manifest.avro"); - tracking.set(MANIFEST_POS_ORDINAL, 3L); - DeletionVectorStruct dv = DeletionVectorStruct.builder() .location("s3://bucket/dv.puffin") @@ -387,20 +380,24 @@ static TrackedFileStruct createFullTrackedFile() { .build(); TrackedFileStruct file = - new TrackedFileStruct( - tracking, - FileContent.DATA, - WRITER_FORMAT_VERSION_V4, - "s3://bucket/data/file.parquet", - FileFormat.PARQUET, - newPartition(7, "music"), - 100L, - 1024L); - file.set(SPEC_ID_ORDINAL, 0); - file.set(SORT_ORDER_ID_ORDINAL, 1); - file.set(DELETION_VECTOR_ORDINAL, dv); - file.set(KEY_METADATA_ORDINAL, ByteBuffer.wrap(new byte[] {1, 2, 3})); - file.set(SPLIT_OFFSETS_ORDINAL, ImmutableList.of(50L)); + (TrackedFileStruct) + TrackedFileBuilder.data(42L) + .formatVersion(FORMAT_VERSION_V4) + .location("s3://bucket/data/file.parquet") + .fileFormat(FileFormat.PARQUET) + .partition(newPartition(7, "music")) + .recordCount(100L) + .fileSizeInBytes(1024L) + .specId(0) + .sortOrderId(1) + .deletionVector(dv) + .keyMetadata(ByteBuffer.wrap(new byte[] {1, 2, 3})) + .splitOffsets(ImmutableList.of(50L)) + .build(); + + TrackingStruct tracking = (TrackingStruct) file.tracking(); + tracking.setManifestLocation("s3://bucket/manifest.avro"); + tracking.set(MANIFEST_POS_ORDINAL, 3L); return file; } @@ -460,19 +457,16 @@ static TrackedFileStruct createTrackedFileWithStats() { .withFieldStats(fieldStatsList) .build(); - TrackedFileStruct file = - new TrackedFileStruct( - null, - FileContent.DATA, - WRITER_FORMAT_VERSION_V4, - "s3://bucket/data/file.parquet", - FileFormat.PARQUET, - new PartitionData(Types.StructType.of()), - 100L, - 1024L); - file.set(SPEC_ID_ORDINAL, 0); - file.set(CONTENT_STATS_ORDINAL, stats); - - return file; + return (TrackedFileStruct) + TrackedFileBuilder.data(0L) + .formatVersion(FORMAT_VERSION_V4) + .location("s3://bucket/data/file.parquet") + .fileFormat(FileFormat.PARQUET) + .partition(new PartitionData(Types.StructType.of())) + .recordCount(100L) + .fileSizeInBytes(1024L) + .specId(0) + .contentStats(stats) + .build(); } } diff --git a/core/src/test/java/org/apache/iceberg/avro/TestSchemaConversions.java b/core/src/test/java/org/apache/iceberg/avro/TestSchemaConversions.java index 0f822d3e4b64..f934e0101b73 100644 --- a/core/src/test/java/org/apache/iceberg/avro/TestSchemaConversions.java +++ b/core/src/test/java/org/apache/iceberg/avro/TestSchemaConversions.java @@ -94,6 +94,12 @@ public void testPrimitiveTypes() { } } + @Test + public void testAvroToIcebergUUIDTypeOnString() { + Schema uuidStringSchema = LogicalTypes.uuid().addToSchema(Schema.create(Schema.Type.STRING)); + assertThat(AvroSchemaUtil.convert(uuidStringSchema)).isEqualTo(Types.UUIDType.get()); + } + @Test public void testAvroToIcebergTimestampTypeWithoutAdjustToUTC() { // Not included in the primitives test because there is not a way to round trip the diff --git a/core/src/test/java/org/apache/iceberg/data/DataTestHelpers.java b/core/src/test/java/org/apache/iceberg/data/DataTestHelpers.java index f2e2b4e7fa34..6321d876a985 100644 --- a/core/src/test/java/org/apache/iceberg/data/DataTestHelpers.java +++ b/core/src/test/java/org/apache/iceberg/data/DataTestHelpers.java @@ -128,6 +128,8 @@ private static void assertEquals(Type type, Object expected, Object actual) { case UUID: case BINARY: case DECIMAL: + case GEOMETRY: + case GEOGRAPHY: assertThat(actual) .as("Primitive value should be equal to expected for type " + type) .isEqualTo(expected); diff --git a/core/src/test/java/org/apache/iceberg/rest/TestRESTScanPlanning.java b/core/src/test/java/org/apache/iceberg/rest/TestRESTScanPlanning.java index 9b42d445f585..117bf7fac24b 100644 --- a/core/src/test/java/org/apache/iceberg/rest/TestRESTScanPlanning.java +++ b/core/src/test/java/org/apache/iceberg/rest/TestRESTScanPlanning.java @@ -75,6 +75,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.ValueSource; import org.mockito.ArgumentCaptor; import org.mockito.Mockito; @@ -1576,4 +1577,84 @@ public void clientRequestsClientAndServerReturnsNothing() { assertThat(table).isNotInstanceOf(RESTTable.class); assertThat(table).isInstanceOf(BaseTable.class); } + + @Test + public void defaultPlanningModeWhenNoneSpecified() { + CatalogWithAdapter catalogWithAdapter = catalogWithModes(null, null); + catalogWithAdapter.catalog.createNamespace(NS); + + Table table = + catalogWithAdapter + .catalog + .buildTable(TableIdentifier.of(NS, "default_mode_test"), SCHEMA) + .create(); + + assertThat(table).isNotInstanceOf(RESTTable.class).isInstanceOf(BaseTable.class); + } + + @Test + public void invalidPlanningModeConfiguredForClient() { + assertThatThrownBy( + () -> + catalogWithModes( + "invalid_mode", RESTCatalogProperties.ScanPlanningMode.CLIENT.modeName())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Invalid scan planning mode: invalid_mode"); + } + + @Test + public void invalidPlanningModeConfiguredForServer() { + CatalogWithAdapter catalogWithAdapter = + catalogWithModes(RESTCatalogProperties.ScanPlanningMode.CLIENT.modeName(), "invalid_mode"); + catalogWithAdapter.catalog.createNamespace(NS); + + assertThatThrownBy( + () -> + catalogWithAdapter + .catalog + .buildTable(TableIdentifier.of(NS, "invalid_server_mode_test"), SCHEMA) + .create()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Invalid scan planning mode: invalid_mode"); + } + + @ParameterizedTest + @ValueSource( + strings = {"client", "CLIENT", "Client", "cLiEnT", "server", "SERVER", "Server", "sErVeR"}) + public void planningModeWithDifferentCasesOnClient(String planningMode) { + CatalogWithAdapter catalogWithAdapter = catalogWithModes(planningMode, null); + catalogWithAdapter.catalog.createNamespace(NS); + + Table table = + catalogWithAdapter + .catalog + .buildTable(TableIdentifier.of(NS, "client_case_test_" + planningMode), SCHEMA) + .create(); + + verifyTableTypeForPlanningMode(planningMode, table); + } + + @ParameterizedTest + @ValueSource( + strings = {"client", "CLIENT", "Client", "cLiEnT", "server", "SERVER", "Server", "sErVeR"}) + public void planningModeWithDifferentCasesOnServer(String serverMode) { + CatalogWithAdapter catalogWithAdapter = catalogWithModes(null, serverMode); + catalogWithAdapter.catalog.createNamespace(NS); + + Table table = + catalogWithAdapter + .catalog + .buildTable(TableIdentifier.of(NS, "server_case_test_" + serverMode), SCHEMA) + .create(); + + verifyTableTypeForPlanningMode(serverMode, table); + } + + private void verifyTableTypeForPlanningMode(String planingMode, Table table) { + if (planingMode.equalsIgnoreCase("client")) { + assertThat(table).isNotInstanceOf(RESTTable.class).isInstanceOf(BaseTable.class); + } else { + assertThat(table).isInstanceOf(RESTTable.class); + } + } } diff --git a/data/src/main/java/org/apache/iceberg/data/IcebergGenerics.java b/data/src/main/java/org/apache/iceberg/data/IcebergGenerics.java index 352658752d98..7bf8f5c7780b 100644 --- a/data/src/main/java/org/apache/iceberg/data/IcebergGenerics.java +++ b/data/src/main/java/org/apache/iceberg/data/IcebergGenerics.java @@ -24,6 +24,7 @@ import org.apache.iceberg.TableScan; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.metrics.MetricsReporter; public class IcebergGenerics { private IcebergGenerics() {} @@ -96,6 +97,11 @@ public ScanBuilder appendsAfter(long fromSnapshotId) { return this; } + public ScanBuilder metricsReporter(MetricsReporter reporter) { + this.tableScan = tableScan.metricsReporter(reporter); + return this; + } + public CloseableIterable build() { return new TableScanIterable(tableScan, reuseContainers); } diff --git a/data/src/test/java/org/apache/iceberg/data/BaseFormatModelTests.java b/data/src/test/java/org/apache/iceberg/data/BaseFormatModelTests.java index 8e9eb55940ee..bde37fdca18d 100644 --- a/data/src/test/java/org/apache/iceberg/data/BaseFormatModelTests.java +++ b/data/src/test/java/org/apache/iceberg/data/BaseFormatModelTests.java @@ -60,7 +60,11 @@ import org.apache.iceberg.deletes.PositionDeleteWriter; import org.apache.iceberg.encryption.EncryptedFiles; import org.apache.iceberg.encryption.EncryptedOutputFile; +import org.apache.iceberg.encryption.EncryptingFileIO; import org.apache.iceberg.encryption.EncryptionKeyMetadata; +import org.apache.iceberg.encryption.EncryptionManager; +import org.apache.iceberg.encryption.EncryptionTestHelpers; +import org.apache.iceberg.encryption.NativeEncryptionKeyMetadata; import org.apache.iceberg.exceptions.AlreadyExistsException; import org.apache.iceberg.exceptions.NotFoundException; import org.apache.iceberg.exceptions.ValidationException; @@ -179,6 +183,8 @@ private static List project(List records, Schema targetSchema) { static final String FEATURE_REUSE_CONTAINERS = "reuseContainers"; static final String FEATURE_COLUMN_LEVEL_METRICS = "columnLevelMetrics"; static final String FEATURE_COLUMN_METRICS_TRUNCATE_BINARY = "columnMetricsTruncateBinary"; + static final String FEATURE_NATIVE_ENCRYPTION = "nativeEncryption"; + static final String FEATURE_AES_STREAM_ENCRYPTION = "aesStreamEncryption"; private static final Map MISSING_FEATURES = Map.of( @@ -188,12 +194,19 @@ private static List project(List records, Schema targetSchema) { FEATURE_CASE_SENSITIVE, FEATURE_SPLIT, FEATURE_COLUMN_LEVEL_METRICS, - FEATURE_COLUMN_METRICS_TRUNCATE_BINARY + FEATURE_COLUMN_METRICS_TRUNCATE_BINARY, + FEATURE_NATIVE_ENCRYPTION }, FileFormat.ORC, new String[] { - FEATURE_REUSE_CONTAINERS, FEATURE_COLUMN_METRICS_TRUNCATE_BINARY, FEATURE_READER_DEFAULT - }); + FEATURE_REUSE_CONTAINERS, + FEATURE_COLUMN_METRICS_TRUNCATE_BINARY, + FEATURE_READER_DEFAULT, + FEATURE_AES_STREAM_ENCRYPTION, + FEATURE_NATIVE_ENCRYPTION + }, + FileFormat.PARQUET, + new String[] {FEATURE_AES_STREAM_ENCRYPTION}); private InMemoryFileIO fileIO; private EncryptedOutputFile encryptedFile; @@ -2067,6 +2080,98 @@ void testDataWriterMetaMap(FileFormat fileFormat) throws IOException { }); } + @ParameterizedTest + @FieldSource("FILE_FORMATS") + void testDataWriterAesStreamEncryption(FileFormat fileFormat) throws IOException { + assumeSupports(fileFormat, FEATURE_AES_STREAM_ENCRYPTION); + + EncryptionManager encryptionManager = EncryptionTestHelpers.createEncryptionManager(); + EncryptingFileIO encryptingFileIO = EncryptingFileIO.combine(fileIO, encryptionManager); + EncryptedOutputFile encryptedOutputFile = + encryptingFileIO.newEncryptingOutputFile("test-file-" + UUID.randomUUID()); + + FileWriterBuilder, ?> writerBuilder = + FormatModelRegistry.dataWriteBuilder(fileFormat, engineType(), encryptedOutputFile) + .keyMetadata(encryptedOutputFile.keyMetadata()); + + writeAndAssertEncryptedDataWriter(fileFormat, encryptingFileIO, writerBuilder); + } + + @ParameterizedTest + @FieldSource("FILE_FORMATS") + void testDataWriterNativeEncryption(FileFormat fileFormat) throws IOException { + assumeSupports(fileFormat, FEATURE_NATIVE_ENCRYPTION); + + EncryptionManager encryptionManager = EncryptionTestHelpers.createEncryptionManager(); + EncryptingFileIO encryptingFileIO = EncryptingFileIO.combine(fileIO, encryptionManager); + String location = "test-file-" + UUID.randomUUID(); + NativeEncryptionKeyMetadata keyMetadata = + (NativeEncryptionKeyMetadata) + encryptingFileIO.newEncryptingOutputFile(location).keyMetadata(); + + // Use a plain encrypted output file so Parquet cannot auto-inject the native encryption key + // and AAD prefix from the output file metadata. This ensures encryption is driven only by + // withFileEncryptionKey and withAADPrefix below. + EncryptedOutputFile encryptedOutputFile = + EncryptedFiles.plainAsEncryptedOutput(fileIO.newOutputFile(location)); + + // keyMetadata is mainly used for parsing during reading, so this call is required here. + FileWriterBuilder, ?> writerBuilder = + FormatModelRegistry.dataWriteBuilder(fileFormat, engineType(), encryptedOutputFile) + .keyMetadata(keyMetadata) + .withFileEncryptionKey(keyMetadata.encryptionKey().duplicate()) + .withAADPrefix(keyMetadata.aadPrefix().duplicate()); + + writeAndAssertEncryptedDataWriter(fileFormat, encryptingFileIO, writerBuilder); + } + + @SuppressWarnings("checkstyle:AssertThatThrownByWithMessageCheck") + private void writeAndAssertEncryptedDataWriter( + FileFormat fileFormat, + EncryptingFileIO encryptingFileIO, + FileWriterBuilder, ?> writerBuilder) + throws IOException { + DataGenerator dataGenerator = new DataGenerators.DefaultSchema(); + Schema schema = dataGenerator.schema(); + List genericRecords = dataGenerator.generateRecords(); + List engineRecords = convertToEngineRecords(genericRecords, schema); + + DataWriter writer = writerBuilder.schema(schema).spec(PartitionSpec.unpartitioned()).build(); + + try (writer) { + engineRecords.forEach(writer::write); + } + + DataFile dataFile = writer.toDataFile(); + assertThat(dataFile).isNotNull(); + assertThat(dataFile.recordCount()).isEqualTo(engineRecords.size()); + assertThat(dataFile.format()).isEqualTo(fileFormat); + assertThat(dataFile.keyMetadata()).isNotNull(); + + assertThatThrownBy( + () -> + readAndAssertGenericRecords( + fileFormat, schema, genericRecords, fileIO.newInputFile(dataFile.location()))) + .isInstanceOf(RuntimeException.class); + + readAndAssertGenericRecords( + fileFormat, schema, genericRecords, encryptingFileIO.newInputFile(dataFile)); + } + + private void readAndAssertGenericRecords( + FileFormat fileFormat, Schema schema, List expected, InputFile inputFile) + throws IOException { + List readRecords; + try (CloseableIterable reader = + FormatModelRegistry.readBuilder(fileFormat, Record.class, inputFile) + .project(schema) + .build()) { + readRecords = ImmutableList.copyOf(reader); + } + + DataTestHelpers.assertEquals(schema.asStruct(), expected, readRecords); + } + private void readAndAssertGenericRecords( FileFormat fileFormat, Schema schema, List expected) throws IOException { InputFile inputFile = encryptedFile.encryptingOutputFile().toInputFile(); diff --git a/data/src/test/java/org/apache/iceberg/data/RandomGenericData.java b/data/src/test/java/org/apache/iceberg/data/RandomGenericData.java index 4963052e0877..583513d3f90d 100644 --- a/data/src/test/java/org/apache/iceberg/data/RandomGenericData.java +++ b/data/src/test/java/org/apache/iceberg/data/RandomGenericData.java @@ -268,6 +268,8 @@ public Object primitive(Type.PrimitiveType primitive) { Object result = randomValue(primitive, random); switch (primitive.typeId()) { case BINARY: + case GEOMETRY: + case GEOGRAPHY: return ByteBuffer.wrap((byte[]) result); case UUID: return UUID.nameUUIDFromBytes((byte[]) result); diff --git a/data/src/test/java/org/apache/iceberg/data/TestLocalScan.java b/data/src/test/java/org/apache/iceberg/data/TestLocalScan.java index 0f85596afbdc..aee5eb5fa76c 100644 --- a/data/src/test/java/org/apache/iceberg/data/TestLocalScan.java +++ b/data/src/test/java/org/apache/iceberg/data/TestLocalScan.java @@ -58,7 +58,10 @@ import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.hadoop.HadoopInputFile; import org.apache.iceberg.hadoop.HadoopTables; +import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.io.DataWriter; +import org.apache.iceberg.metrics.InMemoryMetricsReporter; +import org.apache.iceberg.metrics.ScanReport; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; @@ -289,6 +292,22 @@ public void testFullScan() { assertThat(records).as("Random record set should match").isEqualTo(expected); } + @TestTemplate + public void testScanMetricsReporter() throws IOException { + InMemoryMetricsReporter reporter = new InMemoryMetricsReporter(); + + try (CloseableIterable results = + IcebergGenerics.read(sharedTable).metricsReporter(reporter).build()) { + assertThat(Sets.newHashSet(results)).hasSize(9); + } + + ScanReport scanReport = reporter.scanReport(); + assertThat(scanReport).isNotNull(); + assertThat(scanReport.tableName()).isEqualTo(sharedTable.name()); + assertThat(scanReport.snapshotId()).isEqualTo(sharedTable.currentSnapshot().snapshotId()); + assertThat(scanReport.scanMetrics().resultDataFiles().value()).isEqualTo(3); + } + @TestTemplate public void testFilter() { Iterable result = IcebergGenerics.read(sharedTable).where(lessThan("id", 3)).build(); diff --git a/data/src/test/java/org/apache/iceberg/data/parquet/TestGenericData.java b/data/src/test/java/org/apache/iceberg/data/parquet/TestGenericData.java index 8c0e2e903ab7..63b6e4769631 100644 --- a/data/src/test/java/org/apache/iceberg/data/parquet/TestGenericData.java +++ b/data/src/test/java/org/apache/iceberg/data/parquet/TestGenericData.java @@ -68,6 +68,11 @@ protected boolean supportsRowLineage() { return true; } + @Override + protected boolean supportsGeospatial() { + return true; + } + @Override protected void writeAndValidate(Schema schema) throws IOException { writeAndValidate(schema, schema); diff --git a/data/src/test/java/org/apache/iceberg/orc/TestOrcMetrics.java b/data/src/test/java/org/apache/iceberg/orc/TestOrcMetrics.java index 940cf4522291..c1647bb716c5 100644 --- a/data/src/test/java/org/apache/iceberg/orc/TestOrcMetrics.java +++ b/data/src/test/java/org/apache/iceberg/orc/TestOrcMetrics.java @@ -22,6 +22,8 @@ import java.io.File; import java.io.IOException; +import java.time.LocalDateTime; +import java.time.temporal.ChronoUnit; import java.util.Map; import java.util.UUID; import org.apache.iceberg.FileFormat; @@ -31,6 +33,7 @@ import org.apache.iceberg.ParameterizedTestExtension; import org.apache.iceberg.Schema; import org.apache.iceberg.TestMetrics; +import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.data.Record; import org.apache.iceberg.data.orc.GenericOrcWriter; import org.apache.iceberg.io.FileAppender; @@ -39,7 +42,10 @@ import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.types.Conversions; import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.Types; +import org.junit.jupiter.api.TestTemplate; import org.junit.jupiter.api.extension.ExtendWith; /** Test Metrics for ORC. */ @@ -107,6 +113,35 @@ private boolean isBinaryType(Type type) { return BINARY_TYPES.contains(type.typeId()); } + @TestTemplate + public void timestampNanoBoundsKeepNanoPrecision() throws IOException { + Types.TimestampNanoType nanoType = Types.TimestampNanoType.withoutZone(); + Schema schema = new Schema(Types.NestedField.optional(1, "tsNano", nanoType)); + + // sub-microsecond nanos that would change if truncated to micros + LocalDateTime lower = LocalDateTime.parse("1970-01-01T00:00:00.000001500"); + LocalDateTime upper = LocalDateTime.parse("2024-01-02T03:04:05.123456789"); + + GenericRecord lowerRecord = GenericRecord.create(schema); + lowerRecord.set(0, lower); + GenericRecord upperRecord = GenericRecord.create(schema); + upperRecord.set(0, upper); + + Metrics metrics = getMetrics(schema, lowerRecord, upperRecord); + + LocalDateTime epoch = LocalDateTime.parse("1970-01-01T00:00:00"); + long expectedLower = ChronoUnit.NANOS.between(epoch, lower); + long expectedUpper = ChronoUnit.NANOS.between(epoch, upper); + + long actualLower = Conversions.fromByteBuffer(nanoType, metrics.lowerBounds().get(1)); + long actualUpper = Conversions.fromByteBuffer(nanoType, metrics.upperBounds().get(1)); + + assertThat(actualLower).isEqualTo(expectedLower); + assertThat(actualUpper).isEqualTo(expectedUpper); + // guard against the micros regression: the bound must not be ~1000x smaller + assertThat(actualLower).isEqualTo(1500L); + } + @Override protected void assertBounds( int fieldId, Type type, T lowerBound, T upperBound, MetricsWithStats metricsWithStats) { diff --git a/deploy.gradle b/deploy.gradle index 65836bf1b3f1..fc380daa71e6 100644 --- a/deploy.gradle +++ b/deploy.gradle @@ -47,15 +47,17 @@ subprojects { from sourceSets.test.output } + def sourceArtifactTask = tasks.findByName('shadowSourceJar') ?: sourceJar + artifacts { - archives sourceJar + archives sourceArtifactTask archives javadocJar archives testJar testArtifacts testJar } // add LICENSE and NOTICE - [jar, sourceJar, javadocJar, testJar].each { task -> + [jar, sourceArtifactTask, javadocJar, testJar].each { task -> task.dependsOn rootProject.tasks.buildInfo task.from("${rootDir}/build") { include 'iceberg-build.properties' @@ -81,8 +83,12 @@ subprojects { } else { project.shadow.component(it) } + if (tasks.matching({task -> task.name == 'shadowSourceJar'}).isEmpty()) { + artifact sourceJar + } else { + artifact shadowSourceJar + } - artifact sourceJar artifact javadocJar artifact testJar diff --git a/docker/iceberg-rest-fixture/Dockerfile b/docker/iceberg-rest-fixture/Dockerfile index 5629c959bd5a..1f3fc881a1b2 100644 --- a/docker/iceberg-rest-fixture/Dockerfile +++ b/docker/iceberg-rest-fixture/Dockerfile @@ -30,8 +30,9 @@ RUN set -xeu && \ # Working directory for the application WORKDIR /usr/lib/iceberg-rest -# Copy the JAR file directly to the target location +# Copy runtime files directly to the target location COPY --chown=iceberg:iceberg open-api/build/libs/iceberg-open-api-test-fixtures-runtime-*.jar /usr/lib/iceberg-rest/iceberg-rest-adapter.jar +COPY --chown=iceberg:iceberg docker/iceberg-rest-fixture/entrypoint.sh /usr/lib/iceberg-rest/entrypoint.sh ENV CATALOG_CATALOG__IMPL=org.apache.iceberg.jdbc.JdbcCatalog ENV CATALOG_URI=jdbc:sqlite:/tmp/iceberg_catalog.db @@ -47,4 +48,4 @@ HEALTHCHECK --retries=10 --interval=1s \ EXPOSE $REST_PORT USER iceberg:iceberg ENV LANG=en_US.UTF-8 -CMD ["java", "-jar", "iceberg-rest-adapter.jar"] +CMD ["sh", "/usr/lib/iceberg-rest/entrypoint.sh"] diff --git a/docker/iceberg-rest-fixture/README.md b/docker/iceberg-rest-fixture/README.md index 5e02a2b4712a..928c37fe38dd 100644 --- a/docker/iceberg-rest-fixture/README.md +++ b/docker/iceberg-rest-fixture/README.md @@ -56,6 +56,30 @@ docker run -e CATALOG_CATALOG_NAME=mycatalog -p 8181:8181 apache/iceberg-rest-fi ``` +### Logging + +By default, the fixture logs at `INFO`. To reduce log output, set `LOG_LEVEL` +to another slf4j-simple level such as `WARN`, `ERROR`, or `OFF`: + +```bash +docker run -e LOG_LEVEL=WARN -p 8181:8181 apache/iceberg-rest-fixture +``` + +To use a full slf4j-simple properties file, mount a directory containing +`simplelogger.properties` and set `LOG_CONFIG_DIR` to that directory: + +```bash +docker run \ + -v "$PWD/simplelogger.properties:/etc/iceberg-rest/simplelogger.properties:ro" \ + -e LOG_CONFIG_DIR=/etc/iceberg-rest \ + -p 8181:8181 \ + apache/iceberg-rest-fixture +``` + +If both `LOG_LEVEL` and `LOG_CONFIG_DIR` are set, `LOG_LEVEL` overrides the +default log level from `simplelogger.properties`; the properties file can still +configure other slf4j-simple settings. + ## Build the Docker Image When making changes to the local files and test them out, you can build the image locally: @@ -115,5 +139,3 @@ Snapshots Snapshots Properties write.object-storage.enabled true write.object-storage.path s3://iceberg-test-data/tpc/tpc-ds/3.2.0/1000/iceberg/customer/data ``` - - diff --git a/docker/iceberg-rest-fixture/entrypoint.sh b/docker/iceberg-rest-fixture/entrypoint.sh new file mode 100644 index 000000000000..052bd7b7229b --- /dev/null +++ b/docker/iceberg-rest-fixture/entrypoint.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env sh +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -eu + +CLASSPATH="iceberg-rest-adapter.jar" + +if [ -n "${LOG_CONFIG_DIR:-}" ]; then + CLASSPATH="${LOG_CONFIG_DIR}:${CLASSPATH}" +fi + +set -- java + +if [ -n "${LOG_LEVEL:-}" ]; then + set -- "$@" "-Dorg.slf4j.simpleLogger.defaultLogLevel=${LOG_LEVEL}" +fi + +exec "$@" -cp "$CLASSPATH" org.apache.iceberg.rest.RESTCatalogServer diff --git a/flink/v1.20/build.gradle b/flink/v1.20/build.gradle index 41f2489c8038..069cf2ee623b 100644 --- a/flink/v1.20/build.gradle +++ b/flink/v1.20/build.gradle @@ -68,6 +68,7 @@ project(":iceberg-flink:iceberg-flink-${flinkMajorVersion}") { } implementation libs.datasketches + implementation libs.roaringbitmap testImplementation libs.flink120.connector.test.utils testImplementation libs.flink120.core diff --git a/flink/v1.20/flink-runtime/runtime-deps.txt b/flink/v1.20/flink-runtime/runtime-deps.txt index 8e493d31c510..a295230cef19 100644 --- a/flink/v1.20/flink-runtime/runtime-deps.txt +++ b/flink/v1.20/flink-runtime/runtime-deps.txt @@ -25,7 +25,7 @@ org.apache.parquet:parquet-variant:1.17 org.checkerframework:checker-qual:3.19 org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1 org.locationtech.jts:jts-core:1.20 -org.projectnessie.nessie:nessie-client:0.107 -org.projectnessie.nessie:nessie-model:0.107 +org.projectnessie.nessie:nessie-client:0.108 +org.projectnessie.nessie:nessie-model:0.108 org.roaringbitmap:RoaringBitmap:1.6 org.threeten:threeten-extra:1.7 diff --git a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java index a56c4e0ca6ed..1d1505a28c05 100644 --- a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java +++ b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java @@ -99,7 +99,6 @@ public class FlinkCatalog extends AbstractCatalog { private final Namespace baseNamespace; private final SupportsNamespaces asNamespaceCatalog; private final Closeable closeable; - private final Map catalogProps; private final boolean cacheEnabled; public FlinkCatalog( @@ -107,12 +106,10 @@ public FlinkCatalog( String defaultDatabase, Namespace baseNamespace, CatalogLoader catalogLoader, - Map catalogProps, boolean cacheEnabled, long cacheExpirationIntervalMs) { super(catalogName, defaultDatabase); this.catalogLoader = catalogLoader; - this.catalogProps = catalogProps; this.baseNamespace = baseNamespace; this.cacheEnabled = cacheEnabled; @@ -339,13 +336,12 @@ public CatalogTable getTable(ObjectPath tablePath) Table table = loadIcebergTable(tablePath); // Flink's CREATE TABLE LIKE clause relies on properties sent back here to create new table. - // Inorder to create such table in non iceberg catalog, we need to send across catalog - // properties also. // As Flink API accepts only Map for props, here we are serializing catalog - // props as json string to distinguish between catalog and table properties in createTable. + // name, database, table as json string to distinguish between catalog info + // and table properties in createTable. String srcCatalogProps = FlinkCreateTableOptions.toJson( - getName(), tablePath.getDatabaseName(), tablePath.getObjectName(), catalogProps); + getName(), tablePath.getDatabaseName(), tablePath.getObjectName()); Map tableProps = table.properties(); if (tableProps.containsKey(FlinkCreateTableOptions.CONNECTOR_PROPS_KEY) @@ -427,7 +423,12 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig + "create table without 'connector'='iceberg' related properties in an iceberg table."); } - Preconditions.checkArgument(table instanceof ResolvedCatalogTable, "table should be resolved"); + Preconditions.checkArgument( + table instanceof ResolvedCatalogTable, + "Expected a ResolvedCatalogTable but got: %s. " + + "Iceberg Flink catalog only supports resolved catalog tables " + + "(Materialized tables and other table kinds are not supported).", + table == null ? "null" : table.getClass().getName()); createIcebergTable(tablePath, (ResolvedCatalogTable) table, ignoreIfExists); } diff --git a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalogFactory.java b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalogFactory.java index dd065617bd88..fe4008a13ce5 100644 --- a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalogFactory.java +++ b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalogFactory.java @@ -168,7 +168,6 @@ protected Catalog createCatalog( defaultDatabase, baseNamespace, catalogLoader, - properties, cacheEnabled, cacheExpirationIntervalMs); } diff --git a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/FlinkCreateTableOptions.java b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/FlinkCreateTableOptions.java index 0612260bfe7d..067b42bba954 100644 --- a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/FlinkCreateTableOptions.java +++ b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/FlinkCreateTableOptions.java @@ -18,7 +18,6 @@ */ package org.apache.iceberg.flink; -import java.util.Map; import org.apache.flink.configuration.ConfigOption; import org.apache.flink.configuration.ConfigOptions; import org.apache.iceberg.util.JsonUtil; @@ -27,14 +26,11 @@ class FlinkCreateTableOptions { private final String catalogName; private final String catalogDb; private final String catalogTable; - private final Map catalogProps; - private FlinkCreateTableOptions( - String catalogName, String catalogDb, String catalogTable, Map props) { + private FlinkCreateTableOptions(String catalogName, String catalogDb, String catalogTable) { this.catalogName = catalogName; this.catalogDb = catalogDb; this.catalogTable = catalogTable; - this.catalogProps = props; } public static final ConfigOption CATALOG_NAME = @@ -61,12 +57,6 @@ private FlinkCreateTableOptions( .noDefaultValue() .withDescription("Table name managed in the underlying iceberg catalog and database."); - public static final ConfigOption> CATALOG_PROPS = - ConfigOptions.key("catalog-props") - .mapType() - .noDefaultValue() - .withDescription("Properties for the underlying catalog for iceberg table."); - public static final ConfigOption USE_DYNAMIC_ICEBERG_SINK = ConfigOptions.key("use-dynamic-iceberg-sink") .booleanType() @@ -89,15 +79,13 @@ private FlinkCreateTableOptions( public static final String CONNECTOR_PROPS_KEY = "connector"; public static final String LOCATION_KEY = "location"; - static String toJson( - String catalogName, String catalogDb, String catalogTable, Map catalogProps) { + static String toJson(String catalogName, String catalogDb, String catalogTable) { return JsonUtil.generate( gen -> { gen.writeStartObject(); gen.writeStringField(CATALOG_NAME.key(), catalogName); gen.writeStringField(CATALOG_DATABASE.key(), catalogDb); gen.writeStringField(CATALOG_TABLE.key(), catalogTable); - JsonUtil.writeStringMap(CATALOG_PROPS.key(), catalogProps, gen); gen.writeEndObject(); }, false); @@ -110,9 +98,8 @@ static FlinkCreateTableOptions fromJson(String createTableOptions) { String catalogName = JsonUtil.getString(CATALOG_NAME.key(), node); String catalogDb = JsonUtil.getString(CATALOG_DATABASE.key(), node); String catalogTable = JsonUtil.getString(CATALOG_TABLE.key(), node); - Map catalogProps = JsonUtil.getStringMap(CATALOG_PROPS.key(), node); - return new FlinkCreateTableOptions(catalogName, catalogDb, catalogTable, catalogProps); + return new FlinkCreateTableOptions(catalogName, catalogDb, catalogTable); }); } @@ -127,8 +114,4 @@ String catalogDb() { String catalogTable() { return catalogTable; } - - Map catalogProps() { - return catalogProps; - } } diff --git a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/FlinkDynamicTableFactory.java b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/FlinkDynamicTableFactory.java index b2e2e33b9291..6306ee7a0a1c 100644 --- a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/FlinkDynamicTableFactory.java +++ b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/FlinkDynamicTableFactory.java @@ -237,13 +237,13 @@ private static TableLoader createTableLoader( } /** - * Merges source catalog properties with connector properties. Iceberg Catalog properties are - * serialized as json in FlinkCatalog#getTable to be able to isolate catalog props from iceberg - * table props, Here, we flatten and merge them back to use to create catalog. + * Merges source catalog properties (catalog name, database, table) with connector properties. + * Source catalog name, database, table are serialized as json in FlinkCatalog#getTable to be able + * to isolate them from iceberg table props, Here, we flatten and merge them back. * * @param tableProps the existing table properties - * @return a map of merged properties, with source catalog properties taking precedence when keys - * conflict + * @return a map of merged properties, defaulting to source catalog name, database and table + * unless overridden. */ private static Map mergeSrcCatalogProps(Map tableProps) { String srcCatalogProps = tableProps.get(FlinkCreateTableOptions.SRC_CATALOG_PROPS_KEY); @@ -257,7 +257,6 @@ private static Map mergeSrcCatalogProps(Map tabl FlinkCreateTableOptions.CATALOG_DATABASE.key(), createTableOptions.catalogDb()); mergedProps.put( FlinkCreateTableOptions.CATALOG_TABLE.key(), createTableOptions.catalogTable()); - mergedProps.putAll(createTableOptions.catalogProps()); tableProps.forEach( (k, v) -> { diff --git a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/ConvertEqualityDeletes.java b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/ConvertEqualityDeletes.java new file mode 100644 index 000000000000..f51df3338d02 --- /dev/null +++ b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/ConvertEqualityDeletes.java @@ -0,0 +1,286 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.api; + +import java.util.Collections; +import java.util.List; +import java.util.Set; +import org.apache.flink.annotation.Experimental; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.streaming.api.datastream.BroadcastStream; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; +import org.apache.iceberg.Schema; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableUtil; +import org.apache.iceberg.flink.maintenance.operator.DVPosition; +import org.apache.iceberg.flink.maintenance.operator.DVWriteResult; +import org.apache.iceberg.flink.maintenance.operator.EqualityConvertCommitter; +import org.apache.iceberg.flink.maintenance.operator.EqualityConvertDVWriter; +import org.apache.iceberg.flink.maintenance.operator.EqualityConvertPKIndex; +import org.apache.iceberg.flink.maintenance.operator.EqualityConvertPlan; +import org.apache.iceberg.flink.maintenance.operator.EqualityConvertPlanner; +import org.apache.iceberg.flink.maintenance.operator.EqualityConvertReader; +import org.apache.iceberg.flink.maintenance.operator.IndexCommand; +import org.apache.iceberg.flink.maintenance.operator.ReadCommand; +import org.apache.iceberg.flink.maintenance.operator.SerializedEqualityValues; +import org.apache.iceberg.flink.maintenance.operator.TaskResultAggregator; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.types.Types.NestedField; + +/** + * Creates the equality delete to DV conversion data stream. Runs a single iteration of the + * conversion for every {@link Trigger} event. + * + *

The pipeline reads equality delete files from a staging branch, converts them to deletion + * vectors (DVs) using a primary key index stored in Flink state, and commits the data files and DVs + * to the target branch. + * + *

The conversion is split into parallel stages: + * + *

    + *
  1. Planner (p=1): scans staging branch, emits file-level ReadCommands with phase timestamps + *
  2. Reader (p=N): reads files, emits row-level IndexCommands + *
  3. PKIndex (p=N): maintains PK index shards, resolves equality deletes to DV positions + *
  4. DVWriter (p=N, keyed by data file path): buffers positions per file, writes Puffin DVs + * inline + *
  5. Committer (p=1): commits data files and DVs to the target branch + *
+ * + *

Mutual exclusion with concurrent maintenance tasks (e.g. compaction) is enforced by the Flink + * maintenance framework lock. + */ +@Experimental +public class ConvertEqualityDeletes { + static final String PLANNER_TASK_NAME = "EqConvert Planner"; + static final String READER_TASK_NAME = "EqConvert Reader"; + static final String PK_INDEX_TASK_NAME = "EqConvert PKIndex"; + static final String DV_WRITER_TASK_NAME = "EqConvert DVWriter"; + static final String UPSTREAM_ABORT_TASK_NAME = "EqConvert UpstreamAbort"; + static final String COMMIT_TASK_NAME = "EqConvert Commit"; + static final String AGGREGATOR_TASK_NAME = "EqConvert Aggregator"; + + private ConvertEqualityDeletes() {} + + public static Builder builder() { + return new Builder(); + } + + public static class Builder extends MaintenanceTaskBuilder { + private String stagingBranch; + private String targetBranch = SnapshotRef.MAIN_BRANCH; + private List equalityFieldColumns = Collections.emptyList(); + + @Override + String maintenanceTaskName() { + return "ConvertEqualityDeletes"; + } + + /** Sets the staging branch name that holds the equality delete files and data files. */ + public Builder stagingBranch(String newStagingBranch) { + this.stagingBranch = newStagingBranch; + return this; + } + + /** + * Sets the target branch where converted data files and DVs are committed. Defaults to the main + * branch. + */ + public Builder targetBranch(String newTargetBranch) { + this.targetBranch = newTargetBranch; + return this; + } + + /** + * Sets the equality field columns used by the worker index. Required. Must match the equality + * field columns the writer uses for staging eq-delete files. Mirrors {@link + * org.apache.iceberg.flink.sink.IcebergSink.Builder#equalityFieldColumns}. + * + *

The partition source columns of an equality delete's spec must be a subset of these + * columns. Writes via Flink's IcebergSink already ensure this. + */ + public Builder equalityFieldColumns(List columns) { + Preconditions.checkNotNull(columns, "equalityFieldColumns must not be null"); + Preconditions.checkArgument(!columns.isEmpty(), "equalityFieldColumns must not be empty"); + this.equalityFieldColumns = ImmutableList.copyOf(columns); + return this; + } + + @Override + DataStream append(DataStream trigger) { + Preconditions.checkNotNull(stagingBranch, "stagingBranch must be set"); + Preconditions.checkArgument( + !equalityFieldColumns.isEmpty(), "equalityFieldColumns must be set on the builder"); + Set eqFieldIds = resolveEqualityFieldIds(); + + // Planner (p=1): emits ReadCommands with phase timestamps and watermarks + SingleOutputStreamOperator planned = + setSlotSharingGroup( + trigger + .transform( + operatorName(PLANNER_TASK_NAME), + TypeInformation.of(ReadCommand.class), + new EqualityConvertPlanner( + tableName(), + taskName(), + tableLoader(), + stagingBranch, + targetBranch, + eqFieldIds)) + .uid(PLANNER_TASK_NAME + uidSuffix()) + .forceNonParallel()); + + // Reader (p=N): reads files, emits IndexCommands + SingleOutputStreamOperator index = + setSlotSharingGroup( + planned + .rebalance() + .process( + new EqualityConvertReader( + tableLoader(), eqFieldIds, stagingBranch.equals(targetBranch))) + .name(operatorName(READER_TASK_NAME)) + .uid(READER_TASK_NAME + uidSuffix()) + .setParallelism(parallelism())); + + // Broadcast from the planner to the PKIndex to clear the entire index + BroadcastStream clearIndexBroadcast = + planned + .getSideOutput(EqualityConvertPlanner.CLEAR_BROADCAST_STREAM) + .broadcast(EqualityConvertPKIndex.CLEAR_BROADCAST_DESCRIPTOR); + + // PKIndex (p=N): keyed by full PK, phase-aware buffering. + SingleOutputStreamOperator dvPositions = + setSlotSharingGroup( + index + .keyBy(IndexCommand::key, TypeInformation.of(SerializedEqualityValues.class)) + .connect(clearIndexBroadcast) + .process(new EqualityConvertPKIndex(stagingBranch.equals(targetBranch))) + .name(operatorName(PK_INDEX_TASK_NAME)) + .uid(PK_INDEX_TASK_NAME + uidSuffix()) + .setParallelism(parallelism())); + + // Reader-side abort signals bypass the PKIndex and feed the DVWriter directly, so a reader + // failure can short-circuit the cycle without waiting on a keyed shuffle. This is not a full + // short-circuit: the abort is keyed by data file path (empty for ABORT), so only one resolver + // subtask observes it; the others still write their buffered DVs, which the committer then + // drops. + DataStream readerAborts = + index.getSideOutput(EqualityConvertReader.READER_ABORT_STREAM); + DataStream dvPositionsWithAborts = dvPositions.union(readerAborts); + + // Metadata side output from planner + DataStream metadata = + planned.getSideOutput(EqualityConvertPlanner.METADATA_STREAM); + + // DVWriter (p=N, keyed by data file path): groups positions per file, writes Puffin DV + // files inline, emits a DVWriteResult per cycle. Plan metadata broadcast so every subtask + // sees it. + SingleOutputStreamOperator resolved = + setSlotSharingGroup( + dvPositionsWithAborts + .keyBy(DVPosition::dataFilePath) + .connect(metadata.broadcast()) + .transform( + operatorName(DV_WRITER_TASK_NAME), + TypeInformation.of(DVWriteResult.class), + new EqualityConvertDVWriter( + tableName(), taskName(), tableLoader(), targetBranch)) + .uid(DV_WRITER_TASK_NAME + uidSuffix()) + .setParallelism(parallelism())); + + // Upstream errors become abort signals so a partial read never commits. The same error side + // outputs also feed the aggregator below to surface the exception in TaskResult; the two + // consumers serve different purposes and must both exist. + DataStream upstreamAborts = + setSlotSharingGroup( + index + .getSideOutput(TaskResultAggregator.ERROR_STREAM) + .union(dvPositions.getSideOutput(TaskResultAggregator.ERROR_STREAM)) + .map(e -> DVWriteResult.ABORT) + .returns(TypeInformation.of(DVWriteResult.class)) + .name(operatorName(UPSTREAM_ABORT_TASK_NAME)) + .uid(UPSTREAM_ABORT_TASK_NAME + uidSuffix()) + .forceNonParallel()); + + // Committer (p=1): commits data files + DVs to main. + SingleOutputStreamOperator committed = + setSlotSharingGroup( + resolved + .union(upstreamAborts) + .connect(metadata) + .transform( + operatorName(COMMIT_TASK_NAME), + TypeInformation.of(Trigger.class), + new EqualityConvertCommitter( + tableName(), taskName(), tableLoader(), stagingBranch, targetBranch)) + .uid(COMMIT_TASK_NAME + uidSuffix()) + .forceNonParallel()); + + // Aggregator (p=1): collects errors and emits TaskResult. + return setSlotSharingGroup( + committed + .connect( + planned + .getSideOutput(TaskResultAggregator.ERROR_STREAM) + .union(index.getSideOutput(TaskResultAggregator.ERROR_STREAM)) + .union(dvPositions.getSideOutput(TaskResultAggregator.ERROR_STREAM)) + .union(resolved.getSideOutput(TaskResultAggregator.ERROR_STREAM)) + .union(committed.getSideOutput(TaskResultAggregator.ERROR_STREAM))) + .transform( + operatorName(AGGREGATOR_TASK_NAME), + TypeInformation.of(TaskResult.class), + new TaskResultAggregator(tableName(), taskName(), index())) + .uid(AGGREGATOR_TASK_NAME + uidSuffix()) + .forceNonParallel()); + } + + private Set resolveEqualityFieldIds() { + if (!tableLoader().isOpen()) { + tableLoader().open(); + } + + Table table = tableLoader().loadTable(); + int formatVersion = TableUtil.formatVersion(table); + Preconditions.checkArgument( + formatVersion >= 3, + "ConvertEqualityDeletes requires table format version >= 3 (DVs), " + + "but table '%s' is version %s", + tableName(), + formatVersion); + + Schema schema = table.schema(); + List fieldIds = Lists.newArrayListWithCapacity(equalityFieldColumns.size()); + for (String column : equalityFieldColumns) { + NestedField field = schema.findField(column); + Preconditions.checkArgument( + field != null, + "Equality field column '%s' not found in table schema %s", + column, + schema); + fieldIds.add(field.fieldId()); + } + + return ImmutableSet.copyOf(fieldIds); + } + } +} diff --git a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/TableMaintenance.java b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/TableMaintenance.java index 025a6d17c023..46b38858b8aa 100644 --- a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/TableMaintenance.java +++ b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/TableMaintenance.java @@ -26,6 +26,7 @@ import java.util.UUID; import javax.annotation.Nullable; import org.apache.flink.annotation.Internal; +import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.eventtime.TimestampAssigner; import org.apache.flink.api.common.eventtime.TimestampAssignerSupplier; @@ -378,7 +379,9 @@ private DataStream changeStream(String tableName, TableLoader loade // Create a monitor source to provide the TableChange stream MonitorSource source = new MonitorSource( - loader, RateLimiterStrategy.perSecond(1.0 / rateLimit.getSeconds()), maxReadBack); + loader, + RateLimiterStrategy.perSecond(monitorRatePerSecond(rateLimit.toMillis())), + maxReadBack); return setSlotSharingGroup( env.fromSource( source, @@ -395,6 +398,16 @@ private static String nameFor(MaintenanceTaskBuilder streamBuilder, int taskI return String.format(Locale.ROOT, "%s [%d]", streamBuilder.maintenanceTaskName(), taskIndex); } + /** + * Monitor poll rate per rate-limit interval, in checks/second. We compute from millis instead + * of seconds, otherwise sub-second intervals could be truncated to 0, yielding an infinite rate + * which busy-loops the source. + */ + @VisibleForTesting + static double monitorRatePerSecond(long rateLimitMillis) { + return 1000.0 / rateLimitMillis; + } + private SingleOutputStreamOperator setSlotSharingGroup( SingleOutputStreamOperator operator) { return slotSharingGroup == null ? operator : operator.slotSharingGroup(slotSharingGroup); diff --git a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java new file mode 100644 index 000000000000..95c9ccd5de1c --- /dev/null +++ b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java @@ -0,0 +1,372 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.operator; + +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import org.apache.flink.annotation.Internal; +import org.apache.flink.metrics.Counter; +import org.apache.flink.metrics.MetricGroup; +import org.apache.flink.streaming.api.operators.AbstractStreamOperator; +import org.apache.flink.streaming.api.operators.TwoInputStreamOperator; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.RowDelta; +import org.apache.iceberg.Table; +import org.apache.iceberg.exceptions.CommitStateUnknownException; +import org.apache.iceberg.flink.TableLoader; +import org.apache.iceberg.flink.maintenance.api.Trigger; +import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.apache.iceberg.util.ContentFileUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Commits data files and DVs to the target branch. Receives {@link DVWriteResult}s from parallel + * {@link EqualityConvertDVWriter} instances (input 1) and an {@link EqualityConvertPlan} from the + * planner (input 2). Assembles the final file lists and commits using a {@link RowDelta} operation + * once the plan result and done-timestamp watermark have both arrived. + * + *

The commit is gated on the plan's done-timestamp watermark. + * + *

Watermarks are forwarded only after the cycle commits, never mid-cycle. The {@link + * LockRemover} releases the maintenance lock once a watermark past the trigger's start epoch + * reaches it. The planner emits phase watermarks in the middle of a cycle; forwarding those would + * release the lock before this commit, letting the TriggerManager start a concurrent cycle that + * re-processes the same uncommitted staging snapshot. + * + *

Emits a {@link Trigger} after each cycle (commit, no-op, or error) so the downstream {@link + * TaskResultAggregator} can track task completion. This is the sole source of Trigger records for + * the Aggregator. + * + *

No-op vs error: a no-op cycle (empty plan result from {@code + * EqualityConvertPlanner.emitNoOpResult}) returns early in {@code commitIfNeeded} without writing + * anything; the Trigger emit in {@code processWatermark} still happens, so the maintenance task + * completes cleanly. Errors are reported via {@link TaskResultAggregator#ERROR_STREAM} side output; + * the Aggregator collects them and surfaces failure on its own watermark. + * + *

The committer is intentionally stateless: {@code bufferedResults} and {@code planResult} are + * not checkpointed. The maintenance framework ensures mutual exclusive tasks, and on restart the + * planner re-derives its position from {@link #COMMITTED_STAGING_SNAPSHOT_PROPERTY} on main, so the + * committer never receives a plan for an already-committed staging snapshot. + */ +@Internal +public class EqualityConvertCommitter extends AbstractStreamOperator + implements TwoInputStreamOperator { + + private static final Logger LOG = LoggerFactory.getLogger(EqualityConvertCommitter.class); + + public static final String COMMITTED_STAGING_SNAPSHOT_PROPERTY = + "equality-convert-staging-snapshot"; + + private static final String ADDED_DV_NUM_METRIC = "addedDvNum"; + private static final String COMMIT_DURATION_MS_METRIC = "commitDurationMs"; + + private final String tableName; + private final String taskName; + private final TableLoader tableLoader; + private final String targetBranch; + private final boolean stagingOnTargetBranch; + + private transient Table table; + private transient List bufferedResults; + private transient EqualityConvertPlan planResult; + + private transient Counter errorCounter; + private transient Counter addedDataFileNumCounter; + private transient Counter addedDataFileSizeCounter; + private transient Counter addedDvNumCounter; + private transient Counter commitDurationMsCounter; + + public EqualityConvertCommitter( + String tableName, + String taskName, + TableLoader tableLoader, + String stagingBranch, + String targetBranch) { + this.tableName = tableName; + this.taskName = taskName; + this.tableLoader = tableLoader; + this.targetBranch = targetBranch; + this.stagingOnTargetBranch = stagingBranch.equals(targetBranch); + } + + @Override + public void open() throws Exception { + super.open(); + if (!tableLoader.isOpen()) { + tableLoader.open(); + } + + this.table = tableLoader.loadTable(); + this.bufferedResults = Lists.newArrayList(); + + MetricGroup taskMetricGroup = + TableMaintenanceMetrics.groupFor(getRuntimeContext(), tableName, taskName, 0); + this.errorCounter = taskMetricGroup.counter(TableMaintenanceMetrics.ERROR_COUNTER); + this.addedDataFileNumCounter = + taskMetricGroup.counter(TableMaintenanceMetrics.ADDED_DATA_FILE_NUM_METRIC); + this.addedDataFileSizeCounter = + taskMetricGroup.counter(TableMaintenanceMetrics.ADDED_DATA_FILE_SIZE_METRIC); + this.addedDvNumCounter = taskMetricGroup.counter(ADDED_DV_NUM_METRIC); + this.commitDurationMsCounter = taskMetricGroup.counter(COMMIT_DURATION_MS_METRIC); + } + + @Override + public void processElement1(StreamRecord record) { + bufferedResults.add(record.getValue()); + } + + @Override + public void processElement2(StreamRecord record) { + planResult = record.getValue(); + } + + @Override + public void processWatermark(Watermark mark) throws Exception { + if (planResult == null || mark.getTimestamp() < planResult.doneTimestamp()) { + // Hold back watermarks until the cycle commits so the LockRemover keeps the maintenance lock + // for the whole cycle. Forwarding the planner's mid-cycle phase watermarks will release the + // lock early and could let the next trigger run a concurrent cycle on the same staging + // snapshot. + return; + } + + try { + commitIfNeeded(); + } catch (Exception e) { + LOG.error( + "Failed to commit equality convert result for table {} task {}", tableName, taskName, e); + output.collect(TaskResultAggregator.ERROR_STREAM, new StreamRecord<>(e)); + errorCounter.inc(); + } + + // Emit Trigger for the Aggregator (even on error or no-op). + output.collect(new StreamRecord<>(Trigger.create(planResult.triggerTimestamp(), 0))); + + bufferedResults.clear(); + planResult = null; + + super.processWatermark(mark); + } + + @Override + public void close() throws Exception { + super.close(); + tableLoader.close(); + } + + private void commitIfNeeded() { + for (DVWriteResult result : bufferedResults) { + if (result.isAbort()) { + LOG.warn( + "Skipping commit for table {} task {}: a DV writer reported an error.", + tableName, + taskName); + deleteUncommittedDVs(); + return; + } + } + + // No-op cycle: the planner emitted an empty plan result (see + // EqualityConvertPlanner.emitNoOpResult) because the next staging snapshot was filtered by + // shouldSkip or there was nothing new on staging. processWatermark still forwards a Trigger to + // TaskResultAggregator, so the maintenance task completes cleanly. + if (planResult.noOp()) { + return; + } + + table.refresh(); + + List allDvFiles = Lists.newArrayList(); + List allRewrittenDvFiles = Lists.newArrayList(); + for (DVWriteResult result : bufferedResults) { + allDvFiles.addAll(result.dvFiles()); + allRewrittenDvFiles.addAll(result.rewrittenDvFiles()); + } + + RowDelta rowDelta = buildRowDelta(planResult.dataFiles(), allDvFiles, allRewrittenDvFiles); + + long startNano = System.nanoTime(); + try { + commit(rowDelta); + } catch (CommitStateUnknownException e) { + // Commit outcome unknown: the DVs may already be referenced by a committed snapshot. Leave + // them for Remove Orphan Files rather than risk deleting live data. + throw e; + } catch (Exception e) { + // Commit definitively failed: the DVs this cycle wrote are unreferenced. Delete them so a + // failed cycle does not leak Puffin files. Rewritten DVs stay (still live on the target). + deleteUncommittedDVs(); + throw e; + } + + long durationMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNano); + commitDurationMsCounter.inc(durationMs); + + LOG.info( + "Committed {} data files and {} DV files to branch '{}' for table {} in {} ms. " + + "Processed staging snapshot {}.", + planResult.dataFiles().size(), + allDvFiles.size(), + targetBranch, + tableName, + durationMs, + planResult.stagingSnapshotId()); + + // Only count files actually added by this commit. When sameBranch, the writer already + // committed the data files to target and buildRowDelta does not re-add them. + if (!stagingOnTargetBranch) { + for (DataFile dataFile : planResult.dataFiles()) { + addedDataFileNumCounter.inc(); + addedDataFileSizeCounter.inc(dataFile.fileSizeInBytes()); + } + } + + addedDvNumCounter.inc(allDvFiles.size()); + } + + @VisibleForTesting + void commit(RowDelta rowDelta) { + rowDelta.commit(); + } + + /** + * Deletes the DVs this cycle wrote but did not commit (abort or definite commit failure). Only + * the newly written DVs are removed; rewritten DVs remain referenced on the target branch. Best + * effort: a delete failure is logged, not propagated, so it never masks the original error. + */ + private void deleteUncommittedDVs() { + for (DVWriteResult result : bufferedResults) { + if (result.isAbort()) { + continue; + } + + for (DeleteFile dvFile : result.dvFiles()) { + try { + table.io().deleteFile(dvFile.location()); + } catch (RuntimeException e) { + LOG.warn( + "Failed to delete uncommitted DV {} for table {} task {}", + dvFile.location(), + tableName, + taskName, + e); + } + } + } + } + + private RowDelta buildRowDelta( + List dataFiles, List allDvFiles, List allRewrittenDvFiles) { + RowDelta rowDelta = table.newRowDelta(); + + // Fail the commit on external target-branch activity since the planner's snapshot. The next + // trigger detects the change and reindexes. + if (planResult.mainSnapshotId() != null) { + rowDelta.validateFromSnapshot(planResult.mainSnapshotId()); + } + + rowDelta.validateNoConflictingDataFiles(); + rowDelta.validateNoConflictingDeleteFiles(); + + Set referencedDataFiles = Sets.newHashSet(); + for (DeleteFile dvFile : allDvFiles) { + if (dvFile.referencedDataFile() != null) { + referencedDataFiles.add(dvFile.referencedDataFile()); + } + } + + if (!referencedDataFiles.isEmpty()) { + rowDelta.validateDataFilesExist(referencedDataFiles); + } + + // When stagingBranch == targetBranch, the writer already committed data files and DVs to the + // target branch. Re-adding them here would produce duplicate manifest entries. Skip those + // paths; only the new DVs from the writer need to be added. + if (!stagingOnTargetBranch) { + for (DataFile dataFile : dataFiles) { + rowDelta.addRows(dataFile); + } + } + + for (DeleteFile dvFile : allDvFiles) { + rowDelta.addDeletes(dvFile); + } + + if (!stagingOnTargetBranch) { + addStagingDeletes(rowDelta, referencedDataFiles, planResult.stagingDVFiles()); + } + + removeRewrittenDVs( + rowDelta, allRewrittenDvFiles, planResult.stagingDVFiles(), stagingOnTargetBranch); + + rowDelta.toBranch(targetBranch); + rowDelta.set( + COMMITTED_STAGING_SNAPSHOT_PROPERTY, String.valueOf(planResult.stagingSnapshotId())); + return rowDelta; + } + + /** Adds staging delete files, skipping DVs that overlap with conversion DVs (V3 rule). */ + private static void addStagingDeletes( + RowDelta rowDelta, Set dvCoveredDataFiles, List stagingDVFiles) { + for (DeleteFile stagingDelete : stagingDVFiles) { + // V3 allows one DV per data file. When a staging snapshot contains both a writer-committed + // DV and an eq-delete that resolves to additional positions in the same data file, the + // writer folds the staging DV into a new merged DV (via + // EqualityConvertDVWriter.collectExistingDVs). Skip the superseded staging DV; adding both + // would commit two DVs for the same data file. + if (ContentFileUtil.isDV(stagingDelete) + && stagingDelete.referencedDataFile() != null + && dvCoveredDataFiles.contains(stagingDelete.referencedDataFile())) { + continue; + } + + rowDelta.addDeletes(stagingDelete); + } + } + + /** + * Removes rewritten DVs. On a separate target branch, staging DVs are not yet on target, so they + * are skipped: removeDeletes would fail. On a shared branch they are already on target and must + * be removed like any other rewritten DV; the merged DV that supersedes them would otherwise be a + * second DV for the same data file (V3 allows one). + */ + private static void removeRewrittenDVs( + RowDelta rowDelta, + List allRewrittenDvFiles, + List stagingDVFiles, + boolean stagingOnTargetBranch) { + Set stagingDeleteLocations = Sets.newHashSet(); + for (DeleteFile sd : stagingDVFiles) { + stagingDeleteLocations.add(sd.location()); + } + + for (DeleteFile rewrittenDv : allRewrittenDvFiles) { + if (stagingOnTargetBranch || !stagingDeleteLocations.contains(rewrittenDv.location())) { + rowDelta.removeDeletes(rewrittenDv); + } + } + } +} diff --git a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertDVWriter.java b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertDVWriter.java new file mode 100644 index 000000000000..486b933ee6d7 --- /dev/null +++ b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertDVWriter.java @@ -0,0 +1,354 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.operator; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.flink.annotation.Internal; +import org.apache.flink.streaming.api.operators.AbstractStreamOperator; +import org.apache.flink.streaming.api.operators.TwoInputStreamOperator; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; +import org.apache.iceberg.PartitionField; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.StructLike; +import org.apache.iceberg.Table; +import org.apache.iceberg.data.BaseDeleteLoader; +import org.apache.iceberg.data.DeleteLoader; +import org.apache.iceberg.deletes.BaseDVFileWriter; +import org.apache.iceberg.deletes.PositionDeleteIndex; +import org.apache.iceberg.expressions.Expression; +import org.apache.iceberg.expressions.Expressions; +import org.apache.iceberg.expressions.ManifestEvaluator; +import org.apache.iceberg.flink.TableLoader; +import org.apache.iceberg.io.DeleteWriteResult; +import org.apache.iceberg.io.OutputFileFactory; +import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.ContentFileUtil; +import org.apache.iceberg.util.StructLikeWrapper; +import org.roaringbitmap.longlong.Roaring64Bitmap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Keyed parallel resolver that buffers {@link DVPosition}s per data-file path, then writes Puffin + * DV files directly via {@link BaseDVFileWriter}. Plan metadata arrives broadcast on input 2, so + * every parallel task sees the cycle's metadata and can validate against the main snapshot. + * + *

Each buffered {@link DVPosition} carries the data file's {@code specId} + encoded partition, + * so writing DVs needs no data-manifest scan. Existing DVs are folded into the rewrite (V3 allows + * one DV per data file): delete manifests are pruned by partition summary to the cycle's affected + * partitions, then filtered to entries referencing the affected data files. No cross-cycle state is + * kept; reads are bounded by the pruned manifest set, not the table's full DV history. + * + *

Buffered positions are transient per-task. On failure recovery, upstream replay rebuilds them. + */ +@Internal +public class EqualityConvertDVWriter extends AbstractStreamOperator + implements TwoInputStreamOperator { + + private static final Logger LOG = LoggerFactory.getLogger(EqualityConvertDVWriter.class); + + private final String tableName; + private final String taskName; + private final TableLoader tableLoader; + private final String targetBranch; + + private transient Table table; + private transient OutputFileFactory fileFactory; + private transient DeleteLoader deleteLoader; + private transient Map positionsByFile; + private transient EqualityConvertPlan planResult; + private transient boolean hasUpstreamError; + private transient int manifestsRead; + + public EqualityConvertDVWriter( + String tableName, String taskName, TableLoader tableLoader, String targetBranch) { + this.tableName = tableName; + this.taskName = taskName; + this.tableLoader = tableLoader; + this.targetBranch = targetBranch; + } + + @Override + public void open() throws Exception { + super.open(); + if (!tableLoader.isOpen()) { + tableLoader.open(); + } + + table = tableLoader.loadTable(); + int subtaskIndex = getRuntimeContext().getTaskInfo().getIndexOfThisSubtask(); + fileFactory = + OutputFileFactory.builderFor(table, subtaskIndex, 0L).format(FileFormat.PUFFIN).build(); + deleteLoader = new BaseDeleteLoader(deleteFile -> table.io().newInputFile(deleteFile)); + positionsByFile = Maps.newHashMap(); + } + + @Override + public void processElement1(StreamRecord record) { + DVPosition pos = record.getValue(); + if (pos.isAbort()) { + hasUpstreamError = true; + } + + if (!hasUpstreamError) { + positionsByFile + .computeIfAbsent( + pos.dataFilePath(), k -> new FilePositions(pos.specId(), pos.partition())) + .positions + .addLong(pos.position()); + } + } + + @Override + public void processElement2(StreamRecord record) { + planResult = record.getValue(); + } + + @Override + public void processWatermark(Watermark mark) throws Exception { + if (planResult != null && mark.getTimestamp() >= planResult.doneTimestamp()) { + if (hasUpstreamError) { + output.collect(new StreamRecord<>(DVWriteResult.ABORT)); + } else { + try { + resolveAndWrite(); + } catch (Exception e) { + LOG.error("Error writing DVs for table {} task {}", tableName, taskName, e); + output.collect(TaskResultAggregator.ERROR_STREAM, new StreamRecord<>(e)); + output.collect(new StreamRecord<>(DVWriteResult.ABORT)); + } + } + + positionsByFile.clear(); + hasUpstreamError = false; + planResult = null; + } + + super.processWatermark(mark); + } + + private void resolveAndWrite() throws IOException { + if (positionsByFile.isEmpty()) { + return; + } + + table.refresh(); + + Snapshot mainSnapshot = table.snapshot(targetBranch); + + // Fail fast if the main branch changed since planning, to avoid writing DV files that the + // committer would reject via validateFromSnapshot. The next cycle will reindex. + if (mainSnapshot != null + && planResult.mainSnapshotId() != null + && mainSnapshot.snapshotId() != planResult.mainSnapshotId()) { + throw new IllegalStateException( + "Main branch snapshot changed since planning: expected " + + planResult.mainSnapshotId() + + " but found: " + + mainSnapshot.snapshotId()); + } + + Map dvs = collectExistingDVs(mainSnapshot, positionsByFile.keySet()); + + // Fold staging DVs into the rewrite so the writer emits one DV per data file (V3 rule). Flink + // writes a staging DV only for a newly added data file, so it never collides with a distinct + // existing DV: on a separate target branch collectExistingDVs has not seen it yet; on a shared + // branch it IS that existing DV, so the put is idempotent. + for (DeleteFile sd : planResult.stagingDVFiles()) { + if (ContentFileUtil.isDV(sd) && sd.referencedDataFile() != null) { + dvs.put(sd.referencedDataFile(), sd); + } + } + + BaseDVFileWriter dvWriter = + new BaseDVFileWriter(fileFactory, path -> loadPreviousDV(path, dvs)); + try (dvWriter) { + for (Map.Entry entry : positionsByFile.entrySet()) { + String dataFilePath = entry.getKey(); + FilePositions filePositions = entry.getValue(); + PartitionSpec spec = table.specs().get(filePositions.specId); + StructLike partition = filePositions.partition(spec.partitionType()); + + filePositions.positions.forEach( + (long pos) -> dvWriter.delete(dataFilePath, pos, spec, partition)); + } + } + + DeleteWriteResult result = dvWriter.result(); + LOG.info( + "Wrote {} DV files (rewriting {}) for {} data files in table {} task {}.", + result.deleteFiles().size(), + result.rewrittenDeleteFiles().size(), + positionsByFile.size(), + tableName, + taskName); + + output.collect( + new StreamRecord<>( + new DVWriteResult( + Lists.newArrayList(result.deleteFiles()), + Lists.newArrayList(result.rewrittenDeleteFiles())))); + } + + @Override + public void close() throws Exception { + super.close(); + tableLoader.close(); + } + + private Map collectExistingDVs( + Snapshot mainSnapshot, Set affectedPaths) { + manifestsRead = 0; + Map dvs = Maps.newHashMap(); + if (mainSnapshot == null) { + return dvs; + } + + // Prune delete manifests whose partition summaries cannot cover the cycle's affected + // partitions. A DV inherits its referenced data file's spec and partition, so partition pruning + // works for DV manifests. + Map evaluators = partitionEvaluators(positionsByFile); + for (ManifestFile manifest : mainSnapshot.deleteManifests(table.io())) { + ManifestEvaluator evaluator = evaluators.get(manifest.partitionSpecId()); + if (evaluator == null || !evaluator.eval(manifest)) { + continue; + } + + readDVEntries(manifest, affectedPaths, dvs); + } + + return dvs; + } + + private Map partitionEvaluators( + Map positions) { + Map templatesBySpec = Maps.newHashMap(); + Map> partitionsBySpec = Maps.newHashMap(); + for (FilePositions filePositions : positions.values()) { + PartitionSpec spec = table.specs().get(filePositions.specId); + StructLikeWrapper template = + templatesBySpec.computeIfAbsent( + filePositions.specId, id -> StructLikeWrapper.forType(spec.partitionType())); + partitionsBySpec + .computeIfAbsent(filePositions.specId, k -> Sets.newHashSet()) + .add(template.copyFor(filePositions.partition(spec.partitionType()))); + } + + Map evaluators = Maps.newHashMap(); + for (Map.Entry> entry : partitionsBySpec.entrySet()) { + PartitionSpec spec = table.specs().get(entry.getKey()); + Expression filter = partitionFilter(spec, entry.getValue()); + evaluators.put(entry.getKey(), ManifestEvaluator.forPartitionFilter(filter, spec, false)); + } + + return evaluators; + } + + private static Expression partitionFilter(PartitionSpec spec, Set partitions) { + List fields = spec.fields(); + Expression anyPartition = Expressions.alwaysFalse(); + for (StructLikeWrapper wrapper : partitions) { + StructLike partition = wrapper.get(); + Expression onePartition = Expressions.alwaysTrue(); + for (int i = 0; i < fields.size(); i++) { + String name = fields.get(i).name(); + Object value = partition.get(i, Object.class); + Expression predicate = + value == null ? Expressions.isNull(name) : Expressions.equal(name, value); + onePartition = Expressions.and(onePartition, predicate); + } + + anyPartition = Expressions.or(anyPartition, onePartition); + } + + return anyPartition; + } + + private void readDVEntries( + ManifestFile manifest, Set filterPaths, Map out) { + manifestsRead++; + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { + for (DeleteFile deleteFile : reader) { + if (ContentFileUtil.isDV(deleteFile) + && deleteFile.referencedDataFile() != null + && filterPaths.contains(deleteFile.referencedDataFile())) { + out.put(deleteFile.referencedDataFile(), deleteFile); + } + } + } catch (IOException e) { + throw new UncheckedIOException("Failed to read manifest: " + manifest.path(), e); + } + } + + @VisibleForTesting + int manifestsReadLastCycle() { + return manifestsRead; + } + + @VisibleForTesting + int retainedStateSize() { + return positionsByFile.size(); + } + + private PositionDeleteIndex loadPreviousDV(String dataFilePath, Map dvs) { + DeleteFile existingDV = dvs.get(dataFilePath); + if (existingDV == null) { + return null; + } + + return deleteLoader.loadPositionDeletes(ImmutableList.of(existingDV), dataFilePath); + } + + private static final class FilePositions { + private final int specId; + private final byte[] encodedPartition; + private final Roaring64Bitmap positions = new Roaring64Bitmap(); + private StructLike decodedPartition; + + FilePositions(int specId, byte[] encodedPartition) { + this.specId = specId; + this.encodedPartition = encodedPartition; + } + + StructLike partition(Types.StructType partitionType) { + if (decodedPartition == null) { + decodedPartition = StructLikeSerializer.decodePartition(encodedPartition, partitionType); + } + + return decodedPartition; + } + } +} diff --git a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertPlanner.java b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertPlanner.java new file mode 100644 index 000000000000..89e5510dd848 --- /dev/null +++ b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertPlanner.java @@ -0,0 +1,782 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.operator; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import org.apache.flink.annotation.Internal; +import org.apache.flink.annotation.VisibleForTesting; +import org.apache.flink.api.common.state.ListState; +import org.apache.flink.api.common.state.ListStateDescriptor; +import org.apache.flink.api.common.typeinfo.Types; +import org.apache.flink.metrics.Counter; +import org.apache.flink.metrics.MetricGroup; +import org.apache.flink.runtime.state.StateInitializationContext; +import org.apache.flink.runtime.state.StateSnapshotContext; +import org.apache.flink.streaming.api.operators.AbstractStreamOperator; +import org.apache.flink.streaming.api.operators.OneInputStreamOperator; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.util.OutputTag; +import org.apache.iceberg.ContentFile; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileContent; +import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.PartitionField; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SnapshotChanges; +import org.apache.iceberg.SnapshotSummary; +import org.apache.iceberg.Table; +import org.apache.iceberg.flink.TableLoader; +import org.apache.iceberg.flink.maintenance.api.Trigger; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.apache.iceberg.util.ContentFileUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Planner for the equality delete conversion pipeline. For each trigger, it picks the oldest + * staging snapshot that hasn't been converted yet and emits {@link ReadCommand}s describing the + * files its downstream readers and workers must process. + * + *

Each trigger runs two steps in order: + * + *

    + *
  1. {@link #ensureIndexCurrent}: updates {@link #lastStagingSnapshotId} from main's history, + * bootstraps the worker index from main on first run, and reindexes when external commits + * (e.g. compaction) have advanced main past the currently-indexed snapshot. + *
  2. {@link #processStagingSnapshot}: resolve the chosen staging snapshot's eq deletes against + * the (now-current) index, pass through any DV files, and index the snapshot's new data files + * for the next cycle. + *
+ * + * Watermarks separate phases that gate the worker's keyed state. The contract is documented on + * {@link #advancePhase()}. + * + *

An {@link EqualityConvertPlan} with the current cycle's metadata is emitted via the {@link + * #METADATA_STREAM} side output after the read commands. + * + *

Assumes a single equality-field set supplied via the builder; staging eq-deletes with a + * different {@code equalityFieldIds} fail fast in {@link #retrieveStagingFiles}. Concurrent writes + * on the target branch are handled by {@link #ensureIndexCurrent} reindexing from the new main + * snapshot; commit-time conflicts are caught by {@code RowDelta.validateFromSnapshot}. + */ +@Internal +public class EqualityConvertPlanner extends AbstractStreamOperator + implements OneInputStreamOperator { + + private static final Logger LOG = LoggerFactory.getLogger(EqualityConvertPlanner.class); + + public static final OutputTag METADATA_STREAM = + new OutputTag<>("metadata-stream") {}; + + public static final OutputTag CLEAR_BROADCAST_STREAM = + new OutputTag<>("clear-broadcast-stream") {}; + + private static final String PROCESSED_EQ_DELETE_FILE_NUM_METRIC = "processedEqDeleteFileNum"; + private static final String PROCESSED_STAGING_SNAPSHOT_NUM_METRIC = "processedStagingSnapshotNum"; + private static final String SKIPPED_NO_OP_CYCLES_METRIC = "skippedNoOpCycles"; + private static final String REINDEX_COUNT_METRIC = "reindexCount"; + + private final String tableName; + private final String taskName; + private final TableLoader tableLoader; + private final String stagingBranch; + private final String targetBranch; + private final boolean stagingOnTargetBranch; + // Equality-field-id set the worker keys on. Supplied via the builder; every staging + // eq-delete's equalityFieldIds() must match exactly. + private final Set eqFieldIds; + + // Main snapshot id the worker's index reflects. + private transient ListState indexSnapshotState; + // Main sequence number the worker's index reflects. + private transient ListState indexedSequenceNumberState; + // Equality field IDs the index was built with, allows to detect reconfiguration. + private transient ListState eqFieldIdsState; + + private transient Table table; + + private transient Long lastMainSnapshotId; + private transient Long lastStagingSnapshotId; + private transient Long indexSnapshotId; + private transient Long indexedSequenceNumber; + + private transient long nextPhaseTs; + + private transient Counter processedEqDeleteFileNumCounter; + private transient Counter processedStagingSnapshotNumCounter; + private transient Counter skippedNoOpCyclesCounter; + private transient Counter reindexCounter; + private transient Counter errorCounter; + + public EqualityConvertPlanner( + String tableName, + String taskName, + TableLoader tableLoader, + String stagingBranch, + String targetBranch, + Set eqFieldIds) { + this.tableName = tableName; + this.taskName = taskName; + this.tableLoader = tableLoader; + this.stagingBranch = stagingBranch; + this.targetBranch = targetBranch; + this.stagingOnTargetBranch = stagingBranch.equals(targetBranch); + Preconditions.checkArgument( + eqFieldIds != null && !eqFieldIds.isEmpty(), "eqFieldIds must not be null or empty"); + this.eqFieldIds = ImmutableSet.copyOf(eqFieldIds); + } + + @Override + public void open() throws Exception { + super.open(); + if (!tableLoader.isOpen()) { + tableLoader.open(); + } + + table = tableLoader.loadTable(); + + MetricGroup taskMetricGroup = + TableMaintenanceMetrics.groupFor(getRuntimeContext(), tableName, taskName, 0); + this.processedEqDeleteFileNumCounter = + taskMetricGroup.counter(PROCESSED_EQ_DELETE_FILE_NUM_METRIC); + this.processedStagingSnapshotNumCounter = + taskMetricGroup.counter(PROCESSED_STAGING_SNAPSHOT_NUM_METRIC); + this.skippedNoOpCyclesCounter = taskMetricGroup.counter(SKIPPED_NO_OP_CYCLES_METRIC); + this.reindexCounter = taskMetricGroup.counter(REINDEX_COUNT_METRIC); + this.errorCounter = taskMetricGroup.counter(TableMaintenanceMetrics.ERROR_COUNTER); + } + + @Override + public void initializeState(StateInitializationContext context) throws Exception { + super.initializeState(context); + indexSnapshotState = + context + .getOperatorStateStore() + .getListState(new ListStateDescriptor<>("indexSnapshotId", Types.LONG)); + + indexSnapshotId = null; + for (Long stateValue : indexSnapshotState.get()) { + Preconditions.checkState( + indexSnapshotId == null, "indexSnapshotId state should hold at most one value"); + indexSnapshotId = stateValue; + } + + indexedSequenceNumberState = + context + .getOperatorStateStore() + .getListState(new ListStateDescriptor<>("indexedSequenceNumber", Types.LONG)); + + indexedSequenceNumber = null; + for (Long stateValue : indexedSequenceNumberState.get()) { + Preconditions.checkState( + indexedSequenceNumber == null, + "indexedSequenceNumber state should hold at most one value"); + indexedSequenceNumber = stateValue; + } + + eqFieldIdsState = + context + .getOperatorStateStore() + .getListState(new ListStateDescriptor<>("eqFieldIds", Types.INT)); + Set restoredEqFieldIds = Sets.newHashSet(eqFieldIdsState.get()); + Preconditions.checkState( + restoredEqFieldIds.isEmpty() || restoredEqFieldIds.equals(eqFieldIds), + "Equality field IDs changed across restart: restored=%s, configured=%s. " + + "Reconfiguring equality-field columns is not supported; " + + "restart from a clean state (no savepoint).", + restoredEqFieldIds, + eqFieldIds); + } + + @Override + public void snapshotState(StateSnapshotContext context) throws Exception { + super.snapshotState(context); + indexSnapshotState.clear(); + if (indexSnapshotId != null) { + indexSnapshotState.add(indexSnapshotId); + } + + indexedSequenceNumberState.clear(); + if (indexedSequenceNumber != null) { + indexedSequenceNumberState.add(indexedSequenceNumber); + } + + eqFieldIdsState.clear(); + for (int id : eqFieldIds) { + eqFieldIdsState.add(id); + } + } + + @Override + public void processElement(StreamRecord element) throws Exception { + long triggerTs = element.getTimestamp(); + nextPhaseTs = Math.max(triggerTs, nextPhaseTs + 1); + + Long currentMainSnapshotId = lastMainSnapshotId; + try { + table.refresh(); + Snapshot mainSnapshot = table.snapshot(targetBranch); + currentMainSnapshotId = mainSnapshot != null ? mainSnapshot.snapshotId() : null; + + ensureIndexCurrent(mainSnapshot); + + Snapshot nextToProcess = + nextUnprocessedStagingSnapshot(table.snapshot(stagingBranch), mainSnapshot); + + if (nextToProcess == null) { + LOG.info("Nothing new to convert on staging branch '{}'.", stagingBranch); + emitNoOpResult(triggerTs, currentMainSnapshotId); + return; + } + + processStagingSnapshot(nextToProcess, triggerTs, currentMainSnapshotId); + } catch (Exception e) { + LOG.error("Error processing equality deletes for table {} task {}", tableName, taskName, e); + output.collect(TaskResultAggregator.ERROR_STREAM, new StreamRecord<>(e)); + errorCounter.inc(); + emitDrainResult(triggerTs, currentMainSnapshotId); + } + } + + /** + * Brings the worker's index up to date with the current state of the target branch: + * + *

    + *
  • Updates {@link #lastStagingSnapshotId} from the most recent committer marker on main. + *
  • Bootstraps the index from main on the first trigger with a non-null main snapshot. + *
  • Reindexes from main when external commits (e.g. compaction or direct writes) have + * advanced main past the currently-indexed snapshot. + *
+ * + *

No-op when main hasn't moved since the last trigger. Otherwise the history walk is bounded + * to commits added since {@link #lastMainSnapshotId}. + */ + private void ensureIndexCurrent(Snapshot mainSnapshot) { + Long currentMainSnapshotId = mainSnapshot != null ? mainSnapshot.snapshotId() : null; + + if (Objects.equals(lastMainSnapshotId, currentMainSnapshotId)) { + return; + } + + LastCommittedWork info = discoverLastCommittedWork(mainSnapshot); + updateLastStagingSnapshotId(info); + + boolean bootstrap = mainSnapshot != null && indexSnapshotId == null; + boolean reindex = indexSnapshotId != null && info.externalCommitCount() > 0; + if (bootstrap || reindex) { + LOG.info( + "{} worker index from main snapshot {} for field IDs {}.", + bootstrap ? "Bootstrapping" : "Reindexing", + currentMainSnapshotId, + eqFieldIds); + if (reindex) { + // Evict keyed entries the reindex will not re-add (e.g. data file removed by CoW). + output.collect( + CLEAR_BROADCAST_STREAM, + new StreamRecord<>( + IndexCommand.clearBeforeReindex( + currentMainSnapshotId, mainSnapshot.sequenceNumber()))); + reindexCounter.inc(); + } + + indexSnapshotId = currentMainSnapshotId; + indexedSequenceNumber = mainSnapshot.sequenceNumber(); + emitMainDataReadCommands(mainSnapshot); + } + + lastMainSnapshotId = currentMainSnapshotId; + } + + private void updateLastStagingSnapshotId(LastCommittedWork info) { + if (info.lastCommittedStaging() != null) { + lastStagingSnapshotId = info.lastCommittedStaging(); + return; + } + + Preconditions.checkState( + lastMainSnapshotId == null || lastStagingSnapshotId == null || info.reachedLastInspected(), + "No COMMITTED_STAGING_SNAPSHOT marker reachable on target branch '%s' for table %s, " + + "but a prior marker was seen (lastStagingSnapshotId=%s). Target may have been " + + "rewritten (rollback, replace_main, or snapshot expiration). " + + "Manual intervention required.", + targetBranch, + tableName, + lastStagingSnapshotId); + } + + /** + * Walks main back from head looking for the most recent snapshot tagged with {@link + * EqualityConvertCommitter#COMMITTED_STAGING_SNAPSHOT_PROPERTY}. Returns the staging snapshot id + * recorded there (or {@code null} if not reached) and the count of intervening external commits. + * + *

The walk stops at whichever comes first: + * + *

    + *
  • The first snapshot carrying the committer marker. + *
  • {@link #lastMainSnapshotId} — anything older was inspected on a previous trigger. + *
+ */ + private LastCommittedWork discoverLastCommittedWork(Snapshot mainSnapshot) { + Long lastCommittedStaging = null; + int externalCount = 0; + boolean reachedLastInspected = false; + Snapshot current = mainSnapshot; + while (current != null) { + if (lastMainSnapshotId != null && current.snapshotId() == lastMainSnapshotId) { + reachedLastInspected = true; + break; + } + + String prop = + current.summary().get(EqualityConvertCommitter.COMMITTED_STAGING_SNAPSHOT_PROPERTY); + if (prop != null) { + lastCommittedStaging = Long.parseLong(prop); + break; + } + + externalCount++; + current = parentOf(current); + } + + return new LastCommittedWork(lastCommittedStaging, externalCount, reachedLastInspected); + } + + private record LastCommittedWork( + Long lastCommittedStaging, int externalCommitCount, boolean reachedLastInspected) {} + + /** + * Walks staging history from head back to the stop point and returns the oldest unprocessed + * snapshot to convert this cycle, or {@code null} if there's nothing new. + * + *

The stop point is: + * + *

    + *
  • the last-processed staging snapshot, if known; + *
  • otherwise, on cold start with {@code stagingBranch != targetBranch}, the common ancestor + * with target; + *
  • otherwise, on cold start with {@code stagingBranch == targetBranch}, the root of history + * ({@code null}). {@link #shouldSkip} filters already-converted snapshots in O(1) via the + * {@code COMMITTED_STAGING_SNAPSHOT_PROPERTY} marker, and pure-insert snapshots via the + * same-branch eq-delete predicate. + *
+ * + *

When {@code stagingBranch == targetBranch}, the writer commits eq-deletes and data files + * directly to target; the converter performs in-place eq-delete-to-DV compaction. On cold start + * we walk the full history so eq-deletes committed before the converter started are still picked + * up. + */ + private Snapshot nextUnprocessedStagingSnapshot(Snapshot stagingHead, Snapshot mainSnapshot) { + if (stagingHead == null) { + return null; + } + + Long stopAt; + if (lastStagingSnapshotId != null) { + stopAt = lastStagingSnapshotId; + } else if (stagingOnTargetBranch) { + stopAt = null; + } else { + stopAt = findCommonAncestor(stagingHead, mainSnapshot); + } + + Snapshot current = stagingHead; + Snapshot oldestUnprocessed = null; + while (current != null) { + if (stopAt != null && current.snapshotId() == stopAt) { + break; + } + + if (!shouldSkip(current)) { + oldestUnprocessed = current; + } + + current = parentOf(current); + } + + return oldestUnprocessed; + } + + /** + * Resolves the eq deletes in {@code stagingSnapshot} against the current index and emits the + * cycle's metadata. Phase ordering (separated by watermarks): + * + *

    + *
  1. Eq delete read commands. Eq deletes resolve in the worker. + *
  2. Staging data files. Indexed for the NEXT cycle's eq-delete resolution. + *
+ * + * Cold-start bootstrap of the index from main data is handled separately in {@link + * #processElement}, which runs once before the first cycle. + */ + private void processStagingSnapshot( + Snapshot stagingSnapshot, long triggerTs, Long currentMainSnapshotId) { + + StagingInputs inputs = retrieveStagingFiles(stagingSnapshot); + Preconditions.checkState( + !inputs.isEmpty(), + "Staging snapshot %s has no convertible inputs; shouldSkip should have filtered it.", + stagingSnapshot.snapshotId()); + + emitDeletePhase(inputs.eqDeleteFiles()); + emitSnapshotDataPhase(inputs.newDataFiles()); + + LOG.info( + "Emitted read commands for {} new data files from staging branch '{}'.", + inputs.newDataFiles().size(), + stagingBranch); + + processedStagingSnapshotNumCounter.inc(); + + output.collect( + METADATA_STREAM, + new StreamRecord<>( + new EqualityConvertPlan( + inputs.newDataFiles(), + inputs.stagingDVFiles(), + stagingSnapshot.snapshotId(), + currentMainSnapshotId, + triggerTs, + nextPhaseTs))); + + advancePhase(); + } + + /** + * Classifies the files added by {@code stagingSnapshot} into data files, eq delete files, and DV + * files. Throws if the snapshot: + * + *
    + *
  • Removes data files (rewrites on the staging branch aren't supported). + *
  • Contains V2 positional delete files (the converter expects a V3 staging branch written by + * Flink, which produces only deletion vectors for deletes). + *
  • Contains an eq-delete file whose {@code equalityFieldIds()} doesn't match the + * builder-configured set (silent wrong-key serialization otherwise). + *
+ */ + private StagingInputs retrieveStagingFiles(Snapshot stagingSnapshot) { + SnapshotChanges changes = SnapshotChanges.builderFor(table).snapshot(stagingSnapshot).build(); + + // Rewrites on the staging branch would require rewriting the corresponding DVs against new + // data files on target. Not implemented; fail fast instead of silently dropping work. + if (changes.removedDataFiles().iterator().hasNext()) { + throw new IllegalStateException( + String.format( + "Staging snapshot %s on branch '%s' removes data files; " + + "equality delete conversion does not support rewrites on the staging branch. " + + "Run compaction on the target branch instead.", + stagingSnapshot.snapshotId(), stagingBranch)); + } + + List newDataFiles = Lists.newArrayList(); + List stagingDVFiles = Lists.newArrayList(); + List eqDeleteFiles = Lists.newArrayList(); + + for (DataFile dataFile : changes.addedDataFiles()) { + newDataFiles.add(dataFile); + } + + for (DeleteFile deleteFile : changes.addedDeleteFiles()) { + if (deleteFile.content() == FileContent.EQUALITY_DELETES) { + Set deleteFieldIds = Sets.newHashSet(deleteFile.equalityFieldIds()); + Preconditions.checkState( + deleteFieldIds.equals(eqFieldIds), + "Staging snapshot %s on branch '%s' contains an equality delete file %s with " + + "equalityFieldIds=%s, which does not match the configured eqFieldIds=%s. " + + "The writer must use the same equality field IDs as the converter.", + stagingSnapshot.snapshotId(), + stagingBranch, + deleteFile.location(), + deleteFieldIds, + eqFieldIds); + validateDeleteSpecPartitionColumns(stagingSnapshot, deleteFile); + eqDeleteFiles.add(deleteFile); + } else if (ContentFileUtil.isDV(deleteFile)) { + stagingDVFiles.add(deleteFile); + } else { + throw new IllegalStateException( + String.format( + "Staging snapshot %s on branch '%s' contains a V2 positional delete file (%s); " + + "equality delete conversion expects a V3 staging branch written by Flink, " + + "which produces only deletion vectors for deletes.", + stagingSnapshot.snapshotId(), stagingBranch, deleteFile.location())); + } + } + + return new StagingInputs(newDataFiles, stagingDVFiles, eqDeleteFiles); + } + + private void validateDeleteSpecPartitionColumns(Snapshot stagingSnapshot, DeleteFile deleteFile) { + PartitionSpec spec = table.specs().get(deleteFile.specId()); + for (PartitionField field : spec.fields()) { + Preconditions.checkState( + eqFieldIds.contains(field.sourceId()), + "Staging snapshot %s on branch '%s' contains an equality delete file %s under spec %s, " + + "which partitions by field '%s' (source id %s) that is not an equality field %s. " + + "Partition columns must be a subset of the equality fields.", + stagingSnapshot.snapshotId(), + stagingBranch, + deleteFile.location(), + spec.specId(), + field.name(), + field.sourceId(), + eqFieldIds); + } + } + + /** Files added by one staging snapshot, classified for cycle emission. */ + private record StagingInputs( + List newDataFiles, + List stagingDVFiles, + List eqDeleteFiles) { + + boolean isEmpty() { + return newDataFiles.isEmpty() && eqDeleteFiles.isEmpty() && stagingDVFiles.isEmpty(); + } + } + + private void emitDeletePhase(List eqDeleteFiles) { + for (DeleteFile deleteFile : eqDeleteFiles) { + PartitionSpec spec = table.specs().get(deleteFile.specId()); + output.collect( + new StreamRecord<>( + ReadCommand.eqDeleteFile( + deleteFile, + spec, + indexSnapshotId, + indexedSequenceNumber, + dataSequenceNumber(deleteFile)), + nextPhaseTs)); + processedEqDeleteFileNumCounter.inc(); + } + + advancePhase(); + } + + private void emitSnapshotDataPhase(List snapshotDataFiles) { + // Shared-branch skip: when stagingBranch == targetBranch, these files are already on target + // and were indexed by bootstrap/reindex. Re-emitting would duplicate entries in + // dataRowPositions. + if (!stagingOnTargetBranch) { + for (DataFile dataFile : snapshotDataFiles) { + PartitionSpec spec = table.specs().get(dataFile.specId()); + output.collect( + new StreamRecord<>( + ReadCommand.stagingDataFile( + new FlinkAddedRowsScanTask(dataFile, spec), + indexSnapshotId, + indexedSequenceNumber, + dataSequenceNumber(dataFile)), + nextPhaseTs)); + } + } + + advancePhase(); + } + + private void emitNoOpResult(long triggerTimestamp, Long currentMainSnapshotId) { + skippedNoOpCyclesCounter.inc(); + emitDrainResult(triggerTimestamp, currentMainSnapshotId); + } + + /** Emits an empty plan result and advances the phase so the pipeline drains. */ + private void emitDrainResult(long triggerTimestamp, Long currentMainSnapshotId) { + output.collect( + METADATA_STREAM, + new StreamRecord<>( + EqualityConvertPlan.noOp(currentMainSnapshotId, triggerTimestamp, nextPhaseTs))); + advancePhase(); + } + + /** + * Emits {@link ReadCommand}s for every data file on {@code mainSnapshot} so the worker indexes + * them for the configured equality-field set. Existing DVs attached to a data file are loaded by + * the reader and their positions are skipped. V2 positional deletes are not expected on main; the + * reader throws if it encounters one. Equality deletes attached to the scan task are skipped + * during indexing (they are processed via the planner's eq-delete read commands). + */ + private void emitMainDataReadCommands(Snapshot mainSnapshot) { + long commitSnapshotId = mainSnapshot.snapshotId(); + + try (CloseableIterable tasks = + table.newScan().useSnapshot(commitSnapshotId).planFiles()) { + for (FileScanTask task : tasks) { + output.collect( + new StreamRecord<>( + ReadCommand.dataFile( + task, indexSnapshotId, indexedSequenceNumber, dataSequenceNumber(task.file())), + nextPhaseTs)); + } + } catch (IOException e) { + throw new UncheckedIOException("Failed to plan files for main index", e); + } + + LOG.info( + "Emitted main data read commands for field IDs {} from snapshot {}.", + eqFieldIds, + commitSnapshotId); + + advancePhase(); + } + + /** + * Emits a phase-end watermark and bumps the phase timestamp. Every phase-emitting method must + * call this exactly once after its records; the worker uses these watermarks to gate keyed-state + * transitions. Missing or extra calls silently break ordering. + */ + private void advancePhase() { + output.emitWatermark(new Watermark(nextPhaseTs)); + nextPhaseTs++; + } + + /** + * Returns {@code true} if {@code snapshot} can be skipped: + * + *
    + *
  • It was already committed by us (carries {@link + * EqualityConvertCommitter#COMMITTED_STAGING_SNAPSHOT_PROPERTY}), OR + *
  • it adds no data or delete files (e.g. delete-file-only removals), OR + *
  • {@code stagingBranch == targetBranch} and the snapshot adds no equality-delete files. + * Pure-insert (and data-file-only) commits on the shared branch don't need a conversion + * cycle: their data is already on target, and the worker's index stays fresh via {@link + * #ensureIndexCurrent} when the next eq-delete arrives. + *
+ * + *

Filter checks read per-snapshot counts from {@link Snapshot#summary()}; we don't parse + * manifests here. Manifest parsing happens later in {@link #retrieveStagingFiles} only for the + * chosen snapshot. + */ + private boolean shouldSkip(Snapshot snapshot) { + Map summary = snapshot.summary(); + if (summary.containsKey(EqualityConvertCommitter.COMMITTED_STAGING_SNAPSHOT_PROPERTY)) { + return true; + } + + long addedDataFiles = summaryCount(summary, SnapshotSummary.ADDED_FILES_PROP); + long addedDeleteFiles = summaryCount(summary, SnapshotSummary.ADDED_DELETE_FILES_PROP); + if (addedDataFiles == 0 && addedDeleteFiles == 0) { + LOG.info( + "Skipping staging snapshot {}: no added data or delete files.", snapshot.snapshotId()); + return true; + } + + if (stagingOnTargetBranch + && summaryCount(summary, SnapshotSummary.ADD_EQ_DELETE_FILES_PROP) == 0) { + return true; + } + + return false; + } + + private static long summaryCount(Map summary, String key) { + String value = summary.get(key); + return value == null ? 0L : Long.parseLong(value); + } + + /** + * Returns the id of the newest snapshot that is reachable from both the staging and target branch + * heads (i.e. the most recent common ancestor where the two branches last matched), or {@code + * null} if they share no history. Used on cold start to know where staging diverged from target + * so we can skip converting snapshots that already exist on target. + */ + private Long findCommonAncestor(Snapshot stagingHead, Snapshot mainHead) { + if (mainHead == null) { + return null; + } + + Set stagingSeen = Sets.newHashSet(); + Set mainSeen = Sets.newHashSet(); + Snapshot stagingCurrent = stagingHead; + Snapshot mainCurrent = mainHead; + + while (stagingCurrent != null || mainCurrent != null) { + if (stagingCurrent != null) { + long id = stagingCurrent.snapshotId(); + if (mainSeen.contains(id)) { + return id; + } + + stagingSeen.add(id); + stagingCurrent = parentOf(stagingCurrent); + } + + if (mainCurrent != null) { + long id = mainCurrent.snapshotId(); + if (stagingSeen.contains(id)) { + return id; + } + + mainSeen.add(id); + mainCurrent = parentOf(mainCurrent); + } + } + + return null; + } + + /** Returns the parent snapshot, or {@code null} if {@code snapshot} has no parent. */ + private Snapshot parentOf(Snapshot snapshot) { + Long parentId = snapshot.parentId(); + return parentId != null ? table.snapshot(parentId) : null; + } + + private static long dataSequenceNumber(ContentFile file) { + Long sequenceNumber = file.dataSequenceNumber(); + Preconditions.checkNotNull( + sequenceNumber, "Missing data sequence number for committed file %s", file.location()); + return sequenceNumber; + } + + @Override + public void close() throws Exception { + super.close(); + tableLoader.close(); + } + + @VisibleForTesting + long reindexCount() { + return reindexCounter.getCount(); + } + + @VisibleForTesting + long skippedNoOpCycles() { + return skippedNoOpCyclesCounter.getCount(); + } + + @VisibleForTesting + long processedStagingSnapshotNum() { + return processedStagingSnapshotNumCounter.getCount(); + } + + @VisibleForTesting + long processedEqDeleteFileNum() { + return processedEqDeleteFileNumCounter.getCount(); + } +} diff --git a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityDeleteFileScanTask.java b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityDeleteFileScanTask.java index 2b1fc4096609..649344e35cab 100644 --- a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityDeleteFileScanTask.java +++ b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityDeleteFileScanTask.java @@ -29,10 +29,30 @@ * {@link ContentScanTask} for reading a single equality delete file standalone. The {@link * DeleteFile#equalityFieldIds()} on {@link #file()} carries the PK field IDs the worker resolves * against. + * + *

A plain class rather than a record so it round-trips through Flink's Kryo fallback on Flink + * 1.20, whose bundled Kryo 2.x cannot build a serializer for record classes. */ @Internal -record EqualityDeleteFileScanTask(DeleteFile file, PartitionSpec spec) - implements ContentScanTask { +class EqualityDeleteFileScanTask implements ContentScanTask { + + private final DeleteFile file; + private final PartitionSpec spec; + + EqualityDeleteFileScanTask(DeleteFile file, PartitionSpec spec) { + this.file = file; + this.spec = spec; + } + + @Override + public DeleteFile file() { + return file; + } + + @Override + public PartitionSpec spec() { + return spec; + } @Override public long start() { diff --git a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/FlinkAddedRowsScanTask.java b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/FlinkAddedRowsScanTask.java index 5b1bb016316a..d5787ba9fe9c 100644 --- a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/FlinkAddedRowsScanTask.java +++ b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/FlinkAddedRowsScanTask.java @@ -34,22 +34,45 @@ * staging data file from {@code SnapshotChanges#addedDataFiles}) and need to feed it into the * reader. For data files that already come from a planned scan, we pass the native {@link * FileScanTask} through directly. + * + *

A plain class rather than a record to please Kryo 2.x in Flink 1.20, both of which cannot + * build a serializer for record classes. */ @Internal -record FlinkAddedRowsScanTask(DataFile file, PartitionSpec spec, List deletes) - implements FileScanTask { +class FlinkAddedRowsScanTask implements FileScanTask { + + private final DataFile file; + private final PartitionSpec spec; + private final List deletes; - FlinkAddedRowsScanTask { + FlinkAddedRowsScanTask(DataFile file, PartitionSpec spec, List deletes) { + this.file = file; + this.spec = spec; // Iceberg's scan APIs return Guava ImmutableList (see BaseAddedRowsScanTask#deletes), which - // does not round-trip through Flink's Kryo fallback. Copy into a plain ArrayList so the record + // does not round-trip through Flink's Kryo fallback. Copy into a plain ArrayList so the task // serializes cleanly regardless of what the caller passes. - deletes = Lists.newArrayList(deletes); + this.deletes = Lists.newArrayList(deletes); } FlinkAddedRowsScanTask(DataFile file, PartitionSpec spec) { this(file, spec, Collections.emptyList()); } + @Override + public DataFile file() { + return file; + } + + @Override + public PartitionSpec spec() { + return spec; + } + + @Override + public List deletes() { + return deletes; + } + @Override public long start() { return 0L; diff --git a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/source/DataIterator.java b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/source/DataIterator.java index 3beda960cec8..2cb7667a489a 100644 --- a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/source/DataIterator.java +++ b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/source/DataIterator.java @@ -86,6 +86,7 @@ public void seek(int startingFileOffset, long startingRecordOffset) { combinedTask); for (long i = 0L; i < startingFileOffset; ++i) { tasks.next(); + fileOffset += 1; } updateCurrentIterator(); @@ -103,9 +104,6 @@ public void seek(int startingFileOffset, long startingRecordOffset) { combinedTask)); } } - - fileOffset = startingFileOffset; - recordOffset = startingRecordOffset; } @Override diff --git a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/CatalogTestBase.java b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/CatalogTestBase.java index 062ff68d5d85..e45abe25f919 100644 --- a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/CatalogTestBase.java +++ b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/CatalogTestBase.java @@ -89,6 +89,7 @@ public void before() { config.put(CatalogProperties.URI, getURI(hiveConf)); } config.put(CatalogProperties.WAREHOUSE_LOCATION, String.format("file://%s", warehouseRoot())); + config.put("extra-catalog-prop", "extra-value"); this.flinkDatabase = catalogName + "." + DATABASE; this.icebergNamespace = diff --git a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java index b8c357bbb64a..b4a8965e9bd4 100644 --- a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java +++ b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java @@ -33,7 +33,9 @@ import org.apache.flink.table.api.Schema.UnresolvedPrimaryKey; import org.apache.flink.table.api.TableException; import org.apache.flink.table.api.ValidationException; +import org.apache.flink.table.catalog.CatalogMaterializedTable; import org.apache.flink.table.catalog.CatalogTable; +import org.apache.flink.table.catalog.IntervalFreshness; import org.apache.flink.table.catalog.ObjectPath; import org.apache.flink.table.catalog.exceptions.TableNotExistException; import org.apache.iceberg.BaseTable; @@ -228,13 +230,15 @@ public void testCreateTableLikeInFlinkCatalog() throws TableNotExistException { .column("id", DataTypes.BIGINT()) .build()); - String srcCatalogProps = FlinkCreateTableOptions.toJson(catalogName, DATABASE, "tl", config); + String srcCatalogProps = FlinkCreateTableOptions.toJson(catalogName, DATABASE, "tl"); Map options = catalogTable.getOptions(); assertThat(options) .containsEntry( FlinkCreateTableOptions.CONNECTOR_PROPS_KEY, FlinkDynamicTableFactory.FACTORY_IDENTIFIER) .containsEntry(FlinkCreateTableOptions.SRC_CATALOG_PROPS_KEY, srcCatalogProps); + assertThat(options.get(FlinkCreateTableOptions.SRC_CATALOG_PROPS_KEY)) + .doesNotContain("extra-catalog-prop", "extra-value"); } @TestTemplate @@ -743,6 +747,31 @@ private void validateTableFiles(Table tbl, DataFile... expectedFiles) { assertThat(actualFilePaths).as("Files should match").isEqualTo(expectedFilePaths); } + @TestTemplate + public void testCreateMaterializedTableIsUnsupported() { + CatalogMaterializedTable materializedTable = + CatalogMaterializedTable.newBuilder() + .schema( + org.apache.flink.table.api.Schema.newBuilder() + .column("id", DataTypes.BIGINT()) + .build()) + .definitionQuery("SELECT id FROM tl") + .freshness(IntervalFreshness.ofMinute("5")) + .logicalRefreshMode(CatalogMaterializedTable.LogicalRefreshMode.AUTOMATIC) + .refreshMode(CatalogMaterializedTable.RefreshMode.CONTINUOUS) + .refreshStatus(CatalogMaterializedTable.RefreshStatus.INITIALIZING) + .build(); + + assertThatThrownBy( + () -> + getTableEnv() + .getCatalog(catalogName) + .get() + .createTable(new ObjectPath(DATABASE, "mt_table"), materializedTable, false)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Materialized tables and other table kinds are not supported"); + } + private Table table(String name) { return validationCatalog.loadTable(TableIdentifier.of(icebergNamespace, name)); } diff --git a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestConvertEqualityDeletes.java b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestConvertEqualityDeletes.java new file mode 100644 index 000000000000..1c2f230315be --- /dev/null +++ b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestConvertEqualityDeletes.java @@ -0,0 +1,1333 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.api; + +import static org.apache.iceberg.flink.SimpleDataUtil.createRecord; +import static org.apache.iceberg.flink.maintenance.api.ConvertEqualityDeletes.COMMIT_TASK_NAME; +import static org.apache.iceberg.flink.maintenance.api.ConvertEqualityDeletes.PLANNER_TASK_NAME; +import static org.apache.iceberg.flink.maintenance.operator.EqualityConvertCommitter.COMMITTED_STAGING_SNAPSHOT_PROPERTY; +import static org.apache.iceberg.flink.maintenance.operator.TableMaintenanceMetrics.ADDED_DATA_FILE_NUM_METRIC; +import static org.apache.iceberg.flink.maintenance.operator.TableMaintenanceMetrics.ADDED_DATA_FILE_SIZE_METRIC; +import static org.apache.iceberg.flink.maintenance.operator.TableMaintenanceMetrics.ERROR_COUNTER; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Path; +import java.time.Duration; +import java.util.List; +import java.util.Set; +import java.util.stream.StreamSupport; +import org.apache.flink.core.execution.JobClient; +import org.apache.flink.streaming.api.graph.StreamGraphGenerator; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileContent; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.Files; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; +import org.apache.iceberg.PartitionData; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.RewriteFiles; +import org.apache.iceberg.Schema; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.Table; +import org.apache.iceberg.data.FileHelpers; +import org.apache.iceberg.data.GenericAppenderHelper; +import org.apache.iceberg.data.IcebergGenerics; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.deletes.PositionDelete; +import org.apache.iceberg.flink.SimpleDataUtil; +import org.apache.iceberg.flink.maintenance.operator.MetricsReporterFactoryForTests; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.StructLikeSet; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class TestConvertEqualityDeletes extends MaintenanceTaskTestBase { + + private static final String STAGING_BRANCH = "__flink_staging_test"; + + @TempDir private Path tempDir; + + @Test + void testRejectsFormatVersion2() { + createTableWithDelete(2); + + assertThatThrownBy( + () -> + ConvertEqualityDeletes.builder() + .stagingBranch(STAGING_BRANCH) + .equalityFieldColumns(ImmutableList.of("id", "data")) + .append( + infra.triggerStream(), + DUMMY_TABLE_NAME, + DUMMY_TASK_NAME, + 0, + tableLoader(), + UID_SUFFIX, + StreamGraphGenerator.DEFAULT_SLOT_SHARING_GROUP, + 1)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("format version >= 3"); + } + + @Test + void testRejectsUnknownEqualityFieldColumns() { + createTableWithDelete(3); + + assertThatThrownBy( + () -> + ConvertEqualityDeletes.builder() + .stagingBranch(STAGING_BRANCH) + .equalityFieldColumns(ImmutableList.of("nonexistent")) + .append( + infra.triggerStream(), + DUMMY_TABLE_NAME, + DUMMY_TASK_NAME, + 0, + tableLoader(), + UID_SUFFIX, + StreamGraphGenerator.DEFAULT_SLOT_SHARING_GROUP, + 1)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Equality field column 'nonexistent' not found in table schema"); + } + + @Test + void testConvertEqualityDeletesToDVs() throws Exception { + Table table = createTableWithDelete(3); + + // Insert initial data to main + insert(table, 1, "a"); + insert(table, 2, "b"); + insert(table, 3, "c"); + + assertThat(dataFileCount(table)).isEqualTo(3); + + // Create staging branch from current main state + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Write a new data file (simulating insert of id=4) + DataFile newDataFile = writeDataFile(table, createRecord(4, "d")); + + // Write an equality delete for id=2 (simulating delete of row "b") + DeleteFile eqDelete = writeEqualityDelete(table, 2, "b"); + + // Commit both to the staging branch + table.newRowDelta().addRows(newDataFile).addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Verify staging branch has the eq delete + long stagingEqDeleteCount = + table.snapshot(STAGING_BRANCH).deleteManifests(table.io()).stream() + .flatMap( + m -> + StreamSupport.stream( + ManifestFiles.readDeleteManifest(m, table.io(), table.specs()) + .spliterator(), + false)) + .filter(f -> f.content() == FileContent.EQUALITY_DELETES) + .count(); + assertThat(stagingEqDeleteCount).isEqualTo(1); + + // Wire the ConvertEqualityDeletes maintenance task + appendConvertTask(); + + // Run the maintenance task + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + // Verify main branch now has 4 data files (3 original + 1 new) + table.refresh(); + assertThat(dataFileCount(table)).isEqualTo(4); + + // Verify main branch has exactly one DV (id=2 deleted from its single-row data file) + long mainDvCount = + table.currentSnapshot().deleteManifests(table.io()).stream() + .flatMap( + m -> + StreamSupport.stream( + ManifestFiles.readDeleteManifest(m, table.io(), table.specs()) + .spliterator(), + false)) + .filter(f -> f.content() == FileContent.POSITION_DELETES) + .count(); + assertThat(mainDvCount).isEqualTo(1); + + // Verify no equality deletes on main + long mainEqDeleteCount = + table.currentSnapshot().deleteManifests(table.io()).stream() + .flatMap( + m -> + StreamSupport.stream( + ManifestFiles.readDeleteManifest(m, table.io(), table.specs()) + .spliterator(), + false)) + .filter(f -> f.content() == FileContent.EQUALITY_DELETES) + .count(); + assertThat(mainEqDeleteCount).isEqualTo(0); + + // Verify data correctness: id=2 should be deleted, id=4 should be added + SimpleDataUtil.assertTableRecords( + table, ImmutableList.of(createRecord(1, "a"), createRecord(3, "c"), createRecord(4, "d"))); + } + + @Test + void testMetrics() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // One staging snapshot: a new data file (id=3) plus an eq-delete (id=1). The conversion commits + // exactly one data file and one DV to main. + DataFile newDataFile = writeDataFile(table, createRecord(3, "c")); + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(newDataFile).addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + // Only metrics named on TableMaintenanceMetrics flow through the test reporter. Among the + // converter operators only the planner and committer own an ERROR_COUNTER; the parallel reader, + // PK index, and DV writer report failures through ERROR_STREAM instead. The committer also + // counts the data files it adds. Operator-specific counters (reindexCount, addedDvNum, ...) are + // asserted by the operator unit tests. A -1 expected value means "present, value not checked". + MetricsReporterFactoryForTests.assertCounters( + new ImmutableMap.Builder, Long>() + .put(errorKey(PLANNER_TASK_NAME), 0L) + .put(errorKey(COMMIT_TASK_NAME), 0L) + .put(metricKey(COMMIT_TASK_NAME, ADDED_DATA_FILE_NUM_METRIC), 1L) + .put(metricKey(COMMIT_TASK_NAME, ADDED_DATA_FILE_SIZE_METRIC), -1L) + .build()); + } + + private static List errorKey(String taskName) { + return metricKey(taskName, ERROR_COUNTER); + } + + private static List metricKey(String taskName, String metric) { + return ImmutableList.of(taskName + "[0]", DUMMY_TABLE_NAME, DUMMY_TASK_NAME, "0", metric); + } + + @Test + void testNoOpWhenStagingBranchEmpty() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + // Create staging branch with no new files + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + // Should complete successfully with no changes + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + assertThat(dataFileCount(table)).isEqualTo(1); + SimpleDataUtil.assertTableRecords(table, ImmutableList.of(createRecord(1, "a"))); + } + + @Test + void testMultipleEqualityDeletes() throws Exception { + Table table = createTableWithDelete(3); + + insert(table, 1, "a"); + insert(table, 2, "b"); + insert(table, 3, "c"); + insert(table, 4, "d"); + insert(table, 5, "e"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Delete id=1 and id=4 via equality deletes on staging + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + DeleteFile eqDelete2 = writeEqualityDelete(table, 4, "d"); + + table + .newRowDelta() + .addDeletes(eqDelete1) + .addDeletes(eqDelete2) + .toBranch(STAGING_BRANCH) + .commit(); + table.refresh(); + + appendConvertTask(); + + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + SimpleDataUtil.assertTableRecords( + table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"), createRecord(5, "e"))); + } + + @Test + void testDuplicateKeyAcrossDataFiles() throws Exception { + Table table = createTableWithDelete(3); + + // Two data files with the same key (id=1, data="a") + insert(table, 1, "a"); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Eq delete for id=1 should produce DVs for both data files containing id=1 + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + // Only id=2 should remain + SimpleDataUtil.assertTableRecords(table, ImmutableList.of(createRecord(2, "b"))); + } + + @Test + void testMultiSnapshotStagingWithPerSnapshotScoping() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging snapshot 1: delete id=1 from main + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging snapshot 2: re-insert id=1 (new data file) + DataFile newDataFile = writeDataFile(table, createRecord(1, "a")); + table.newAppend().appendFile(newDataFile).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + // Cycle 1: processes S1 (delete id=1) + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.success()).isTrue(); + + // Cycle 2: processes S2 (re-insert id=1) + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.success()).isTrue(); + + table.refresh(); + // id=1 should still exist: the delete from S1 removed the original, + // but S2 re-inserted it. Per-snapshot scoping ensures S1's delete + // doesn't affect S2's data. + assertRecords(table, ImmutableList.of(createRecord(1, "a"))); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testInsertThenDeleteAcrossCycles() throws Exception { + Table table = createTableWithDelete(3); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging snapshot 1: insert id=1 (data-only, no eq deletes) + DataFile insertS1 = writeDataFile(table, createRecord(1, "a")); + table.newAppend().appendFile(insertS1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging snapshot 2: eq delete the row written in S1 + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + // Cycle 1: processes S1 (insert id=1), commits data file to main + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.exceptions()).isEmpty(); + assertThat(result1.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "a"))); + + // Cycle 2: processes S2 (eq delete id=1), must find id=1 on main + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of()); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testInsertUpdateDeleteInsertUpdateChain() throws Exception { + Table table = createTableWithDelete(3); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S1: insert K=1, V=A + DataFile s1 = writeDataFile(table, createRecord(1, "a")); + table.newAppend().appendFile(s1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S2: update K=1 to V=B (eq delete + insert in same commit) + DataFile s2 = writeDataFile(table, createRecord(1, "b")); + DeleteFile e2 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(s2).addDeletes(e2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S3: delete K=1 + DeleteFile e3 = writeEqualityDelete(table, 1, "b"); + table.newRowDelta().addDeletes(e3).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S4: insert K=1, V=C + DataFile s4 = writeDataFile(table, createRecord(1, "c")); + table.newAppend().appendFile(s4).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S5: update K=1 to V=D (eq delete + insert in same commit) + DataFile s5 = writeDataFile(table, createRecord(1, "d")); + DeleteFile e5 = writeEqualityDelete(table, 1, "c"); + table.newRowDelta().addRows(s5).addDeletes(e5).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + long time = System.currentTimeMillis(); + + // Cycle 1: S1 inserts K=1, V=A + long time1 = time; + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.exceptions()).isEmpty(); + assertThat(result1.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "a"))); + + // Cycle 2: S2 updates K=1 from V=A to V=B (eq delete + insert) + long time2 = time + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "b"))); + + // Cycle 3: S3 deletes K=1 + long time3 = time + 2; + infra.source().sendRecord(Trigger.create(time3, 0), time3); + TaskResult result3 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result3.exceptions()).isEmpty(); + assertThat(result3.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of()); + + // Cycle 4: S4 inserts K=1, V=C + long time4 = time + 3; + infra.source().sendRecord(Trigger.create(time4, 0), time4); + TaskResult result4 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result4.exceptions()).isEmpty(); + assertThat(result4.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "c"))); + + // Cycle 5: S5 updates K=1 from V=C to V=D (eq delete + insert) + long time5 = time + 4; + infra.source().sendRecord(Trigger.create(time5, 0), time5); + TaskResult result5 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result5.exceptions()).isEmpty(); + assertThat(result5.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "d"))); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testParallelInsertOfToBeDeletedKeySurvives() throws Exception { + Table table = createTableWithDelete(3); + + // Main holds the original (1, "a"); the staging eq-delete below removes this copy. + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // One staging snapshot updates id=1 in place: re-insert (1, "a") plus an eq-delete (1, "a") in + // the same commit. The re-insert shares the equality key with the delete and carries the + // delete's sequence, so it must survive: the delete only removes the lower-sequence main copy. + // At parallelism > 1 the staging-data ADD can reach the index before the eq-delete resolves. + // Event-time phase ordering is what keeps the re-insert from being accidentally deleted. + DataFile reinsert = writeDataFile(table, createRecord(1, "a")); + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(reinsert).addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(STAGING_BRANCH); + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "a"))); + } + + @Test + void testDVMergeAcrossConversionCycles() throws Exception { + Table table = createTableWithDelete(3); + + // Single data file with 3 rows so DV merge applies to the same file + insert( + table, ImmutableList.of(createRecord(1, "a"), createRecord(2, "b"), createRecord(3, "c"))); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Cycle 1 setup: eq delete for id=1 + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + // Cycle 1: convert eq delete for id=1 + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"))); + + // Cycle 2 setup: eq delete for id=2 (committed while job is running) + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Cycle 2: convert eq delete for id=2, should merge DV with existing + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(3, "c"))); + + // Verify: main has DVs, no equality deletes + assertNoEqualityDeletesOnMain(table, 0); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testConversionCorrectAfterCompaction() throws Exception { + Table table = createTableWithDelete(3); + + // Three separate data files + insert(table, 1, "a"); + insert(table, 2, "b"); + insert(table, 3, "c"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Cycle 1: delete id=1 + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"))); + + // Compact file2 and file3 on main into one file (leave file1 + its DV untouched) + Set allDataFiles = Sets.newHashSet(); + for (ManifestFile manifest : table.currentSnapshot().dataManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.read(manifest, table.io(), table.specs())) { + for (DataFile df : reader) { + allDataFiles.add(df.copy()); + } + } + } + + // Find file1 (contains id=1) by checking which file has a DV against it + Set dvReferencedFiles = Sets.newHashSet(); + for (ManifestFile manifest : table.currentSnapshot().deleteManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { + for (DeleteFile df : reader) { + if (df.referencedDataFile() != null) { + dvReferencedFiles.add(df.referencedDataFile()); + } + } + } + } + + Set filesToCompact = Sets.newHashSet(); + for (DataFile df : allDataFiles) { + if (!dvReferencedFiles.contains(df.location())) { + filesToCompact.add(df); + } + } + + assertThat(filesToCompact).hasSize(2); + + DataFile compactedFile = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"))); + RewriteFiles rewrite = table.newRewrite(); + for (DataFile old : filesToCompact) { + rewrite.deleteFile(old); + } + + rewrite.addFile(compactedFile); + rewrite.commit(); + table.refresh(); + + assertRecords(table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"))); + + // Cycle 2: delete id=2 (should target the compacted file) + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(3, "c"))); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testConvertEqualityDeletesPartitionedTable() throws Exception { + Table table = createPartitionedTableWithDelete(3); + + // Insert data into two partitions + insertPartitioned(table, 1, "a"); + insertPartitioned(table, 2, "b"); + insertPartitioned(table, 3, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Partition-scoped equality delete for id=1 in partition data="a" + DeleteFile eqDelete = writePartitionedEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + // id=1 deleted from partition "a", id=2 in partition "b" and id=3 in partition "a" remain + assertRecords(table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "a"))); + } + + @Test + void testEqualityDeleteIsScopedToItsPartition() throws Exception { + Table table = createPartitionedTableWithDelete(3); + + // Same PK (id=1) exists in two partitions. An eq delete in one partition must not delete + // rows in the other. + insertPartitioned(table, 1, "a"); + insertPartitioned(table, 1, "b"); + insertPartitioned(table, 2, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete = writePartitionedEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + // Only (1, "a") is deleted; (1, "b") remains because the eq delete was scoped to partition + // "a", and (2, "a") remains because its equality field values don't match. + assertRecords(table, ImmutableList.of(createRecord(1, "b"), createRecord(2, "a"))); + } + + @Test + void testStagingPositionDeleteMergedIntoConversionDV() throws Exception { + Table table = createTableWithDelete(3); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S1: write a data file with two rows (id=1 at pos 0, id=2 at pos 1). + DataFile data = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(1, "a"), createRecord(2, "b"))); + table.newAppend().appendFile(data).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S2: eq delete matches row 0 (will produce a conversion DV at pos 0) + a position + // delete DV referencing the same data file at pos 1. Both DVs target the same data + // file and must be merged into a single DV (V3 invariant). + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + DeleteFile stagingDV = writeStagingDV(table, data.location(), 1L); + table + .newRowDelta() + .addDeletes(eqDelete) + .addDeletes(stagingDV) + .toBranch(STAGING_BRANCH) + .commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + // Cycle 1: commits S1's data file to main + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.success()).isTrue(); + + // Cycle 2: converts S2's eq delete to DV, merges with staging DV + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + table.refresh(); + // Both rows from S1 must be masked: pos 0 by the conversion DV, pos 1 by the staging DV. + assertRecords(table, ImmutableList.of()); + + // Exactly one DV per data file (V3 invariant): the resolver must have folded the + // staging DV's positions into the conversion DV. + long dvCount = + table.currentSnapshot().deleteManifests(table.io()).stream() + .flatMap( + m -> + StreamSupport.stream( + ManifestFiles.readDeleteManifest(m, table.io(), table.specs()) + .spliterator(), + false)) + .filter(f -> f.content() == FileContent.POSITION_DELETES) + .filter(f -> data.location().equals(f.referencedDataFile())) + .count(); + assertThat(dvCount).isEqualTo(1); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testStagingDataFilesOnlyNoEqDeletes() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging has only a new data file, no equality deletes + DataFile newDataFile = writeDataFile(table, createRecord(2, "b")); + table.newAppend().appendFile(newDataFile).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + assertThat(dataFileCount(table)).isEqualTo(2); + SimpleDataUtil.assertTableRecords( + table, ImmutableList.of(createRecord(1, "a"), createRecord(2, "b"))); + } + + @Test + void testReindexAfterExternalCommit() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Cycle 1: delete id=1 + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(2, "b"))); + + // External commit: insert id=3 directly to main (not via staging) + insert(table, 3, "c"); + + // Cycle 2: delete id=2 (should reindex because of external commit) + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(3, "c"))); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testReindexEvictsGhostKeyAfterExternalDataFileRemoval() throws Exception { + // CoW removal case: an external commit removes a data file, leaving a stale + // PK in the worker's index. A later staging eq-delete for that PK must NOT produce a DV + // referencing the removed file. The external commit advances main, so the next cycle reindexes + // and the worker clears the ghost key before resolving the delete. + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + table.refresh(); + DataFile file1 = table.currentSnapshot().addedDataFiles(table.io()).iterator().next().copy(); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Cycle 1: delete id=2. Bootstraps the worker index from main (id=1 -> file1, id=2 -> file2). + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "a"))); + + // External CoW-style removal: drop file1 (id=1) from main without re-adding the row. The + // worker's index still holds the ghost id=1 -> file1 until the next reindex clears it. + table.newDelete().deleteFile(file1).commit(); + table.refresh(); + assertRecords(table, ImmutableList.of()); + + // Cycle 2: stage an eq-delete for the removed key id=1. Without ghost eviction the worker + // would emit a DV position against the now-absent file1. + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + // No deletion vector may reference the removed file1. + table.refresh(); + assertThat(dvReferencedDataFiles(table)).doesNotContain(file1.location()); + assertRecords(table, ImmutableList.of()); + } finally { + closeJobClient(jobClient); + } + } + + private static Set dvReferencedDataFiles(Table table) { + Set referenced = Sets.newHashSet(); + for (ManifestFile manifest : table.currentSnapshot().deleteManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { + for (DeleteFile df : reader) { + if (df.referencedDataFile() != null) { + referenced.add(df.referencedDataFile()); + } + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + return referenced; + } + + private DataFile writeDataFile(Table table, Record record) throws IOException { + return new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(record)); + } + + private DeleteFile writeStagingDV(Table table, String dataFilePath, long position) + throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + + PositionDelete delete = PositionDelete.create(); + delete.set(dataFilePath, position, null); + return FileHelpers.writePosDeleteFile( + table, + Files.localOutput(file), + new PartitionData(PartitionSpec.unpartitioned().partitionType()), + Lists.newArrayList(delete), + 3); + } + + private DeleteFile writeEqualityDelete(Table table, Integer id, String data) throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + + Schema eqDeleteSchema = table.schema(); + return FileHelpers.writeDeleteFile( + table, + Files.localOutput(file), + new PartitionData(PartitionSpec.unpartitioned().partitionType()), + Lists.newArrayList(createRecord(id, data)), + eqDeleteSchema); + } + + private DeleteFile writePartitionedEqualityDelete(Table table, Integer id, String data) + throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + + PartitionData partition = new PartitionData(table.spec().partitionType()); + partition.set(0, data); + return FileHelpers.writeDeleteFile( + table, + Files.localOutput(file), + partition, + Lists.newArrayList(createRecord(id, data)), + table.schema()); + } + + private static long dataFileCount(Table table) { + table.refresh(); + long count = 0; + for (ManifestFile manifest : table.currentSnapshot().dataManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.read(manifest, table.io(), table.specs())) { + for (DataFile ignored : reader) { + count++; + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + return count; + } + + @Test + void testStagingEqualsTargetBranch() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + // Write eq delete directly to main (no separate staging branch) + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + DataFile newData = writeDataFile(table, createRecord(3, "c")); + table.newRowDelta().addRows(newData).addDeletes(eqDelete).commit(); + table.refresh(); + + appendConvertTask(SnapshotRef.MAIN_BRANCH); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + // Cycle 1: process the eq delete for id=1 + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"))); + long dataFilesAfterCycle1 = dataFileCount(table); + // Expect 3: two from insert() + one from the writer's rowDelta.addRows(newData). + // When stagingBranch == targetBranch, the committer must NOT re-add newData via + // rowDelta.addRows(...) — that would duplicate (count=4). + assertThat(dataFilesAfterCycle1).isEqualTo(3); + + // Cycle 2: no-op (converter's own commit must be skipped) + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"))); + assertThat(dataFileCount(table)).isEqualTo(dataFilesAfterCycle1); + + // New eq delete for id=2 committed directly to main between cycles + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + DataFile newData2 = writeDataFile(table, createRecord(4, "d")); + table.newRowDelta().addRows(newData2).addDeletes(eqDelete2).commit(); + table.refresh(); + + // Cycle 3: process the new eq delete + long time3 = time2 + 1; + infra.source().sendRecord(Trigger.create(time3, 0), time3); + TaskResult result3 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result3.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(3, "c"), createRecord(4, "d"))); + long dataFilesAfterCycle3 = dataFileCount(table); + + // Cycle 4: no-op again + long time4 = time3 + 1; + infra.source().sendRecord(Trigger.create(time4, 0), time4); + TaskResult result4 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result4.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(3, "c"), createRecord(4, "d"))); + assertThat(dataFileCount(table)).isEqualTo(dataFilesAfterCycle3); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testStagingEqualsTargetBranchColdStartCatchUp() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + insert(table, 3, "c"); + + // Writer commits three eq-deletes to main BEFORE the converter starts. + // Cold start must pick up the unconverted history, not just the head snapshot. + table.newRowDelta().addDeletes(writeEqualityDelete(table, 1, "a")).commit(); + table.newRowDelta().addDeletes(writeEqualityDelete(table, 2, "b")).commit(); + table.newRowDelta().addDeletes(writeEqualityDelete(table, 3, "c")).commit(); + table.refresh(); + + appendConvertTask(SnapshotRef.MAIN_BRANCH); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + // One unconverted snapshot per cycle, oldest first. After three cycles every eq-delete + // commit has its own committer commit carrying the marker. + for (int cycle = 1; cycle <= 3; cycle++) { + long ts = System.currentTimeMillis() + cycle; + infra.source().sendRecord(Trigger.create(ts, 0), ts); + TaskResult result = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result.success()).isTrue(); + } + + table.refresh(); + long convertedCount = + StreamSupport.stream(table.snapshots().spliterator(), false) + .filter(s -> s.summary().containsKey(COMMITTED_STAGING_SNAPSHOT_PROPERTY)) + .count(); + assertThat(convertedCount).isEqualTo(3); + assertRecords(table, ImmutableList.of()); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testStagingEqualsTargetBranchReinsertAfterDeleteSurvives() throws Exception { + Table table = createTableWithDelete(3); + + // Shared branch: insert id=1, eq-delete id=1, then re-insert id=1. The re-insert has a higher + // sequence than the delete and must survive the conversion (sequence-aware resolution). + insert(table, 1, "a"); + table.newRowDelta().addDeletes(writeEqualityDelete(table, 1, "a")).commit(); + table.newAppend().appendFile(writeDataFile(table, createRecord(1, "a"))).commit(); + table.refresh(); + + appendConvertTask(SnapshotRef.MAIN_BRANCH); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + // One cycle converts the eq-delete: the original row is deleted, the newer re-insert is + // not (its sequence is at or above the delete's). + long ts = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(ts, 0), ts); + TaskResult result = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result.exceptions()).isEmpty(); + assertThat(result.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "a"))); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testStagingEqualsTargetBranchMergesStagingDvIntoSingleDv() throws Exception { + Table table = createTableWithDelete(3); + + // One data file with two rows: id=1 at pos 0, id=2 at pos 1, committed to main. + DataFile data = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(1, "a"), createRecord(2, "b"))); + table.newAppend().appendFile(data).commit(); + table.refresh(); + + // Same-branch commit: an eq-delete for id=1 (resolves to a conversion DV at pos 0) plus a + // writer DV at pos 1 on the same data file. The resolver folds the staging DV into the + // conversion DV; on a shared branch the committer must remove the superseded staging DV. + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + DeleteFile stagingDV = writeStagingDV(table, data.location(), 1L); + table.newRowDelta().addDeletes(eqDelete).addDeletes(stagingDV).commit(); + table.refresh(); + + appendConvertTask(SnapshotRef.MAIN_BRANCH); + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + // Both rows masked: pos 0 by the conversion DV, pos 1 by the merged-in staging DV. + assertRecords(table, ImmutableList.of()); + + // Exactly one DV per data file (V3 invariant). Without removing the rewritten staging DV on a + // shared branch, the data file would carry two DVs. + long dvCount = + table.currentSnapshot().deleteManifests(table.io()).stream() + .flatMap( + m -> + StreamSupport.stream( + ManifestFiles.readDeleteManifest(m, table.io(), table.specs()) + .spliterator(), + false)) + .filter(f -> f.content() == FileContent.POSITION_DELETES) + .filter(f -> data.location().equals(f.referencedDataFile())) + .count(); + assertThat(dvCount).isEqualTo(1); + } + + @Test + void testReaderErrorSkipsCommit() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + long mainSnapshotBeforeStaging = table.currentSnapshot().snapshotId(); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging data file + eq delete file, both referenced by the staging commit. + DataFile newDataFile = writeDataFile(table, createRecord(2, "b")); + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(newDataFile).addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + File eqDeleteLocalFile = new File(eqDelete.location().replace("file:", "")); + + // Delete the eq delete file; the committer must abort rather than committing data without its + // DV. + assertThat(eqDeleteLocalFile.delete()).isTrue(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + + assertThat(result1.success()).isFalse(); + assertThat(result1.exceptions()).isNotEmpty(); + + table.refresh(); + // Main must not have advanced (no commit happened). + assertThat(table.currentSnapshot().snapshotId()).isEqualTo(mainSnapshotBeforeStaging); + SimpleDataUtil.assertTableRecords(table, ImmutableList.of(createRecord(1, "a"))); + + // Restore the eq delete file content by rewriting an identical delete, and retry: + // the planner must re-process the same staging snapshot (cursor didn't advance on failure). + DeleteFile recreated = + FileHelpers.writeDeleteFile( + table, + Files.localOutput(eqDeleteLocalFile), + new PartitionData(PartitionSpec.unpartitioned().partitionType()), + Lists.newArrayList(createRecord(1, "a")), + table.schema()); + assertThat(recreated.location()).isEqualTo(eqDelete.location()); + + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + table.refresh(); + // Staging data file committed with DV for id=1: should see id=2 only. + SimpleDataUtil.assertTableRecords(table, ImmutableList.of(createRecord(2, "b"))); + } finally { + closeJobClient(jobClient); + } + } + + private void appendConvertTask() { + appendConvertTask(STAGING_BRANCH); + } + + private void appendConvertTask(String stagingBranch) { + ConvertEqualityDeletes.builder() + .stagingBranch(stagingBranch) + .equalityFieldColumns(ImmutableList.of("id", "data")) + .parallelism(2) + .append( + infra.triggerStream(), + DUMMY_TABLE_NAME, + DUMMY_TASK_NAME, + 0, + tableLoader(), + UID_SUFFIX, + StreamGraphGenerator.DEFAULT_SLOT_SHARING_GROUP, + 1) + .sinkTo(infra.sink()); + } + + private static void assertRecords(Table table, List expected) throws IOException { + table.refresh(); + Types.StructType type = SimpleDataUtil.SCHEMA.asStruct(); + + StructLikeSet expectedSet = StructLikeSet.create(type); + expectedSet.addAll(expected); + + try (CloseableIterable iterable = + IcebergGenerics.read(table) + .useSnapshot(table.currentSnapshot().snapshotId()) + .project(SimpleDataUtil.SCHEMA) + .build()) { + StructLikeSet actualSet = StructLikeSet.create(type); + for (Record record : iterable) { + actualSet.add(record); + } + + assertThat(actualSet).isEqualTo(expectedSet); + } + } + + private static void assertNoEqualityDeletesOnMain(Table table, long expectedEqDeleteCount) { + long mainEqDeleteCount = 0; + for (ManifestFile manifest : table.currentSnapshot().deleteManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { + for (DeleteFile f : reader) { + if (f.content() == FileContent.EQUALITY_DELETES) { + mainEqDeleteCount++; + } + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + assertThat(mainEqDeleteCount).isEqualTo(expectedEqDeleteCount); + } +} diff --git a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestConvertEqualityDeletesE2E.java b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestConvertEqualityDeletesE2E.java new file mode 100644 index 000000000000..f6d1fd9464e2 --- /dev/null +++ b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestConvertEqualityDeletesE2E.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.api; + +import static org.apache.iceberg.flink.SimpleDataUtil.createRecord; +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.time.Duration; +import org.apache.flink.core.execution.JobClient; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.Files; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; +import org.apache.iceberg.PartitionData; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.Table; +import org.apache.iceberg.data.FileHelpers; +import org.apache.iceberg.data.GenericAppenderHelper; +import org.apache.iceberg.flink.SimpleDataUtil; +import org.apache.iceberg.flink.maintenance.operator.OperatorTestBase; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.util.ContentFileUtil; +import org.awaitility.Awaitility; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +/** + * End-to-end test for {@link ConvertEqualityDeletes} wired through the {@link TableMaintenance} + * framework. Verifies that the converter actually runs and commits a DV when the framework triggers + * it, exercising the full operator graph including the framework's monitor source, trigger manager, + * and lock remover. + */ +class TestConvertEqualityDeletesE2E extends OperatorTestBase { + private static final String STAGING_BRANCH = "staging"; + + @TempDir private Path tempDir; + private StreamExecutionEnvironment env; + + @BeforeEach + public void beforeEach() { + this.env = StreamExecutionEnvironment.getExecutionEnvironment(); + } + + @ParameterizedTest + @ValueSource(strings = {STAGING_BRANCH, SnapshotRef.MAIN_BRANCH}) + void testConvertEqualityDeletesE2E(String stagingBranch) throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + // When staging is a separate branch, fork it from main first. + if (!stagingBranch.equals(SnapshotRef.MAIN_BRANCH)) { + table.manageSnapshots().createBranch(stagingBranch).commit(); + table.refresh(); + } + + // Commit a new data file + eq delete to staging. This pre-job snapshot exercises both the + // "new data file" and "eq delete" paths in one cycle. + DataFile newData = writeDataFile(table, 3, "c"); + DeleteFile firstDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(newData).addDeletes(firstDelete).toBranch(stagingBranch).commit(); + table.refresh(); + assertThat(dvCountOnMain(table)).isZero(); + + TableMaintenance.forTable(env, tableLoader(), LOCK_FACTORY) + .uidSuffix("ConvertEqualityDeletesE2EUID-" + stagingBranch) + .rateLimit(Duration.ofMillis(50)) + .lockCheckDelay(Duration.ofMillis(50)) + .add( + ConvertEqualityDeletes.builder() + .scheduleOnInterval(Duration.ofMillis(100)) + .stagingBranch(stagingBranch) + .targetBranch(SnapshotRef.MAIN_BRANCH) + .equalityFieldColumns(ImmutableList.of("id", "data")) + .parallelism(2)) + .append(); + + JobClient jobClient = null; + try { + jobClient = env.executeAsync(); + + // Cycle 1: row 1 deleted by the converted DV; row 3 added on staging and committed to main. + Awaitility.await() + .atMost(Duration.ofMinutes(5)) + .pollInterval(Duration.ofMillis(200)) + .untilAsserted(() -> assertThat(dvCountOnMain(table)).isEqualTo(1)); + SimpleDataUtil.assertTableRecords( + table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"))); + + // Cycle 2: commit a second staging snapshot while the job is still running. The framework's + // next interval-trigger should pick it up and produce a second DV against the data file + // holding id=2. + table.refresh(); + DeleteFile secondDelete = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(secondDelete).toBranch(stagingBranch).commit(); + + Awaitility.await() + .atMost(Duration.ofMinutes(5)) + .pollInterval(Duration.ofMillis(200)) + .untilAsserted(() -> assertThat(dvCountOnMain(table)).isEqualTo(2)); + SimpleDataUtil.assertTableRecords(table, ImmutableList.of(createRecord(3, "c"))); + } finally { + closeJobClient(jobClient); + } + } + + private DataFile writeDataFile(Table table, Integer id, String data) throws IOException { + return new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(SimpleDataUtil.createRecord(id, data))); + } + + private DeleteFile writeEqualityDelete(Table table, Integer id, String data) throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + return FileHelpers.writeDeleteFile( + table, + Files.localOutput(file), + new PartitionData(PartitionSpec.unpartitioned().partitionType()), + Lists.newArrayList(SimpleDataUtil.createRecord(id, data)), + table.schema()); + } + + private static long dvCountOnMain(Table table) throws IOException { + table.refresh(); + if (table.currentSnapshot() == null) { + return 0; + } + + long count = 0; + for (ManifestFile manifest : table.currentSnapshot().deleteManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { + for (DeleteFile file : reader) { + if (ContentFileUtil.isDV(file)) { + count++; + } + } + } + } + + return count; + } +} diff --git a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestMaintenanceE2E.java b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestMaintenanceE2E.java index fe8457167a1f..f786f1cdb29d 100644 --- a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestMaintenanceE2E.java +++ b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestMaintenanceE2E.java @@ -24,8 +24,11 @@ import java.time.Duration; import org.apache.flink.core.execution.JobClient; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.SnapshotRef; import org.apache.iceberg.Table; import org.apache.iceberg.flink.maintenance.operator.OperatorTestBase; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -116,4 +119,34 @@ void testE2eUseCoordinator() throws Exception { closeJobClient(jobClient); } } + + @Test + void testE2eConvertEqualityDeletes() throws Exception { + // Converter requires V3 (DV support); replace the V2 table created in @BeforeEach. + dropTable(); + createTable(3, FileFormat.PARQUET); + + TableMaintenance.forTable(env, tableLoader(), LOCK_FACTORY) + .uidSuffix("E2eConvertEqualityDeletesUID") + .rateLimit(Duration.ofMinutes(10)) + .lockCheckDelay(Duration.ofSeconds(10)) + .add( + ConvertEqualityDeletes.builder() + .scheduleOnEqDeleteFileCount(1) + .stagingBranch("staging") + .targetBranch(SnapshotRef.MAIN_BRANCH) + .equalityFieldColumns(ImmutableList.of("id")) + .parallelism(2)) + .append(); + + JobClient jobClient = null; + try { + jobClient = env.executeAsync(); + + // Just make sure that we are able to instantiate the flow + assertThat(jobClient).isNotNull(); + } finally { + closeJobClient(jobClient); + } + } } diff --git a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestTableMaintenance.java b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestTableMaintenance.java index 49219d5b4698..96b2fe04d66e 100644 --- a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestTableMaintenance.java +++ b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestTableMaintenance.java @@ -350,6 +350,14 @@ void testUidAndSlotSharingGroupForMonitorSource() throws IOException { checkSlotSharingGroupsAreSet(env, SLOT_SHARING_GROUP); } + @Test + void testMonitorRatePerSecond() { + // Sub-second rate limits must not yield infinite (unthrottled) monitor rates. + assertThat(TableMaintenance.Builder.monitorRatePerSecond(50)).isEqualTo(20.0); + assertThat(TableMaintenance.Builder.monitorRatePerSecond(100)).isEqualTo(10.0); + assertThat(TableMaintenance.Builder.monitorRatePerSecond(60_000)).isEqualTo(1.0 / 60); + } + /** * Sends the events though the {@link ManualSource} provided, and waits until the given number of * records are processed. diff --git a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/OperatorTestBase.java b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/OperatorTestBase.java index c21518c6d29c..a45198c6ea9d 100644 --- a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/OperatorTestBase.java +++ b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/OperatorTestBase.java @@ -188,6 +188,18 @@ protected static Table createTableWithDelete(int formatVersion) { "format-version", String.valueOf(formatVersion), "write.upsert.enabled", "true")); } + protected static Table createPartitionedTableWithDelete(int formatVersion) { + return CATALOG_EXTENSION + .catalog() + .createTable( + TestFixtures.TABLE_IDENTIFIER, + SCHEMA_WITH_PRIMARY_KEY, + PartitionSpec.builderFor(SCHEMA_WITH_PRIMARY_KEY).identity("data").build(), + null, + ImmutableMap.of( + "format-version", String.valueOf(formatVersion), "write.upsert.enabled", "true")); + } + protected static Table createPartitionedTable(int formatVersion, FileFormat fileFormat) { return CATALOG_EXTENSION .catalog() diff --git a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertCommitter.java b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertCommitter.java new file mode 100644 index 000000000000..54f0b1758d90 --- /dev/null +++ b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertCommitter.java @@ -0,0 +1,569 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.operator; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.IOException; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.util.TwoInputStreamOperatorTestHarness; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; +import org.apache.iceberg.RowDelta; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.Table; +import org.apache.iceberg.data.FileHelpers; +import org.apache.iceberg.data.GenericRecord; +import org.apache.iceberg.deletes.PositionDelete; +import org.apache.iceberg.exceptions.CommitStateUnknownException; +import org.apache.iceberg.exceptions.ValidationException; +import org.apache.iceberg.flink.maintenance.api.Trigger; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.junit.jupiter.api.Test; + +class TestEqualityConvertCommitter extends OperatorTestBase { + + @Test + void commitsDataFilesToMainBranch() throws Exception { + Table table = createTable(3, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long snapshotIdBefore = table.currentSnapshot().snapshotId(); + long stagingSnapshotId = 42L; + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + stagingSnapshotId, + snapshotIdBefore, + doneTs - 1, + doneTs); + + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + harness.processBothWatermarks(new Watermark(doneTs)); + + assertThat(harness.extractOutputValues()).hasSize(1); + table.refresh(); + assertThat(table.currentSnapshot().snapshotId()).isNotEqualTo(snapshotIdBefore); + assertThat( + table + .currentSnapshot() + .summary() + .get(EqualityConvertCommitter.COMMITTED_STAGING_SNAPSHOT_PROPERTY)) + .isEqualTo(String.valueOf(stagingSnapshotId)); + } + } + + @Test + void holdsBackWatermarkUntilCommit() throws Exception { + Table table = createTable(3, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long snapshotIdBefore = table.currentSnapshot().snapshotId(); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + 42L, + snapshotIdBefore, + doneTs - 2, + doneTs); + + harness.processElement2(new StreamRecord<>(planResult, doneTs - 2)); + + // A phase watermark before the plan's done timestamp must not be forwarded: it would let the + // LockRemover release the maintenance lock before this cycle commits. + harness.processBothWatermarks(new Watermark(doneTs - 1)); + assertThat(harness.extractOutputValues()).isEmpty(); + assertThat(watermarks(harness)).isEmpty(); + + // The done-timestamp watermark commits the cycle; the watermark is forwarded only now. + harness.processBothWatermarks(new Watermark(doneTs)); + assertThat(harness.extractOutputValues()).hasSize(1); + assertThat(watermarks(harness)).containsExactly(new Watermark(doneTs)); + } + } + + @Test + void skipsCommitForEmptyCycle() throws Exception { + Table table = createTable(3, FileFormat.PARQUET); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = EqualityConvertPlan.noOp(null, doneTs - 1, doneTs); + + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + harness.processBothWatermarks(new Watermark(doneTs)); + + assertThat(harness.extractOutputValues()).hasSize(1); + table.refresh(); + assertThat(table.currentSnapshot()).isNull(); + } + } + + @Test + void abortsCommitWhenDVWriterFailed() throws Exception { + Table table = createTable(3, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long snapshotIdBefore = table.currentSnapshot().snapshotId(); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + 42L, + snapshotIdBefore, + doneTs - 1, + doneTs); + + // Receive an abort signal from the DVWriter followed by the planning planResult. + harness.processElement1(new StreamRecord<>(DVWriteResult.ABORT, doneTs)); + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + + harness.processBothWatermarks(new Watermark(doneTs)); + + // The cycle must complete (Trigger emitted) but nothing must be committed to the table. + assertThat(harness.extractOutputValues()).hasSize(1); + table.refresh(); + assertThat(table.currentSnapshot().snapshotId()).isEqualTo(snapshotIdBefore); + } + } + + @Test + void failsCommitWhenExternalCommitLandsAfterPlan() throws Exception { + Table table = createTable(3, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long mainSnapshotIdAtPlan = table.currentSnapshot().snapshotId(); + long stagingSnapshotId = 555L; + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + stagingSnapshotId, + mainSnapshotIdAtPlan, + doneTs - 1, + doneTs); + + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + + // External writer appends to main between plan and commit. + insert(table, 2, "b"); + table.refresh(); + long mainSnapshotIdAfterExternal = table.currentSnapshot().snapshotId(); + assertThat(mainSnapshotIdAfterExternal).isNotEqualTo(mainSnapshotIdAtPlan); + + harness.processBothWatermarks(new Watermark(doneTs)); + + // Trigger still fires (so the aggregator records the cycle), but the commit + // must have been rejected by validateNoConflictingDataFiles. + assertThat(harness.extractOutputValues()).hasSize(1); + + List> errors = + Lists.newArrayList(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)); + assertThat(errors).hasSize(1); + Exception failure = errors.get(0).getValue(); + assertThat(failure).isInstanceOf(ValidationException.class); + assertThat(failure) + .hasMessageContaining("Found conflicting files that can contain records matching"); + + // Target head is still the external commit, not our cycle's commit. + table.refresh(); + assertThat(table.currentSnapshot().snapshotId()).isEqualTo(mainSnapshotIdAfterExternal); + assertThat( + table + .currentSnapshot() + .summary() + .get(EqualityConvertCommitter.COMMITTED_STAGING_SNAPSHOT_PROPERTY)) + .isNull(); + } + } + + @Test + void failsReplayOfCommittedPlanFromOlderState() throws Exception { + Table table = createTable(3, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long mainSnapshotIdAtPlan = table.currentSnapshot().snapshotId(); + long stagingSnapshotId = 909L; + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + stagingSnapshotId, + mainSnapshotIdAtPlan, + doneTs - 1, + doneTs); + + // Cycle 1 commits, advancing the target past mainSnapshotIdAtPlan and writing the marker. + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + harness.processBothWatermarks(new Watermark(doneTs)); + assertThat(harness.extractOutputValues()).hasSize(1); + + table.refresh(); + long committedSnapshotId = table.currentSnapshot().snapshotId(); + assertThat(committedSnapshotId).isNotEqualTo(mainSnapshotIdAtPlan); + + // Restart from older state: the identical plan with its stale mainSnapshotId is replayed. + // The committer is stateless, so validateFromSnapshot is the only guard against a duplicate + // commit. It must reject the replay. + long doneTs2 = doneTs + 1; + EqualityConvertPlan replay = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + stagingSnapshotId, + mainSnapshotIdAtPlan, + doneTs2 - 1, + doneTs2); + + harness.processElement2(new StreamRecord<>(replay, doneTs2 - 1)); + harness.processBothWatermarks(new Watermark(doneTs2)); + + // Trigger still fires (so the aggregator records the cycle), but the commit was rejected. + assertThat(harness.extractOutputValues()).hasSize(2); + + List> errors = + Lists.newArrayList(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)); + assertThat(errors).hasSize(1); + Exception failure = errors.get(0).getValue(); + assertThat(failure).isInstanceOf(ValidationException.class); + assertThat(failure) + .hasMessageContaining("Found conflicting files that can contain records matching"); + + // No duplicate commit: target head is still cycle 1's commit. + table.refresh(); + assertThat(table.currentSnapshot().snapshotId()).isEqualTo(committedSnapshotId); + } + } + + @Test + void deletesWrittenDvsWhenCommitFails() throws Exception { + Table table = createTable(2, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long mainSnapshotIdAtPlan = table.currentSnapshot().snapshotId(); + + DeleteFile writtenDv = writePosDeleteFile(table, stagingDataFile.location(), 0L); + assertThat(table.io().newInputFile(writtenDv.location()).exists()).isTrue(); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + 777L, + mainSnapshotIdAtPlan, + doneTs - 1, + doneTs); + + harness.processElement1( + new StreamRecord<>( + new DVWriteResult(Lists.newArrayList(writtenDv), Lists.newArrayList()), doneTs)); + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + + // External writer appends to main between plan and commit, so the commit is rejected. + insert(table, 2, "b"); + table.refresh(); + + harness.processBothWatermarks(new Watermark(doneTs)); + + assertThat(harness.extractOutputValues()).hasSize(1); + + List> errors = + Lists.newArrayList(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)); + assertThat(errors).hasSize(1); + assertThat(errors.get(0).getValue()).isInstanceOf(ValidationException.class); + + // The DV written this cycle is unreferenced after the failed commit and must be deleted. + assertThat(table.io().newInputFile(writtenDv.location()).exists()).isFalse(); + } + } + + @Test + void deletesSiblingDvsOnAbortButKeepsRewrittenDvs() throws Exception { + Table table = createTable(2, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long snapshotIdBefore = table.currentSnapshot().snapshotId(); + + DeleteFile writtenDv = writePosDeleteFile(table, stagingDataFile.location(), 0L); + DeleteFile rewrittenDv = writePosDeleteFile(table, stagingDataFile.location(), 1L); + assertThat(table.io().newInputFile(writtenDv.location()).exists()).isTrue(); + assertThat(table.io().newInputFile(rewrittenDv.location()).exists()).isTrue(); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + 42L, + snapshotIdBefore, + doneTs - 1, + doneTs); + + // One writer task wrote a DV (rewriting an existing one); a sibling task aborted. + harness.processElement1( + new StreamRecord<>( + new DVWriteResult(Lists.newArrayList(writtenDv), Lists.newArrayList(rewrittenDv)), + doneTs)); + harness.processElement1(new StreamRecord<>(DVWriteResult.ABORT, doneTs)); + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + harness.processBothWatermarks(new Watermark(doneTs)); + + assertThat(harness.extractOutputValues()).hasSize(1); + table.refresh(); + assertThat(table.currentSnapshot().snapshotId()).isEqualTo(snapshotIdBefore); + + // The freshly written DV is removed; the rewritten DV is still live on target and stays. + assertThat(table.io().newInputFile(writtenDv.location()).exists()).isFalse(); + assertThat(table.io().newInputFile(rewrittenDv.location()).exists()).isTrue(); + } + } + + @Test + void retainsWrittenDvsWhenCommitStateUnknown() throws Exception { + Table table = createTable(2, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long mainSnapshotIdAtPlan = table.currentSnapshot().snapshotId(); + + DeleteFile writtenDv = writePosDeleteFile(table, stagingDataFile.location(), 0L); + assertThat(table.io().newInputFile(writtenDv.location()).exists()).isTrue(); + + EqualityConvertCommitter committer = + new EqualityConvertCommitter( + DUMMY_TABLE_NAME, DUMMY_TASK_NAME, tableLoader(), "staging", SnapshotRef.MAIN_BRANCH) { + @Override + void commit(RowDelta rowDelta) { + throw new CommitStateUnknownException( + new RuntimeException("simulated unknown commit state")); + } + }; + + try (TwoInputStreamOperatorTestHarness harness = + new TwoInputStreamOperatorTestHarness<>(committer)) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + 888L, + mainSnapshotIdAtPlan, + doneTs - 1, + doneTs); + + harness.processElement1( + new StreamRecord<>( + new DVWriteResult(Lists.newArrayList(writtenDv), Lists.newArrayList()), doneTs)); + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + harness.processBothWatermarks(new Watermark(doneTs)); + + assertThat(harness.extractOutputValues()).hasSize(1); + + List> errors = + Lists.newArrayList(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)); + assertThat(errors).hasSize(1); + assertThat(errors.get(0).getValue()).isInstanceOf(CommitStateUnknownException.class); + + // Commit outcome unknown: the DV may be live, so it must not be deleted. + assertThat(table.io().newInputFile(writtenDv.location()).exists()).isTrue(); + } + } + + @Test + void removesRewrittenStagingDvOnSharedBranch() throws Exception { + // Shared branch: stagingBranch == targetBranch == main, so the writer's DVs are already on + // target. A staging DV folded into a merged conversion DV must be removed, else the data file + // would carry two DVs (V3 allows one per data file). + Table table = createTable(3, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile dataFile = getFirstDataFile(table); + + // Staging DV already committed on the shared target branch; the writer rewrites it by folding + // it into a merged DV for the same data file. + DeleteFile stagingDv = writeDV(table, dataFile.location(), 0L); + table.newRowDelta().addDeletes(stagingDv).commit(); + table.refresh(); + long mainSnapshotId = table.currentSnapshot().snapshotId(); + + DeleteFile mergedDv = writeDV(table, dataFile.location(), 0L); + + try (TwoInputStreamOperatorTestHarness harness = + sharedBranchHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(), + Lists.newArrayList(stagingDv), + 123L, + mainSnapshotId, + doneTs - 1, + doneTs); + + harness.processElement1( + new StreamRecord<>( + new DVWriteResult(Lists.newArrayList(mergedDv), Lists.newArrayList(stagingDv)), + doneTs)); + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + harness.processBothWatermarks(new Watermark(doneTs)); + + assertThat(harness.extractOutputValues()).hasSize(1); + + // Exactly one DV references the data file: the merged DV, with the rewritten staging DV + // removed rather than left as a second DV on the same data file. + table.refresh(); + List dvs = deletesForDataFile(table, dataFile.location()); + assertThat(dvs).hasSize(1); + assertThat(dvs.get(0).location()).isEqualTo(mergedDv.location()); + } + } + + private TwoInputStreamOperatorTestHarness + createHarness() throws Exception { + return new TwoInputStreamOperatorTestHarness<>( + new EqualityConvertCommitter( + DUMMY_TABLE_NAME, DUMMY_TASK_NAME, tableLoader(), "staging", SnapshotRef.MAIN_BRANCH)); + } + + private TwoInputStreamOperatorTestHarness + sharedBranchHarness() throws Exception { + return new TwoInputStreamOperatorTestHarness<>( + new EqualityConvertCommitter( + DUMMY_TABLE_NAME, + DUMMY_TASK_NAME, + tableLoader(), + SnapshotRef.MAIN_BRANCH, + SnapshotRef.MAIN_BRANCH)); + } + + private static List watermarks(TwoInputStreamOperatorTestHarness harness) { + return harness.getOutput().stream() + .filter(Watermark.class::isInstance) + .map(Watermark.class::cast) + .collect(Collectors.toList()); + } + + private static List deletesForDataFile(Table table, String dataFilePath) { + List deletes = Lists.newArrayList(); + for (ManifestFile manifest : table.currentSnapshot().deleteManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { + for (DeleteFile file : reader) { + if (dataFilePath.equals(file.referencedDataFile())) { + deletes.add(file.copy()); + } + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + return deletes; + } + + private static DeleteFile writeDV(Table table, String dataFilePath, long... positions) + throws IOException { + List> deletes = Lists.newArrayList(); + GenericRecord nested = GenericRecord.create(table.schema()); + for (long pos : positions) { + PositionDelete delete = PositionDelete.create(); + delete.set(dataFilePath, pos, nested); + deletes.add(delete); + } + + return FileHelpers.writePosDeleteFile(table, null, null, deletes, 3); + } + + private static DataFile getFirstDataFile(Table table) { + for (ManifestFile manifest : table.currentSnapshot().dataManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.read(manifest, table.io(), table.specs())) { + for (DataFile file : reader) { + return file.copy(); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + throw new IllegalStateException("No data files found"); + } +} diff --git a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertDVWriter.java b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertDVWriter.java new file mode 100644 index 000000000000..a0c2f3568baf --- /dev/null +++ b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertDVWriter.java @@ -0,0 +1,472 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.operator; + +import static org.apache.iceberg.flink.SimpleDataUtil.createRecord; +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.util.TwoInputStreamOperatorTestHarness; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.RowDelta; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.Table; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.junit.jupiter.api.Test; + +class TestEqualityConvertDVWriter extends OperatorTestBase { + + private static final byte[] EMPTY_PARTITION = new byte[0]; + + @Test + void writesDVFileForSinglePosition() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + String dataFilePath = getFirstDataFilePath(table); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time = System.currentTimeMillis(); + harness.processElement1( + new StreamRecord<>(new DVPosition(dataFilePath, 0, 0, EMPTY_PARTITION, 0L), time)); + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time)); + + harness.processBothWatermarks(new Watermark(time)); + + List output = harness.extractOutputValues(); + assertThat(output).hasSize(1); + assertThat(output.get(0).hasError()).isFalse(); + assertThat(output.get(0).dvFiles()).hasSize(1); + } + } + + @Test + void writesSingleDVFileForMultiplePositionsOnSameDataFile() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + insert(table, 3, "c"); + + String dataFilePath = getFirstDataFilePath(table); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time = System.currentTimeMillis(); + harness.processElement1( + new StreamRecord<>(new DVPosition(dataFilePath, 0, 0, EMPTY_PARTITION, 0L), time)); + harness.processElement1( + new StreamRecord<>(new DVPosition(dataFilePath, 1, 0, EMPTY_PARTITION, 0L), time)); + harness.processElement1( + new StreamRecord<>(new DVPosition(dataFilePath, 2, 0, EMPTY_PARTITION, 0L), time)); + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time)); + + harness.processBothWatermarks(new Watermark(time)); + + List output = harness.extractOutputValues(); + assertThat(output).hasSize(1); + assertThat(output.get(0).hasError()).isFalse(); + assertThat(output.get(0).dvFiles()).hasSize(1); + assertThat(output.get(0).dvFiles().get(0).recordCount()).isEqualTo(3); + assertThat(output.get(0).rewrittenDvFiles()).isEmpty(); + } + } + + @Test + void emptyPositionsProducesNoOutput() throws Exception { + createTableWithDelete(3); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time = System.currentTimeMillis(); + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time)); + + harness.processBothWatermarks(new Watermark(time)); + + assertThat(harness.extractOutputValues()).isEmpty(); + } + } + + @Test + void noOutputWithoutPlanResult() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + String dataFilePath = getFirstDataFilePath(table); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time = System.currentTimeMillis(); + harness.processElement1( + new StreamRecord<>(new DVPosition(dataFilePath, 0, 0, EMPTY_PARTITION, 0L), time)); + + harness.processBothWatermarks(new Watermark(time)); + + assertThat(harness.extractOutputValues()).isEmpty(); + } + } + + @Test + void writesDVFilesForMultipleDataFiles() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + List dataFilePaths = getDataFilePaths(table); + assertThat(dataFilePaths).hasSize(2); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time = System.currentTimeMillis(); + harness.processElement1( + new StreamRecord<>( + new DVPosition(dataFilePaths.get(0), 0, 0, EMPTY_PARTITION, 0L), time)); + harness.processElement1( + new StreamRecord<>( + new DVPosition(dataFilePaths.get(1), 0, 0, EMPTY_PARTITION, 0L), time)); + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time)); + + harness.processBothWatermarks(new Watermark(time)); + + List output = harness.extractOutputValues(); + assertThat(output).hasSize(1); + assertThat(output.get(0).dvFiles()).hasSize(2); + } + } + + @Test + void routesErrorToErrorStream() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + String dataFilePath = getFirstDataFilePath(table); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time = System.currentTimeMillis(); + harness.processElement1( + new StreamRecord<>(new DVPosition(dataFilePath, 0, 0, EMPTY_PARTITION, 0L), time)); + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time)); + + dropTable(); + + harness.processBothWatermarks(new Watermark(time)); + + assertThat(harness.extractOutputValues()).hasSize(1); + assertThat(harness.extractOutputValues().get(0).hasError()).isTrue(); + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).hasSize(1); + } + } + + @Test + void mergesStagingDVIntoWrittenDV() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + String dataFilePath = getFirstDataFilePath(table); + + // Staging DV for the data file at position 0, produced but not committed to the target branch. + DeleteFile stagingDV = writeStagingDV(dataFilePath, 0); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(), Lists.newArrayList(stagingDV), 1L, null, 0L, 0L); + + // New position 1 in the same file. Must merge it with the staged position 0. + harness.processElement1( + new StreamRecord<>(new DVPosition(dataFilePath, 1, 0, EMPTY_PARTITION, 0L), time)); + harness.processElement2(new StreamRecord<>(planResult, time)); + + harness.processBothWatermarks(new Watermark(time)); + + List output = harness.extractOutputValues(); + assertThat(output).hasSize(1); + assertThat(output.get(0).hasError()).isFalse(); + assertThat(output.get(0).dvFiles()).hasSize(1); + assertThat(output.get(0).dvFiles().get(0).recordCount()).isEqualTo(2); + assertThat(output.get(0).rewrittenDvFiles()).hasSize(1); + assertThat(output.get(0).rewrittenDvFiles().get(0).location()) + .isEqualTo(stagingDV.location()); + } + } + + @Test + void abortsWhenMainSnapshotChangedSincePlanning() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + String dataFilePath = getFirstDataFilePath(table); + long plannedSnapshotId = table.currentSnapshot().snapshotId(); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(), Lists.newArrayList(), 1L, plannedSnapshotId, 0L, 0L); + + harness.processElement1( + new StreamRecord<>(new DVPosition(dataFilePath, 0, 0, EMPTY_PARTITION, 0L), time)); + harness.processElement2(new StreamRecord<>(planResult, time)); + + // Advance the target branch after planning. The writer must refuse to write stale DVs. + insert(table, 2, "b"); + + harness.processBothWatermarks(new Watermark(time)); + + List output = harness.extractOutputValues(); + assertThat(output).hasSize(1); + assertThat(output.get(0).hasError()).isTrue(); + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).hasSize(1); + } + } + + @Test + void stateResetBetweenBatches() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + String dataFilePath = getFirstDataFilePath(table); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time1 = System.currentTimeMillis(); + harness.processElement1( + new StreamRecord<>(new DVPosition(dataFilePath, 0, 0, EMPTY_PARTITION, 0L), time1)); + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time1)); + harness.processBothWatermarks(new Watermark(time1)); + + assertThat(harness.extractOutputValues()).hasSize(1); + + // Second batch with no positions, plan only. + long time2 = time1 + 1; + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time2)); + harness.processBothWatermarks(new Watermark(time2)); + + // Cumulative output: still hasSize(1) since the empty batch produced nothing. + assertThat(harness.extractOutputValues()).hasSize(1); + } + } + + @Test + void abortsOnUpstreamAbortPosition() throws Exception { + createTableWithDelete(3); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time = System.currentTimeMillis(); + harness.processElement1(new StreamRecord<>(DVPosition.ABORT, time)); + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time)); + harness.processBothWatermarks(new Watermark(time)); + + List output = harness.extractOutputValues(); + assertThat(output).hasSize(1); + assertThat(output.get(0).hasError()).isTrue(); + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).isNull(); + } + } + + @Test + void doesNotRetainStateAcrossCycles() throws Exception { + Table table = createTableWithDelete(3); + int cycles = 5; + for (int i = 0; i < cycles; i++) { + insert(table, i, "v" + i); + } + + List dataFilePaths = getDataFilePaths(table); + assertThat(dataFilePaths).hasSize(cycles); + + EqualityConvertDVWriter resolver = newResolver(); + try (TwoInputStreamOperatorTestHarness harness = + new TwoInputStreamOperatorTestHarness<>(resolver)) { + harness.open(); + + long time = System.currentTimeMillis(); + for (int i = 0; i < cycles; i++) { + // Each cycle resolves a position in a distinct data file, growing the set of files seen. + harness.processElement1( + new StreamRecord<>( + new DVPosition(dataFilePaths.get(i), 0, 0, EMPTY_PARTITION, 0L), time + i)); + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time + i)); + harness.processBothWatermarks(new Watermark(time + i)); + + // No cross-cycle state: the buffer clears every cycle regardless of how many distinct data + // files have been processed. + assertThat(resolver.retainedStateSize()).isZero(); + assertThat(harness.extractOutputValues()).hasSize(i + 1); + } + } + } + + @Test + void prunesDeleteManifestsByPartition() throws Exception { + Table table = createPartitionedTableWithDelete(3); + insertPartitioned(table, Lists.newArrayList(createRecord(1, "a")), "a"); + DataFile fileA = dataFile(table, "a"); + insertPartitioned(table, Lists.newArrayList(createRecord(2, "b")), "b"); + DataFile fileB = dataFile(table, "b"); + + // Commit one DV per partition: two separate delete manifests, summaries {a} and {b}. + commitDV(table, fileA); + commitDV(table, fileB); + assertThat(table.currentSnapshot().deleteManifests(table.io())).hasSize(2); + + // A new cycle resolves another position in partition a's file. Only partition a's delete + // manifest overlaps; partition b's manifest must be pruned. + EqualityConvertDVWriter resolver = newResolver(); + try (TwoInputStreamOperatorTestHarness harness = + new TwoInputStreamOperatorTestHarness<>(resolver)) { + harness.open(); + + long time = System.currentTimeMillis(); + harness.processElement1(new StreamRecord<>(dvPosition(table, fileA, 0), time)); + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time)); + harness.processBothWatermarks(new Watermark(time)); + + List output = harness.extractOutputValues(); + assertThat(output).hasSize(1); + // Folded partition a's existing DV: the matching manifest was read. + assertThat(output.get(0).rewrittenDvFiles()).hasSize(1); + assertThat(output.get(0).dvFiles()).hasSize(1); + // Partition b's manifest was skipped: only one manifest read this cycle. + assertThat(resolver.manifestsReadLastCycle()).isEqualTo(1); + } + } + + private void commitDV(Table table, DataFile dataFile) throws Exception { + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time = System.currentTimeMillis(); + harness.processElement1(new StreamRecord<>(dvPosition(table, dataFile, 0), time)); + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time)); + harness.processBothWatermarks(new Watermark(time)); + + RowDelta rowDelta = table.newRowDelta(); + harness.extractOutputValues().get(0).dvFiles().forEach(rowDelta::addDeletes); + rowDelta.commit(); + table.refresh(); + } + } + + private static DVPosition dvPosition(Table table, DataFile dataFile, long position) { + PartitionSpec spec = table.specs().get(dataFile.specId()); + byte[] partition = + new StructLikeSerializer().encodePartition(dataFile.partition(), spec.partitionType()); + return new DVPosition(dataFile.location(), position, dataFile.specId(), partition, 0L); + } + + private static DataFile dataFile(Table table, String partitionValue) { + for (ManifestFile manifest : table.currentSnapshot().dataManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.read(manifest, table.io(), table.specs())) { + for (DataFile file : reader) { + if (partitionValue.equals(String.valueOf(file.partition().get(0, Object.class)))) { + return file; + } + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + throw new IllegalStateException("No data file for partition: " + partitionValue); + } + + private DeleteFile writeStagingDV(String dataFilePath, long position) throws Exception { + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time = System.currentTimeMillis(); + harness.processElement1( + new StreamRecord<>(new DVPosition(dataFilePath, position, 0, EMPTY_PARTITION, 0L), time)); + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time)); + harness.processBothWatermarks(new Watermark(time)); + + return harness.extractOutputValues().get(0).dvFiles().get(0); + } + } + + private EqualityConvertDVWriter newResolver() { + return new EqualityConvertDVWriter( + DUMMY_TABLE_NAME, DUMMY_TASK_NAME, tableLoader(), SnapshotRef.MAIN_BRANCH); + } + + private TwoInputStreamOperatorTestHarness + createHarness() throws Exception { + return new TwoInputStreamOperatorTestHarness<>(newResolver()); + } + + private static EqualityConvertPlan emptyEqualityConvertPlan() { + return new EqualityConvertPlan(Lists.newArrayList(), Lists.newArrayList(), 1L, null, 0L, 0L); + } + + private static String getFirstDataFilePath(Table table) { + return getDataFilePaths(table).get(0); + } + + private static List getDataFilePaths(Table table) { + List paths = Lists.newArrayList(); + for (ManifestFile manifest : table.currentSnapshot().dataManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.read(manifest, table.io(), table.specs())) { + for (DataFile file : reader) { + paths.add(file.location()); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + return paths; + } +} diff --git a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertPlanner.java b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertPlanner.java new file mode 100644 index 000000000000..1ee5c5d0d5fd --- /dev/null +++ b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertPlanner.java @@ -0,0 +1,1155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.operator; + +import static org.apache.iceberg.flink.SimpleDataUtil.createRecord; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.flink.runtime.checkpoint.OperatorSubtaskState; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.Files; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; +import org.apache.iceberg.PartitionData; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.RewriteFiles; +import org.apache.iceberg.Schema; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.Table; +import org.apache.iceberg.data.FileHelpers; +import org.apache.iceberg.data.GenericAppenderHelper; +import org.apache.iceberg.data.GenericRecord; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.deletes.PositionDelete; +import org.apache.iceberg.flink.maintenance.api.Trigger; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class TestEqualityConvertPlanner extends OperatorTestBase { + + private static final String STAGING_BRANCH = "__flink_staging_test"; + + @TempDir private Path tempDir; + + @Test + void bootstrapsIndexFromBuilderEqualityFieldIds() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + // Staging branch exists but contains no eq-delete files yet. The planner still populates the + // worker index from main using the builder-configured equality field set so the index is ready + // when the first delete arrives. + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH, Lists.newArrayList(1, 2))) { + harness.open(); + sendTrigger(harness); + + List commands = harness.extractOutputValues(); + assertThat(countDataFileTasks(commands)).isEqualTo(2); + assertThat(countEqDeleteTasks(commands)).isZero(); + + assertThat(harness.getSideOutput(EqualityConvertPlanner.METADATA_STREAM)).hasSize(1); + } + } + + @Test + void failsWhenStagingEqDeleteFieldIdsMismatchBuilder() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Builder configured for [1, 2], but writer produces an eq delete with [1] only. + DeleteFile mismatched = writeIdOnlyEqualityDelete(table, 1); + table.newRowDelta().addDeletes(mismatched).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH, Lists.newArrayList(1, 2))) { + harness.open(); + sendTrigger(harness); + + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).hasSize(1); + } + } + + @Test + void failsWhenStagingEqDeleteSpecPartitionsByNonEqualityColumn() throws Exception { + Table table = createPartitionedTableWithDelete(3); + insertPartitioned(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete = writeIdOnlyPartitionedEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH, Lists.newArrayList(1))) { + harness.open(); + sendTrigger(harness); + + List> errOutput = + Lists.newArrayList(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)); + + assertThat(errOutput).hasSize(1); + assertThat(errOutput.get(0).getValue()) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Partition columns must be a subset of the equality fields."); + } + } + + @Test + void doesNotDuplicateNewDataFilesWhenStagingEqualsTarget() throws Exception { + // When stagingBranch == targetBranch, the writer commits new data files directly to main. + // Bootstrap scans the main snapshot (which already includes those files) and indexes them. + // The planner must NOT also emit the staging-data phase for the same files — that would + // duplicate ADD_DATA_ROW commands for the same (PK, file, position) in the worker's index. + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + // Writer commits (new-data id=3) + (eq-delete id=1) in one RowDelta directly to main. + DataFile newDataFile = writeDataFile(table, createRecord(3, "c")); + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(newDataFile).addDeletes(eqDelete).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(SnapshotRef.MAIN_BRANCH)) { + harness.open(); + sendTrigger(harness); + + List commands = harness.extractOutputValues(); + + // Bootstrap from main emits one data-file command per file (id=1, id=2, id=3) = 3. + // Without the same-branch guard, emitSnapshotDataPhase would also emit newDataFile → 4. + assertThat(countDataFileTasks(commands)).isEqualTo(3); + assertThat(countEqDeleteTasks(commands)).isEqualTo(1); + + long newDataFileCount = + commands.stream() + .filter(c -> c.task() instanceof FileScanTask) + .filter(c -> c.task().file().location().equals(newDataFile.location())) + .count(); + assertThat(newDataFileCount).isEqualTo(1); + } + } + + @Test + void emitsReadCommandsForEqualityDeletes() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + insert(table, 3, "c"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + List commands = harness.extractOutputValues(); + // 3 DATA_FILE (from main) + 1 EQ_DELETE_FILE + assertThat(countDataFileTasks(commands)).isEqualTo(3); + assertThat(countEqDeleteTasks(commands)).isEqualTo(1); + assertThat(planner(harness).processedStagingSnapshotNum()).isEqualTo(1); + assertThat(planner(harness).processedEqDeleteFileNum()).isEqualTo(1); + + List> metadata = + Lists.newArrayList(harness.getSideOutput(EqualityConvertPlanner.METADATA_STREAM)); + assertThat(metadata).hasSize(1); + assertThat(metadata.get(0).getValue().dataFiles()).isEmpty(); + } + } + + @Test + void emitsDataFileFromInsertOnlyStagingSnapshot() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S1: eq delete targeting main data + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + long s1SnapshotId = table.snapshot(STAGING_BRANCH).snapshotId(); + + // S2: insert-only (no eq deletes), but its data must still be indexed for the configured + // equality fields + DataFile insertS2 = writeDataFile(table, createRecord(2, "b")); + table.newAppend().appendFile(insertS2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + // Trigger 1: processes S1 (eq delete) + sendTrigger(harness); + int afterFirst = harness.extractOutputValues().size(); + assertThat(countEqDeleteTasks(harness.extractOutputValues())).isEqualTo(1); + + // Record S1's conversion on main, so the planner walks past S1 before processing S2. + simulateConvertCommit(table, s1SnapshotId); + + // Trigger 2: processes S2 (insert-only). Must emit its data file so the configured equality + // fields stay indexed. + sendTrigger(harness); + List allCommands = harness.extractOutputValues(); + List trigger2Commands = allCommands.subList(afterFirst, allCommands.size()); + + Set dataFilePaths = + trigger2Commands.stream() + .filter(c -> c.task() instanceof FileScanTask) + .map(TestEqualityConvertPlanner::filePath) + .collect(Collectors.toSet()); + + assertThat(dataFilePaths).contains(insertS2.location()); + } + } + + @Test + void includesNewDataFilesFromStaging() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DataFile newDataFile = writeDataFile(table, createRecord(2, "b")); + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(newDataFile).addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + List commands = harness.extractOutputValues(); + // 1 main data file + 1 eq delete + 1 staging data file + assertThat(countDataFileTasks(commands)).isEqualTo(2); + assertThat(countEqDeleteTasks(commands)).isEqualTo(1); + + List> metadata = + Lists.newArrayList(harness.getSideOutput(EqualityConvertPlanner.METADATA_STREAM)); + assertThat(metadata).hasSize(1); + assertThat(metadata.get(0).getValue().dataFiles()).hasSize(1); + } + } + + @Test + void findsIntersectionWhenMainAdvancedAfterStagingFork() throws Exception { + Table table = createTableWithDelete(3); + // Pre-fork main: two snapshots. + insert(table, 1, "a"); + insert(table, 2, "b"); + + // Fork staging from current main head. + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Main advances with two more snapshots after the fork. These are on main's + // lineage but not on staging's. + insert(table, 3, "c"); + insert(table, 4, "d"); + + // Staging gets one new snapshot containing an equality delete. + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + // Planner identifies the fork point and processes only the staging-only + // commit (the eq delete), and emits all four current main data files + // for the index refresh. + List commands = harness.extractOutputValues(); + assertThat(countEqDeleteTasks(commands)).isEqualTo(1); + assertThat(countDataFileTasks(commands)).isEqualTo(4); + } + } + + @Test + void skipsAlreadyProcessedStagingSnapshot() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + long stagingSnapshotId = table.snapshot(STAGING_BRANCH).snapshotId(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + sendTrigger(harness); + int firstTriggerCount = harness.extractOutputValues().size(); + assertThat(firstTriggerCount).isGreaterThan(0); + assertThat(planner(harness).skippedNoOpCycles()).isZero(); + + // Simulate the committer committing to main with the staging snapshot property + // (the planner promotes pending only after confirming the commit landed on main). + simulateConvertCommit(table, stagingSnapshotId); + + sendTrigger(harness); + assertThat(harness.extractOutputValues()).hasSize(firstTriggerCount); + assertThat(planner(harness).skippedNoOpCycles()).isEqualTo(1); + } + } + + @Test + void noOutputWhenStagingBranchEmpty() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + // Bootstrap from main for the configured field set: 1 main DATA_FILE; no-op metadata still + // emitted because there's nothing on staging to convert. + assertThat(countDataFileTasks(harness.extractOutputValues())).isEqualTo(1); + assertThat(countEqDeleteTasks(harness.extractOutputValues())).isZero(); + assertThat(harness.getSideOutput(EqualityConvertPlanner.METADATA_STREAM)).hasSize(1); + assertThat(planner(harness).skippedNoOpCycles()).isEqualTo(1); + assertThat(planner(harness).reindexCount()).isZero(); + } + } + + @Test + void propagatesStagingOnlyPositionalDeletes() throws Exception { + Table table = createTableWithDelete(3); + DataFile mainData = writeDataFile(table, createRecord(1, "a")); + table.newAppend().appendFile(mainData).commit(); + table.refresh(); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging snapshot has only a DV (no eq deletes, no new data files). Without the + // planner forwarding stagingDVFiles in this case, the committer would never + // see the DV and it would be lost. + DeleteFile stagingDV = writeStagingDV(table, mainData.location(), 0L); + table.newRowDelta().addDeletes(stagingDV).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + List commands = harness.extractOutputValues(); + assertThat(countDataFileTasks(commands)).isEqualTo(1); + assertThat(countEqDeleteTasks(commands)).isZero(); + + List> metadata = + Lists.newArrayList(harness.getSideOutput(EqualityConvertPlanner.METADATA_STREAM)); + assertThat(metadata).hasSize(1); + EqualityConvertPlan result = metadata.get(0).getValue(); + assertThat(result.dataFiles()).isEmpty(); + assertThat(result.stagingDVFiles()).hasSize(1); + assertThat(result.stagingDVFiles().get(0).location()).isEqualTo(stagingDV.location()); + } + } + + @Test + void phaseTimestampsAreMonotonicallyIncreasing() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging: eq delete + new data file (triggers 3 phases: main data, eq delete, staging data) + DataFile newDataFile = writeDataFile(table, createRecord(2, "b")); + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(newDataFile).addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + long triggerTs = 100L; + sendTrigger(harness, triggerTs); + + List output = Lists.newArrayList(harness.getOutput()); + assertThat(((StreamRecord) output.get(0)).getValue().task()) + .isInstanceOf(FileScanTask.class); + assertThat(output.get(1)).isEqualTo(new Watermark(triggerTs)); + + assertThat(((StreamRecord) output.get(2)).getValue().task()) + .isInstanceOf(EqualityDeleteFileScanTask.class); + assertThat(output.get(3)).isEqualTo(new Watermark(triggerTs + 1)); + + assertThat(((StreamRecord) output.get(4)).getValue().task()) + .isInstanceOf(FileScanTask.class); + assertThat(output.get(5)).isEqualTo(new Watermark(triggerTs + 2)); + } + } + + @Test + void routesExceptionToErrorStream() throws Exception { + createTableWithDelete(3); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + dropTable(); + + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).isNull(); + sendTrigger(harness); + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).hasSize(1); + } + } + + @Test + void failsOnRemovedDataFilesOnStagingBranch() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + Set oldDataFiles = Sets.newHashSet(); + for (ManifestFile manifest : table.currentSnapshot().dataManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.read(manifest, table.io(), table.specs())) { + for (DataFile df : reader) { + oldDataFiles.add(df.copy()); + } + } + } + + assertThat(oldDataFiles).hasSize(2); + + // Rewrite on the staging branch (not main). Equality delete conversion does not support + // rewrites on staging; the planner must fail the cycle instead of silently dropping the + // removed files. + DataFile compactedFile = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(1, "a"), createRecord(2, "b"))); + RewriteFiles rewrite = table.newRewrite(); + for (DataFile old : oldDataFiles) { + rewrite.deleteFile(old); + } + + rewrite.addFile(compactedFile); + rewrite.toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).hasSize(1); + // Bootstrap runs before processCycle's failure, so the main data commands are already on + // the wire. Only the cycle itself fails. + assertThat(countDataFileTasks(harness.extractOutputValues())).isEqualTo(2); + assertThat(countEqDeleteTasks(harness.extractOutputValues())).isZero(); + } + } + + @Test + void reEmitsMainDataAfterCompaction() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + sendTrigger(harness); + int firstTriggerCount = harness.extractOutputValues().size(); + // 2 DATA_FILE (main) + 1 EQ_DELETE_FILE + assertThat(firstTriggerCount).isEqualTo(3); + + // Compact data files on main: rewrite 2 files into 1 + Set oldDataFiles = Sets.newHashSet(); + for (ManifestFile manifest : table.currentSnapshot().dataManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.read(manifest, table.io(), table.specs())) { + for (DataFile df : reader) { + oldDataFiles.add(df.copy()); + } + } + } + + assertThat(oldDataFiles).hasSize(2); + + DataFile compactedFile = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(1, "a"), createRecord(2, "b"))); + RewriteFiles rewrite = table.newRewrite(); + for (DataFile old : oldDataFiles) { + rewrite.deleteFile(old); + } + + rewrite.addFile(compactedFile); + rewrite.commit(); + table.refresh(); + + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + sendTrigger(harness); + List allCommands = harness.extractOutputValues(); + List trigger2Commands = + allCommands.subList(firstTriggerCount, allCommands.size()); + + // 1 DATA_FILE (compacted main) + 1 EQ_DELETE_FILE + assertThat(countDataFileTasks(trigger2Commands)).isEqualTo(1); + assertThat(countEqDeleteTasks(trigger2Commands)).isEqualTo(1); + + // DATA_FILE should reference the compacted file + List dataCmds = + trigger2Commands.stream() + .filter(c -> c.task() instanceof FileScanTask) + .collect(Collectors.toList()); + for (ReadCommand cmd : dataCmds) { + assertThat(cmd.task().file().location()).isEqualTo(compactedFile.location()); + } + } + } + + @Test + void refreshesIndexBeforeFirstCycleProcessesEqDeletes() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Stage MULTIPLE eq-delete snapshots before the first trigger arrives. The first trigger + // bootstraps the index from main for the configured equality fields before processing any eq + // deletes, then processes one snapshot per trigger (oldest first). + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + long s2SnapshotId = table.snapshot(STAGING_BRANCH).snapshotId(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + sendTrigger(harness); + int afterFirst = harness.extractOutputValues().size(); + + // First trigger processes the OLDER staging snapshot (eqDelete1). Bootstrap built the index + // from main once: 2 main DATA_FILE + 1 EQ_DELETE_FILE = 3 commands. + assertThat(afterFirst).isEqualTo(3); + assertThat(countDataFileTasks(harness.extractOutputValues())).isEqualTo(2); + assertThat(countEqDeleteTasks(harness.extractOutputValues())).isEqualTo(1); + + simulateConvertCommit(table, table.snapshot(STAGING_BRANCH).parentId()); + + // Second trigger processes the newer staging snapshot. The index is already up-to-date. + sendTrigger(harness); + List allCommands = harness.extractOutputValues(); + List trigger2Commands = allCommands.subList(afterFirst, allCommands.size()); + assertThat(countDataFileTasks(trigger2Commands)).isZero(); + assertThat(countEqDeleteTasks(trigger2Commands)).isEqualTo(1); + + simulateConvertCommit(table, s2SnapshotId); + } + } + + @Test + void noMainReEmitWhenUnchanged() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + sendTrigger(harness); + int firstTriggerCount = harness.extractOutputValues().size(); + // 2 DATA_FILE (main) + 1 EQ_DELETE_FILE + assertThat(firstTriggerCount).isEqualTo(3); + assertThat(harness.getSideOutput(EqualityConvertPlanner.METADATA_STREAM)).hasSize(1); + + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + sendTrigger(harness); + List allCommands = harness.extractOutputValues(); + List trigger2Commands = + allCommands.subList(firstTriggerCount, allCommands.size()); + + // Only 1 EQ_DELETE_FILE, no main re-emission + assertThat(countDataFileTasks(trigger2Commands)).isEqualTo(0); + assertThat(countEqDeleteTasks(trigger2Commands)).isEqualTo(1); + + assertThat(harness.getSideOutput(EqualityConvertPlanner.METADATA_STREAM)).hasSize(2); + } + } + + @Test + void noMainReEmitAfterOwnCommit() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + long indexedStagingSnapshotId = table.snapshot(STAGING_BRANCH).snapshotId(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + sendTrigger(harness); + int firstTriggerCount = harness.extractOutputValues().size(); + + // Record the converter's own commit (COMMITTED_STAGING_SNAPSHOT property) for the staging + // snapshot the first trigger processed. The next trigger reads the property off main and + // advances lastStagingSnapshotId. No reindex should be triggered. + simulateConvertCommit(table, indexedStagingSnapshotId); + + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + sendTrigger(harness); + List allCommands = harness.extractOutputValues(); + List trigger2Commands = + allCommands.subList(firstTriggerCount, allCommands.size()); + + // Own commits don't trigger index rebuild. + assertThat(countDataFileTasks(trigger2Commands)).isEqualTo(0); + assertThat(countEqDeleteTasks(trigger2Commands)).isEqualTo(1); + } + } + + @Test + void reIndexesAfterExternalCommit() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + sendTrigger(harness); + int firstTriggerCount = harness.extractOutputValues().size(); + // 1 DATA_FILE (main) + 1 EQ_DELETE_FILE + assertThat(firstTriggerCount).isEqualTo(2); + assertThat(planner(harness).reindexCount()).isZero(); + assertThat(planner(harness).processedStagingSnapshotNum()).isEqualTo(1); + assertThat(planner(harness).processedEqDeleteFileNum()).isEqualTo(1); + + // External commit: no COMMITTED_STAGING_SNAPSHOT_PROPERTY + DataFile externalFile = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(10, "z"))); + table.newAppend().appendFile(externalFile).commit(); + table.refresh(); + + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + sendTrigger(harness); + List allCommands = harness.extractOutputValues(); + List trigger2Commands = + allCommands.subList(firstTriggerCount, allCommands.size()); + + // External commit triggers re-index: main data files are re-emitted + assertThat(countDataFileTasks(trigger2Commands)).isGreaterThan(0); + assertThat(countEqDeleteTasks(trigger2Commands)).isEqualTo(1); + assertThat(planner(harness).reindexCount()).isEqualTo(1); + assertThat(planner(harness).processedStagingSnapshotNum()).isEqualTo(2); + assertThat(planner(harness).processedEqDeleteFileNum()).isEqualTo(2); + } + } + + @Test + void emitsClearIndexBroadcastOnReindex() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + // Trigger 1 bootstraps the index. Bootstrap does not broadcast CLEAR_INDEX. + sendTrigger(harness); + assertThat(harness.getSideOutput(EqualityConvertPlanner.CLEAR_BROADCAST_STREAM)) + .isNullOrEmpty(); + + // External commit advances main. The next trigger reindexes and must broadcast CLEAR_INDEX + // so workers evict ghost keys (e.g. a PK whose data file was removed by external CoW). + DataFile externalFile = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(10, "z"))); + table.newAppend().appendFile(externalFile).commit(); + table.refresh(); + long mainAfterExternal = table.snapshot(SnapshotRef.MAIN_BRANCH).snapshotId(); + long mainSeqAfterExternal = table.snapshot(SnapshotRef.MAIN_BRANCH).sequenceNumber(); + + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + sendTrigger(harness); + + List clears = + harness.getSideOutput(EqualityConvertPlanner.CLEAR_BROADCAST_STREAM).stream() + .map(StreamRecord::getValue) + .collect(Collectors.toList()); + assertThat(clears).hasSize(1); + assertThat(clears.get(0).type()).isEqualTo(IndexCommand.Type.CLEAR_INDEX); + assertThat(clears.get(0).mainSnapshotId()).isEqualTo(mainAfterExternal); + assertThat(clears.get(0).mainSequenceNumber()).isEqualTo(mainSeqAfterExternal); + } + } + + @Test + void detectsMainBranchChangeWithoutNewStagingSnapshots() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + // Trigger 1: build the index and process the existing eq delete. + sendTrigger(harness); + int afterFirst = harness.extractOutputValues().size(); + assertThat(afterFirst).isGreaterThan(0); + + // External commit on main (no COMMITTED_STAGING_SNAPSHOT_PROPERTY). + DataFile externalFile = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(10, "z"))); + table.newAppend().appendFile(externalFile).commit(); + table.refresh(); + + // Trigger 2: nothing new on staging, but the planner must still notice the + // external main change and immediately re-emit main data for index rebuild. + sendTrigger(harness); + List allAfterTrigger2 = harness.extractOutputValues(); + List trigger2Commands = + allAfterTrigger2.subList(afterFirst, allAfterTrigger2.size()); + assertThat(countDataFileTasks(trigger2Commands)).isGreaterThan(0); + + // A subsequent staging eq delete should NOT re-emit main data because the index + // was already rebuilt in trigger 2. + int afterSecond = allAfterTrigger2.size(); + DeleteFile eqDelete2 = writeEqualityDelete(table, 10, "z"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + sendTrigger(harness); + List all = harness.extractOutputValues(); + List trigger3 = all.subList(afterSecond, all.size()); + assertThat(countDataFileTasks(trigger3)).isEqualTo(0); + assertThat(countEqDeleteTasks(trigger3)).isEqualTo(1); + } + } + + @Test + void stateRestoredAfterCheckpoint() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + long stagingSnapshotId = table.snapshot(STAGING_BRANCH).snapshotId(); + + OperatorSubtaskState state; + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + // First trigger bootstraps the index and processes the staging snapshot. + sendTrigger(harness); + int commandCount = harness.extractOutputValues().size(); + assertThat(commandCount).isGreaterThan(0); + + // Record S1's conversion on main (committer marker) so the planner advances + // lastStagingSnapshotId on the second trigger. + simulateConvertCommit(table, stagingSnapshotId); + + // Second trigger reads the marker and advances lastStagingSnapshotId. + sendTrigger(harness); + assertThat(harness.extractOutputValues()).hasSize(commandCount); + + state = harness.snapshot(1, System.currentTimeMillis()); + } + + // Restore from checkpoint: the restored index is not re-indexed because its state matches the + // snapshot COMMITTED_STAGING_SNAPSHOT_PROPERTY property. + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.initializeState(state); + harness.open(); + + sendTrigger(harness); + assertThat(harness.extractOutputValues()).isEmpty(); + } + } + + @Test + void failsOnV2PosDeleteOnStagingBranch() throws Exception { + Table table = createTableWithDelete(2); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Flink-written staging branches are V3 and only produce DVs for deletes. A V2 positional + // delete on staging is a writer-side misconfiguration and should fail the cycle rather than + // be silently absorbed. + DataFile dataFile = writeDataFile(table, createRecord(2, "b")); + DeleteFile posDelete = writePosDeleteFile(table, dataFile.location(), 0); + table.newRowDelta().addRows(dataFile).addDeletes(posDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).hasSize(1); + } + } + + @Test + void skipsCommittedSnapshotAfterCommitLandsButCheckpointDoesNot() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + long stagingSnapshotId = table.snapshot(STAGING_BRANCH).snapshotId(); + + // Capture the planner's checkpoint before any cycle has run. + OperatorSubtaskState state; + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + state = harness.snapshot(1, System.currentTimeMillis()); + } + + // The staging commit landed on main but lastStagingSnapshotId was not checkpointed. + simulateConvertCommit(table, stagingSnapshotId); + + // On restore, the planner walks main, finds the COMMITTED_STAGING_SNAPSHOT_PROPERTY, + // advances lastStagingSnapshotId, and skips re-emitting the already-committed snapshot. + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.initializeState(state); + harness.open(); + + sendTrigger(harness); + + List commands = harness.extractOutputValues(); + assertThat(countEqDeleteTasks(commands)).isEqualTo(0); + // Bootstrap from main creates the index from the main data files on the first trigger even + // though the staging snapshot itself is already committed and skipped. Main now has the + // original insert plus simulateConvertCommit's marker file = 2 data files. + assertThat(countDataFileTasks(commands)).isEqualTo(2); + } + } + + @Test + void failsWhenCommitMarkerDisappears() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + long startSnapshotId = table.currentSnapshot().snapshotId(); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + long stagingSnapshotId = table.snapshot(STAGING_BRANCH).snapshotId(); + + simulateConvertCommit(table, stagingSnapshotId); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + table.manageSnapshots().rollbackTo(startSnapshotId).commit(); + table.refresh(); + + sendTrigger(harness); + + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).hasSize(1); + assertThat( + harness + .getSideOutput(TaskResultAggregator.ERROR_STREAM) + .poll() + .getValue() + .getMessage()) + .contains("No COMMITTED_STAGING_SNAPSHOT marker reachable"); + } + } + + @Test + void failsOnEqFieldIdsChangeAcrossRestart() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + OperatorSubtaskState state; + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH, Lists.newArrayList(1, 2))) { + harness.open(); + state = harness.snapshot(1, System.currentTimeMillis()); + } + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH, Lists.newArrayList(1))) { + assertThatThrownBy(() -> harness.initializeState(state)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Equality field IDs changed across restart"); + } + } + + private OneInputStreamOperatorTestHarness createHarness( + String stagingBranch) throws Exception { + // Default matches the [1, 2] equality-field-id list produced by writeEqualityDelete. + return createHarness(stagingBranch, Lists.newArrayList(1, 2)); + } + + private OneInputStreamOperatorTestHarness createHarness( + String stagingBranch, List eqFieldIds) throws Exception { + return new OneInputStreamOperatorTestHarness<>( + new EqualityConvertPlanner( + DUMMY_TABLE_NAME, + DUMMY_TASK_NAME, + tableLoader(), + stagingBranch, + SnapshotRef.MAIN_BRANCH, + Sets.newHashSet(eqFieldIds))); + } + + private static void sendTrigger(OneInputStreamOperatorTestHarness harness) + throws Exception { + sendTrigger(harness, System.currentTimeMillis()); + } + + private static void sendTrigger(OneInputStreamOperatorTestHarness harness, long time) + throws Exception { + harness.processElement(new StreamRecord<>(Trigger.create(time, 0), time)); + } + + private DataFile writeDataFile(Table table, Record record) throws IOException { + return new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(record)); + } + + private DeleteFile writeEqualityDelete(Table table, Integer id, String data) throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + return FileHelpers.writeDeleteFile( + table, + Files.localOutput(file), + new PartitionData(PartitionSpec.unpartitioned().partitionType()), + Lists.newArrayList(createRecord(id, data)), + table.schema()); + } + + private DeleteFile writeIdOnlyEqualityDelete(Table table, int id) throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + Schema idOnly = table.schema().select("id"); + Record record = GenericRecord.create(idOnly); + record.setField("id", id); + return FileHelpers.writeDeleteFile( + table, + Files.localOutput(file), + new PartitionData(PartitionSpec.unpartitioned().partitionType()), + Lists.newArrayList(record), + idOnly); + } + + private DeleteFile writeIdOnlyPartitionedEqualityDelete(Table table, int id, String data) + throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + Schema idOnly = table.schema().select("id"); + Record record = GenericRecord.create(idOnly); + record.setField("id", id); + PartitionData partition = new PartitionData(table.spec().partitionType()); + partition.set(0, data); + return FileHelpers.writeDeleteFile( + table, Files.localOutput(file), partition, Lists.newArrayList(record), idOnly); + } + + private DeleteFile writeStagingDV(Table table, String dataFilePath, long position) + throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + + PositionDelete delete = PositionDelete.create(); + delete.set(dataFilePath, position, null); + return FileHelpers.writePosDeleteFile( + table, + Files.localOutput(file), + new PartitionData(PartitionSpec.unpartitioned().partitionType()), + Lists.newArrayList(delete), + 3); + } + + private static long countDataFileTasks(List commands) { + return commands.stream().filter(c -> c.task() instanceof FileScanTask).count(); + } + + private static long countEqDeleteTasks(List commands) { + return commands.stream().filter(TestEqualityConvertPlanner::isEqDelete).count(); + } + + private static EqualityConvertPlanner planner( + OneInputStreamOperatorTestHarness harness) { + return (EqualityConvertPlanner) harness.getOperator(); + } + + private static boolean isEqDelete(ReadCommand cmd) { + return cmd.task() instanceof EqualityDeleteFileScanTask; + } + + private static String filePath(ReadCommand cmd) { + return cmd.task().file().location(); + } + + private void simulateConvertCommit(Table table, long stagingSnapshotId) throws IOException { + DataFile dummy = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(-1, "marker" + stagingSnapshotId))); + table + .newRowDelta() + .addRows(dummy) + .set( + EqualityConvertCommitter.COMMITTED_STAGING_SNAPSHOT_PROPERTY, + String.valueOf(stagingSnapshotId)) + .commit(); + table.refresh(); + } +} diff --git a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceSql.java b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceSql.java index 0cdaf8371cbd..6cfc18868af0 100644 --- a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceSql.java +++ b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceSql.java @@ -179,7 +179,10 @@ public void testReadFlinkDynamicTable() throws Exception { List expected = generateExpectedRecords(false); SqlHelpers.sql( getTableEnv(), - "create table `default_catalog`.`default_database`.flink_table LIKE iceberg_catalog.`default`.%s", + "create table `default_catalog`.`default_database`.flink_table " + + "WITH ('catalog-type'='hadoop', 'warehouse'='%s') " + + "LIKE iceberg_catalog.`default`.%s", + CATALOG_EXTENSION.warehouse(), TestFixtures.TABLE); // Read from table in flink catalog @@ -199,8 +202,11 @@ public void testWatermarkInvalidConfig() { getStreamingTableEnv(), "CREATE TABLE %s " + "(eventTS AS CAST(t1 AS TIMESTAMP(3)), " - + "WATERMARK FOR eventTS AS SOURCE_WATERMARK()) LIKE iceberg_catalog.`default`.%s", + + "WATERMARK FOR eventTS AS SOURCE_WATERMARK()) " + + "WITH ('catalog-type'='hadoop', 'warehouse'='%s') " + + "LIKE iceberg_catalog.`default`.%s", flinkTable, + CATALOG_EXTENSION.warehouse(), TestFixtures.TABLE); assertThatThrownBy(() -> SqlHelpers.sql(getStreamingTableEnv(), "SELECT * FROM %s", flinkTable)) @@ -218,8 +224,11 @@ public void testWatermarkValidConfig() throws Exception { getStreamingTableEnv(), "CREATE TABLE %s " + "(eventTS AS CAST(t1 AS TIMESTAMP(3)), " - + "WATERMARK FOR eventTS AS SOURCE_WATERMARK()) WITH ('watermark-column'='t1') LIKE iceberg_catalog.`default`.%s", + + "WATERMARK FOR eventTS AS SOURCE_WATERMARK()) " + + "WITH ('catalog-type'='hadoop', 'warehouse'='%s', 'watermark-column'='t1') " + + "LIKE iceberg_catalog.`default`.%s", flinkTable, + CATALOG_EXTENSION.warehouse(), TestFixtures.TABLE); TestHelpers.assertRecordsWithOrder( diff --git a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/source/reader/ReaderUtil.java b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/source/reader/ReaderUtil.java index 3b094ba02298..5a6787001170 100644 --- a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/source/reader/ReaderUtil.java +++ b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/source/reader/ReaderUtil.java @@ -57,6 +57,21 @@ private ReaderUtil() {} public static FileScanTask createFileTask( List records, File file, FileFormat fileFormat, Schema schema) throws IOException { + return createFileTask( + records, + file, + fileFormat, + schema, + ResidualEvaluator.unpartitioned(Expressions.alwaysTrue())); + } + + public static FileScanTask createFileTask( + List records, + File file, + FileFormat fileFormat, + Schema schema, + ResidualEvaluator residuals) + throws IOException { DataWriter writer = new GenericFileWriterFactory.Builder() .dataSchema(schema) @@ -69,7 +84,6 @@ public static FileScanTask createFileTask( DataFile dataFile = writer.toDataFile(); - ResidualEvaluator residuals = ResidualEvaluator.unpartitioned(Expressions.alwaysTrue()); return new BaseFileScanTask( dataFile, null, diff --git a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/source/reader/TestArrayPoolDataIteratorBatcherRowData.java b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/source/reader/TestArrayPoolDataIteratorBatcherRowData.java index 4f6d182bb3c1..994227f52bcd 100644 --- a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/source/reader/TestArrayPoolDataIteratorBatcherRowData.java +++ b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/source/reader/TestArrayPoolDataIteratorBatcherRowData.java @@ -21,6 +21,7 @@ import static org.assertj.core.api.Assertions.assertThat; import java.io.File; +import java.io.IOException; import java.nio.file.Path; import java.util.Arrays; import java.util.List; @@ -32,13 +33,18 @@ import org.apache.iceberg.CombinedScanTask; import org.apache.iceberg.FileFormat; import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.data.RandomGenericData; import org.apache.iceberg.data.Record; +import org.apache.iceberg.expressions.Expression; +import org.apache.iceberg.expressions.Expressions; +import org.apache.iceberg.expressions.ResidualEvaluator; import org.apache.iceberg.flink.FlinkConfigOptions; import org.apache.iceberg.flink.TestFixtures; import org.apache.iceberg.flink.TestHelpers; import org.apache.iceberg.flink.source.DataIterator; import org.apache.iceberg.io.CloseableIterator; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -354,4 +360,168 @@ public void testMultipleFilesWithSeekPosition() throws Exception { assertThat(recordBatchIterator).isExhausted(); } + + @Test + public void testDataIteratorWithResidualFilter() throws IOException { + GenericRecord record0 = GenericRecord.create(TestFixtures.SCHEMA); + record0.setField("data", "file-1"); + record0.setField("id", 0L); + record0.setField("dt", "-"); + + GenericRecord record1 = GenericRecord.create(TestFixtures.SCHEMA); + record1.setField("data", "file-2"); + record1.setField("id", 1L); + record1.setField("dt", "-"); + + List fileRecords0 = ImmutableList.of(record0); + List fileRecords1 = ImmutableList.of(record1); + + validateFileOffsetWithResidualFilter(fileRecords0, fileRecords1, Expressions.alwaysTrue()); + + validateFileOffsetWithResidualFilter( + fileRecords0, fileRecords1, Expressions.greaterThan("id", 0)); + } + + private void validateFileOffsetWithResidualFilter( + List fileRecords0, List fileRecords1, Expression residualFilter) + throws IOException { + ResidualEvaluator residualEvaluator = ResidualEvaluator.unpartitioned(residualFilter); + + FileScanTask fileTask0 = + ReaderUtil.createFileTask( + fileRecords0, + File.createTempFile("junit", null, temporaryFolder.toFile()), + FILE_FORMAT, + TestFixtures.SCHEMA, + residualEvaluator); + + FileScanTask fileTask1 = + ReaderUtil.createFileTask( + fileRecords1, + File.createTempFile("junit", null, temporaryFolder.toFile()), + FILE_FORMAT, + TestFixtures.SCHEMA, + residualEvaluator); + + CombinedScanTask combinedTask = new BaseCombinedScanTask(Arrays.asList(fileTask0, fileTask1)); + + DataIterator dataIterator = ReaderUtil.createDataIterator(combinedTask); + + dataIterator.seek(0, 0); + + while (dataIterator.hasNext()) { + assertThat(dataIterator.fileOffset()).isEqualTo(dataIterator.next().getLong(1)); + } + } + + @Test + public void testInitializationWithHeadFilesSkipped() throws IOException { + GenericRecord record0 = GenericRecord.create(TestFixtures.SCHEMA); + record0.setField("data", "a"); + record0.setField("id", 1L); + record0.setField("dt", "-"); + + GenericRecord record1 = GenericRecord.create(TestFixtures.SCHEMA); + record1.setField("data", "a"); + record1.setField("id", 10L); + record1.setField("dt", "-"); + + ResidualEvaluator residuals = ResidualEvaluator.unpartitioned(Expressions.greaterThan("id", 5)); + + FileScanTask fileTask0 = + ReaderUtil.createFileTask( + ImmutableList.of(record0), + File.createTempFile("junit", null, temporaryFolder.toFile()), + FILE_FORMAT, + TestFixtures.SCHEMA, + residuals); + + FileScanTask fileTask1 = + ReaderUtil.createFileTask( + ImmutableList.of(record1), + File.createTempFile("junit", null, temporaryFolder.toFile()), + FILE_FORMAT, + TestFixtures.SCHEMA, + residuals); + + CombinedScanTask combinedTask = new BaseCombinedScanTask(Arrays.asList(fileTask0, fileTask1)); + + DataIterator dataIterator = ReaderUtil.createDataIterator(combinedTask); + + dataIterator.seek(0, 0); + + assertThat(dataIterator.fileOffset()) + .as("File offset should be 1 because file 0 should be skipped") + .isEqualTo(1); + + TestHelpers.assertRowData(TestFixtures.SCHEMA, record1, dataIterator.next()); + } + + @Test + void testSeekResumesCorrectlyAfterHeadFilesSkipped() throws IOException { + GenericRecord filteredRecord = GenericRecord.create(TestFixtures.SCHEMA); + filteredRecord.setField("data", "a"); + filteredRecord.setField("id", 1L); + filteredRecord.setField("dt", "-"); + + GenericRecord f1Record0 = GenericRecord.create(TestFixtures.SCHEMA); + f1Record0.setField("data", "b"); + f1Record0.setField("id", 10L); + f1Record0.setField("dt", "-"); + + GenericRecord f2Record0 = GenericRecord.create(TestFixtures.SCHEMA); + f2Record0.setField("data", "d"); + f2Record0.setField("id", 20L); + f2Record0.setField("dt", "-"); + + GenericRecord f2Record1 = GenericRecord.create(TestFixtures.SCHEMA); + f2Record1.setField("data", "e"); + f2Record1.setField("id", 21L); + f2Record1.setField("dt", "-"); + + ResidualEvaluator residuals = ResidualEvaluator.unpartitioned(Expressions.greaterThan("id", 5)); + + FileScanTask fileTask0 = + ReaderUtil.createFileTask( + ImmutableList.of(filteredRecord), + File.createTempFile("junit", null, temporaryFolder.toFile()), + FILE_FORMAT, + TestFixtures.SCHEMA, + residuals); + + FileScanTask fileTask1 = + ReaderUtil.createFileTask( + ImmutableList.of(f1Record0), + File.createTempFile("junit", null, temporaryFolder.toFile()), + FILE_FORMAT, + TestFixtures.SCHEMA, + residuals); + + FileScanTask fileTask2 = + ReaderUtil.createFileTask( + ImmutableList.of(f2Record0, f2Record1), + File.createTempFile("junit", null, temporaryFolder.toFile()), + FILE_FORMAT, + TestFixtures.SCHEMA, + residuals); + + CombinedScanTask combinedTask = + new BaseCombinedScanTask(Arrays.asList(fileTask0, fileTask1, fileTask2)); + + DataIterator dataIterator = ReaderUtil.createDataIterator(combinedTask); + dataIterator.seek(0, 0); + + TestHelpers.assertRowData(TestFixtures.SCHEMA, f1Record0, dataIterator.next()); + TestHelpers.assertRowData(TestFixtures.SCHEMA, f2Record0, dataIterator.next()); + + int checkpointFileOffset = dataIterator.fileOffset(); + long checkpointRecordOffset = dataIterator.recordOffset(); + + DataIterator restoredIterator = ReaderUtil.createDataIterator(combinedTask); + restoredIterator.seek(checkpointFileOffset, checkpointRecordOffset); + + assertThat(restoredIterator.hasNext()).isTrue(); + TestHelpers.assertRowData(TestFixtures.SCHEMA, f2Record1, restoredIterator.next()); + assertThat(restoredIterator.hasNext()).isFalse(); + } } diff --git a/flink/v2.0/build.gradle b/flink/v2.0/build.gradle index 7bc37b30e5a1..b34eb6d06537 100644 --- a/flink/v2.0/build.gradle +++ b/flink/v2.0/build.gradle @@ -68,6 +68,7 @@ project(":iceberg-flink:iceberg-flink-${flinkMajorVersion}") { } implementation libs.datasketches + implementation libs.roaringbitmap testImplementation libs.flink20.connector.test.utils testImplementation libs.flink20.core diff --git a/flink/v2.0/flink-runtime/runtime-deps.txt b/flink/v2.0/flink-runtime/runtime-deps.txt index 8e493d31c510..a295230cef19 100644 --- a/flink/v2.0/flink-runtime/runtime-deps.txt +++ b/flink/v2.0/flink-runtime/runtime-deps.txt @@ -25,7 +25,7 @@ org.apache.parquet:parquet-variant:1.17 org.checkerframework:checker-qual:3.19 org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1 org.locationtech.jts:jts-core:1.20 -org.projectnessie.nessie:nessie-client:0.107 -org.projectnessie.nessie:nessie-model:0.107 +org.projectnessie.nessie:nessie-client:0.108 +org.projectnessie.nessie:nessie-model:0.108 org.roaringbitmap:RoaringBitmap:1.6 org.threeten:threeten-extra:1.7 diff --git a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java index a56c4e0ca6ed..1d1505a28c05 100644 --- a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java +++ b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java @@ -99,7 +99,6 @@ public class FlinkCatalog extends AbstractCatalog { private final Namespace baseNamespace; private final SupportsNamespaces asNamespaceCatalog; private final Closeable closeable; - private final Map catalogProps; private final boolean cacheEnabled; public FlinkCatalog( @@ -107,12 +106,10 @@ public FlinkCatalog( String defaultDatabase, Namespace baseNamespace, CatalogLoader catalogLoader, - Map catalogProps, boolean cacheEnabled, long cacheExpirationIntervalMs) { super(catalogName, defaultDatabase); this.catalogLoader = catalogLoader; - this.catalogProps = catalogProps; this.baseNamespace = baseNamespace; this.cacheEnabled = cacheEnabled; @@ -339,13 +336,12 @@ public CatalogTable getTable(ObjectPath tablePath) Table table = loadIcebergTable(tablePath); // Flink's CREATE TABLE LIKE clause relies on properties sent back here to create new table. - // Inorder to create such table in non iceberg catalog, we need to send across catalog - // properties also. // As Flink API accepts only Map for props, here we are serializing catalog - // props as json string to distinguish between catalog and table properties in createTable. + // name, database, table as json string to distinguish between catalog info + // and table properties in createTable. String srcCatalogProps = FlinkCreateTableOptions.toJson( - getName(), tablePath.getDatabaseName(), tablePath.getObjectName(), catalogProps); + getName(), tablePath.getDatabaseName(), tablePath.getObjectName()); Map tableProps = table.properties(); if (tableProps.containsKey(FlinkCreateTableOptions.CONNECTOR_PROPS_KEY) @@ -427,7 +423,12 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig + "create table without 'connector'='iceberg' related properties in an iceberg table."); } - Preconditions.checkArgument(table instanceof ResolvedCatalogTable, "table should be resolved"); + Preconditions.checkArgument( + table instanceof ResolvedCatalogTable, + "Expected a ResolvedCatalogTable but got: %s. " + + "Iceberg Flink catalog only supports resolved catalog tables " + + "(Materialized tables and other table kinds are not supported).", + table == null ? "null" : table.getClass().getName()); createIcebergTable(tablePath, (ResolvedCatalogTable) table, ignoreIfExists); } diff --git a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalogFactory.java b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalogFactory.java index 33cbc92ddeec..c1889dc6bd03 100644 --- a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalogFactory.java +++ b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalogFactory.java @@ -170,7 +170,6 @@ protected Catalog createCatalog( defaultDatabase, baseNamespace, catalogLoader, - properties, cacheEnabled, cacheExpirationIntervalMs); } diff --git a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/FlinkCreateTableOptions.java b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/FlinkCreateTableOptions.java index 0612260bfe7d..067b42bba954 100644 --- a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/FlinkCreateTableOptions.java +++ b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/FlinkCreateTableOptions.java @@ -18,7 +18,6 @@ */ package org.apache.iceberg.flink; -import java.util.Map; import org.apache.flink.configuration.ConfigOption; import org.apache.flink.configuration.ConfigOptions; import org.apache.iceberg.util.JsonUtil; @@ -27,14 +26,11 @@ class FlinkCreateTableOptions { private final String catalogName; private final String catalogDb; private final String catalogTable; - private final Map catalogProps; - private FlinkCreateTableOptions( - String catalogName, String catalogDb, String catalogTable, Map props) { + private FlinkCreateTableOptions(String catalogName, String catalogDb, String catalogTable) { this.catalogName = catalogName; this.catalogDb = catalogDb; this.catalogTable = catalogTable; - this.catalogProps = props; } public static final ConfigOption CATALOG_NAME = @@ -61,12 +57,6 @@ private FlinkCreateTableOptions( .noDefaultValue() .withDescription("Table name managed in the underlying iceberg catalog and database."); - public static final ConfigOption> CATALOG_PROPS = - ConfigOptions.key("catalog-props") - .mapType() - .noDefaultValue() - .withDescription("Properties for the underlying catalog for iceberg table."); - public static final ConfigOption USE_DYNAMIC_ICEBERG_SINK = ConfigOptions.key("use-dynamic-iceberg-sink") .booleanType() @@ -89,15 +79,13 @@ private FlinkCreateTableOptions( public static final String CONNECTOR_PROPS_KEY = "connector"; public static final String LOCATION_KEY = "location"; - static String toJson( - String catalogName, String catalogDb, String catalogTable, Map catalogProps) { + static String toJson(String catalogName, String catalogDb, String catalogTable) { return JsonUtil.generate( gen -> { gen.writeStartObject(); gen.writeStringField(CATALOG_NAME.key(), catalogName); gen.writeStringField(CATALOG_DATABASE.key(), catalogDb); gen.writeStringField(CATALOG_TABLE.key(), catalogTable); - JsonUtil.writeStringMap(CATALOG_PROPS.key(), catalogProps, gen); gen.writeEndObject(); }, false); @@ -110,9 +98,8 @@ static FlinkCreateTableOptions fromJson(String createTableOptions) { String catalogName = JsonUtil.getString(CATALOG_NAME.key(), node); String catalogDb = JsonUtil.getString(CATALOG_DATABASE.key(), node); String catalogTable = JsonUtil.getString(CATALOG_TABLE.key(), node); - Map catalogProps = JsonUtil.getStringMap(CATALOG_PROPS.key(), node); - return new FlinkCreateTableOptions(catalogName, catalogDb, catalogTable, catalogProps); + return new FlinkCreateTableOptions(catalogName, catalogDb, catalogTable); }); } @@ -127,8 +114,4 @@ String catalogDb() { String catalogTable() { return catalogTable; } - - Map catalogProps() { - return catalogProps; - } } diff --git a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/FlinkDynamicTableFactory.java b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/FlinkDynamicTableFactory.java index b2e2e33b9291..6306ee7a0a1c 100644 --- a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/FlinkDynamicTableFactory.java +++ b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/FlinkDynamicTableFactory.java @@ -237,13 +237,13 @@ private static TableLoader createTableLoader( } /** - * Merges source catalog properties with connector properties. Iceberg Catalog properties are - * serialized as json in FlinkCatalog#getTable to be able to isolate catalog props from iceberg - * table props, Here, we flatten and merge them back to use to create catalog. + * Merges source catalog properties (catalog name, database, table) with connector properties. + * Source catalog name, database, table are serialized as json in FlinkCatalog#getTable to be able + * to isolate them from iceberg table props, Here, we flatten and merge them back. * * @param tableProps the existing table properties - * @return a map of merged properties, with source catalog properties taking precedence when keys - * conflict + * @return a map of merged properties, defaulting to source catalog name, database and table + * unless overridden. */ private static Map mergeSrcCatalogProps(Map tableProps) { String srcCatalogProps = tableProps.get(FlinkCreateTableOptions.SRC_CATALOG_PROPS_KEY); @@ -257,7 +257,6 @@ private static Map mergeSrcCatalogProps(Map tabl FlinkCreateTableOptions.CATALOG_DATABASE.key(), createTableOptions.catalogDb()); mergedProps.put( FlinkCreateTableOptions.CATALOG_TABLE.key(), createTableOptions.catalogTable()); - mergedProps.putAll(createTableOptions.catalogProps()); tableProps.forEach( (k, v) -> { diff --git a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/ConvertEqualityDeletes.java b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/ConvertEqualityDeletes.java new file mode 100644 index 000000000000..f51df3338d02 --- /dev/null +++ b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/ConvertEqualityDeletes.java @@ -0,0 +1,286 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.api; + +import java.util.Collections; +import java.util.List; +import java.util.Set; +import org.apache.flink.annotation.Experimental; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.streaming.api.datastream.BroadcastStream; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; +import org.apache.iceberg.Schema; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableUtil; +import org.apache.iceberg.flink.maintenance.operator.DVPosition; +import org.apache.iceberg.flink.maintenance.operator.DVWriteResult; +import org.apache.iceberg.flink.maintenance.operator.EqualityConvertCommitter; +import org.apache.iceberg.flink.maintenance.operator.EqualityConvertDVWriter; +import org.apache.iceberg.flink.maintenance.operator.EqualityConvertPKIndex; +import org.apache.iceberg.flink.maintenance.operator.EqualityConvertPlan; +import org.apache.iceberg.flink.maintenance.operator.EqualityConvertPlanner; +import org.apache.iceberg.flink.maintenance.operator.EqualityConvertReader; +import org.apache.iceberg.flink.maintenance.operator.IndexCommand; +import org.apache.iceberg.flink.maintenance.operator.ReadCommand; +import org.apache.iceberg.flink.maintenance.operator.SerializedEqualityValues; +import org.apache.iceberg.flink.maintenance.operator.TaskResultAggregator; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.types.Types.NestedField; + +/** + * Creates the equality delete to DV conversion data stream. Runs a single iteration of the + * conversion for every {@link Trigger} event. + * + *

The pipeline reads equality delete files from a staging branch, converts them to deletion + * vectors (DVs) using a primary key index stored in Flink state, and commits the data files and DVs + * to the target branch. + * + *

The conversion is split into parallel stages: + * + *

    + *
  1. Planner (p=1): scans staging branch, emits file-level ReadCommands with phase timestamps + *
  2. Reader (p=N): reads files, emits row-level IndexCommands + *
  3. PKIndex (p=N): maintains PK index shards, resolves equality deletes to DV positions + *
  4. DVWriter (p=N, keyed by data file path): buffers positions per file, writes Puffin DVs + * inline + *
  5. Committer (p=1): commits data files and DVs to the target branch + *
+ * + *

Mutual exclusion with concurrent maintenance tasks (e.g. compaction) is enforced by the Flink + * maintenance framework lock. + */ +@Experimental +public class ConvertEqualityDeletes { + static final String PLANNER_TASK_NAME = "EqConvert Planner"; + static final String READER_TASK_NAME = "EqConvert Reader"; + static final String PK_INDEX_TASK_NAME = "EqConvert PKIndex"; + static final String DV_WRITER_TASK_NAME = "EqConvert DVWriter"; + static final String UPSTREAM_ABORT_TASK_NAME = "EqConvert UpstreamAbort"; + static final String COMMIT_TASK_NAME = "EqConvert Commit"; + static final String AGGREGATOR_TASK_NAME = "EqConvert Aggregator"; + + private ConvertEqualityDeletes() {} + + public static Builder builder() { + return new Builder(); + } + + public static class Builder extends MaintenanceTaskBuilder { + private String stagingBranch; + private String targetBranch = SnapshotRef.MAIN_BRANCH; + private List equalityFieldColumns = Collections.emptyList(); + + @Override + String maintenanceTaskName() { + return "ConvertEqualityDeletes"; + } + + /** Sets the staging branch name that holds the equality delete files and data files. */ + public Builder stagingBranch(String newStagingBranch) { + this.stagingBranch = newStagingBranch; + return this; + } + + /** + * Sets the target branch where converted data files and DVs are committed. Defaults to the main + * branch. + */ + public Builder targetBranch(String newTargetBranch) { + this.targetBranch = newTargetBranch; + return this; + } + + /** + * Sets the equality field columns used by the worker index. Required. Must match the equality + * field columns the writer uses for staging eq-delete files. Mirrors {@link + * org.apache.iceberg.flink.sink.IcebergSink.Builder#equalityFieldColumns}. + * + *

The partition source columns of an equality delete's spec must be a subset of these + * columns. Writes via Flink's IcebergSink already ensure this. + */ + public Builder equalityFieldColumns(List columns) { + Preconditions.checkNotNull(columns, "equalityFieldColumns must not be null"); + Preconditions.checkArgument(!columns.isEmpty(), "equalityFieldColumns must not be empty"); + this.equalityFieldColumns = ImmutableList.copyOf(columns); + return this; + } + + @Override + DataStream append(DataStream trigger) { + Preconditions.checkNotNull(stagingBranch, "stagingBranch must be set"); + Preconditions.checkArgument( + !equalityFieldColumns.isEmpty(), "equalityFieldColumns must be set on the builder"); + Set eqFieldIds = resolveEqualityFieldIds(); + + // Planner (p=1): emits ReadCommands with phase timestamps and watermarks + SingleOutputStreamOperator planned = + setSlotSharingGroup( + trigger + .transform( + operatorName(PLANNER_TASK_NAME), + TypeInformation.of(ReadCommand.class), + new EqualityConvertPlanner( + tableName(), + taskName(), + tableLoader(), + stagingBranch, + targetBranch, + eqFieldIds)) + .uid(PLANNER_TASK_NAME + uidSuffix()) + .forceNonParallel()); + + // Reader (p=N): reads files, emits IndexCommands + SingleOutputStreamOperator index = + setSlotSharingGroup( + planned + .rebalance() + .process( + new EqualityConvertReader( + tableLoader(), eqFieldIds, stagingBranch.equals(targetBranch))) + .name(operatorName(READER_TASK_NAME)) + .uid(READER_TASK_NAME + uidSuffix()) + .setParallelism(parallelism())); + + // Broadcast from the planner to the PKIndex to clear the entire index + BroadcastStream clearIndexBroadcast = + planned + .getSideOutput(EqualityConvertPlanner.CLEAR_BROADCAST_STREAM) + .broadcast(EqualityConvertPKIndex.CLEAR_BROADCAST_DESCRIPTOR); + + // PKIndex (p=N): keyed by full PK, phase-aware buffering. + SingleOutputStreamOperator dvPositions = + setSlotSharingGroup( + index + .keyBy(IndexCommand::key, TypeInformation.of(SerializedEqualityValues.class)) + .connect(clearIndexBroadcast) + .process(new EqualityConvertPKIndex(stagingBranch.equals(targetBranch))) + .name(operatorName(PK_INDEX_TASK_NAME)) + .uid(PK_INDEX_TASK_NAME + uidSuffix()) + .setParallelism(parallelism())); + + // Reader-side abort signals bypass the PKIndex and feed the DVWriter directly, so a reader + // failure can short-circuit the cycle without waiting on a keyed shuffle. This is not a full + // short-circuit: the abort is keyed by data file path (empty for ABORT), so only one resolver + // subtask observes it; the others still write their buffered DVs, which the committer then + // drops. + DataStream readerAborts = + index.getSideOutput(EqualityConvertReader.READER_ABORT_STREAM); + DataStream dvPositionsWithAborts = dvPositions.union(readerAborts); + + // Metadata side output from planner + DataStream metadata = + planned.getSideOutput(EqualityConvertPlanner.METADATA_STREAM); + + // DVWriter (p=N, keyed by data file path): groups positions per file, writes Puffin DV + // files inline, emits a DVWriteResult per cycle. Plan metadata broadcast so every subtask + // sees it. + SingleOutputStreamOperator resolved = + setSlotSharingGroup( + dvPositionsWithAborts + .keyBy(DVPosition::dataFilePath) + .connect(metadata.broadcast()) + .transform( + operatorName(DV_WRITER_TASK_NAME), + TypeInformation.of(DVWriteResult.class), + new EqualityConvertDVWriter( + tableName(), taskName(), tableLoader(), targetBranch)) + .uid(DV_WRITER_TASK_NAME + uidSuffix()) + .setParallelism(parallelism())); + + // Upstream errors become abort signals so a partial read never commits. The same error side + // outputs also feed the aggregator below to surface the exception in TaskResult; the two + // consumers serve different purposes and must both exist. + DataStream upstreamAborts = + setSlotSharingGroup( + index + .getSideOutput(TaskResultAggregator.ERROR_STREAM) + .union(dvPositions.getSideOutput(TaskResultAggregator.ERROR_STREAM)) + .map(e -> DVWriteResult.ABORT) + .returns(TypeInformation.of(DVWriteResult.class)) + .name(operatorName(UPSTREAM_ABORT_TASK_NAME)) + .uid(UPSTREAM_ABORT_TASK_NAME + uidSuffix()) + .forceNonParallel()); + + // Committer (p=1): commits data files + DVs to main. + SingleOutputStreamOperator committed = + setSlotSharingGroup( + resolved + .union(upstreamAborts) + .connect(metadata) + .transform( + operatorName(COMMIT_TASK_NAME), + TypeInformation.of(Trigger.class), + new EqualityConvertCommitter( + tableName(), taskName(), tableLoader(), stagingBranch, targetBranch)) + .uid(COMMIT_TASK_NAME + uidSuffix()) + .forceNonParallel()); + + // Aggregator (p=1): collects errors and emits TaskResult. + return setSlotSharingGroup( + committed + .connect( + planned + .getSideOutput(TaskResultAggregator.ERROR_STREAM) + .union(index.getSideOutput(TaskResultAggregator.ERROR_STREAM)) + .union(dvPositions.getSideOutput(TaskResultAggregator.ERROR_STREAM)) + .union(resolved.getSideOutput(TaskResultAggregator.ERROR_STREAM)) + .union(committed.getSideOutput(TaskResultAggregator.ERROR_STREAM))) + .transform( + operatorName(AGGREGATOR_TASK_NAME), + TypeInformation.of(TaskResult.class), + new TaskResultAggregator(tableName(), taskName(), index())) + .uid(AGGREGATOR_TASK_NAME + uidSuffix()) + .forceNonParallel()); + } + + private Set resolveEqualityFieldIds() { + if (!tableLoader().isOpen()) { + tableLoader().open(); + } + + Table table = tableLoader().loadTable(); + int formatVersion = TableUtil.formatVersion(table); + Preconditions.checkArgument( + formatVersion >= 3, + "ConvertEqualityDeletes requires table format version >= 3 (DVs), " + + "but table '%s' is version %s", + tableName(), + formatVersion); + + Schema schema = table.schema(); + List fieldIds = Lists.newArrayListWithCapacity(equalityFieldColumns.size()); + for (String column : equalityFieldColumns) { + NestedField field = schema.findField(column); + Preconditions.checkArgument( + field != null, + "Equality field column '%s' not found in table schema %s", + column, + schema); + fieldIds.add(field.fieldId()); + } + + return ImmutableSet.copyOf(fieldIds); + } + } +} diff --git a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/TableMaintenance.java b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/TableMaintenance.java index 025a6d17c023..46b38858b8aa 100644 --- a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/TableMaintenance.java +++ b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/TableMaintenance.java @@ -26,6 +26,7 @@ import java.util.UUID; import javax.annotation.Nullable; import org.apache.flink.annotation.Internal; +import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.eventtime.TimestampAssigner; import org.apache.flink.api.common.eventtime.TimestampAssignerSupplier; @@ -378,7 +379,9 @@ private DataStream changeStream(String tableName, TableLoader loade // Create a monitor source to provide the TableChange stream MonitorSource source = new MonitorSource( - loader, RateLimiterStrategy.perSecond(1.0 / rateLimit.getSeconds()), maxReadBack); + loader, + RateLimiterStrategy.perSecond(monitorRatePerSecond(rateLimit.toMillis())), + maxReadBack); return setSlotSharingGroup( env.fromSource( source, @@ -395,6 +398,16 @@ private static String nameFor(MaintenanceTaskBuilder streamBuilder, int taskI return String.format(Locale.ROOT, "%s [%d]", streamBuilder.maintenanceTaskName(), taskIndex); } + /** + * Monitor poll rate per rate-limit interval, in checks/second. We compute from millis instead + * of seconds, otherwise sub-second intervals could be truncated to 0, yielding an infinite rate + * which busy-loops the source. + */ + @VisibleForTesting + static double monitorRatePerSecond(long rateLimitMillis) { + return 1000.0 / rateLimitMillis; + } + private SingleOutputStreamOperator setSlotSharingGroup( SingleOutputStreamOperator operator) { return slotSharingGroup == null ? operator : operator.slotSharingGroup(slotSharingGroup); diff --git a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java new file mode 100644 index 000000000000..95c9ccd5de1c --- /dev/null +++ b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java @@ -0,0 +1,372 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.operator; + +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import org.apache.flink.annotation.Internal; +import org.apache.flink.metrics.Counter; +import org.apache.flink.metrics.MetricGroup; +import org.apache.flink.streaming.api.operators.AbstractStreamOperator; +import org.apache.flink.streaming.api.operators.TwoInputStreamOperator; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.RowDelta; +import org.apache.iceberg.Table; +import org.apache.iceberg.exceptions.CommitStateUnknownException; +import org.apache.iceberg.flink.TableLoader; +import org.apache.iceberg.flink.maintenance.api.Trigger; +import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.apache.iceberg.util.ContentFileUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Commits data files and DVs to the target branch. Receives {@link DVWriteResult}s from parallel + * {@link EqualityConvertDVWriter} instances (input 1) and an {@link EqualityConvertPlan} from the + * planner (input 2). Assembles the final file lists and commits using a {@link RowDelta} operation + * once the plan result and done-timestamp watermark have both arrived. + * + *

The commit is gated on the plan's done-timestamp watermark. + * + *

Watermarks are forwarded only after the cycle commits, never mid-cycle. The {@link + * LockRemover} releases the maintenance lock once a watermark past the trigger's start epoch + * reaches it. The planner emits phase watermarks in the middle of a cycle; forwarding those would + * release the lock before this commit, letting the TriggerManager start a concurrent cycle that + * re-processes the same uncommitted staging snapshot. + * + *

Emits a {@link Trigger} after each cycle (commit, no-op, or error) so the downstream {@link + * TaskResultAggregator} can track task completion. This is the sole source of Trigger records for + * the Aggregator. + * + *

No-op vs error: a no-op cycle (empty plan result from {@code + * EqualityConvertPlanner.emitNoOpResult}) returns early in {@code commitIfNeeded} without writing + * anything; the Trigger emit in {@code processWatermark} still happens, so the maintenance task + * completes cleanly. Errors are reported via {@link TaskResultAggregator#ERROR_STREAM} side output; + * the Aggregator collects them and surfaces failure on its own watermark. + * + *

The committer is intentionally stateless: {@code bufferedResults} and {@code planResult} are + * not checkpointed. The maintenance framework ensures mutual exclusive tasks, and on restart the + * planner re-derives its position from {@link #COMMITTED_STAGING_SNAPSHOT_PROPERTY} on main, so the + * committer never receives a plan for an already-committed staging snapshot. + */ +@Internal +public class EqualityConvertCommitter extends AbstractStreamOperator + implements TwoInputStreamOperator { + + private static final Logger LOG = LoggerFactory.getLogger(EqualityConvertCommitter.class); + + public static final String COMMITTED_STAGING_SNAPSHOT_PROPERTY = + "equality-convert-staging-snapshot"; + + private static final String ADDED_DV_NUM_METRIC = "addedDvNum"; + private static final String COMMIT_DURATION_MS_METRIC = "commitDurationMs"; + + private final String tableName; + private final String taskName; + private final TableLoader tableLoader; + private final String targetBranch; + private final boolean stagingOnTargetBranch; + + private transient Table table; + private transient List bufferedResults; + private transient EqualityConvertPlan planResult; + + private transient Counter errorCounter; + private transient Counter addedDataFileNumCounter; + private transient Counter addedDataFileSizeCounter; + private transient Counter addedDvNumCounter; + private transient Counter commitDurationMsCounter; + + public EqualityConvertCommitter( + String tableName, + String taskName, + TableLoader tableLoader, + String stagingBranch, + String targetBranch) { + this.tableName = tableName; + this.taskName = taskName; + this.tableLoader = tableLoader; + this.targetBranch = targetBranch; + this.stagingOnTargetBranch = stagingBranch.equals(targetBranch); + } + + @Override + public void open() throws Exception { + super.open(); + if (!tableLoader.isOpen()) { + tableLoader.open(); + } + + this.table = tableLoader.loadTable(); + this.bufferedResults = Lists.newArrayList(); + + MetricGroup taskMetricGroup = + TableMaintenanceMetrics.groupFor(getRuntimeContext(), tableName, taskName, 0); + this.errorCounter = taskMetricGroup.counter(TableMaintenanceMetrics.ERROR_COUNTER); + this.addedDataFileNumCounter = + taskMetricGroup.counter(TableMaintenanceMetrics.ADDED_DATA_FILE_NUM_METRIC); + this.addedDataFileSizeCounter = + taskMetricGroup.counter(TableMaintenanceMetrics.ADDED_DATA_FILE_SIZE_METRIC); + this.addedDvNumCounter = taskMetricGroup.counter(ADDED_DV_NUM_METRIC); + this.commitDurationMsCounter = taskMetricGroup.counter(COMMIT_DURATION_MS_METRIC); + } + + @Override + public void processElement1(StreamRecord record) { + bufferedResults.add(record.getValue()); + } + + @Override + public void processElement2(StreamRecord record) { + planResult = record.getValue(); + } + + @Override + public void processWatermark(Watermark mark) throws Exception { + if (planResult == null || mark.getTimestamp() < planResult.doneTimestamp()) { + // Hold back watermarks until the cycle commits so the LockRemover keeps the maintenance lock + // for the whole cycle. Forwarding the planner's mid-cycle phase watermarks will release the + // lock early and could let the next trigger run a concurrent cycle on the same staging + // snapshot. + return; + } + + try { + commitIfNeeded(); + } catch (Exception e) { + LOG.error( + "Failed to commit equality convert result for table {} task {}", tableName, taskName, e); + output.collect(TaskResultAggregator.ERROR_STREAM, new StreamRecord<>(e)); + errorCounter.inc(); + } + + // Emit Trigger for the Aggregator (even on error or no-op). + output.collect(new StreamRecord<>(Trigger.create(planResult.triggerTimestamp(), 0))); + + bufferedResults.clear(); + planResult = null; + + super.processWatermark(mark); + } + + @Override + public void close() throws Exception { + super.close(); + tableLoader.close(); + } + + private void commitIfNeeded() { + for (DVWriteResult result : bufferedResults) { + if (result.isAbort()) { + LOG.warn( + "Skipping commit for table {} task {}: a DV writer reported an error.", + tableName, + taskName); + deleteUncommittedDVs(); + return; + } + } + + // No-op cycle: the planner emitted an empty plan result (see + // EqualityConvertPlanner.emitNoOpResult) because the next staging snapshot was filtered by + // shouldSkip or there was nothing new on staging. processWatermark still forwards a Trigger to + // TaskResultAggregator, so the maintenance task completes cleanly. + if (planResult.noOp()) { + return; + } + + table.refresh(); + + List allDvFiles = Lists.newArrayList(); + List allRewrittenDvFiles = Lists.newArrayList(); + for (DVWriteResult result : bufferedResults) { + allDvFiles.addAll(result.dvFiles()); + allRewrittenDvFiles.addAll(result.rewrittenDvFiles()); + } + + RowDelta rowDelta = buildRowDelta(planResult.dataFiles(), allDvFiles, allRewrittenDvFiles); + + long startNano = System.nanoTime(); + try { + commit(rowDelta); + } catch (CommitStateUnknownException e) { + // Commit outcome unknown: the DVs may already be referenced by a committed snapshot. Leave + // them for Remove Orphan Files rather than risk deleting live data. + throw e; + } catch (Exception e) { + // Commit definitively failed: the DVs this cycle wrote are unreferenced. Delete them so a + // failed cycle does not leak Puffin files. Rewritten DVs stay (still live on the target). + deleteUncommittedDVs(); + throw e; + } + + long durationMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNano); + commitDurationMsCounter.inc(durationMs); + + LOG.info( + "Committed {} data files and {} DV files to branch '{}' for table {} in {} ms. " + + "Processed staging snapshot {}.", + planResult.dataFiles().size(), + allDvFiles.size(), + targetBranch, + tableName, + durationMs, + planResult.stagingSnapshotId()); + + // Only count files actually added by this commit. When sameBranch, the writer already + // committed the data files to target and buildRowDelta does not re-add them. + if (!stagingOnTargetBranch) { + for (DataFile dataFile : planResult.dataFiles()) { + addedDataFileNumCounter.inc(); + addedDataFileSizeCounter.inc(dataFile.fileSizeInBytes()); + } + } + + addedDvNumCounter.inc(allDvFiles.size()); + } + + @VisibleForTesting + void commit(RowDelta rowDelta) { + rowDelta.commit(); + } + + /** + * Deletes the DVs this cycle wrote but did not commit (abort or definite commit failure). Only + * the newly written DVs are removed; rewritten DVs remain referenced on the target branch. Best + * effort: a delete failure is logged, not propagated, so it never masks the original error. + */ + private void deleteUncommittedDVs() { + for (DVWriteResult result : bufferedResults) { + if (result.isAbort()) { + continue; + } + + for (DeleteFile dvFile : result.dvFiles()) { + try { + table.io().deleteFile(dvFile.location()); + } catch (RuntimeException e) { + LOG.warn( + "Failed to delete uncommitted DV {} for table {} task {}", + dvFile.location(), + tableName, + taskName, + e); + } + } + } + } + + private RowDelta buildRowDelta( + List dataFiles, List allDvFiles, List allRewrittenDvFiles) { + RowDelta rowDelta = table.newRowDelta(); + + // Fail the commit on external target-branch activity since the planner's snapshot. The next + // trigger detects the change and reindexes. + if (planResult.mainSnapshotId() != null) { + rowDelta.validateFromSnapshot(planResult.mainSnapshotId()); + } + + rowDelta.validateNoConflictingDataFiles(); + rowDelta.validateNoConflictingDeleteFiles(); + + Set referencedDataFiles = Sets.newHashSet(); + for (DeleteFile dvFile : allDvFiles) { + if (dvFile.referencedDataFile() != null) { + referencedDataFiles.add(dvFile.referencedDataFile()); + } + } + + if (!referencedDataFiles.isEmpty()) { + rowDelta.validateDataFilesExist(referencedDataFiles); + } + + // When stagingBranch == targetBranch, the writer already committed data files and DVs to the + // target branch. Re-adding them here would produce duplicate manifest entries. Skip those + // paths; only the new DVs from the writer need to be added. + if (!stagingOnTargetBranch) { + for (DataFile dataFile : dataFiles) { + rowDelta.addRows(dataFile); + } + } + + for (DeleteFile dvFile : allDvFiles) { + rowDelta.addDeletes(dvFile); + } + + if (!stagingOnTargetBranch) { + addStagingDeletes(rowDelta, referencedDataFiles, planResult.stagingDVFiles()); + } + + removeRewrittenDVs( + rowDelta, allRewrittenDvFiles, planResult.stagingDVFiles(), stagingOnTargetBranch); + + rowDelta.toBranch(targetBranch); + rowDelta.set( + COMMITTED_STAGING_SNAPSHOT_PROPERTY, String.valueOf(planResult.stagingSnapshotId())); + return rowDelta; + } + + /** Adds staging delete files, skipping DVs that overlap with conversion DVs (V3 rule). */ + private static void addStagingDeletes( + RowDelta rowDelta, Set dvCoveredDataFiles, List stagingDVFiles) { + for (DeleteFile stagingDelete : stagingDVFiles) { + // V3 allows one DV per data file. When a staging snapshot contains both a writer-committed + // DV and an eq-delete that resolves to additional positions in the same data file, the + // writer folds the staging DV into a new merged DV (via + // EqualityConvertDVWriter.collectExistingDVs). Skip the superseded staging DV; adding both + // would commit two DVs for the same data file. + if (ContentFileUtil.isDV(stagingDelete) + && stagingDelete.referencedDataFile() != null + && dvCoveredDataFiles.contains(stagingDelete.referencedDataFile())) { + continue; + } + + rowDelta.addDeletes(stagingDelete); + } + } + + /** + * Removes rewritten DVs. On a separate target branch, staging DVs are not yet on target, so they + * are skipped: removeDeletes would fail. On a shared branch they are already on target and must + * be removed like any other rewritten DV; the merged DV that supersedes them would otherwise be a + * second DV for the same data file (V3 allows one). + */ + private static void removeRewrittenDVs( + RowDelta rowDelta, + List allRewrittenDvFiles, + List stagingDVFiles, + boolean stagingOnTargetBranch) { + Set stagingDeleteLocations = Sets.newHashSet(); + for (DeleteFile sd : stagingDVFiles) { + stagingDeleteLocations.add(sd.location()); + } + + for (DeleteFile rewrittenDv : allRewrittenDvFiles) { + if (stagingOnTargetBranch || !stagingDeleteLocations.contains(rewrittenDv.location())) { + rowDelta.removeDeletes(rewrittenDv); + } + } + } +} diff --git a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertDVWriter.java b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertDVWriter.java new file mode 100644 index 000000000000..486b933ee6d7 --- /dev/null +++ b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertDVWriter.java @@ -0,0 +1,354 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.operator; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.flink.annotation.Internal; +import org.apache.flink.streaming.api.operators.AbstractStreamOperator; +import org.apache.flink.streaming.api.operators.TwoInputStreamOperator; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; +import org.apache.iceberg.PartitionField; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.StructLike; +import org.apache.iceberg.Table; +import org.apache.iceberg.data.BaseDeleteLoader; +import org.apache.iceberg.data.DeleteLoader; +import org.apache.iceberg.deletes.BaseDVFileWriter; +import org.apache.iceberg.deletes.PositionDeleteIndex; +import org.apache.iceberg.expressions.Expression; +import org.apache.iceberg.expressions.Expressions; +import org.apache.iceberg.expressions.ManifestEvaluator; +import org.apache.iceberg.flink.TableLoader; +import org.apache.iceberg.io.DeleteWriteResult; +import org.apache.iceberg.io.OutputFileFactory; +import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.ContentFileUtil; +import org.apache.iceberg.util.StructLikeWrapper; +import org.roaringbitmap.longlong.Roaring64Bitmap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Keyed parallel resolver that buffers {@link DVPosition}s per data-file path, then writes Puffin + * DV files directly via {@link BaseDVFileWriter}. Plan metadata arrives broadcast on input 2, so + * every parallel task sees the cycle's metadata and can validate against the main snapshot. + * + *

Each buffered {@link DVPosition} carries the data file's {@code specId} + encoded partition, + * so writing DVs needs no data-manifest scan. Existing DVs are folded into the rewrite (V3 allows + * one DV per data file): delete manifests are pruned by partition summary to the cycle's affected + * partitions, then filtered to entries referencing the affected data files. No cross-cycle state is + * kept; reads are bounded by the pruned manifest set, not the table's full DV history. + * + *

Buffered positions are transient per-task. On failure recovery, upstream replay rebuilds them. + */ +@Internal +public class EqualityConvertDVWriter extends AbstractStreamOperator + implements TwoInputStreamOperator { + + private static final Logger LOG = LoggerFactory.getLogger(EqualityConvertDVWriter.class); + + private final String tableName; + private final String taskName; + private final TableLoader tableLoader; + private final String targetBranch; + + private transient Table table; + private transient OutputFileFactory fileFactory; + private transient DeleteLoader deleteLoader; + private transient Map positionsByFile; + private transient EqualityConvertPlan planResult; + private transient boolean hasUpstreamError; + private transient int manifestsRead; + + public EqualityConvertDVWriter( + String tableName, String taskName, TableLoader tableLoader, String targetBranch) { + this.tableName = tableName; + this.taskName = taskName; + this.tableLoader = tableLoader; + this.targetBranch = targetBranch; + } + + @Override + public void open() throws Exception { + super.open(); + if (!tableLoader.isOpen()) { + tableLoader.open(); + } + + table = tableLoader.loadTable(); + int subtaskIndex = getRuntimeContext().getTaskInfo().getIndexOfThisSubtask(); + fileFactory = + OutputFileFactory.builderFor(table, subtaskIndex, 0L).format(FileFormat.PUFFIN).build(); + deleteLoader = new BaseDeleteLoader(deleteFile -> table.io().newInputFile(deleteFile)); + positionsByFile = Maps.newHashMap(); + } + + @Override + public void processElement1(StreamRecord record) { + DVPosition pos = record.getValue(); + if (pos.isAbort()) { + hasUpstreamError = true; + } + + if (!hasUpstreamError) { + positionsByFile + .computeIfAbsent( + pos.dataFilePath(), k -> new FilePositions(pos.specId(), pos.partition())) + .positions + .addLong(pos.position()); + } + } + + @Override + public void processElement2(StreamRecord record) { + planResult = record.getValue(); + } + + @Override + public void processWatermark(Watermark mark) throws Exception { + if (planResult != null && mark.getTimestamp() >= planResult.doneTimestamp()) { + if (hasUpstreamError) { + output.collect(new StreamRecord<>(DVWriteResult.ABORT)); + } else { + try { + resolveAndWrite(); + } catch (Exception e) { + LOG.error("Error writing DVs for table {} task {}", tableName, taskName, e); + output.collect(TaskResultAggregator.ERROR_STREAM, new StreamRecord<>(e)); + output.collect(new StreamRecord<>(DVWriteResult.ABORT)); + } + } + + positionsByFile.clear(); + hasUpstreamError = false; + planResult = null; + } + + super.processWatermark(mark); + } + + private void resolveAndWrite() throws IOException { + if (positionsByFile.isEmpty()) { + return; + } + + table.refresh(); + + Snapshot mainSnapshot = table.snapshot(targetBranch); + + // Fail fast if the main branch changed since planning, to avoid writing DV files that the + // committer would reject via validateFromSnapshot. The next cycle will reindex. + if (mainSnapshot != null + && planResult.mainSnapshotId() != null + && mainSnapshot.snapshotId() != planResult.mainSnapshotId()) { + throw new IllegalStateException( + "Main branch snapshot changed since planning: expected " + + planResult.mainSnapshotId() + + " but found: " + + mainSnapshot.snapshotId()); + } + + Map dvs = collectExistingDVs(mainSnapshot, positionsByFile.keySet()); + + // Fold staging DVs into the rewrite so the writer emits one DV per data file (V3 rule). Flink + // writes a staging DV only for a newly added data file, so it never collides with a distinct + // existing DV: on a separate target branch collectExistingDVs has not seen it yet; on a shared + // branch it IS that existing DV, so the put is idempotent. + for (DeleteFile sd : planResult.stagingDVFiles()) { + if (ContentFileUtil.isDV(sd) && sd.referencedDataFile() != null) { + dvs.put(sd.referencedDataFile(), sd); + } + } + + BaseDVFileWriter dvWriter = + new BaseDVFileWriter(fileFactory, path -> loadPreviousDV(path, dvs)); + try (dvWriter) { + for (Map.Entry entry : positionsByFile.entrySet()) { + String dataFilePath = entry.getKey(); + FilePositions filePositions = entry.getValue(); + PartitionSpec spec = table.specs().get(filePositions.specId); + StructLike partition = filePositions.partition(spec.partitionType()); + + filePositions.positions.forEach( + (long pos) -> dvWriter.delete(dataFilePath, pos, spec, partition)); + } + } + + DeleteWriteResult result = dvWriter.result(); + LOG.info( + "Wrote {} DV files (rewriting {}) for {} data files in table {} task {}.", + result.deleteFiles().size(), + result.rewrittenDeleteFiles().size(), + positionsByFile.size(), + tableName, + taskName); + + output.collect( + new StreamRecord<>( + new DVWriteResult( + Lists.newArrayList(result.deleteFiles()), + Lists.newArrayList(result.rewrittenDeleteFiles())))); + } + + @Override + public void close() throws Exception { + super.close(); + tableLoader.close(); + } + + private Map collectExistingDVs( + Snapshot mainSnapshot, Set affectedPaths) { + manifestsRead = 0; + Map dvs = Maps.newHashMap(); + if (mainSnapshot == null) { + return dvs; + } + + // Prune delete manifests whose partition summaries cannot cover the cycle's affected + // partitions. A DV inherits its referenced data file's spec and partition, so partition pruning + // works for DV manifests. + Map evaluators = partitionEvaluators(positionsByFile); + for (ManifestFile manifest : mainSnapshot.deleteManifests(table.io())) { + ManifestEvaluator evaluator = evaluators.get(manifest.partitionSpecId()); + if (evaluator == null || !evaluator.eval(manifest)) { + continue; + } + + readDVEntries(manifest, affectedPaths, dvs); + } + + return dvs; + } + + private Map partitionEvaluators( + Map positions) { + Map templatesBySpec = Maps.newHashMap(); + Map> partitionsBySpec = Maps.newHashMap(); + for (FilePositions filePositions : positions.values()) { + PartitionSpec spec = table.specs().get(filePositions.specId); + StructLikeWrapper template = + templatesBySpec.computeIfAbsent( + filePositions.specId, id -> StructLikeWrapper.forType(spec.partitionType())); + partitionsBySpec + .computeIfAbsent(filePositions.specId, k -> Sets.newHashSet()) + .add(template.copyFor(filePositions.partition(spec.partitionType()))); + } + + Map evaluators = Maps.newHashMap(); + for (Map.Entry> entry : partitionsBySpec.entrySet()) { + PartitionSpec spec = table.specs().get(entry.getKey()); + Expression filter = partitionFilter(spec, entry.getValue()); + evaluators.put(entry.getKey(), ManifestEvaluator.forPartitionFilter(filter, spec, false)); + } + + return evaluators; + } + + private static Expression partitionFilter(PartitionSpec spec, Set partitions) { + List fields = spec.fields(); + Expression anyPartition = Expressions.alwaysFalse(); + for (StructLikeWrapper wrapper : partitions) { + StructLike partition = wrapper.get(); + Expression onePartition = Expressions.alwaysTrue(); + for (int i = 0; i < fields.size(); i++) { + String name = fields.get(i).name(); + Object value = partition.get(i, Object.class); + Expression predicate = + value == null ? Expressions.isNull(name) : Expressions.equal(name, value); + onePartition = Expressions.and(onePartition, predicate); + } + + anyPartition = Expressions.or(anyPartition, onePartition); + } + + return anyPartition; + } + + private void readDVEntries( + ManifestFile manifest, Set filterPaths, Map out) { + manifestsRead++; + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { + for (DeleteFile deleteFile : reader) { + if (ContentFileUtil.isDV(deleteFile) + && deleteFile.referencedDataFile() != null + && filterPaths.contains(deleteFile.referencedDataFile())) { + out.put(deleteFile.referencedDataFile(), deleteFile); + } + } + } catch (IOException e) { + throw new UncheckedIOException("Failed to read manifest: " + manifest.path(), e); + } + } + + @VisibleForTesting + int manifestsReadLastCycle() { + return manifestsRead; + } + + @VisibleForTesting + int retainedStateSize() { + return positionsByFile.size(); + } + + private PositionDeleteIndex loadPreviousDV(String dataFilePath, Map dvs) { + DeleteFile existingDV = dvs.get(dataFilePath); + if (existingDV == null) { + return null; + } + + return deleteLoader.loadPositionDeletes(ImmutableList.of(existingDV), dataFilePath); + } + + private static final class FilePositions { + private final int specId; + private final byte[] encodedPartition; + private final Roaring64Bitmap positions = new Roaring64Bitmap(); + private StructLike decodedPartition; + + FilePositions(int specId, byte[] encodedPartition) { + this.specId = specId; + this.encodedPartition = encodedPartition; + } + + StructLike partition(Types.StructType partitionType) { + if (decodedPartition == null) { + decodedPartition = StructLikeSerializer.decodePartition(encodedPartition, partitionType); + } + + return decodedPartition; + } + } +} diff --git a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertPlanner.java b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertPlanner.java new file mode 100644 index 000000000000..89e5510dd848 --- /dev/null +++ b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertPlanner.java @@ -0,0 +1,782 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.operator; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import org.apache.flink.annotation.Internal; +import org.apache.flink.annotation.VisibleForTesting; +import org.apache.flink.api.common.state.ListState; +import org.apache.flink.api.common.state.ListStateDescriptor; +import org.apache.flink.api.common.typeinfo.Types; +import org.apache.flink.metrics.Counter; +import org.apache.flink.metrics.MetricGroup; +import org.apache.flink.runtime.state.StateInitializationContext; +import org.apache.flink.runtime.state.StateSnapshotContext; +import org.apache.flink.streaming.api.operators.AbstractStreamOperator; +import org.apache.flink.streaming.api.operators.OneInputStreamOperator; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.util.OutputTag; +import org.apache.iceberg.ContentFile; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileContent; +import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.PartitionField; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SnapshotChanges; +import org.apache.iceberg.SnapshotSummary; +import org.apache.iceberg.Table; +import org.apache.iceberg.flink.TableLoader; +import org.apache.iceberg.flink.maintenance.api.Trigger; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.apache.iceberg.util.ContentFileUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Planner for the equality delete conversion pipeline. For each trigger, it picks the oldest + * staging snapshot that hasn't been converted yet and emits {@link ReadCommand}s describing the + * files its downstream readers and workers must process. + * + *

Each trigger runs two steps in order: + * + *

    + *
  1. {@link #ensureIndexCurrent}: updates {@link #lastStagingSnapshotId} from main's history, + * bootstraps the worker index from main on first run, and reindexes when external commits + * (e.g. compaction) have advanced main past the currently-indexed snapshot. + *
  2. {@link #processStagingSnapshot}: resolve the chosen staging snapshot's eq deletes against + * the (now-current) index, pass through any DV files, and index the snapshot's new data files + * for the next cycle. + *
+ * + * Watermarks separate phases that gate the worker's keyed state. The contract is documented on + * {@link #advancePhase()}. + * + *

An {@link EqualityConvertPlan} with the current cycle's metadata is emitted via the {@link + * #METADATA_STREAM} side output after the read commands. + * + *

Assumes a single equality-field set supplied via the builder; staging eq-deletes with a + * different {@code equalityFieldIds} fail fast in {@link #retrieveStagingFiles}. Concurrent writes + * on the target branch are handled by {@link #ensureIndexCurrent} reindexing from the new main + * snapshot; commit-time conflicts are caught by {@code RowDelta.validateFromSnapshot}. + */ +@Internal +public class EqualityConvertPlanner extends AbstractStreamOperator + implements OneInputStreamOperator { + + private static final Logger LOG = LoggerFactory.getLogger(EqualityConvertPlanner.class); + + public static final OutputTag METADATA_STREAM = + new OutputTag<>("metadata-stream") {}; + + public static final OutputTag CLEAR_BROADCAST_STREAM = + new OutputTag<>("clear-broadcast-stream") {}; + + private static final String PROCESSED_EQ_DELETE_FILE_NUM_METRIC = "processedEqDeleteFileNum"; + private static final String PROCESSED_STAGING_SNAPSHOT_NUM_METRIC = "processedStagingSnapshotNum"; + private static final String SKIPPED_NO_OP_CYCLES_METRIC = "skippedNoOpCycles"; + private static final String REINDEX_COUNT_METRIC = "reindexCount"; + + private final String tableName; + private final String taskName; + private final TableLoader tableLoader; + private final String stagingBranch; + private final String targetBranch; + private final boolean stagingOnTargetBranch; + // Equality-field-id set the worker keys on. Supplied via the builder; every staging + // eq-delete's equalityFieldIds() must match exactly. + private final Set eqFieldIds; + + // Main snapshot id the worker's index reflects. + private transient ListState indexSnapshotState; + // Main sequence number the worker's index reflects. + private transient ListState indexedSequenceNumberState; + // Equality field IDs the index was built with, allows to detect reconfiguration. + private transient ListState eqFieldIdsState; + + private transient Table table; + + private transient Long lastMainSnapshotId; + private transient Long lastStagingSnapshotId; + private transient Long indexSnapshotId; + private transient Long indexedSequenceNumber; + + private transient long nextPhaseTs; + + private transient Counter processedEqDeleteFileNumCounter; + private transient Counter processedStagingSnapshotNumCounter; + private transient Counter skippedNoOpCyclesCounter; + private transient Counter reindexCounter; + private transient Counter errorCounter; + + public EqualityConvertPlanner( + String tableName, + String taskName, + TableLoader tableLoader, + String stagingBranch, + String targetBranch, + Set eqFieldIds) { + this.tableName = tableName; + this.taskName = taskName; + this.tableLoader = tableLoader; + this.stagingBranch = stagingBranch; + this.targetBranch = targetBranch; + this.stagingOnTargetBranch = stagingBranch.equals(targetBranch); + Preconditions.checkArgument( + eqFieldIds != null && !eqFieldIds.isEmpty(), "eqFieldIds must not be null or empty"); + this.eqFieldIds = ImmutableSet.copyOf(eqFieldIds); + } + + @Override + public void open() throws Exception { + super.open(); + if (!tableLoader.isOpen()) { + tableLoader.open(); + } + + table = tableLoader.loadTable(); + + MetricGroup taskMetricGroup = + TableMaintenanceMetrics.groupFor(getRuntimeContext(), tableName, taskName, 0); + this.processedEqDeleteFileNumCounter = + taskMetricGroup.counter(PROCESSED_EQ_DELETE_FILE_NUM_METRIC); + this.processedStagingSnapshotNumCounter = + taskMetricGroup.counter(PROCESSED_STAGING_SNAPSHOT_NUM_METRIC); + this.skippedNoOpCyclesCounter = taskMetricGroup.counter(SKIPPED_NO_OP_CYCLES_METRIC); + this.reindexCounter = taskMetricGroup.counter(REINDEX_COUNT_METRIC); + this.errorCounter = taskMetricGroup.counter(TableMaintenanceMetrics.ERROR_COUNTER); + } + + @Override + public void initializeState(StateInitializationContext context) throws Exception { + super.initializeState(context); + indexSnapshotState = + context + .getOperatorStateStore() + .getListState(new ListStateDescriptor<>("indexSnapshotId", Types.LONG)); + + indexSnapshotId = null; + for (Long stateValue : indexSnapshotState.get()) { + Preconditions.checkState( + indexSnapshotId == null, "indexSnapshotId state should hold at most one value"); + indexSnapshotId = stateValue; + } + + indexedSequenceNumberState = + context + .getOperatorStateStore() + .getListState(new ListStateDescriptor<>("indexedSequenceNumber", Types.LONG)); + + indexedSequenceNumber = null; + for (Long stateValue : indexedSequenceNumberState.get()) { + Preconditions.checkState( + indexedSequenceNumber == null, + "indexedSequenceNumber state should hold at most one value"); + indexedSequenceNumber = stateValue; + } + + eqFieldIdsState = + context + .getOperatorStateStore() + .getListState(new ListStateDescriptor<>("eqFieldIds", Types.INT)); + Set restoredEqFieldIds = Sets.newHashSet(eqFieldIdsState.get()); + Preconditions.checkState( + restoredEqFieldIds.isEmpty() || restoredEqFieldIds.equals(eqFieldIds), + "Equality field IDs changed across restart: restored=%s, configured=%s. " + + "Reconfiguring equality-field columns is not supported; " + + "restart from a clean state (no savepoint).", + restoredEqFieldIds, + eqFieldIds); + } + + @Override + public void snapshotState(StateSnapshotContext context) throws Exception { + super.snapshotState(context); + indexSnapshotState.clear(); + if (indexSnapshotId != null) { + indexSnapshotState.add(indexSnapshotId); + } + + indexedSequenceNumberState.clear(); + if (indexedSequenceNumber != null) { + indexedSequenceNumberState.add(indexedSequenceNumber); + } + + eqFieldIdsState.clear(); + for (int id : eqFieldIds) { + eqFieldIdsState.add(id); + } + } + + @Override + public void processElement(StreamRecord element) throws Exception { + long triggerTs = element.getTimestamp(); + nextPhaseTs = Math.max(triggerTs, nextPhaseTs + 1); + + Long currentMainSnapshotId = lastMainSnapshotId; + try { + table.refresh(); + Snapshot mainSnapshot = table.snapshot(targetBranch); + currentMainSnapshotId = mainSnapshot != null ? mainSnapshot.snapshotId() : null; + + ensureIndexCurrent(mainSnapshot); + + Snapshot nextToProcess = + nextUnprocessedStagingSnapshot(table.snapshot(stagingBranch), mainSnapshot); + + if (nextToProcess == null) { + LOG.info("Nothing new to convert on staging branch '{}'.", stagingBranch); + emitNoOpResult(triggerTs, currentMainSnapshotId); + return; + } + + processStagingSnapshot(nextToProcess, triggerTs, currentMainSnapshotId); + } catch (Exception e) { + LOG.error("Error processing equality deletes for table {} task {}", tableName, taskName, e); + output.collect(TaskResultAggregator.ERROR_STREAM, new StreamRecord<>(e)); + errorCounter.inc(); + emitDrainResult(triggerTs, currentMainSnapshotId); + } + } + + /** + * Brings the worker's index up to date with the current state of the target branch: + * + *

    + *
  • Updates {@link #lastStagingSnapshotId} from the most recent committer marker on main. + *
  • Bootstraps the index from main on the first trigger with a non-null main snapshot. + *
  • Reindexes from main when external commits (e.g. compaction or direct writes) have + * advanced main past the currently-indexed snapshot. + *
+ * + *

No-op when main hasn't moved since the last trigger. Otherwise the history walk is bounded + * to commits added since {@link #lastMainSnapshotId}. + */ + private void ensureIndexCurrent(Snapshot mainSnapshot) { + Long currentMainSnapshotId = mainSnapshot != null ? mainSnapshot.snapshotId() : null; + + if (Objects.equals(lastMainSnapshotId, currentMainSnapshotId)) { + return; + } + + LastCommittedWork info = discoverLastCommittedWork(mainSnapshot); + updateLastStagingSnapshotId(info); + + boolean bootstrap = mainSnapshot != null && indexSnapshotId == null; + boolean reindex = indexSnapshotId != null && info.externalCommitCount() > 0; + if (bootstrap || reindex) { + LOG.info( + "{} worker index from main snapshot {} for field IDs {}.", + bootstrap ? "Bootstrapping" : "Reindexing", + currentMainSnapshotId, + eqFieldIds); + if (reindex) { + // Evict keyed entries the reindex will not re-add (e.g. data file removed by CoW). + output.collect( + CLEAR_BROADCAST_STREAM, + new StreamRecord<>( + IndexCommand.clearBeforeReindex( + currentMainSnapshotId, mainSnapshot.sequenceNumber()))); + reindexCounter.inc(); + } + + indexSnapshotId = currentMainSnapshotId; + indexedSequenceNumber = mainSnapshot.sequenceNumber(); + emitMainDataReadCommands(mainSnapshot); + } + + lastMainSnapshotId = currentMainSnapshotId; + } + + private void updateLastStagingSnapshotId(LastCommittedWork info) { + if (info.lastCommittedStaging() != null) { + lastStagingSnapshotId = info.lastCommittedStaging(); + return; + } + + Preconditions.checkState( + lastMainSnapshotId == null || lastStagingSnapshotId == null || info.reachedLastInspected(), + "No COMMITTED_STAGING_SNAPSHOT marker reachable on target branch '%s' for table %s, " + + "but a prior marker was seen (lastStagingSnapshotId=%s). Target may have been " + + "rewritten (rollback, replace_main, or snapshot expiration). " + + "Manual intervention required.", + targetBranch, + tableName, + lastStagingSnapshotId); + } + + /** + * Walks main back from head looking for the most recent snapshot tagged with {@link + * EqualityConvertCommitter#COMMITTED_STAGING_SNAPSHOT_PROPERTY}. Returns the staging snapshot id + * recorded there (or {@code null} if not reached) and the count of intervening external commits. + * + *

The walk stops at whichever comes first: + * + *

    + *
  • The first snapshot carrying the committer marker. + *
  • {@link #lastMainSnapshotId} — anything older was inspected on a previous trigger. + *
+ */ + private LastCommittedWork discoverLastCommittedWork(Snapshot mainSnapshot) { + Long lastCommittedStaging = null; + int externalCount = 0; + boolean reachedLastInspected = false; + Snapshot current = mainSnapshot; + while (current != null) { + if (lastMainSnapshotId != null && current.snapshotId() == lastMainSnapshotId) { + reachedLastInspected = true; + break; + } + + String prop = + current.summary().get(EqualityConvertCommitter.COMMITTED_STAGING_SNAPSHOT_PROPERTY); + if (prop != null) { + lastCommittedStaging = Long.parseLong(prop); + break; + } + + externalCount++; + current = parentOf(current); + } + + return new LastCommittedWork(lastCommittedStaging, externalCount, reachedLastInspected); + } + + private record LastCommittedWork( + Long lastCommittedStaging, int externalCommitCount, boolean reachedLastInspected) {} + + /** + * Walks staging history from head back to the stop point and returns the oldest unprocessed + * snapshot to convert this cycle, or {@code null} if there's nothing new. + * + *

The stop point is: + * + *

    + *
  • the last-processed staging snapshot, if known; + *
  • otherwise, on cold start with {@code stagingBranch != targetBranch}, the common ancestor + * with target; + *
  • otherwise, on cold start with {@code stagingBranch == targetBranch}, the root of history + * ({@code null}). {@link #shouldSkip} filters already-converted snapshots in O(1) via the + * {@code COMMITTED_STAGING_SNAPSHOT_PROPERTY} marker, and pure-insert snapshots via the + * same-branch eq-delete predicate. + *
+ * + *

When {@code stagingBranch == targetBranch}, the writer commits eq-deletes and data files + * directly to target; the converter performs in-place eq-delete-to-DV compaction. On cold start + * we walk the full history so eq-deletes committed before the converter started are still picked + * up. + */ + private Snapshot nextUnprocessedStagingSnapshot(Snapshot stagingHead, Snapshot mainSnapshot) { + if (stagingHead == null) { + return null; + } + + Long stopAt; + if (lastStagingSnapshotId != null) { + stopAt = lastStagingSnapshotId; + } else if (stagingOnTargetBranch) { + stopAt = null; + } else { + stopAt = findCommonAncestor(stagingHead, mainSnapshot); + } + + Snapshot current = stagingHead; + Snapshot oldestUnprocessed = null; + while (current != null) { + if (stopAt != null && current.snapshotId() == stopAt) { + break; + } + + if (!shouldSkip(current)) { + oldestUnprocessed = current; + } + + current = parentOf(current); + } + + return oldestUnprocessed; + } + + /** + * Resolves the eq deletes in {@code stagingSnapshot} against the current index and emits the + * cycle's metadata. Phase ordering (separated by watermarks): + * + *

    + *
  1. Eq delete read commands. Eq deletes resolve in the worker. + *
  2. Staging data files. Indexed for the NEXT cycle's eq-delete resolution. + *
+ * + * Cold-start bootstrap of the index from main data is handled separately in {@link + * #processElement}, which runs once before the first cycle. + */ + private void processStagingSnapshot( + Snapshot stagingSnapshot, long triggerTs, Long currentMainSnapshotId) { + + StagingInputs inputs = retrieveStagingFiles(stagingSnapshot); + Preconditions.checkState( + !inputs.isEmpty(), + "Staging snapshot %s has no convertible inputs; shouldSkip should have filtered it.", + stagingSnapshot.snapshotId()); + + emitDeletePhase(inputs.eqDeleteFiles()); + emitSnapshotDataPhase(inputs.newDataFiles()); + + LOG.info( + "Emitted read commands for {} new data files from staging branch '{}'.", + inputs.newDataFiles().size(), + stagingBranch); + + processedStagingSnapshotNumCounter.inc(); + + output.collect( + METADATA_STREAM, + new StreamRecord<>( + new EqualityConvertPlan( + inputs.newDataFiles(), + inputs.stagingDVFiles(), + stagingSnapshot.snapshotId(), + currentMainSnapshotId, + triggerTs, + nextPhaseTs))); + + advancePhase(); + } + + /** + * Classifies the files added by {@code stagingSnapshot} into data files, eq delete files, and DV + * files. Throws if the snapshot: + * + *
    + *
  • Removes data files (rewrites on the staging branch aren't supported). + *
  • Contains V2 positional delete files (the converter expects a V3 staging branch written by + * Flink, which produces only deletion vectors for deletes). + *
  • Contains an eq-delete file whose {@code equalityFieldIds()} doesn't match the + * builder-configured set (silent wrong-key serialization otherwise). + *
+ */ + private StagingInputs retrieveStagingFiles(Snapshot stagingSnapshot) { + SnapshotChanges changes = SnapshotChanges.builderFor(table).snapshot(stagingSnapshot).build(); + + // Rewrites on the staging branch would require rewriting the corresponding DVs against new + // data files on target. Not implemented; fail fast instead of silently dropping work. + if (changes.removedDataFiles().iterator().hasNext()) { + throw new IllegalStateException( + String.format( + "Staging snapshot %s on branch '%s' removes data files; " + + "equality delete conversion does not support rewrites on the staging branch. " + + "Run compaction on the target branch instead.", + stagingSnapshot.snapshotId(), stagingBranch)); + } + + List newDataFiles = Lists.newArrayList(); + List stagingDVFiles = Lists.newArrayList(); + List eqDeleteFiles = Lists.newArrayList(); + + for (DataFile dataFile : changes.addedDataFiles()) { + newDataFiles.add(dataFile); + } + + for (DeleteFile deleteFile : changes.addedDeleteFiles()) { + if (deleteFile.content() == FileContent.EQUALITY_DELETES) { + Set deleteFieldIds = Sets.newHashSet(deleteFile.equalityFieldIds()); + Preconditions.checkState( + deleteFieldIds.equals(eqFieldIds), + "Staging snapshot %s on branch '%s' contains an equality delete file %s with " + + "equalityFieldIds=%s, which does not match the configured eqFieldIds=%s. " + + "The writer must use the same equality field IDs as the converter.", + stagingSnapshot.snapshotId(), + stagingBranch, + deleteFile.location(), + deleteFieldIds, + eqFieldIds); + validateDeleteSpecPartitionColumns(stagingSnapshot, deleteFile); + eqDeleteFiles.add(deleteFile); + } else if (ContentFileUtil.isDV(deleteFile)) { + stagingDVFiles.add(deleteFile); + } else { + throw new IllegalStateException( + String.format( + "Staging snapshot %s on branch '%s' contains a V2 positional delete file (%s); " + + "equality delete conversion expects a V3 staging branch written by Flink, " + + "which produces only deletion vectors for deletes.", + stagingSnapshot.snapshotId(), stagingBranch, deleteFile.location())); + } + } + + return new StagingInputs(newDataFiles, stagingDVFiles, eqDeleteFiles); + } + + private void validateDeleteSpecPartitionColumns(Snapshot stagingSnapshot, DeleteFile deleteFile) { + PartitionSpec spec = table.specs().get(deleteFile.specId()); + for (PartitionField field : spec.fields()) { + Preconditions.checkState( + eqFieldIds.contains(field.sourceId()), + "Staging snapshot %s on branch '%s' contains an equality delete file %s under spec %s, " + + "which partitions by field '%s' (source id %s) that is not an equality field %s. " + + "Partition columns must be a subset of the equality fields.", + stagingSnapshot.snapshotId(), + stagingBranch, + deleteFile.location(), + spec.specId(), + field.name(), + field.sourceId(), + eqFieldIds); + } + } + + /** Files added by one staging snapshot, classified for cycle emission. */ + private record StagingInputs( + List newDataFiles, + List stagingDVFiles, + List eqDeleteFiles) { + + boolean isEmpty() { + return newDataFiles.isEmpty() && eqDeleteFiles.isEmpty() && stagingDVFiles.isEmpty(); + } + } + + private void emitDeletePhase(List eqDeleteFiles) { + for (DeleteFile deleteFile : eqDeleteFiles) { + PartitionSpec spec = table.specs().get(deleteFile.specId()); + output.collect( + new StreamRecord<>( + ReadCommand.eqDeleteFile( + deleteFile, + spec, + indexSnapshotId, + indexedSequenceNumber, + dataSequenceNumber(deleteFile)), + nextPhaseTs)); + processedEqDeleteFileNumCounter.inc(); + } + + advancePhase(); + } + + private void emitSnapshotDataPhase(List snapshotDataFiles) { + // Shared-branch skip: when stagingBranch == targetBranch, these files are already on target + // and were indexed by bootstrap/reindex. Re-emitting would duplicate entries in + // dataRowPositions. + if (!stagingOnTargetBranch) { + for (DataFile dataFile : snapshotDataFiles) { + PartitionSpec spec = table.specs().get(dataFile.specId()); + output.collect( + new StreamRecord<>( + ReadCommand.stagingDataFile( + new FlinkAddedRowsScanTask(dataFile, spec), + indexSnapshotId, + indexedSequenceNumber, + dataSequenceNumber(dataFile)), + nextPhaseTs)); + } + } + + advancePhase(); + } + + private void emitNoOpResult(long triggerTimestamp, Long currentMainSnapshotId) { + skippedNoOpCyclesCounter.inc(); + emitDrainResult(triggerTimestamp, currentMainSnapshotId); + } + + /** Emits an empty plan result and advances the phase so the pipeline drains. */ + private void emitDrainResult(long triggerTimestamp, Long currentMainSnapshotId) { + output.collect( + METADATA_STREAM, + new StreamRecord<>( + EqualityConvertPlan.noOp(currentMainSnapshotId, triggerTimestamp, nextPhaseTs))); + advancePhase(); + } + + /** + * Emits {@link ReadCommand}s for every data file on {@code mainSnapshot} so the worker indexes + * them for the configured equality-field set. Existing DVs attached to a data file are loaded by + * the reader and their positions are skipped. V2 positional deletes are not expected on main; the + * reader throws if it encounters one. Equality deletes attached to the scan task are skipped + * during indexing (they are processed via the planner's eq-delete read commands). + */ + private void emitMainDataReadCommands(Snapshot mainSnapshot) { + long commitSnapshotId = mainSnapshot.snapshotId(); + + try (CloseableIterable tasks = + table.newScan().useSnapshot(commitSnapshotId).planFiles()) { + for (FileScanTask task : tasks) { + output.collect( + new StreamRecord<>( + ReadCommand.dataFile( + task, indexSnapshotId, indexedSequenceNumber, dataSequenceNumber(task.file())), + nextPhaseTs)); + } + } catch (IOException e) { + throw new UncheckedIOException("Failed to plan files for main index", e); + } + + LOG.info( + "Emitted main data read commands for field IDs {} from snapshot {}.", + eqFieldIds, + commitSnapshotId); + + advancePhase(); + } + + /** + * Emits a phase-end watermark and bumps the phase timestamp. Every phase-emitting method must + * call this exactly once after its records; the worker uses these watermarks to gate keyed-state + * transitions. Missing or extra calls silently break ordering. + */ + private void advancePhase() { + output.emitWatermark(new Watermark(nextPhaseTs)); + nextPhaseTs++; + } + + /** + * Returns {@code true} if {@code snapshot} can be skipped: + * + *
    + *
  • It was already committed by us (carries {@link + * EqualityConvertCommitter#COMMITTED_STAGING_SNAPSHOT_PROPERTY}), OR + *
  • it adds no data or delete files (e.g. delete-file-only removals), OR + *
  • {@code stagingBranch == targetBranch} and the snapshot adds no equality-delete files. + * Pure-insert (and data-file-only) commits on the shared branch don't need a conversion + * cycle: their data is already on target, and the worker's index stays fresh via {@link + * #ensureIndexCurrent} when the next eq-delete arrives. + *
+ * + *

Filter checks read per-snapshot counts from {@link Snapshot#summary()}; we don't parse + * manifests here. Manifest parsing happens later in {@link #retrieveStagingFiles} only for the + * chosen snapshot. + */ + private boolean shouldSkip(Snapshot snapshot) { + Map summary = snapshot.summary(); + if (summary.containsKey(EqualityConvertCommitter.COMMITTED_STAGING_SNAPSHOT_PROPERTY)) { + return true; + } + + long addedDataFiles = summaryCount(summary, SnapshotSummary.ADDED_FILES_PROP); + long addedDeleteFiles = summaryCount(summary, SnapshotSummary.ADDED_DELETE_FILES_PROP); + if (addedDataFiles == 0 && addedDeleteFiles == 0) { + LOG.info( + "Skipping staging snapshot {}: no added data or delete files.", snapshot.snapshotId()); + return true; + } + + if (stagingOnTargetBranch + && summaryCount(summary, SnapshotSummary.ADD_EQ_DELETE_FILES_PROP) == 0) { + return true; + } + + return false; + } + + private static long summaryCount(Map summary, String key) { + String value = summary.get(key); + return value == null ? 0L : Long.parseLong(value); + } + + /** + * Returns the id of the newest snapshot that is reachable from both the staging and target branch + * heads (i.e. the most recent common ancestor where the two branches last matched), or {@code + * null} if they share no history. Used on cold start to know where staging diverged from target + * so we can skip converting snapshots that already exist on target. + */ + private Long findCommonAncestor(Snapshot stagingHead, Snapshot mainHead) { + if (mainHead == null) { + return null; + } + + Set stagingSeen = Sets.newHashSet(); + Set mainSeen = Sets.newHashSet(); + Snapshot stagingCurrent = stagingHead; + Snapshot mainCurrent = mainHead; + + while (stagingCurrent != null || mainCurrent != null) { + if (stagingCurrent != null) { + long id = stagingCurrent.snapshotId(); + if (mainSeen.contains(id)) { + return id; + } + + stagingSeen.add(id); + stagingCurrent = parentOf(stagingCurrent); + } + + if (mainCurrent != null) { + long id = mainCurrent.snapshotId(); + if (stagingSeen.contains(id)) { + return id; + } + + mainSeen.add(id); + mainCurrent = parentOf(mainCurrent); + } + } + + return null; + } + + /** Returns the parent snapshot, or {@code null} if {@code snapshot} has no parent. */ + private Snapshot parentOf(Snapshot snapshot) { + Long parentId = snapshot.parentId(); + return parentId != null ? table.snapshot(parentId) : null; + } + + private static long dataSequenceNumber(ContentFile file) { + Long sequenceNumber = file.dataSequenceNumber(); + Preconditions.checkNotNull( + sequenceNumber, "Missing data sequence number for committed file %s", file.location()); + return sequenceNumber; + } + + @Override + public void close() throws Exception { + super.close(); + tableLoader.close(); + } + + @VisibleForTesting + long reindexCount() { + return reindexCounter.getCount(); + } + + @VisibleForTesting + long skippedNoOpCycles() { + return skippedNoOpCyclesCounter.getCount(); + } + + @VisibleForTesting + long processedStagingSnapshotNum() { + return processedStagingSnapshotNumCounter.getCount(); + } + + @VisibleForTesting + long processedEqDeleteFileNum() { + return processedEqDeleteFileNumCounter.getCount(); + } +} diff --git a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/source/DataIterator.java b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/source/DataIterator.java index 3beda960cec8..2cb7667a489a 100644 --- a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/source/DataIterator.java +++ b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/source/DataIterator.java @@ -86,6 +86,7 @@ public void seek(int startingFileOffset, long startingRecordOffset) { combinedTask); for (long i = 0L; i < startingFileOffset; ++i) { tasks.next(); + fileOffset += 1; } updateCurrentIterator(); @@ -103,9 +104,6 @@ public void seek(int startingFileOffset, long startingRecordOffset) { combinedTask)); } } - - fileOffset = startingFileOffset; - recordOffset = startingRecordOffset; } @Override diff --git a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/CatalogTestBase.java b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/CatalogTestBase.java index 062ff68d5d85..e45abe25f919 100644 --- a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/CatalogTestBase.java +++ b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/CatalogTestBase.java @@ -89,6 +89,7 @@ public void before() { config.put(CatalogProperties.URI, getURI(hiveConf)); } config.put(CatalogProperties.WAREHOUSE_LOCATION, String.format("file://%s", warehouseRoot())); + config.put("extra-catalog-prop", "extra-value"); this.flinkDatabase = catalogName + "." + DATABASE; this.icebergNamespace = diff --git a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java index 020663a7de02..30daf6d55ef2 100644 --- a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java +++ b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java @@ -33,8 +33,9 @@ import org.apache.flink.table.api.Schema.UnresolvedPrimaryKey; import org.apache.flink.table.api.TableException; import org.apache.flink.table.api.ValidationException; +import org.apache.flink.table.catalog.CatalogMaterializedTable; import org.apache.flink.table.catalog.CatalogTable; -import org.apache.flink.table.catalog.CommonCatalogOptions; +import org.apache.flink.table.catalog.IntervalFreshness; import org.apache.flink.table.catalog.ObjectPath; import org.apache.flink.table.catalog.exceptions.TableNotExistException; import org.apache.iceberg.BaseTable; @@ -229,19 +230,15 @@ public void testCreateTableLikeInFlinkCatalog() throws TableNotExistException { .column("id", DataTypes.BIGINT()) .build()); - // `type` option is filtered out by Flink - // https://github.com/apache/flink/blob/edc3d68736de73665440f4313ddcfd9142d8d42b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/factories/FactoryUtil.java#L378 - Map filteredOptions = Maps.newHashMap(config); - filteredOptions.remove(CommonCatalogOptions.CATALOG_TYPE.key()); - - String srcCatalogProps = - FlinkCreateTableOptions.toJson(catalogName, DATABASE, "tl", filteredOptions); + String srcCatalogProps = FlinkCreateTableOptions.toJson(catalogName, DATABASE, "tl"); Map options = catalogTable.getOptions(); assertThat(options) .containsEntry( FlinkCreateTableOptions.CONNECTOR_PROPS_KEY, FlinkDynamicTableFactory.FACTORY_IDENTIFIER) .containsEntry(FlinkCreateTableOptions.SRC_CATALOG_PROPS_KEY, srcCatalogProps); + assertThat(options.get(FlinkCreateTableOptions.SRC_CATALOG_PROPS_KEY)) + .doesNotContain("extra-catalog-prop", "extra-value"); } @TestTemplate @@ -750,6 +747,31 @@ private void validateTableFiles(Table tbl, DataFile... expectedFiles) { assertThat(actualFilePaths).as("Files should match").isEqualTo(expectedFilePaths); } + @TestTemplate + public void testCreateMaterializedTableIsUnsupported() { + CatalogMaterializedTable materializedTable = + CatalogMaterializedTable.newBuilder() + .schema( + org.apache.flink.table.api.Schema.newBuilder() + .column("id", DataTypes.BIGINT()) + .build()) + .definitionQuery("SELECT id FROM tl") + .freshness(IntervalFreshness.ofMinute("5")) + .logicalRefreshMode(CatalogMaterializedTable.LogicalRefreshMode.AUTOMATIC) + .refreshMode(CatalogMaterializedTable.RefreshMode.CONTINUOUS) + .refreshStatus(CatalogMaterializedTable.RefreshStatus.INITIALIZING) + .build(); + + assertThatThrownBy( + () -> + getTableEnv() + .getCatalog(catalogName) + .get() + .createTable(new ObjectPath(DATABASE, "mt_table"), materializedTable, false)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Materialized tables and other table kinds are not supported"); + } + private Table table(String name) { return validationCatalog.loadTable(TableIdentifier.of(icebergNamespace, name)); } diff --git a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestConvertEqualityDeletes.java b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestConvertEqualityDeletes.java new file mode 100644 index 000000000000..1c2f230315be --- /dev/null +++ b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestConvertEqualityDeletes.java @@ -0,0 +1,1333 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.api; + +import static org.apache.iceberg.flink.SimpleDataUtil.createRecord; +import static org.apache.iceberg.flink.maintenance.api.ConvertEqualityDeletes.COMMIT_TASK_NAME; +import static org.apache.iceberg.flink.maintenance.api.ConvertEqualityDeletes.PLANNER_TASK_NAME; +import static org.apache.iceberg.flink.maintenance.operator.EqualityConvertCommitter.COMMITTED_STAGING_SNAPSHOT_PROPERTY; +import static org.apache.iceberg.flink.maintenance.operator.TableMaintenanceMetrics.ADDED_DATA_FILE_NUM_METRIC; +import static org.apache.iceberg.flink.maintenance.operator.TableMaintenanceMetrics.ADDED_DATA_FILE_SIZE_METRIC; +import static org.apache.iceberg.flink.maintenance.operator.TableMaintenanceMetrics.ERROR_COUNTER; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Path; +import java.time.Duration; +import java.util.List; +import java.util.Set; +import java.util.stream.StreamSupport; +import org.apache.flink.core.execution.JobClient; +import org.apache.flink.streaming.api.graph.StreamGraphGenerator; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileContent; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.Files; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; +import org.apache.iceberg.PartitionData; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.RewriteFiles; +import org.apache.iceberg.Schema; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.Table; +import org.apache.iceberg.data.FileHelpers; +import org.apache.iceberg.data.GenericAppenderHelper; +import org.apache.iceberg.data.IcebergGenerics; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.deletes.PositionDelete; +import org.apache.iceberg.flink.SimpleDataUtil; +import org.apache.iceberg.flink.maintenance.operator.MetricsReporterFactoryForTests; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.StructLikeSet; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class TestConvertEqualityDeletes extends MaintenanceTaskTestBase { + + private static final String STAGING_BRANCH = "__flink_staging_test"; + + @TempDir private Path tempDir; + + @Test + void testRejectsFormatVersion2() { + createTableWithDelete(2); + + assertThatThrownBy( + () -> + ConvertEqualityDeletes.builder() + .stagingBranch(STAGING_BRANCH) + .equalityFieldColumns(ImmutableList.of("id", "data")) + .append( + infra.triggerStream(), + DUMMY_TABLE_NAME, + DUMMY_TASK_NAME, + 0, + tableLoader(), + UID_SUFFIX, + StreamGraphGenerator.DEFAULT_SLOT_SHARING_GROUP, + 1)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("format version >= 3"); + } + + @Test + void testRejectsUnknownEqualityFieldColumns() { + createTableWithDelete(3); + + assertThatThrownBy( + () -> + ConvertEqualityDeletes.builder() + .stagingBranch(STAGING_BRANCH) + .equalityFieldColumns(ImmutableList.of("nonexistent")) + .append( + infra.triggerStream(), + DUMMY_TABLE_NAME, + DUMMY_TASK_NAME, + 0, + tableLoader(), + UID_SUFFIX, + StreamGraphGenerator.DEFAULT_SLOT_SHARING_GROUP, + 1)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Equality field column 'nonexistent' not found in table schema"); + } + + @Test + void testConvertEqualityDeletesToDVs() throws Exception { + Table table = createTableWithDelete(3); + + // Insert initial data to main + insert(table, 1, "a"); + insert(table, 2, "b"); + insert(table, 3, "c"); + + assertThat(dataFileCount(table)).isEqualTo(3); + + // Create staging branch from current main state + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Write a new data file (simulating insert of id=4) + DataFile newDataFile = writeDataFile(table, createRecord(4, "d")); + + // Write an equality delete for id=2 (simulating delete of row "b") + DeleteFile eqDelete = writeEqualityDelete(table, 2, "b"); + + // Commit both to the staging branch + table.newRowDelta().addRows(newDataFile).addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Verify staging branch has the eq delete + long stagingEqDeleteCount = + table.snapshot(STAGING_BRANCH).deleteManifests(table.io()).stream() + .flatMap( + m -> + StreamSupport.stream( + ManifestFiles.readDeleteManifest(m, table.io(), table.specs()) + .spliterator(), + false)) + .filter(f -> f.content() == FileContent.EQUALITY_DELETES) + .count(); + assertThat(stagingEqDeleteCount).isEqualTo(1); + + // Wire the ConvertEqualityDeletes maintenance task + appendConvertTask(); + + // Run the maintenance task + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + // Verify main branch now has 4 data files (3 original + 1 new) + table.refresh(); + assertThat(dataFileCount(table)).isEqualTo(4); + + // Verify main branch has exactly one DV (id=2 deleted from its single-row data file) + long mainDvCount = + table.currentSnapshot().deleteManifests(table.io()).stream() + .flatMap( + m -> + StreamSupport.stream( + ManifestFiles.readDeleteManifest(m, table.io(), table.specs()) + .spliterator(), + false)) + .filter(f -> f.content() == FileContent.POSITION_DELETES) + .count(); + assertThat(mainDvCount).isEqualTo(1); + + // Verify no equality deletes on main + long mainEqDeleteCount = + table.currentSnapshot().deleteManifests(table.io()).stream() + .flatMap( + m -> + StreamSupport.stream( + ManifestFiles.readDeleteManifest(m, table.io(), table.specs()) + .spliterator(), + false)) + .filter(f -> f.content() == FileContent.EQUALITY_DELETES) + .count(); + assertThat(mainEqDeleteCount).isEqualTo(0); + + // Verify data correctness: id=2 should be deleted, id=4 should be added + SimpleDataUtil.assertTableRecords( + table, ImmutableList.of(createRecord(1, "a"), createRecord(3, "c"), createRecord(4, "d"))); + } + + @Test + void testMetrics() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // One staging snapshot: a new data file (id=3) plus an eq-delete (id=1). The conversion commits + // exactly one data file and one DV to main. + DataFile newDataFile = writeDataFile(table, createRecord(3, "c")); + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(newDataFile).addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + // Only metrics named on TableMaintenanceMetrics flow through the test reporter. Among the + // converter operators only the planner and committer own an ERROR_COUNTER; the parallel reader, + // PK index, and DV writer report failures through ERROR_STREAM instead. The committer also + // counts the data files it adds. Operator-specific counters (reindexCount, addedDvNum, ...) are + // asserted by the operator unit tests. A -1 expected value means "present, value not checked". + MetricsReporterFactoryForTests.assertCounters( + new ImmutableMap.Builder, Long>() + .put(errorKey(PLANNER_TASK_NAME), 0L) + .put(errorKey(COMMIT_TASK_NAME), 0L) + .put(metricKey(COMMIT_TASK_NAME, ADDED_DATA_FILE_NUM_METRIC), 1L) + .put(metricKey(COMMIT_TASK_NAME, ADDED_DATA_FILE_SIZE_METRIC), -1L) + .build()); + } + + private static List errorKey(String taskName) { + return metricKey(taskName, ERROR_COUNTER); + } + + private static List metricKey(String taskName, String metric) { + return ImmutableList.of(taskName + "[0]", DUMMY_TABLE_NAME, DUMMY_TASK_NAME, "0", metric); + } + + @Test + void testNoOpWhenStagingBranchEmpty() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + // Create staging branch with no new files + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + // Should complete successfully with no changes + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + assertThat(dataFileCount(table)).isEqualTo(1); + SimpleDataUtil.assertTableRecords(table, ImmutableList.of(createRecord(1, "a"))); + } + + @Test + void testMultipleEqualityDeletes() throws Exception { + Table table = createTableWithDelete(3); + + insert(table, 1, "a"); + insert(table, 2, "b"); + insert(table, 3, "c"); + insert(table, 4, "d"); + insert(table, 5, "e"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Delete id=1 and id=4 via equality deletes on staging + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + DeleteFile eqDelete2 = writeEqualityDelete(table, 4, "d"); + + table + .newRowDelta() + .addDeletes(eqDelete1) + .addDeletes(eqDelete2) + .toBranch(STAGING_BRANCH) + .commit(); + table.refresh(); + + appendConvertTask(); + + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + SimpleDataUtil.assertTableRecords( + table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"), createRecord(5, "e"))); + } + + @Test + void testDuplicateKeyAcrossDataFiles() throws Exception { + Table table = createTableWithDelete(3); + + // Two data files with the same key (id=1, data="a") + insert(table, 1, "a"); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Eq delete for id=1 should produce DVs for both data files containing id=1 + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + // Only id=2 should remain + SimpleDataUtil.assertTableRecords(table, ImmutableList.of(createRecord(2, "b"))); + } + + @Test + void testMultiSnapshotStagingWithPerSnapshotScoping() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging snapshot 1: delete id=1 from main + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging snapshot 2: re-insert id=1 (new data file) + DataFile newDataFile = writeDataFile(table, createRecord(1, "a")); + table.newAppend().appendFile(newDataFile).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + // Cycle 1: processes S1 (delete id=1) + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.success()).isTrue(); + + // Cycle 2: processes S2 (re-insert id=1) + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.success()).isTrue(); + + table.refresh(); + // id=1 should still exist: the delete from S1 removed the original, + // but S2 re-inserted it. Per-snapshot scoping ensures S1's delete + // doesn't affect S2's data. + assertRecords(table, ImmutableList.of(createRecord(1, "a"))); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testInsertThenDeleteAcrossCycles() throws Exception { + Table table = createTableWithDelete(3); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging snapshot 1: insert id=1 (data-only, no eq deletes) + DataFile insertS1 = writeDataFile(table, createRecord(1, "a")); + table.newAppend().appendFile(insertS1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging snapshot 2: eq delete the row written in S1 + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + // Cycle 1: processes S1 (insert id=1), commits data file to main + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.exceptions()).isEmpty(); + assertThat(result1.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "a"))); + + // Cycle 2: processes S2 (eq delete id=1), must find id=1 on main + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of()); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testInsertUpdateDeleteInsertUpdateChain() throws Exception { + Table table = createTableWithDelete(3); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S1: insert K=1, V=A + DataFile s1 = writeDataFile(table, createRecord(1, "a")); + table.newAppend().appendFile(s1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S2: update K=1 to V=B (eq delete + insert in same commit) + DataFile s2 = writeDataFile(table, createRecord(1, "b")); + DeleteFile e2 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(s2).addDeletes(e2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S3: delete K=1 + DeleteFile e3 = writeEqualityDelete(table, 1, "b"); + table.newRowDelta().addDeletes(e3).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S4: insert K=1, V=C + DataFile s4 = writeDataFile(table, createRecord(1, "c")); + table.newAppend().appendFile(s4).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S5: update K=1 to V=D (eq delete + insert in same commit) + DataFile s5 = writeDataFile(table, createRecord(1, "d")); + DeleteFile e5 = writeEqualityDelete(table, 1, "c"); + table.newRowDelta().addRows(s5).addDeletes(e5).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + long time = System.currentTimeMillis(); + + // Cycle 1: S1 inserts K=1, V=A + long time1 = time; + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.exceptions()).isEmpty(); + assertThat(result1.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "a"))); + + // Cycle 2: S2 updates K=1 from V=A to V=B (eq delete + insert) + long time2 = time + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "b"))); + + // Cycle 3: S3 deletes K=1 + long time3 = time + 2; + infra.source().sendRecord(Trigger.create(time3, 0), time3); + TaskResult result3 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result3.exceptions()).isEmpty(); + assertThat(result3.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of()); + + // Cycle 4: S4 inserts K=1, V=C + long time4 = time + 3; + infra.source().sendRecord(Trigger.create(time4, 0), time4); + TaskResult result4 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result4.exceptions()).isEmpty(); + assertThat(result4.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "c"))); + + // Cycle 5: S5 updates K=1 from V=C to V=D (eq delete + insert) + long time5 = time + 4; + infra.source().sendRecord(Trigger.create(time5, 0), time5); + TaskResult result5 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result5.exceptions()).isEmpty(); + assertThat(result5.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "d"))); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testParallelInsertOfToBeDeletedKeySurvives() throws Exception { + Table table = createTableWithDelete(3); + + // Main holds the original (1, "a"); the staging eq-delete below removes this copy. + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // One staging snapshot updates id=1 in place: re-insert (1, "a") plus an eq-delete (1, "a") in + // the same commit. The re-insert shares the equality key with the delete and carries the + // delete's sequence, so it must survive: the delete only removes the lower-sequence main copy. + // At parallelism > 1 the staging-data ADD can reach the index before the eq-delete resolves. + // Event-time phase ordering is what keeps the re-insert from being accidentally deleted. + DataFile reinsert = writeDataFile(table, createRecord(1, "a")); + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(reinsert).addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(STAGING_BRANCH); + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "a"))); + } + + @Test + void testDVMergeAcrossConversionCycles() throws Exception { + Table table = createTableWithDelete(3); + + // Single data file with 3 rows so DV merge applies to the same file + insert( + table, ImmutableList.of(createRecord(1, "a"), createRecord(2, "b"), createRecord(3, "c"))); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Cycle 1 setup: eq delete for id=1 + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + // Cycle 1: convert eq delete for id=1 + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"))); + + // Cycle 2 setup: eq delete for id=2 (committed while job is running) + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Cycle 2: convert eq delete for id=2, should merge DV with existing + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(3, "c"))); + + // Verify: main has DVs, no equality deletes + assertNoEqualityDeletesOnMain(table, 0); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testConversionCorrectAfterCompaction() throws Exception { + Table table = createTableWithDelete(3); + + // Three separate data files + insert(table, 1, "a"); + insert(table, 2, "b"); + insert(table, 3, "c"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Cycle 1: delete id=1 + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"))); + + // Compact file2 and file3 on main into one file (leave file1 + its DV untouched) + Set allDataFiles = Sets.newHashSet(); + for (ManifestFile manifest : table.currentSnapshot().dataManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.read(manifest, table.io(), table.specs())) { + for (DataFile df : reader) { + allDataFiles.add(df.copy()); + } + } + } + + // Find file1 (contains id=1) by checking which file has a DV against it + Set dvReferencedFiles = Sets.newHashSet(); + for (ManifestFile manifest : table.currentSnapshot().deleteManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { + for (DeleteFile df : reader) { + if (df.referencedDataFile() != null) { + dvReferencedFiles.add(df.referencedDataFile()); + } + } + } + } + + Set filesToCompact = Sets.newHashSet(); + for (DataFile df : allDataFiles) { + if (!dvReferencedFiles.contains(df.location())) { + filesToCompact.add(df); + } + } + + assertThat(filesToCompact).hasSize(2); + + DataFile compactedFile = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"))); + RewriteFiles rewrite = table.newRewrite(); + for (DataFile old : filesToCompact) { + rewrite.deleteFile(old); + } + + rewrite.addFile(compactedFile); + rewrite.commit(); + table.refresh(); + + assertRecords(table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"))); + + // Cycle 2: delete id=2 (should target the compacted file) + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(3, "c"))); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testConvertEqualityDeletesPartitionedTable() throws Exception { + Table table = createPartitionedTableWithDelete(3); + + // Insert data into two partitions + insertPartitioned(table, 1, "a"); + insertPartitioned(table, 2, "b"); + insertPartitioned(table, 3, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Partition-scoped equality delete for id=1 in partition data="a" + DeleteFile eqDelete = writePartitionedEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + // id=1 deleted from partition "a", id=2 in partition "b" and id=3 in partition "a" remain + assertRecords(table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "a"))); + } + + @Test + void testEqualityDeleteIsScopedToItsPartition() throws Exception { + Table table = createPartitionedTableWithDelete(3); + + // Same PK (id=1) exists in two partitions. An eq delete in one partition must not delete + // rows in the other. + insertPartitioned(table, 1, "a"); + insertPartitioned(table, 1, "b"); + insertPartitioned(table, 2, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete = writePartitionedEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + // Only (1, "a") is deleted; (1, "b") remains because the eq delete was scoped to partition + // "a", and (2, "a") remains because its equality field values don't match. + assertRecords(table, ImmutableList.of(createRecord(1, "b"), createRecord(2, "a"))); + } + + @Test + void testStagingPositionDeleteMergedIntoConversionDV() throws Exception { + Table table = createTableWithDelete(3); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S1: write a data file with two rows (id=1 at pos 0, id=2 at pos 1). + DataFile data = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(1, "a"), createRecord(2, "b"))); + table.newAppend().appendFile(data).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S2: eq delete matches row 0 (will produce a conversion DV at pos 0) + a position + // delete DV referencing the same data file at pos 1. Both DVs target the same data + // file and must be merged into a single DV (V3 invariant). + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + DeleteFile stagingDV = writeStagingDV(table, data.location(), 1L); + table + .newRowDelta() + .addDeletes(eqDelete) + .addDeletes(stagingDV) + .toBranch(STAGING_BRANCH) + .commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + // Cycle 1: commits S1's data file to main + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.success()).isTrue(); + + // Cycle 2: converts S2's eq delete to DV, merges with staging DV + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + table.refresh(); + // Both rows from S1 must be masked: pos 0 by the conversion DV, pos 1 by the staging DV. + assertRecords(table, ImmutableList.of()); + + // Exactly one DV per data file (V3 invariant): the resolver must have folded the + // staging DV's positions into the conversion DV. + long dvCount = + table.currentSnapshot().deleteManifests(table.io()).stream() + .flatMap( + m -> + StreamSupport.stream( + ManifestFiles.readDeleteManifest(m, table.io(), table.specs()) + .spliterator(), + false)) + .filter(f -> f.content() == FileContent.POSITION_DELETES) + .filter(f -> data.location().equals(f.referencedDataFile())) + .count(); + assertThat(dvCount).isEqualTo(1); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testStagingDataFilesOnlyNoEqDeletes() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging has only a new data file, no equality deletes + DataFile newDataFile = writeDataFile(table, createRecord(2, "b")); + table.newAppend().appendFile(newDataFile).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + assertThat(dataFileCount(table)).isEqualTo(2); + SimpleDataUtil.assertTableRecords( + table, ImmutableList.of(createRecord(1, "a"), createRecord(2, "b"))); + } + + @Test + void testReindexAfterExternalCommit() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Cycle 1: delete id=1 + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(2, "b"))); + + // External commit: insert id=3 directly to main (not via staging) + insert(table, 3, "c"); + + // Cycle 2: delete id=2 (should reindex because of external commit) + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(3, "c"))); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testReindexEvictsGhostKeyAfterExternalDataFileRemoval() throws Exception { + // CoW removal case: an external commit removes a data file, leaving a stale + // PK in the worker's index. A later staging eq-delete for that PK must NOT produce a DV + // referencing the removed file. The external commit advances main, so the next cycle reindexes + // and the worker clears the ghost key before resolving the delete. + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + table.refresh(); + DataFile file1 = table.currentSnapshot().addedDataFiles(table.io()).iterator().next().copy(); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Cycle 1: delete id=2. Bootstraps the worker index from main (id=1 -> file1, id=2 -> file2). + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "a"))); + + // External CoW-style removal: drop file1 (id=1) from main without re-adding the row. The + // worker's index still holds the ghost id=1 -> file1 until the next reindex clears it. + table.newDelete().deleteFile(file1).commit(); + table.refresh(); + assertRecords(table, ImmutableList.of()); + + // Cycle 2: stage an eq-delete for the removed key id=1. Without ghost eviction the worker + // would emit a DV position against the now-absent file1. + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + // No deletion vector may reference the removed file1. + table.refresh(); + assertThat(dvReferencedDataFiles(table)).doesNotContain(file1.location()); + assertRecords(table, ImmutableList.of()); + } finally { + closeJobClient(jobClient); + } + } + + private static Set dvReferencedDataFiles(Table table) { + Set referenced = Sets.newHashSet(); + for (ManifestFile manifest : table.currentSnapshot().deleteManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { + for (DeleteFile df : reader) { + if (df.referencedDataFile() != null) { + referenced.add(df.referencedDataFile()); + } + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + return referenced; + } + + private DataFile writeDataFile(Table table, Record record) throws IOException { + return new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(record)); + } + + private DeleteFile writeStagingDV(Table table, String dataFilePath, long position) + throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + + PositionDelete delete = PositionDelete.create(); + delete.set(dataFilePath, position, null); + return FileHelpers.writePosDeleteFile( + table, + Files.localOutput(file), + new PartitionData(PartitionSpec.unpartitioned().partitionType()), + Lists.newArrayList(delete), + 3); + } + + private DeleteFile writeEqualityDelete(Table table, Integer id, String data) throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + + Schema eqDeleteSchema = table.schema(); + return FileHelpers.writeDeleteFile( + table, + Files.localOutput(file), + new PartitionData(PartitionSpec.unpartitioned().partitionType()), + Lists.newArrayList(createRecord(id, data)), + eqDeleteSchema); + } + + private DeleteFile writePartitionedEqualityDelete(Table table, Integer id, String data) + throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + + PartitionData partition = new PartitionData(table.spec().partitionType()); + partition.set(0, data); + return FileHelpers.writeDeleteFile( + table, + Files.localOutput(file), + partition, + Lists.newArrayList(createRecord(id, data)), + table.schema()); + } + + private static long dataFileCount(Table table) { + table.refresh(); + long count = 0; + for (ManifestFile manifest : table.currentSnapshot().dataManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.read(manifest, table.io(), table.specs())) { + for (DataFile ignored : reader) { + count++; + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + return count; + } + + @Test + void testStagingEqualsTargetBranch() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + // Write eq delete directly to main (no separate staging branch) + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + DataFile newData = writeDataFile(table, createRecord(3, "c")); + table.newRowDelta().addRows(newData).addDeletes(eqDelete).commit(); + table.refresh(); + + appendConvertTask(SnapshotRef.MAIN_BRANCH); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + // Cycle 1: process the eq delete for id=1 + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"))); + long dataFilesAfterCycle1 = dataFileCount(table); + // Expect 3: two from insert() + one from the writer's rowDelta.addRows(newData). + // When stagingBranch == targetBranch, the committer must NOT re-add newData via + // rowDelta.addRows(...) — that would duplicate (count=4). + assertThat(dataFilesAfterCycle1).isEqualTo(3); + + // Cycle 2: no-op (converter's own commit must be skipped) + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"))); + assertThat(dataFileCount(table)).isEqualTo(dataFilesAfterCycle1); + + // New eq delete for id=2 committed directly to main between cycles + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + DataFile newData2 = writeDataFile(table, createRecord(4, "d")); + table.newRowDelta().addRows(newData2).addDeletes(eqDelete2).commit(); + table.refresh(); + + // Cycle 3: process the new eq delete + long time3 = time2 + 1; + infra.source().sendRecord(Trigger.create(time3, 0), time3); + TaskResult result3 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result3.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(3, "c"), createRecord(4, "d"))); + long dataFilesAfterCycle3 = dataFileCount(table); + + // Cycle 4: no-op again + long time4 = time3 + 1; + infra.source().sendRecord(Trigger.create(time4, 0), time4); + TaskResult result4 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result4.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(3, "c"), createRecord(4, "d"))); + assertThat(dataFileCount(table)).isEqualTo(dataFilesAfterCycle3); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testStagingEqualsTargetBranchColdStartCatchUp() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + insert(table, 3, "c"); + + // Writer commits three eq-deletes to main BEFORE the converter starts. + // Cold start must pick up the unconverted history, not just the head snapshot. + table.newRowDelta().addDeletes(writeEqualityDelete(table, 1, "a")).commit(); + table.newRowDelta().addDeletes(writeEqualityDelete(table, 2, "b")).commit(); + table.newRowDelta().addDeletes(writeEqualityDelete(table, 3, "c")).commit(); + table.refresh(); + + appendConvertTask(SnapshotRef.MAIN_BRANCH); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + // One unconverted snapshot per cycle, oldest first. After three cycles every eq-delete + // commit has its own committer commit carrying the marker. + for (int cycle = 1; cycle <= 3; cycle++) { + long ts = System.currentTimeMillis() + cycle; + infra.source().sendRecord(Trigger.create(ts, 0), ts); + TaskResult result = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result.success()).isTrue(); + } + + table.refresh(); + long convertedCount = + StreamSupport.stream(table.snapshots().spliterator(), false) + .filter(s -> s.summary().containsKey(COMMITTED_STAGING_SNAPSHOT_PROPERTY)) + .count(); + assertThat(convertedCount).isEqualTo(3); + assertRecords(table, ImmutableList.of()); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testStagingEqualsTargetBranchReinsertAfterDeleteSurvives() throws Exception { + Table table = createTableWithDelete(3); + + // Shared branch: insert id=1, eq-delete id=1, then re-insert id=1. The re-insert has a higher + // sequence than the delete and must survive the conversion (sequence-aware resolution). + insert(table, 1, "a"); + table.newRowDelta().addDeletes(writeEqualityDelete(table, 1, "a")).commit(); + table.newAppend().appendFile(writeDataFile(table, createRecord(1, "a"))).commit(); + table.refresh(); + + appendConvertTask(SnapshotRef.MAIN_BRANCH); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + // One cycle converts the eq-delete: the original row is deleted, the newer re-insert is + // not (its sequence is at or above the delete's). + long ts = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(ts, 0), ts); + TaskResult result = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result.exceptions()).isEmpty(); + assertThat(result.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "a"))); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testStagingEqualsTargetBranchMergesStagingDvIntoSingleDv() throws Exception { + Table table = createTableWithDelete(3); + + // One data file with two rows: id=1 at pos 0, id=2 at pos 1, committed to main. + DataFile data = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(1, "a"), createRecord(2, "b"))); + table.newAppend().appendFile(data).commit(); + table.refresh(); + + // Same-branch commit: an eq-delete for id=1 (resolves to a conversion DV at pos 0) plus a + // writer DV at pos 1 on the same data file. The resolver folds the staging DV into the + // conversion DV; on a shared branch the committer must remove the superseded staging DV. + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + DeleteFile stagingDV = writeStagingDV(table, data.location(), 1L); + table.newRowDelta().addDeletes(eqDelete).addDeletes(stagingDV).commit(); + table.refresh(); + + appendConvertTask(SnapshotRef.MAIN_BRANCH); + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + // Both rows masked: pos 0 by the conversion DV, pos 1 by the merged-in staging DV. + assertRecords(table, ImmutableList.of()); + + // Exactly one DV per data file (V3 invariant). Without removing the rewritten staging DV on a + // shared branch, the data file would carry two DVs. + long dvCount = + table.currentSnapshot().deleteManifests(table.io()).stream() + .flatMap( + m -> + StreamSupport.stream( + ManifestFiles.readDeleteManifest(m, table.io(), table.specs()) + .spliterator(), + false)) + .filter(f -> f.content() == FileContent.POSITION_DELETES) + .filter(f -> data.location().equals(f.referencedDataFile())) + .count(); + assertThat(dvCount).isEqualTo(1); + } + + @Test + void testReaderErrorSkipsCommit() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + long mainSnapshotBeforeStaging = table.currentSnapshot().snapshotId(); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging data file + eq delete file, both referenced by the staging commit. + DataFile newDataFile = writeDataFile(table, createRecord(2, "b")); + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(newDataFile).addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + File eqDeleteLocalFile = new File(eqDelete.location().replace("file:", "")); + + // Delete the eq delete file; the committer must abort rather than committing data without its + // DV. + assertThat(eqDeleteLocalFile.delete()).isTrue(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + + assertThat(result1.success()).isFalse(); + assertThat(result1.exceptions()).isNotEmpty(); + + table.refresh(); + // Main must not have advanced (no commit happened). + assertThat(table.currentSnapshot().snapshotId()).isEqualTo(mainSnapshotBeforeStaging); + SimpleDataUtil.assertTableRecords(table, ImmutableList.of(createRecord(1, "a"))); + + // Restore the eq delete file content by rewriting an identical delete, and retry: + // the planner must re-process the same staging snapshot (cursor didn't advance on failure). + DeleteFile recreated = + FileHelpers.writeDeleteFile( + table, + Files.localOutput(eqDeleteLocalFile), + new PartitionData(PartitionSpec.unpartitioned().partitionType()), + Lists.newArrayList(createRecord(1, "a")), + table.schema()); + assertThat(recreated.location()).isEqualTo(eqDelete.location()); + + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + table.refresh(); + // Staging data file committed with DV for id=1: should see id=2 only. + SimpleDataUtil.assertTableRecords(table, ImmutableList.of(createRecord(2, "b"))); + } finally { + closeJobClient(jobClient); + } + } + + private void appendConvertTask() { + appendConvertTask(STAGING_BRANCH); + } + + private void appendConvertTask(String stagingBranch) { + ConvertEqualityDeletes.builder() + .stagingBranch(stagingBranch) + .equalityFieldColumns(ImmutableList.of("id", "data")) + .parallelism(2) + .append( + infra.triggerStream(), + DUMMY_TABLE_NAME, + DUMMY_TASK_NAME, + 0, + tableLoader(), + UID_SUFFIX, + StreamGraphGenerator.DEFAULT_SLOT_SHARING_GROUP, + 1) + .sinkTo(infra.sink()); + } + + private static void assertRecords(Table table, List expected) throws IOException { + table.refresh(); + Types.StructType type = SimpleDataUtil.SCHEMA.asStruct(); + + StructLikeSet expectedSet = StructLikeSet.create(type); + expectedSet.addAll(expected); + + try (CloseableIterable iterable = + IcebergGenerics.read(table) + .useSnapshot(table.currentSnapshot().snapshotId()) + .project(SimpleDataUtil.SCHEMA) + .build()) { + StructLikeSet actualSet = StructLikeSet.create(type); + for (Record record : iterable) { + actualSet.add(record); + } + + assertThat(actualSet).isEqualTo(expectedSet); + } + } + + private static void assertNoEqualityDeletesOnMain(Table table, long expectedEqDeleteCount) { + long mainEqDeleteCount = 0; + for (ManifestFile manifest : table.currentSnapshot().deleteManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { + for (DeleteFile f : reader) { + if (f.content() == FileContent.EQUALITY_DELETES) { + mainEqDeleteCount++; + } + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + assertThat(mainEqDeleteCount).isEqualTo(expectedEqDeleteCount); + } +} diff --git a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestConvertEqualityDeletesE2E.java b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestConvertEqualityDeletesE2E.java new file mode 100644 index 000000000000..f6d1fd9464e2 --- /dev/null +++ b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestConvertEqualityDeletesE2E.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.api; + +import static org.apache.iceberg.flink.SimpleDataUtil.createRecord; +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.time.Duration; +import org.apache.flink.core.execution.JobClient; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.Files; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; +import org.apache.iceberg.PartitionData; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.Table; +import org.apache.iceberg.data.FileHelpers; +import org.apache.iceberg.data.GenericAppenderHelper; +import org.apache.iceberg.flink.SimpleDataUtil; +import org.apache.iceberg.flink.maintenance.operator.OperatorTestBase; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.util.ContentFileUtil; +import org.awaitility.Awaitility; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +/** + * End-to-end test for {@link ConvertEqualityDeletes} wired through the {@link TableMaintenance} + * framework. Verifies that the converter actually runs and commits a DV when the framework triggers + * it, exercising the full operator graph including the framework's monitor source, trigger manager, + * and lock remover. + */ +class TestConvertEqualityDeletesE2E extends OperatorTestBase { + private static final String STAGING_BRANCH = "staging"; + + @TempDir private Path tempDir; + private StreamExecutionEnvironment env; + + @BeforeEach + public void beforeEach() { + this.env = StreamExecutionEnvironment.getExecutionEnvironment(); + } + + @ParameterizedTest + @ValueSource(strings = {STAGING_BRANCH, SnapshotRef.MAIN_BRANCH}) + void testConvertEqualityDeletesE2E(String stagingBranch) throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + // When staging is a separate branch, fork it from main first. + if (!stagingBranch.equals(SnapshotRef.MAIN_BRANCH)) { + table.manageSnapshots().createBranch(stagingBranch).commit(); + table.refresh(); + } + + // Commit a new data file + eq delete to staging. This pre-job snapshot exercises both the + // "new data file" and "eq delete" paths in one cycle. + DataFile newData = writeDataFile(table, 3, "c"); + DeleteFile firstDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(newData).addDeletes(firstDelete).toBranch(stagingBranch).commit(); + table.refresh(); + assertThat(dvCountOnMain(table)).isZero(); + + TableMaintenance.forTable(env, tableLoader(), LOCK_FACTORY) + .uidSuffix("ConvertEqualityDeletesE2EUID-" + stagingBranch) + .rateLimit(Duration.ofMillis(50)) + .lockCheckDelay(Duration.ofMillis(50)) + .add( + ConvertEqualityDeletes.builder() + .scheduleOnInterval(Duration.ofMillis(100)) + .stagingBranch(stagingBranch) + .targetBranch(SnapshotRef.MAIN_BRANCH) + .equalityFieldColumns(ImmutableList.of("id", "data")) + .parallelism(2)) + .append(); + + JobClient jobClient = null; + try { + jobClient = env.executeAsync(); + + // Cycle 1: row 1 deleted by the converted DV; row 3 added on staging and committed to main. + Awaitility.await() + .atMost(Duration.ofMinutes(5)) + .pollInterval(Duration.ofMillis(200)) + .untilAsserted(() -> assertThat(dvCountOnMain(table)).isEqualTo(1)); + SimpleDataUtil.assertTableRecords( + table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"))); + + // Cycle 2: commit a second staging snapshot while the job is still running. The framework's + // next interval-trigger should pick it up and produce a second DV against the data file + // holding id=2. + table.refresh(); + DeleteFile secondDelete = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(secondDelete).toBranch(stagingBranch).commit(); + + Awaitility.await() + .atMost(Duration.ofMinutes(5)) + .pollInterval(Duration.ofMillis(200)) + .untilAsserted(() -> assertThat(dvCountOnMain(table)).isEqualTo(2)); + SimpleDataUtil.assertTableRecords(table, ImmutableList.of(createRecord(3, "c"))); + } finally { + closeJobClient(jobClient); + } + } + + private DataFile writeDataFile(Table table, Integer id, String data) throws IOException { + return new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(SimpleDataUtil.createRecord(id, data))); + } + + private DeleteFile writeEqualityDelete(Table table, Integer id, String data) throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + return FileHelpers.writeDeleteFile( + table, + Files.localOutput(file), + new PartitionData(PartitionSpec.unpartitioned().partitionType()), + Lists.newArrayList(SimpleDataUtil.createRecord(id, data)), + table.schema()); + } + + private static long dvCountOnMain(Table table) throws IOException { + table.refresh(); + if (table.currentSnapshot() == null) { + return 0; + } + + long count = 0; + for (ManifestFile manifest : table.currentSnapshot().deleteManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { + for (DeleteFile file : reader) { + if (ContentFileUtil.isDV(file)) { + count++; + } + } + } + } + + return count; + } +} diff --git a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestMaintenanceE2E.java b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestMaintenanceE2E.java index fe8457167a1f..f786f1cdb29d 100644 --- a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestMaintenanceE2E.java +++ b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestMaintenanceE2E.java @@ -24,8 +24,11 @@ import java.time.Duration; import org.apache.flink.core.execution.JobClient; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.SnapshotRef; import org.apache.iceberg.Table; import org.apache.iceberg.flink.maintenance.operator.OperatorTestBase; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -116,4 +119,34 @@ void testE2eUseCoordinator() throws Exception { closeJobClient(jobClient); } } + + @Test + void testE2eConvertEqualityDeletes() throws Exception { + // Converter requires V3 (DV support); replace the V2 table created in @BeforeEach. + dropTable(); + createTable(3, FileFormat.PARQUET); + + TableMaintenance.forTable(env, tableLoader(), LOCK_FACTORY) + .uidSuffix("E2eConvertEqualityDeletesUID") + .rateLimit(Duration.ofMinutes(10)) + .lockCheckDelay(Duration.ofSeconds(10)) + .add( + ConvertEqualityDeletes.builder() + .scheduleOnEqDeleteFileCount(1) + .stagingBranch("staging") + .targetBranch(SnapshotRef.MAIN_BRANCH) + .equalityFieldColumns(ImmutableList.of("id")) + .parallelism(2)) + .append(); + + JobClient jobClient = null; + try { + jobClient = env.executeAsync(); + + // Just make sure that we are able to instantiate the flow + assertThat(jobClient).isNotNull(); + } finally { + closeJobClient(jobClient); + } + } } diff --git a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestTableMaintenance.java b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestTableMaintenance.java index 49219d5b4698..96b2fe04d66e 100644 --- a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestTableMaintenance.java +++ b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestTableMaintenance.java @@ -350,6 +350,14 @@ void testUidAndSlotSharingGroupForMonitorSource() throws IOException { checkSlotSharingGroupsAreSet(env, SLOT_SHARING_GROUP); } + @Test + void testMonitorRatePerSecond() { + // Sub-second rate limits must not yield infinite (unthrottled) monitor rates. + assertThat(TableMaintenance.Builder.monitorRatePerSecond(50)).isEqualTo(20.0); + assertThat(TableMaintenance.Builder.monitorRatePerSecond(100)).isEqualTo(10.0); + assertThat(TableMaintenance.Builder.monitorRatePerSecond(60_000)).isEqualTo(1.0 / 60); + } + /** * Sends the events though the {@link ManualSource} provided, and waits until the given number of * records are processed. diff --git a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/OperatorTestBase.java b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/OperatorTestBase.java index b4f6ef95fc0f..a80e5c1bb2c0 100644 --- a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/OperatorTestBase.java +++ b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/OperatorTestBase.java @@ -188,6 +188,18 @@ protected static Table createTableWithDelete(int formatVersion) { "format-version", String.valueOf(formatVersion), "write.upsert.enabled", "true")); } + protected static Table createPartitionedTableWithDelete(int formatVersion) { + return CATALOG_EXTENSION + .catalog() + .createTable( + TestFixtures.TABLE_IDENTIFIER, + SCHEMA_WITH_PRIMARY_KEY, + PartitionSpec.builderFor(SCHEMA_WITH_PRIMARY_KEY).identity("data").build(), + null, + ImmutableMap.of( + "format-version", String.valueOf(formatVersion), "write.upsert.enabled", "true")); + } + protected static Table createPartitionedTable(int formatVersion, FileFormat fileFormat) { return CATALOG_EXTENSION .catalog() diff --git a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertCommitter.java b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertCommitter.java new file mode 100644 index 000000000000..54f0b1758d90 --- /dev/null +++ b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertCommitter.java @@ -0,0 +1,569 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.operator; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.IOException; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.util.TwoInputStreamOperatorTestHarness; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; +import org.apache.iceberg.RowDelta; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.Table; +import org.apache.iceberg.data.FileHelpers; +import org.apache.iceberg.data.GenericRecord; +import org.apache.iceberg.deletes.PositionDelete; +import org.apache.iceberg.exceptions.CommitStateUnknownException; +import org.apache.iceberg.exceptions.ValidationException; +import org.apache.iceberg.flink.maintenance.api.Trigger; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.junit.jupiter.api.Test; + +class TestEqualityConvertCommitter extends OperatorTestBase { + + @Test + void commitsDataFilesToMainBranch() throws Exception { + Table table = createTable(3, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long snapshotIdBefore = table.currentSnapshot().snapshotId(); + long stagingSnapshotId = 42L; + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + stagingSnapshotId, + snapshotIdBefore, + doneTs - 1, + doneTs); + + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + harness.processBothWatermarks(new Watermark(doneTs)); + + assertThat(harness.extractOutputValues()).hasSize(1); + table.refresh(); + assertThat(table.currentSnapshot().snapshotId()).isNotEqualTo(snapshotIdBefore); + assertThat( + table + .currentSnapshot() + .summary() + .get(EqualityConvertCommitter.COMMITTED_STAGING_SNAPSHOT_PROPERTY)) + .isEqualTo(String.valueOf(stagingSnapshotId)); + } + } + + @Test + void holdsBackWatermarkUntilCommit() throws Exception { + Table table = createTable(3, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long snapshotIdBefore = table.currentSnapshot().snapshotId(); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + 42L, + snapshotIdBefore, + doneTs - 2, + doneTs); + + harness.processElement2(new StreamRecord<>(planResult, doneTs - 2)); + + // A phase watermark before the plan's done timestamp must not be forwarded: it would let the + // LockRemover release the maintenance lock before this cycle commits. + harness.processBothWatermarks(new Watermark(doneTs - 1)); + assertThat(harness.extractOutputValues()).isEmpty(); + assertThat(watermarks(harness)).isEmpty(); + + // The done-timestamp watermark commits the cycle; the watermark is forwarded only now. + harness.processBothWatermarks(new Watermark(doneTs)); + assertThat(harness.extractOutputValues()).hasSize(1); + assertThat(watermarks(harness)).containsExactly(new Watermark(doneTs)); + } + } + + @Test + void skipsCommitForEmptyCycle() throws Exception { + Table table = createTable(3, FileFormat.PARQUET); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = EqualityConvertPlan.noOp(null, doneTs - 1, doneTs); + + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + harness.processBothWatermarks(new Watermark(doneTs)); + + assertThat(harness.extractOutputValues()).hasSize(1); + table.refresh(); + assertThat(table.currentSnapshot()).isNull(); + } + } + + @Test + void abortsCommitWhenDVWriterFailed() throws Exception { + Table table = createTable(3, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long snapshotIdBefore = table.currentSnapshot().snapshotId(); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + 42L, + snapshotIdBefore, + doneTs - 1, + doneTs); + + // Receive an abort signal from the DVWriter followed by the planning planResult. + harness.processElement1(new StreamRecord<>(DVWriteResult.ABORT, doneTs)); + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + + harness.processBothWatermarks(new Watermark(doneTs)); + + // The cycle must complete (Trigger emitted) but nothing must be committed to the table. + assertThat(harness.extractOutputValues()).hasSize(1); + table.refresh(); + assertThat(table.currentSnapshot().snapshotId()).isEqualTo(snapshotIdBefore); + } + } + + @Test + void failsCommitWhenExternalCommitLandsAfterPlan() throws Exception { + Table table = createTable(3, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long mainSnapshotIdAtPlan = table.currentSnapshot().snapshotId(); + long stagingSnapshotId = 555L; + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + stagingSnapshotId, + mainSnapshotIdAtPlan, + doneTs - 1, + doneTs); + + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + + // External writer appends to main between plan and commit. + insert(table, 2, "b"); + table.refresh(); + long mainSnapshotIdAfterExternal = table.currentSnapshot().snapshotId(); + assertThat(mainSnapshotIdAfterExternal).isNotEqualTo(mainSnapshotIdAtPlan); + + harness.processBothWatermarks(new Watermark(doneTs)); + + // Trigger still fires (so the aggregator records the cycle), but the commit + // must have been rejected by validateNoConflictingDataFiles. + assertThat(harness.extractOutputValues()).hasSize(1); + + List> errors = + Lists.newArrayList(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)); + assertThat(errors).hasSize(1); + Exception failure = errors.get(0).getValue(); + assertThat(failure).isInstanceOf(ValidationException.class); + assertThat(failure) + .hasMessageContaining("Found conflicting files that can contain records matching"); + + // Target head is still the external commit, not our cycle's commit. + table.refresh(); + assertThat(table.currentSnapshot().snapshotId()).isEqualTo(mainSnapshotIdAfterExternal); + assertThat( + table + .currentSnapshot() + .summary() + .get(EqualityConvertCommitter.COMMITTED_STAGING_SNAPSHOT_PROPERTY)) + .isNull(); + } + } + + @Test + void failsReplayOfCommittedPlanFromOlderState() throws Exception { + Table table = createTable(3, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long mainSnapshotIdAtPlan = table.currentSnapshot().snapshotId(); + long stagingSnapshotId = 909L; + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + stagingSnapshotId, + mainSnapshotIdAtPlan, + doneTs - 1, + doneTs); + + // Cycle 1 commits, advancing the target past mainSnapshotIdAtPlan and writing the marker. + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + harness.processBothWatermarks(new Watermark(doneTs)); + assertThat(harness.extractOutputValues()).hasSize(1); + + table.refresh(); + long committedSnapshotId = table.currentSnapshot().snapshotId(); + assertThat(committedSnapshotId).isNotEqualTo(mainSnapshotIdAtPlan); + + // Restart from older state: the identical plan with its stale mainSnapshotId is replayed. + // The committer is stateless, so validateFromSnapshot is the only guard against a duplicate + // commit. It must reject the replay. + long doneTs2 = doneTs + 1; + EqualityConvertPlan replay = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + stagingSnapshotId, + mainSnapshotIdAtPlan, + doneTs2 - 1, + doneTs2); + + harness.processElement2(new StreamRecord<>(replay, doneTs2 - 1)); + harness.processBothWatermarks(new Watermark(doneTs2)); + + // Trigger still fires (so the aggregator records the cycle), but the commit was rejected. + assertThat(harness.extractOutputValues()).hasSize(2); + + List> errors = + Lists.newArrayList(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)); + assertThat(errors).hasSize(1); + Exception failure = errors.get(0).getValue(); + assertThat(failure).isInstanceOf(ValidationException.class); + assertThat(failure) + .hasMessageContaining("Found conflicting files that can contain records matching"); + + // No duplicate commit: target head is still cycle 1's commit. + table.refresh(); + assertThat(table.currentSnapshot().snapshotId()).isEqualTo(committedSnapshotId); + } + } + + @Test + void deletesWrittenDvsWhenCommitFails() throws Exception { + Table table = createTable(2, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long mainSnapshotIdAtPlan = table.currentSnapshot().snapshotId(); + + DeleteFile writtenDv = writePosDeleteFile(table, stagingDataFile.location(), 0L); + assertThat(table.io().newInputFile(writtenDv.location()).exists()).isTrue(); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + 777L, + mainSnapshotIdAtPlan, + doneTs - 1, + doneTs); + + harness.processElement1( + new StreamRecord<>( + new DVWriteResult(Lists.newArrayList(writtenDv), Lists.newArrayList()), doneTs)); + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + + // External writer appends to main between plan and commit, so the commit is rejected. + insert(table, 2, "b"); + table.refresh(); + + harness.processBothWatermarks(new Watermark(doneTs)); + + assertThat(harness.extractOutputValues()).hasSize(1); + + List> errors = + Lists.newArrayList(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)); + assertThat(errors).hasSize(1); + assertThat(errors.get(0).getValue()).isInstanceOf(ValidationException.class); + + // The DV written this cycle is unreferenced after the failed commit and must be deleted. + assertThat(table.io().newInputFile(writtenDv.location()).exists()).isFalse(); + } + } + + @Test + void deletesSiblingDvsOnAbortButKeepsRewrittenDvs() throws Exception { + Table table = createTable(2, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long snapshotIdBefore = table.currentSnapshot().snapshotId(); + + DeleteFile writtenDv = writePosDeleteFile(table, stagingDataFile.location(), 0L); + DeleteFile rewrittenDv = writePosDeleteFile(table, stagingDataFile.location(), 1L); + assertThat(table.io().newInputFile(writtenDv.location()).exists()).isTrue(); + assertThat(table.io().newInputFile(rewrittenDv.location()).exists()).isTrue(); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + 42L, + snapshotIdBefore, + doneTs - 1, + doneTs); + + // One writer task wrote a DV (rewriting an existing one); a sibling task aborted. + harness.processElement1( + new StreamRecord<>( + new DVWriteResult(Lists.newArrayList(writtenDv), Lists.newArrayList(rewrittenDv)), + doneTs)); + harness.processElement1(new StreamRecord<>(DVWriteResult.ABORT, doneTs)); + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + harness.processBothWatermarks(new Watermark(doneTs)); + + assertThat(harness.extractOutputValues()).hasSize(1); + table.refresh(); + assertThat(table.currentSnapshot().snapshotId()).isEqualTo(snapshotIdBefore); + + // The freshly written DV is removed; the rewritten DV is still live on target and stays. + assertThat(table.io().newInputFile(writtenDv.location()).exists()).isFalse(); + assertThat(table.io().newInputFile(rewrittenDv.location()).exists()).isTrue(); + } + } + + @Test + void retainsWrittenDvsWhenCommitStateUnknown() throws Exception { + Table table = createTable(2, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long mainSnapshotIdAtPlan = table.currentSnapshot().snapshotId(); + + DeleteFile writtenDv = writePosDeleteFile(table, stagingDataFile.location(), 0L); + assertThat(table.io().newInputFile(writtenDv.location()).exists()).isTrue(); + + EqualityConvertCommitter committer = + new EqualityConvertCommitter( + DUMMY_TABLE_NAME, DUMMY_TASK_NAME, tableLoader(), "staging", SnapshotRef.MAIN_BRANCH) { + @Override + void commit(RowDelta rowDelta) { + throw new CommitStateUnknownException( + new RuntimeException("simulated unknown commit state")); + } + }; + + try (TwoInputStreamOperatorTestHarness harness = + new TwoInputStreamOperatorTestHarness<>(committer)) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + 888L, + mainSnapshotIdAtPlan, + doneTs - 1, + doneTs); + + harness.processElement1( + new StreamRecord<>( + new DVWriteResult(Lists.newArrayList(writtenDv), Lists.newArrayList()), doneTs)); + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + harness.processBothWatermarks(new Watermark(doneTs)); + + assertThat(harness.extractOutputValues()).hasSize(1); + + List> errors = + Lists.newArrayList(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)); + assertThat(errors).hasSize(1); + assertThat(errors.get(0).getValue()).isInstanceOf(CommitStateUnknownException.class); + + // Commit outcome unknown: the DV may be live, so it must not be deleted. + assertThat(table.io().newInputFile(writtenDv.location()).exists()).isTrue(); + } + } + + @Test + void removesRewrittenStagingDvOnSharedBranch() throws Exception { + // Shared branch: stagingBranch == targetBranch == main, so the writer's DVs are already on + // target. A staging DV folded into a merged conversion DV must be removed, else the data file + // would carry two DVs (V3 allows one per data file). + Table table = createTable(3, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile dataFile = getFirstDataFile(table); + + // Staging DV already committed on the shared target branch; the writer rewrites it by folding + // it into a merged DV for the same data file. + DeleteFile stagingDv = writeDV(table, dataFile.location(), 0L); + table.newRowDelta().addDeletes(stagingDv).commit(); + table.refresh(); + long mainSnapshotId = table.currentSnapshot().snapshotId(); + + DeleteFile mergedDv = writeDV(table, dataFile.location(), 0L); + + try (TwoInputStreamOperatorTestHarness harness = + sharedBranchHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(), + Lists.newArrayList(stagingDv), + 123L, + mainSnapshotId, + doneTs - 1, + doneTs); + + harness.processElement1( + new StreamRecord<>( + new DVWriteResult(Lists.newArrayList(mergedDv), Lists.newArrayList(stagingDv)), + doneTs)); + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + harness.processBothWatermarks(new Watermark(doneTs)); + + assertThat(harness.extractOutputValues()).hasSize(1); + + // Exactly one DV references the data file: the merged DV, with the rewritten staging DV + // removed rather than left as a second DV on the same data file. + table.refresh(); + List dvs = deletesForDataFile(table, dataFile.location()); + assertThat(dvs).hasSize(1); + assertThat(dvs.get(0).location()).isEqualTo(mergedDv.location()); + } + } + + private TwoInputStreamOperatorTestHarness + createHarness() throws Exception { + return new TwoInputStreamOperatorTestHarness<>( + new EqualityConvertCommitter( + DUMMY_TABLE_NAME, DUMMY_TASK_NAME, tableLoader(), "staging", SnapshotRef.MAIN_BRANCH)); + } + + private TwoInputStreamOperatorTestHarness + sharedBranchHarness() throws Exception { + return new TwoInputStreamOperatorTestHarness<>( + new EqualityConvertCommitter( + DUMMY_TABLE_NAME, + DUMMY_TASK_NAME, + tableLoader(), + SnapshotRef.MAIN_BRANCH, + SnapshotRef.MAIN_BRANCH)); + } + + private static List watermarks(TwoInputStreamOperatorTestHarness harness) { + return harness.getOutput().stream() + .filter(Watermark.class::isInstance) + .map(Watermark.class::cast) + .collect(Collectors.toList()); + } + + private static List deletesForDataFile(Table table, String dataFilePath) { + List deletes = Lists.newArrayList(); + for (ManifestFile manifest : table.currentSnapshot().deleteManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { + for (DeleteFile file : reader) { + if (dataFilePath.equals(file.referencedDataFile())) { + deletes.add(file.copy()); + } + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + return deletes; + } + + private static DeleteFile writeDV(Table table, String dataFilePath, long... positions) + throws IOException { + List> deletes = Lists.newArrayList(); + GenericRecord nested = GenericRecord.create(table.schema()); + for (long pos : positions) { + PositionDelete delete = PositionDelete.create(); + delete.set(dataFilePath, pos, nested); + deletes.add(delete); + } + + return FileHelpers.writePosDeleteFile(table, null, null, deletes, 3); + } + + private static DataFile getFirstDataFile(Table table) { + for (ManifestFile manifest : table.currentSnapshot().dataManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.read(manifest, table.io(), table.specs())) { + for (DataFile file : reader) { + return file.copy(); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + throw new IllegalStateException("No data files found"); + } +} diff --git a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertDVWriter.java b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertDVWriter.java new file mode 100644 index 000000000000..a0c2f3568baf --- /dev/null +++ b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertDVWriter.java @@ -0,0 +1,472 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.operator; + +import static org.apache.iceberg.flink.SimpleDataUtil.createRecord; +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.util.TwoInputStreamOperatorTestHarness; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.RowDelta; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.Table; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.junit.jupiter.api.Test; + +class TestEqualityConvertDVWriter extends OperatorTestBase { + + private static final byte[] EMPTY_PARTITION = new byte[0]; + + @Test + void writesDVFileForSinglePosition() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + String dataFilePath = getFirstDataFilePath(table); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time = System.currentTimeMillis(); + harness.processElement1( + new StreamRecord<>(new DVPosition(dataFilePath, 0, 0, EMPTY_PARTITION, 0L), time)); + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time)); + + harness.processBothWatermarks(new Watermark(time)); + + List output = harness.extractOutputValues(); + assertThat(output).hasSize(1); + assertThat(output.get(0).hasError()).isFalse(); + assertThat(output.get(0).dvFiles()).hasSize(1); + } + } + + @Test + void writesSingleDVFileForMultiplePositionsOnSameDataFile() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + insert(table, 3, "c"); + + String dataFilePath = getFirstDataFilePath(table); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time = System.currentTimeMillis(); + harness.processElement1( + new StreamRecord<>(new DVPosition(dataFilePath, 0, 0, EMPTY_PARTITION, 0L), time)); + harness.processElement1( + new StreamRecord<>(new DVPosition(dataFilePath, 1, 0, EMPTY_PARTITION, 0L), time)); + harness.processElement1( + new StreamRecord<>(new DVPosition(dataFilePath, 2, 0, EMPTY_PARTITION, 0L), time)); + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time)); + + harness.processBothWatermarks(new Watermark(time)); + + List output = harness.extractOutputValues(); + assertThat(output).hasSize(1); + assertThat(output.get(0).hasError()).isFalse(); + assertThat(output.get(0).dvFiles()).hasSize(1); + assertThat(output.get(0).dvFiles().get(0).recordCount()).isEqualTo(3); + assertThat(output.get(0).rewrittenDvFiles()).isEmpty(); + } + } + + @Test + void emptyPositionsProducesNoOutput() throws Exception { + createTableWithDelete(3); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time = System.currentTimeMillis(); + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time)); + + harness.processBothWatermarks(new Watermark(time)); + + assertThat(harness.extractOutputValues()).isEmpty(); + } + } + + @Test + void noOutputWithoutPlanResult() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + String dataFilePath = getFirstDataFilePath(table); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time = System.currentTimeMillis(); + harness.processElement1( + new StreamRecord<>(new DVPosition(dataFilePath, 0, 0, EMPTY_PARTITION, 0L), time)); + + harness.processBothWatermarks(new Watermark(time)); + + assertThat(harness.extractOutputValues()).isEmpty(); + } + } + + @Test + void writesDVFilesForMultipleDataFiles() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + List dataFilePaths = getDataFilePaths(table); + assertThat(dataFilePaths).hasSize(2); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time = System.currentTimeMillis(); + harness.processElement1( + new StreamRecord<>( + new DVPosition(dataFilePaths.get(0), 0, 0, EMPTY_PARTITION, 0L), time)); + harness.processElement1( + new StreamRecord<>( + new DVPosition(dataFilePaths.get(1), 0, 0, EMPTY_PARTITION, 0L), time)); + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time)); + + harness.processBothWatermarks(new Watermark(time)); + + List output = harness.extractOutputValues(); + assertThat(output).hasSize(1); + assertThat(output.get(0).dvFiles()).hasSize(2); + } + } + + @Test + void routesErrorToErrorStream() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + String dataFilePath = getFirstDataFilePath(table); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time = System.currentTimeMillis(); + harness.processElement1( + new StreamRecord<>(new DVPosition(dataFilePath, 0, 0, EMPTY_PARTITION, 0L), time)); + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time)); + + dropTable(); + + harness.processBothWatermarks(new Watermark(time)); + + assertThat(harness.extractOutputValues()).hasSize(1); + assertThat(harness.extractOutputValues().get(0).hasError()).isTrue(); + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).hasSize(1); + } + } + + @Test + void mergesStagingDVIntoWrittenDV() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + String dataFilePath = getFirstDataFilePath(table); + + // Staging DV for the data file at position 0, produced but not committed to the target branch. + DeleteFile stagingDV = writeStagingDV(dataFilePath, 0); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(), Lists.newArrayList(stagingDV), 1L, null, 0L, 0L); + + // New position 1 in the same file. Must merge it with the staged position 0. + harness.processElement1( + new StreamRecord<>(new DVPosition(dataFilePath, 1, 0, EMPTY_PARTITION, 0L), time)); + harness.processElement2(new StreamRecord<>(planResult, time)); + + harness.processBothWatermarks(new Watermark(time)); + + List output = harness.extractOutputValues(); + assertThat(output).hasSize(1); + assertThat(output.get(0).hasError()).isFalse(); + assertThat(output.get(0).dvFiles()).hasSize(1); + assertThat(output.get(0).dvFiles().get(0).recordCount()).isEqualTo(2); + assertThat(output.get(0).rewrittenDvFiles()).hasSize(1); + assertThat(output.get(0).rewrittenDvFiles().get(0).location()) + .isEqualTo(stagingDV.location()); + } + } + + @Test + void abortsWhenMainSnapshotChangedSincePlanning() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + String dataFilePath = getFirstDataFilePath(table); + long plannedSnapshotId = table.currentSnapshot().snapshotId(); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(), Lists.newArrayList(), 1L, plannedSnapshotId, 0L, 0L); + + harness.processElement1( + new StreamRecord<>(new DVPosition(dataFilePath, 0, 0, EMPTY_PARTITION, 0L), time)); + harness.processElement2(new StreamRecord<>(planResult, time)); + + // Advance the target branch after planning. The writer must refuse to write stale DVs. + insert(table, 2, "b"); + + harness.processBothWatermarks(new Watermark(time)); + + List output = harness.extractOutputValues(); + assertThat(output).hasSize(1); + assertThat(output.get(0).hasError()).isTrue(); + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).hasSize(1); + } + } + + @Test + void stateResetBetweenBatches() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + String dataFilePath = getFirstDataFilePath(table); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time1 = System.currentTimeMillis(); + harness.processElement1( + new StreamRecord<>(new DVPosition(dataFilePath, 0, 0, EMPTY_PARTITION, 0L), time1)); + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time1)); + harness.processBothWatermarks(new Watermark(time1)); + + assertThat(harness.extractOutputValues()).hasSize(1); + + // Second batch with no positions, plan only. + long time2 = time1 + 1; + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time2)); + harness.processBothWatermarks(new Watermark(time2)); + + // Cumulative output: still hasSize(1) since the empty batch produced nothing. + assertThat(harness.extractOutputValues()).hasSize(1); + } + } + + @Test + void abortsOnUpstreamAbortPosition() throws Exception { + createTableWithDelete(3); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time = System.currentTimeMillis(); + harness.processElement1(new StreamRecord<>(DVPosition.ABORT, time)); + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time)); + harness.processBothWatermarks(new Watermark(time)); + + List output = harness.extractOutputValues(); + assertThat(output).hasSize(1); + assertThat(output.get(0).hasError()).isTrue(); + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).isNull(); + } + } + + @Test + void doesNotRetainStateAcrossCycles() throws Exception { + Table table = createTableWithDelete(3); + int cycles = 5; + for (int i = 0; i < cycles; i++) { + insert(table, i, "v" + i); + } + + List dataFilePaths = getDataFilePaths(table); + assertThat(dataFilePaths).hasSize(cycles); + + EqualityConvertDVWriter resolver = newResolver(); + try (TwoInputStreamOperatorTestHarness harness = + new TwoInputStreamOperatorTestHarness<>(resolver)) { + harness.open(); + + long time = System.currentTimeMillis(); + for (int i = 0; i < cycles; i++) { + // Each cycle resolves a position in a distinct data file, growing the set of files seen. + harness.processElement1( + new StreamRecord<>( + new DVPosition(dataFilePaths.get(i), 0, 0, EMPTY_PARTITION, 0L), time + i)); + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time + i)); + harness.processBothWatermarks(new Watermark(time + i)); + + // No cross-cycle state: the buffer clears every cycle regardless of how many distinct data + // files have been processed. + assertThat(resolver.retainedStateSize()).isZero(); + assertThat(harness.extractOutputValues()).hasSize(i + 1); + } + } + } + + @Test + void prunesDeleteManifestsByPartition() throws Exception { + Table table = createPartitionedTableWithDelete(3); + insertPartitioned(table, Lists.newArrayList(createRecord(1, "a")), "a"); + DataFile fileA = dataFile(table, "a"); + insertPartitioned(table, Lists.newArrayList(createRecord(2, "b")), "b"); + DataFile fileB = dataFile(table, "b"); + + // Commit one DV per partition: two separate delete manifests, summaries {a} and {b}. + commitDV(table, fileA); + commitDV(table, fileB); + assertThat(table.currentSnapshot().deleteManifests(table.io())).hasSize(2); + + // A new cycle resolves another position in partition a's file. Only partition a's delete + // manifest overlaps; partition b's manifest must be pruned. + EqualityConvertDVWriter resolver = newResolver(); + try (TwoInputStreamOperatorTestHarness harness = + new TwoInputStreamOperatorTestHarness<>(resolver)) { + harness.open(); + + long time = System.currentTimeMillis(); + harness.processElement1(new StreamRecord<>(dvPosition(table, fileA, 0), time)); + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time)); + harness.processBothWatermarks(new Watermark(time)); + + List output = harness.extractOutputValues(); + assertThat(output).hasSize(1); + // Folded partition a's existing DV: the matching manifest was read. + assertThat(output.get(0).rewrittenDvFiles()).hasSize(1); + assertThat(output.get(0).dvFiles()).hasSize(1); + // Partition b's manifest was skipped: only one manifest read this cycle. + assertThat(resolver.manifestsReadLastCycle()).isEqualTo(1); + } + } + + private void commitDV(Table table, DataFile dataFile) throws Exception { + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time = System.currentTimeMillis(); + harness.processElement1(new StreamRecord<>(dvPosition(table, dataFile, 0), time)); + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time)); + harness.processBothWatermarks(new Watermark(time)); + + RowDelta rowDelta = table.newRowDelta(); + harness.extractOutputValues().get(0).dvFiles().forEach(rowDelta::addDeletes); + rowDelta.commit(); + table.refresh(); + } + } + + private static DVPosition dvPosition(Table table, DataFile dataFile, long position) { + PartitionSpec spec = table.specs().get(dataFile.specId()); + byte[] partition = + new StructLikeSerializer().encodePartition(dataFile.partition(), spec.partitionType()); + return new DVPosition(dataFile.location(), position, dataFile.specId(), partition, 0L); + } + + private static DataFile dataFile(Table table, String partitionValue) { + for (ManifestFile manifest : table.currentSnapshot().dataManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.read(manifest, table.io(), table.specs())) { + for (DataFile file : reader) { + if (partitionValue.equals(String.valueOf(file.partition().get(0, Object.class)))) { + return file; + } + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + throw new IllegalStateException("No data file for partition: " + partitionValue); + } + + private DeleteFile writeStagingDV(String dataFilePath, long position) throws Exception { + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long time = System.currentTimeMillis(); + harness.processElement1( + new StreamRecord<>(new DVPosition(dataFilePath, position, 0, EMPTY_PARTITION, 0L), time)); + harness.processElement2(new StreamRecord<>(emptyEqualityConvertPlan(), time)); + harness.processBothWatermarks(new Watermark(time)); + + return harness.extractOutputValues().get(0).dvFiles().get(0); + } + } + + private EqualityConvertDVWriter newResolver() { + return new EqualityConvertDVWriter( + DUMMY_TABLE_NAME, DUMMY_TASK_NAME, tableLoader(), SnapshotRef.MAIN_BRANCH); + } + + private TwoInputStreamOperatorTestHarness + createHarness() throws Exception { + return new TwoInputStreamOperatorTestHarness<>(newResolver()); + } + + private static EqualityConvertPlan emptyEqualityConvertPlan() { + return new EqualityConvertPlan(Lists.newArrayList(), Lists.newArrayList(), 1L, null, 0L, 0L); + } + + private static String getFirstDataFilePath(Table table) { + return getDataFilePaths(table).get(0); + } + + private static List getDataFilePaths(Table table) { + List paths = Lists.newArrayList(); + for (ManifestFile manifest : table.currentSnapshot().dataManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.read(manifest, table.io(), table.specs())) { + for (DataFile file : reader) { + paths.add(file.location()); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + return paths; + } +} diff --git a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertPlanner.java b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertPlanner.java new file mode 100644 index 000000000000..1ee5c5d0d5fd --- /dev/null +++ b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertPlanner.java @@ -0,0 +1,1155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.operator; + +import static org.apache.iceberg.flink.SimpleDataUtil.createRecord; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.flink.runtime.checkpoint.OperatorSubtaskState; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.Files; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; +import org.apache.iceberg.PartitionData; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.RewriteFiles; +import org.apache.iceberg.Schema; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.Table; +import org.apache.iceberg.data.FileHelpers; +import org.apache.iceberg.data.GenericAppenderHelper; +import org.apache.iceberg.data.GenericRecord; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.deletes.PositionDelete; +import org.apache.iceberg.flink.maintenance.api.Trigger; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class TestEqualityConvertPlanner extends OperatorTestBase { + + private static final String STAGING_BRANCH = "__flink_staging_test"; + + @TempDir private Path tempDir; + + @Test + void bootstrapsIndexFromBuilderEqualityFieldIds() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + // Staging branch exists but contains no eq-delete files yet. The planner still populates the + // worker index from main using the builder-configured equality field set so the index is ready + // when the first delete arrives. + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH, Lists.newArrayList(1, 2))) { + harness.open(); + sendTrigger(harness); + + List commands = harness.extractOutputValues(); + assertThat(countDataFileTasks(commands)).isEqualTo(2); + assertThat(countEqDeleteTasks(commands)).isZero(); + + assertThat(harness.getSideOutput(EqualityConvertPlanner.METADATA_STREAM)).hasSize(1); + } + } + + @Test + void failsWhenStagingEqDeleteFieldIdsMismatchBuilder() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Builder configured for [1, 2], but writer produces an eq delete with [1] only. + DeleteFile mismatched = writeIdOnlyEqualityDelete(table, 1); + table.newRowDelta().addDeletes(mismatched).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH, Lists.newArrayList(1, 2))) { + harness.open(); + sendTrigger(harness); + + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).hasSize(1); + } + } + + @Test + void failsWhenStagingEqDeleteSpecPartitionsByNonEqualityColumn() throws Exception { + Table table = createPartitionedTableWithDelete(3); + insertPartitioned(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete = writeIdOnlyPartitionedEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH, Lists.newArrayList(1))) { + harness.open(); + sendTrigger(harness); + + List> errOutput = + Lists.newArrayList(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)); + + assertThat(errOutput).hasSize(1); + assertThat(errOutput.get(0).getValue()) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Partition columns must be a subset of the equality fields."); + } + } + + @Test + void doesNotDuplicateNewDataFilesWhenStagingEqualsTarget() throws Exception { + // When stagingBranch == targetBranch, the writer commits new data files directly to main. + // Bootstrap scans the main snapshot (which already includes those files) and indexes them. + // The planner must NOT also emit the staging-data phase for the same files — that would + // duplicate ADD_DATA_ROW commands for the same (PK, file, position) in the worker's index. + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + // Writer commits (new-data id=3) + (eq-delete id=1) in one RowDelta directly to main. + DataFile newDataFile = writeDataFile(table, createRecord(3, "c")); + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(newDataFile).addDeletes(eqDelete).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(SnapshotRef.MAIN_BRANCH)) { + harness.open(); + sendTrigger(harness); + + List commands = harness.extractOutputValues(); + + // Bootstrap from main emits one data-file command per file (id=1, id=2, id=3) = 3. + // Without the same-branch guard, emitSnapshotDataPhase would also emit newDataFile → 4. + assertThat(countDataFileTasks(commands)).isEqualTo(3); + assertThat(countEqDeleteTasks(commands)).isEqualTo(1); + + long newDataFileCount = + commands.stream() + .filter(c -> c.task() instanceof FileScanTask) + .filter(c -> c.task().file().location().equals(newDataFile.location())) + .count(); + assertThat(newDataFileCount).isEqualTo(1); + } + } + + @Test + void emitsReadCommandsForEqualityDeletes() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + insert(table, 3, "c"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + List commands = harness.extractOutputValues(); + // 3 DATA_FILE (from main) + 1 EQ_DELETE_FILE + assertThat(countDataFileTasks(commands)).isEqualTo(3); + assertThat(countEqDeleteTasks(commands)).isEqualTo(1); + assertThat(planner(harness).processedStagingSnapshotNum()).isEqualTo(1); + assertThat(planner(harness).processedEqDeleteFileNum()).isEqualTo(1); + + List> metadata = + Lists.newArrayList(harness.getSideOutput(EqualityConvertPlanner.METADATA_STREAM)); + assertThat(metadata).hasSize(1); + assertThat(metadata.get(0).getValue().dataFiles()).isEmpty(); + } + } + + @Test + void emitsDataFileFromInsertOnlyStagingSnapshot() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S1: eq delete targeting main data + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + long s1SnapshotId = table.snapshot(STAGING_BRANCH).snapshotId(); + + // S2: insert-only (no eq deletes), but its data must still be indexed for the configured + // equality fields + DataFile insertS2 = writeDataFile(table, createRecord(2, "b")); + table.newAppend().appendFile(insertS2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + // Trigger 1: processes S1 (eq delete) + sendTrigger(harness); + int afterFirst = harness.extractOutputValues().size(); + assertThat(countEqDeleteTasks(harness.extractOutputValues())).isEqualTo(1); + + // Record S1's conversion on main, so the planner walks past S1 before processing S2. + simulateConvertCommit(table, s1SnapshotId); + + // Trigger 2: processes S2 (insert-only). Must emit its data file so the configured equality + // fields stay indexed. + sendTrigger(harness); + List allCommands = harness.extractOutputValues(); + List trigger2Commands = allCommands.subList(afterFirst, allCommands.size()); + + Set dataFilePaths = + trigger2Commands.stream() + .filter(c -> c.task() instanceof FileScanTask) + .map(TestEqualityConvertPlanner::filePath) + .collect(Collectors.toSet()); + + assertThat(dataFilePaths).contains(insertS2.location()); + } + } + + @Test + void includesNewDataFilesFromStaging() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DataFile newDataFile = writeDataFile(table, createRecord(2, "b")); + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(newDataFile).addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + List commands = harness.extractOutputValues(); + // 1 main data file + 1 eq delete + 1 staging data file + assertThat(countDataFileTasks(commands)).isEqualTo(2); + assertThat(countEqDeleteTasks(commands)).isEqualTo(1); + + List> metadata = + Lists.newArrayList(harness.getSideOutput(EqualityConvertPlanner.METADATA_STREAM)); + assertThat(metadata).hasSize(1); + assertThat(metadata.get(0).getValue().dataFiles()).hasSize(1); + } + } + + @Test + void findsIntersectionWhenMainAdvancedAfterStagingFork() throws Exception { + Table table = createTableWithDelete(3); + // Pre-fork main: two snapshots. + insert(table, 1, "a"); + insert(table, 2, "b"); + + // Fork staging from current main head. + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Main advances with two more snapshots after the fork. These are on main's + // lineage but not on staging's. + insert(table, 3, "c"); + insert(table, 4, "d"); + + // Staging gets one new snapshot containing an equality delete. + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + // Planner identifies the fork point and processes only the staging-only + // commit (the eq delete), and emits all four current main data files + // for the index refresh. + List commands = harness.extractOutputValues(); + assertThat(countEqDeleteTasks(commands)).isEqualTo(1); + assertThat(countDataFileTasks(commands)).isEqualTo(4); + } + } + + @Test + void skipsAlreadyProcessedStagingSnapshot() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + long stagingSnapshotId = table.snapshot(STAGING_BRANCH).snapshotId(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + sendTrigger(harness); + int firstTriggerCount = harness.extractOutputValues().size(); + assertThat(firstTriggerCount).isGreaterThan(0); + assertThat(planner(harness).skippedNoOpCycles()).isZero(); + + // Simulate the committer committing to main with the staging snapshot property + // (the planner promotes pending only after confirming the commit landed on main). + simulateConvertCommit(table, stagingSnapshotId); + + sendTrigger(harness); + assertThat(harness.extractOutputValues()).hasSize(firstTriggerCount); + assertThat(planner(harness).skippedNoOpCycles()).isEqualTo(1); + } + } + + @Test + void noOutputWhenStagingBranchEmpty() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + // Bootstrap from main for the configured field set: 1 main DATA_FILE; no-op metadata still + // emitted because there's nothing on staging to convert. + assertThat(countDataFileTasks(harness.extractOutputValues())).isEqualTo(1); + assertThat(countEqDeleteTasks(harness.extractOutputValues())).isZero(); + assertThat(harness.getSideOutput(EqualityConvertPlanner.METADATA_STREAM)).hasSize(1); + assertThat(planner(harness).skippedNoOpCycles()).isEqualTo(1); + assertThat(planner(harness).reindexCount()).isZero(); + } + } + + @Test + void propagatesStagingOnlyPositionalDeletes() throws Exception { + Table table = createTableWithDelete(3); + DataFile mainData = writeDataFile(table, createRecord(1, "a")); + table.newAppend().appendFile(mainData).commit(); + table.refresh(); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging snapshot has only a DV (no eq deletes, no new data files). Without the + // planner forwarding stagingDVFiles in this case, the committer would never + // see the DV and it would be lost. + DeleteFile stagingDV = writeStagingDV(table, mainData.location(), 0L); + table.newRowDelta().addDeletes(stagingDV).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + List commands = harness.extractOutputValues(); + assertThat(countDataFileTasks(commands)).isEqualTo(1); + assertThat(countEqDeleteTasks(commands)).isZero(); + + List> metadata = + Lists.newArrayList(harness.getSideOutput(EqualityConvertPlanner.METADATA_STREAM)); + assertThat(metadata).hasSize(1); + EqualityConvertPlan result = metadata.get(0).getValue(); + assertThat(result.dataFiles()).isEmpty(); + assertThat(result.stagingDVFiles()).hasSize(1); + assertThat(result.stagingDVFiles().get(0).location()).isEqualTo(stagingDV.location()); + } + } + + @Test + void phaseTimestampsAreMonotonicallyIncreasing() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging: eq delete + new data file (triggers 3 phases: main data, eq delete, staging data) + DataFile newDataFile = writeDataFile(table, createRecord(2, "b")); + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(newDataFile).addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + long triggerTs = 100L; + sendTrigger(harness, triggerTs); + + List output = Lists.newArrayList(harness.getOutput()); + assertThat(((StreamRecord) output.get(0)).getValue().task()) + .isInstanceOf(FileScanTask.class); + assertThat(output.get(1)).isEqualTo(new Watermark(triggerTs)); + + assertThat(((StreamRecord) output.get(2)).getValue().task()) + .isInstanceOf(EqualityDeleteFileScanTask.class); + assertThat(output.get(3)).isEqualTo(new Watermark(triggerTs + 1)); + + assertThat(((StreamRecord) output.get(4)).getValue().task()) + .isInstanceOf(FileScanTask.class); + assertThat(output.get(5)).isEqualTo(new Watermark(triggerTs + 2)); + } + } + + @Test + void routesExceptionToErrorStream() throws Exception { + createTableWithDelete(3); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + dropTable(); + + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).isNull(); + sendTrigger(harness); + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).hasSize(1); + } + } + + @Test + void failsOnRemovedDataFilesOnStagingBranch() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + Set oldDataFiles = Sets.newHashSet(); + for (ManifestFile manifest : table.currentSnapshot().dataManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.read(manifest, table.io(), table.specs())) { + for (DataFile df : reader) { + oldDataFiles.add(df.copy()); + } + } + } + + assertThat(oldDataFiles).hasSize(2); + + // Rewrite on the staging branch (not main). Equality delete conversion does not support + // rewrites on staging; the planner must fail the cycle instead of silently dropping the + // removed files. + DataFile compactedFile = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(1, "a"), createRecord(2, "b"))); + RewriteFiles rewrite = table.newRewrite(); + for (DataFile old : oldDataFiles) { + rewrite.deleteFile(old); + } + + rewrite.addFile(compactedFile); + rewrite.toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).hasSize(1); + // Bootstrap runs before processCycle's failure, so the main data commands are already on + // the wire. Only the cycle itself fails. + assertThat(countDataFileTasks(harness.extractOutputValues())).isEqualTo(2); + assertThat(countEqDeleteTasks(harness.extractOutputValues())).isZero(); + } + } + + @Test + void reEmitsMainDataAfterCompaction() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + sendTrigger(harness); + int firstTriggerCount = harness.extractOutputValues().size(); + // 2 DATA_FILE (main) + 1 EQ_DELETE_FILE + assertThat(firstTriggerCount).isEqualTo(3); + + // Compact data files on main: rewrite 2 files into 1 + Set oldDataFiles = Sets.newHashSet(); + for (ManifestFile manifest : table.currentSnapshot().dataManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.read(manifest, table.io(), table.specs())) { + for (DataFile df : reader) { + oldDataFiles.add(df.copy()); + } + } + } + + assertThat(oldDataFiles).hasSize(2); + + DataFile compactedFile = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(1, "a"), createRecord(2, "b"))); + RewriteFiles rewrite = table.newRewrite(); + for (DataFile old : oldDataFiles) { + rewrite.deleteFile(old); + } + + rewrite.addFile(compactedFile); + rewrite.commit(); + table.refresh(); + + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + sendTrigger(harness); + List allCommands = harness.extractOutputValues(); + List trigger2Commands = + allCommands.subList(firstTriggerCount, allCommands.size()); + + // 1 DATA_FILE (compacted main) + 1 EQ_DELETE_FILE + assertThat(countDataFileTasks(trigger2Commands)).isEqualTo(1); + assertThat(countEqDeleteTasks(trigger2Commands)).isEqualTo(1); + + // DATA_FILE should reference the compacted file + List dataCmds = + trigger2Commands.stream() + .filter(c -> c.task() instanceof FileScanTask) + .collect(Collectors.toList()); + for (ReadCommand cmd : dataCmds) { + assertThat(cmd.task().file().location()).isEqualTo(compactedFile.location()); + } + } + } + + @Test + void refreshesIndexBeforeFirstCycleProcessesEqDeletes() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Stage MULTIPLE eq-delete snapshots before the first trigger arrives. The first trigger + // bootstraps the index from main for the configured equality fields before processing any eq + // deletes, then processes one snapshot per trigger (oldest first). + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + long s2SnapshotId = table.snapshot(STAGING_BRANCH).snapshotId(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + sendTrigger(harness); + int afterFirst = harness.extractOutputValues().size(); + + // First trigger processes the OLDER staging snapshot (eqDelete1). Bootstrap built the index + // from main once: 2 main DATA_FILE + 1 EQ_DELETE_FILE = 3 commands. + assertThat(afterFirst).isEqualTo(3); + assertThat(countDataFileTasks(harness.extractOutputValues())).isEqualTo(2); + assertThat(countEqDeleteTasks(harness.extractOutputValues())).isEqualTo(1); + + simulateConvertCommit(table, table.snapshot(STAGING_BRANCH).parentId()); + + // Second trigger processes the newer staging snapshot. The index is already up-to-date. + sendTrigger(harness); + List allCommands = harness.extractOutputValues(); + List trigger2Commands = allCommands.subList(afterFirst, allCommands.size()); + assertThat(countDataFileTasks(trigger2Commands)).isZero(); + assertThat(countEqDeleteTasks(trigger2Commands)).isEqualTo(1); + + simulateConvertCommit(table, s2SnapshotId); + } + } + + @Test + void noMainReEmitWhenUnchanged() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + sendTrigger(harness); + int firstTriggerCount = harness.extractOutputValues().size(); + // 2 DATA_FILE (main) + 1 EQ_DELETE_FILE + assertThat(firstTriggerCount).isEqualTo(3); + assertThat(harness.getSideOutput(EqualityConvertPlanner.METADATA_STREAM)).hasSize(1); + + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + sendTrigger(harness); + List allCommands = harness.extractOutputValues(); + List trigger2Commands = + allCommands.subList(firstTriggerCount, allCommands.size()); + + // Only 1 EQ_DELETE_FILE, no main re-emission + assertThat(countDataFileTasks(trigger2Commands)).isEqualTo(0); + assertThat(countEqDeleteTasks(trigger2Commands)).isEqualTo(1); + + assertThat(harness.getSideOutput(EqualityConvertPlanner.METADATA_STREAM)).hasSize(2); + } + } + + @Test + void noMainReEmitAfterOwnCommit() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + long indexedStagingSnapshotId = table.snapshot(STAGING_BRANCH).snapshotId(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + sendTrigger(harness); + int firstTriggerCount = harness.extractOutputValues().size(); + + // Record the converter's own commit (COMMITTED_STAGING_SNAPSHOT property) for the staging + // snapshot the first trigger processed. The next trigger reads the property off main and + // advances lastStagingSnapshotId. No reindex should be triggered. + simulateConvertCommit(table, indexedStagingSnapshotId); + + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + sendTrigger(harness); + List allCommands = harness.extractOutputValues(); + List trigger2Commands = + allCommands.subList(firstTriggerCount, allCommands.size()); + + // Own commits don't trigger index rebuild. + assertThat(countDataFileTasks(trigger2Commands)).isEqualTo(0); + assertThat(countEqDeleteTasks(trigger2Commands)).isEqualTo(1); + } + } + + @Test + void reIndexesAfterExternalCommit() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + sendTrigger(harness); + int firstTriggerCount = harness.extractOutputValues().size(); + // 1 DATA_FILE (main) + 1 EQ_DELETE_FILE + assertThat(firstTriggerCount).isEqualTo(2); + assertThat(planner(harness).reindexCount()).isZero(); + assertThat(planner(harness).processedStagingSnapshotNum()).isEqualTo(1); + assertThat(planner(harness).processedEqDeleteFileNum()).isEqualTo(1); + + // External commit: no COMMITTED_STAGING_SNAPSHOT_PROPERTY + DataFile externalFile = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(10, "z"))); + table.newAppend().appendFile(externalFile).commit(); + table.refresh(); + + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + sendTrigger(harness); + List allCommands = harness.extractOutputValues(); + List trigger2Commands = + allCommands.subList(firstTriggerCount, allCommands.size()); + + // External commit triggers re-index: main data files are re-emitted + assertThat(countDataFileTasks(trigger2Commands)).isGreaterThan(0); + assertThat(countEqDeleteTasks(trigger2Commands)).isEqualTo(1); + assertThat(planner(harness).reindexCount()).isEqualTo(1); + assertThat(planner(harness).processedStagingSnapshotNum()).isEqualTo(2); + assertThat(planner(harness).processedEqDeleteFileNum()).isEqualTo(2); + } + } + + @Test + void emitsClearIndexBroadcastOnReindex() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + // Trigger 1 bootstraps the index. Bootstrap does not broadcast CLEAR_INDEX. + sendTrigger(harness); + assertThat(harness.getSideOutput(EqualityConvertPlanner.CLEAR_BROADCAST_STREAM)) + .isNullOrEmpty(); + + // External commit advances main. The next trigger reindexes and must broadcast CLEAR_INDEX + // so workers evict ghost keys (e.g. a PK whose data file was removed by external CoW). + DataFile externalFile = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(10, "z"))); + table.newAppend().appendFile(externalFile).commit(); + table.refresh(); + long mainAfterExternal = table.snapshot(SnapshotRef.MAIN_BRANCH).snapshotId(); + long mainSeqAfterExternal = table.snapshot(SnapshotRef.MAIN_BRANCH).sequenceNumber(); + + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + sendTrigger(harness); + + List clears = + harness.getSideOutput(EqualityConvertPlanner.CLEAR_BROADCAST_STREAM).stream() + .map(StreamRecord::getValue) + .collect(Collectors.toList()); + assertThat(clears).hasSize(1); + assertThat(clears.get(0).type()).isEqualTo(IndexCommand.Type.CLEAR_INDEX); + assertThat(clears.get(0).mainSnapshotId()).isEqualTo(mainAfterExternal); + assertThat(clears.get(0).mainSequenceNumber()).isEqualTo(mainSeqAfterExternal); + } + } + + @Test + void detectsMainBranchChangeWithoutNewStagingSnapshots() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + // Trigger 1: build the index and process the existing eq delete. + sendTrigger(harness); + int afterFirst = harness.extractOutputValues().size(); + assertThat(afterFirst).isGreaterThan(0); + + // External commit on main (no COMMITTED_STAGING_SNAPSHOT_PROPERTY). + DataFile externalFile = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(10, "z"))); + table.newAppend().appendFile(externalFile).commit(); + table.refresh(); + + // Trigger 2: nothing new on staging, but the planner must still notice the + // external main change and immediately re-emit main data for index rebuild. + sendTrigger(harness); + List allAfterTrigger2 = harness.extractOutputValues(); + List trigger2Commands = + allAfterTrigger2.subList(afterFirst, allAfterTrigger2.size()); + assertThat(countDataFileTasks(trigger2Commands)).isGreaterThan(0); + + // A subsequent staging eq delete should NOT re-emit main data because the index + // was already rebuilt in trigger 2. + int afterSecond = allAfterTrigger2.size(); + DeleteFile eqDelete2 = writeEqualityDelete(table, 10, "z"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + sendTrigger(harness); + List all = harness.extractOutputValues(); + List trigger3 = all.subList(afterSecond, all.size()); + assertThat(countDataFileTasks(trigger3)).isEqualTo(0); + assertThat(countEqDeleteTasks(trigger3)).isEqualTo(1); + } + } + + @Test + void stateRestoredAfterCheckpoint() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + long stagingSnapshotId = table.snapshot(STAGING_BRANCH).snapshotId(); + + OperatorSubtaskState state; + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + // First trigger bootstraps the index and processes the staging snapshot. + sendTrigger(harness); + int commandCount = harness.extractOutputValues().size(); + assertThat(commandCount).isGreaterThan(0); + + // Record S1's conversion on main (committer marker) so the planner advances + // lastStagingSnapshotId on the second trigger. + simulateConvertCommit(table, stagingSnapshotId); + + // Second trigger reads the marker and advances lastStagingSnapshotId. + sendTrigger(harness); + assertThat(harness.extractOutputValues()).hasSize(commandCount); + + state = harness.snapshot(1, System.currentTimeMillis()); + } + + // Restore from checkpoint: the restored index is not re-indexed because its state matches the + // snapshot COMMITTED_STAGING_SNAPSHOT_PROPERTY property. + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.initializeState(state); + harness.open(); + + sendTrigger(harness); + assertThat(harness.extractOutputValues()).isEmpty(); + } + } + + @Test + void failsOnV2PosDeleteOnStagingBranch() throws Exception { + Table table = createTableWithDelete(2); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Flink-written staging branches are V3 and only produce DVs for deletes. A V2 positional + // delete on staging is a writer-side misconfiguration and should fail the cycle rather than + // be silently absorbed. + DataFile dataFile = writeDataFile(table, createRecord(2, "b")); + DeleteFile posDelete = writePosDeleteFile(table, dataFile.location(), 0); + table.newRowDelta().addRows(dataFile).addDeletes(posDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).hasSize(1); + } + } + + @Test + void skipsCommittedSnapshotAfterCommitLandsButCheckpointDoesNot() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + long stagingSnapshotId = table.snapshot(STAGING_BRANCH).snapshotId(); + + // Capture the planner's checkpoint before any cycle has run. + OperatorSubtaskState state; + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + state = harness.snapshot(1, System.currentTimeMillis()); + } + + // The staging commit landed on main but lastStagingSnapshotId was not checkpointed. + simulateConvertCommit(table, stagingSnapshotId); + + // On restore, the planner walks main, finds the COMMITTED_STAGING_SNAPSHOT_PROPERTY, + // advances lastStagingSnapshotId, and skips re-emitting the already-committed snapshot. + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.initializeState(state); + harness.open(); + + sendTrigger(harness); + + List commands = harness.extractOutputValues(); + assertThat(countEqDeleteTasks(commands)).isEqualTo(0); + // Bootstrap from main creates the index from the main data files on the first trigger even + // though the staging snapshot itself is already committed and skipped. Main now has the + // original insert plus simulateConvertCommit's marker file = 2 data files. + assertThat(countDataFileTasks(commands)).isEqualTo(2); + } + } + + @Test + void failsWhenCommitMarkerDisappears() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + long startSnapshotId = table.currentSnapshot().snapshotId(); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + long stagingSnapshotId = table.snapshot(STAGING_BRANCH).snapshotId(); + + simulateConvertCommit(table, stagingSnapshotId); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + table.manageSnapshots().rollbackTo(startSnapshotId).commit(); + table.refresh(); + + sendTrigger(harness); + + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).hasSize(1); + assertThat( + harness + .getSideOutput(TaskResultAggregator.ERROR_STREAM) + .poll() + .getValue() + .getMessage()) + .contains("No COMMITTED_STAGING_SNAPSHOT marker reachable"); + } + } + + @Test + void failsOnEqFieldIdsChangeAcrossRestart() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + OperatorSubtaskState state; + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH, Lists.newArrayList(1, 2))) { + harness.open(); + state = harness.snapshot(1, System.currentTimeMillis()); + } + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH, Lists.newArrayList(1))) { + assertThatThrownBy(() -> harness.initializeState(state)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Equality field IDs changed across restart"); + } + } + + private OneInputStreamOperatorTestHarness createHarness( + String stagingBranch) throws Exception { + // Default matches the [1, 2] equality-field-id list produced by writeEqualityDelete. + return createHarness(stagingBranch, Lists.newArrayList(1, 2)); + } + + private OneInputStreamOperatorTestHarness createHarness( + String stagingBranch, List eqFieldIds) throws Exception { + return new OneInputStreamOperatorTestHarness<>( + new EqualityConvertPlanner( + DUMMY_TABLE_NAME, + DUMMY_TASK_NAME, + tableLoader(), + stagingBranch, + SnapshotRef.MAIN_BRANCH, + Sets.newHashSet(eqFieldIds))); + } + + private static void sendTrigger(OneInputStreamOperatorTestHarness harness) + throws Exception { + sendTrigger(harness, System.currentTimeMillis()); + } + + private static void sendTrigger(OneInputStreamOperatorTestHarness harness, long time) + throws Exception { + harness.processElement(new StreamRecord<>(Trigger.create(time, 0), time)); + } + + private DataFile writeDataFile(Table table, Record record) throws IOException { + return new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(record)); + } + + private DeleteFile writeEqualityDelete(Table table, Integer id, String data) throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + return FileHelpers.writeDeleteFile( + table, + Files.localOutput(file), + new PartitionData(PartitionSpec.unpartitioned().partitionType()), + Lists.newArrayList(createRecord(id, data)), + table.schema()); + } + + private DeleteFile writeIdOnlyEqualityDelete(Table table, int id) throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + Schema idOnly = table.schema().select("id"); + Record record = GenericRecord.create(idOnly); + record.setField("id", id); + return FileHelpers.writeDeleteFile( + table, + Files.localOutput(file), + new PartitionData(PartitionSpec.unpartitioned().partitionType()), + Lists.newArrayList(record), + idOnly); + } + + private DeleteFile writeIdOnlyPartitionedEqualityDelete(Table table, int id, String data) + throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + Schema idOnly = table.schema().select("id"); + Record record = GenericRecord.create(idOnly); + record.setField("id", id); + PartitionData partition = new PartitionData(table.spec().partitionType()); + partition.set(0, data); + return FileHelpers.writeDeleteFile( + table, Files.localOutput(file), partition, Lists.newArrayList(record), idOnly); + } + + private DeleteFile writeStagingDV(Table table, String dataFilePath, long position) + throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + + PositionDelete delete = PositionDelete.create(); + delete.set(dataFilePath, position, null); + return FileHelpers.writePosDeleteFile( + table, + Files.localOutput(file), + new PartitionData(PartitionSpec.unpartitioned().partitionType()), + Lists.newArrayList(delete), + 3); + } + + private static long countDataFileTasks(List commands) { + return commands.stream().filter(c -> c.task() instanceof FileScanTask).count(); + } + + private static long countEqDeleteTasks(List commands) { + return commands.stream().filter(TestEqualityConvertPlanner::isEqDelete).count(); + } + + private static EqualityConvertPlanner planner( + OneInputStreamOperatorTestHarness harness) { + return (EqualityConvertPlanner) harness.getOperator(); + } + + private static boolean isEqDelete(ReadCommand cmd) { + return cmd.task() instanceof EqualityDeleteFileScanTask; + } + + private static String filePath(ReadCommand cmd) { + return cmd.task().file().location(); + } + + private void simulateConvertCommit(Table table, long stagingSnapshotId) throws IOException { + DataFile dummy = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(-1, "marker" + stagingSnapshotId))); + table + .newRowDelta() + .addRows(dummy) + .set( + EqualityConvertCommitter.COMMITTED_STAGING_SNAPSHOT_PROPERTY, + String.valueOf(stagingSnapshotId)) + .commit(); + table.refresh(); + } +} diff --git a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceSql.java b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceSql.java index 0cdaf8371cbd..6cfc18868af0 100644 --- a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceSql.java +++ b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceSql.java @@ -179,7 +179,10 @@ public void testReadFlinkDynamicTable() throws Exception { List expected = generateExpectedRecords(false); SqlHelpers.sql( getTableEnv(), - "create table `default_catalog`.`default_database`.flink_table LIKE iceberg_catalog.`default`.%s", + "create table `default_catalog`.`default_database`.flink_table " + + "WITH ('catalog-type'='hadoop', 'warehouse'='%s') " + + "LIKE iceberg_catalog.`default`.%s", + CATALOG_EXTENSION.warehouse(), TestFixtures.TABLE); // Read from table in flink catalog @@ -199,8 +202,11 @@ public void testWatermarkInvalidConfig() { getStreamingTableEnv(), "CREATE TABLE %s " + "(eventTS AS CAST(t1 AS TIMESTAMP(3)), " - + "WATERMARK FOR eventTS AS SOURCE_WATERMARK()) LIKE iceberg_catalog.`default`.%s", + + "WATERMARK FOR eventTS AS SOURCE_WATERMARK()) " + + "WITH ('catalog-type'='hadoop', 'warehouse'='%s') " + + "LIKE iceberg_catalog.`default`.%s", flinkTable, + CATALOG_EXTENSION.warehouse(), TestFixtures.TABLE); assertThatThrownBy(() -> SqlHelpers.sql(getStreamingTableEnv(), "SELECT * FROM %s", flinkTable)) @@ -218,8 +224,11 @@ public void testWatermarkValidConfig() throws Exception { getStreamingTableEnv(), "CREATE TABLE %s " + "(eventTS AS CAST(t1 AS TIMESTAMP(3)), " - + "WATERMARK FOR eventTS AS SOURCE_WATERMARK()) WITH ('watermark-column'='t1') LIKE iceberg_catalog.`default`.%s", + + "WATERMARK FOR eventTS AS SOURCE_WATERMARK()) " + + "WITH ('catalog-type'='hadoop', 'warehouse'='%s', 'watermark-column'='t1') " + + "LIKE iceberg_catalog.`default`.%s", flinkTable, + CATALOG_EXTENSION.warehouse(), TestFixtures.TABLE); TestHelpers.assertRecordsWithOrder( diff --git a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/source/reader/ReaderUtil.java b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/source/reader/ReaderUtil.java index 3b094ba02298..5a6787001170 100644 --- a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/source/reader/ReaderUtil.java +++ b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/source/reader/ReaderUtil.java @@ -57,6 +57,21 @@ private ReaderUtil() {} public static FileScanTask createFileTask( List records, File file, FileFormat fileFormat, Schema schema) throws IOException { + return createFileTask( + records, + file, + fileFormat, + schema, + ResidualEvaluator.unpartitioned(Expressions.alwaysTrue())); + } + + public static FileScanTask createFileTask( + List records, + File file, + FileFormat fileFormat, + Schema schema, + ResidualEvaluator residuals) + throws IOException { DataWriter writer = new GenericFileWriterFactory.Builder() .dataSchema(schema) @@ -69,7 +84,6 @@ public static FileScanTask createFileTask( DataFile dataFile = writer.toDataFile(); - ResidualEvaluator residuals = ResidualEvaluator.unpartitioned(Expressions.alwaysTrue()); return new BaseFileScanTask( dataFile, null, diff --git a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/source/reader/TestArrayPoolDataIteratorBatcherRowData.java b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/source/reader/TestArrayPoolDataIteratorBatcherRowData.java index 4f6d182bb3c1..994227f52bcd 100644 --- a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/source/reader/TestArrayPoolDataIteratorBatcherRowData.java +++ b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/source/reader/TestArrayPoolDataIteratorBatcherRowData.java @@ -21,6 +21,7 @@ import static org.assertj.core.api.Assertions.assertThat; import java.io.File; +import java.io.IOException; import java.nio.file.Path; import java.util.Arrays; import java.util.List; @@ -32,13 +33,18 @@ import org.apache.iceberg.CombinedScanTask; import org.apache.iceberg.FileFormat; import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.data.RandomGenericData; import org.apache.iceberg.data.Record; +import org.apache.iceberg.expressions.Expression; +import org.apache.iceberg.expressions.Expressions; +import org.apache.iceberg.expressions.ResidualEvaluator; import org.apache.iceberg.flink.FlinkConfigOptions; import org.apache.iceberg.flink.TestFixtures; import org.apache.iceberg.flink.TestHelpers; import org.apache.iceberg.flink.source.DataIterator; import org.apache.iceberg.io.CloseableIterator; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -354,4 +360,168 @@ public void testMultipleFilesWithSeekPosition() throws Exception { assertThat(recordBatchIterator).isExhausted(); } + + @Test + public void testDataIteratorWithResidualFilter() throws IOException { + GenericRecord record0 = GenericRecord.create(TestFixtures.SCHEMA); + record0.setField("data", "file-1"); + record0.setField("id", 0L); + record0.setField("dt", "-"); + + GenericRecord record1 = GenericRecord.create(TestFixtures.SCHEMA); + record1.setField("data", "file-2"); + record1.setField("id", 1L); + record1.setField("dt", "-"); + + List fileRecords0 = ImmutableList.of(record0); + List fileRecords1 = ImmutableList.of(record1); + + validateFileOffsetWithResidualFilter(fileRecords0, fileRecords1, Expressions.alwaysTrue()); + + validateFileOffsetWithResidualFilter( + fileRecords0, fileRecords1, Expressions.greaterThan("id", 0)); + } + + private void validateFileOffsetWithResidualFilter( + List fileRecords0, List fileRecords1, Expression residualFilter) + throws IOException { + ResidualEvaluator residualEvaluator = ResidualEvaluator.unpartitioned(residualFilter); + + FileScanTask fileTask0 = + ReaderUtil.createFileTask( + fileRecords0, + File.createTempFile("junit", null, temporaryFolder.toFile()), + FILE_FORMAT, + TestFixtures.SCHEMA, + residualEvaluator); + + FileScanTask fileTask1 = + ReaderUtil.createFileTask( + fileRecords1, + File.createTempFile("junit", null, temporaryFolder.toFile()), + FILE_FORMAT, + TestFixtures.SCHEMA, + residualEvaluator); + + CombinedScanTask combinedTask = new BaseCombinedScanTask(Arrays.asList(fileTask0, fileTask1)); + + DataIterator dataIterator = ReaderUtil.createDataIterator(combinedTask); + + dataIterator.seek(0, 0); + + while (dataIterator.hasNext()) { + assertThat(dataIterator.fileOffset()).isEqualTo(dataIterator.next().getLong(1)); + } + } + + @Test + public void testInitializationWithHeadFilesSkipped() throws IOException { + GenericRecord record0 = GenericRecord.create(TestFixtures.SCHEMA); + record0.setField("data", "a"); + record0.setField("id", 1L); + record0.setField("dt", "-"); + + GenericRecord record1 = GenericRecord.create(TestFixtures.SCHEMA); + record1.setField("data", "a"); + record1.setField("id", 10L); + record1.setField("dt", "-"); + + ResidualEvaluator residuals = ResidualEvaluator.unpartitioned(Expressions.greaterThan("id", 5)); + + FileScanTask fileTask0 = + ReaderUtil.createFileTask( + ImmutableList.of(record0), + File.createTempFile("junit", null, temporaryFolder.toFile()), + FILE_FORMAT, + TestFixtures.SCHEMA, + residuals); + + FileScanTask fileTask1 = + ReaderUtil.createFileTask( + ImmutableList.of(record1), + File.createTempFile("junit", null, temporaryFolder.toFile()), + FILE_FORMAT, + TestFixtures.SCHEMA, + residuals); + + CombinedScanTask combinedTask = new BaseCombinedScanTask(Arrays.asList(fileTask0, fileTask1)); + + DataIterator dataIterator = ReaderUtil.createDataIterator(combinedTask); + + dataIterator.seek(0, 0); + + assertThat(dataIterator.fileOffset()) + .as("File offset should be 1 because file 0 should be skipped") + .isEqualTo(1); + + TestHelpers.assertRowData(TestFixtures.SCHEMA, record1, dataIterator.next()); + } + + @Test + void testSeekResumesCorrectlyAfterHeadFilesSkipped() throws IOException { + GenericRecord filteredRecord = GenericRecord.create(TestFixtures.SCHEMA); + filteredRecord.setField("data", "a"); + filteredRecord.setField("id", 1L); + filteredRecord.setField("dt", "-"); + + GenericRecord f1Record0 = GenericRecord.create(TestFixtures.SCHEMA); + f1Record0.setField("data", "b"); + f1Record0.setField("id", 10L); + f1Record0.setField("dt", "-"); + + GenericRecord f2Record0 = GenericRecord.create(TestFixtures.SCHEMA); + f2Record0.setField("data", "d"); + f2Record0.setField("id", 20L); + f2Record0.setField("dt", "-"); + + GenericRecord f2Record1 = GenericRecord.create(TestFixtures.SCHEMA); + f2Record1.setField("data", "e"); + f2Record1.setField("id", 21L); + f2Record1.setField("dt", "-"); + + ResidualEvaluator residuals = ResidualEvaluator.unpartitioned(Expressions.greaterThan("id", 5)); + + FileScanTask fileTask0 = + ReaderUtil.createFileTask( + ImmutableList.of(filteredRecord), + File.createTempFile("junit", null, temporaryFolder.toFile()), + FILE_FORMAT, + TestFixtures.SCHEMA, + residuals); + + FileScanTask fileTask1 = + ReaderUtil.createFileTask( + ImmutableList.of(f1Record0), + File.createTempFile("junit", null, temporaryFolder.toFile()), + FILE_FORMAT, + TestFixtures.SCHEMA, + residuals); + + FileScanTask fileTask2 = + ReaderUtil.createFileTask( + ImmutableList.of(f2Record0, f2Record1), + File.createTempFile("junit", null, temporaryFolder.toFile()), + FILE_FORMAT, + TestFixtures.SCHEMA, + residuals); + + CombinedScanTask combinedTask = + new BaseCombinedScanTask(Arrays.asList(fileTask0, fileTask1, fileTask2)); + + DataIterator dataIterator = ReaderUtil.createDataIterator(combinedTask); + dataIterator.seek(0, 0); + + TestHelpers.assertRowData(TestFixtures.SCHEMA, f1Record0, dataIterator.next()); + TestHelpers.assertRowData(TestFixtures.SCHEMA, f2Record0, dataIterator.next()); + + int checkpointFileOffset = dataIterator.fileOffset(); + long checkpointRecordOffset = dataIterator.recordOffset(); + + DataIterator restoredIterator = ReaderUtil.createDataIterator(combinedTask); + restoredIterator.seek(checkpointFileOffset, checkpointRecordOffset); + + assertThat(restoredIterator.hasNext()).isTrue(); + TestHelpers.assertRowData(TestFixtures.SCHEMA, f2Record1, restoredIterator.next()); + assertThat(restoredIterator.hasNext()).isFalse(); + } } diff --git a/flink/v2.1/flink-runtime/runtime-deps.txt b/flink/v2.1/flink-runtime/runtime-deps.txt index 8e493d31c510..a295230cef19 100644 --- a/flink/v2.1/flink-runtime/runtime-deps.txt +++ b/flink/v2.1/flink-runtime/runtime-deps.txt @@ -25,7 +25,7 @@ org.apache.parquet:parquet-variant:1.17 org.checkerframework:checker-qual:3.19 org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1 org.locationtech.jts:jts-core:1.20 -org.projectnessie.nessie:nessie-client:0.107 -org.projectnessie.nessie:nessie-model:0.107 +org.projectnessie.nessie:nessie-client:0.108 +org.projectnessie.nessie:nessie-model:0.108 org.roaringbitmap:RoaringBitmap:1.6 org.threeten:threeten-extra:1.7 diff --git a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java index a56c4e0ca6ed..1d1505a28c05 100644 --- a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java +++ b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java @@ -99,7 +99,6 @@ public class FlinkCatalog extends AbstractCatalog { private final Namespace baseNamespace; private final SupportsNamespaces asNamespaceCatalog; private final Closeable closeable; - private final Map catalogProps; private final boolean cacheEnabled; public FlinkCatalog( @@ -107,12 +106,10 @@ public FlinkCatalog( String defaultDatabase, Namespace baseNamespace, CatalogLoader catalogLoader, - Map catalogProps, boolean cacheEnabled, long cacheExpirationIntervalMs) { super(catalogName, defaultDatabase); this.catalogLoader = catalogLoader; - this.catalogProps = catalogProps; this.baseNamespace = baseNamespace; this.cacheEnabled = cacheEnabled; @@ -339,13 +336,12 @@ public CatalogTable getTable(ObjectPath tablePath) Table table = loadIcebergTable(tablePath); // Flink's CREATE TABLE LIKE clause relies on properties sent back here to create new table. - // Inorder to create such table in non iceberg catalog, we need to send across catalog - // properties also. // As Flink API accepts only Map for props, here we are serializing catalog - // props as json string to distinguish between catalog and table properties in createTable. + // name, database, table as json string to distinguish between catalog info + // and table properties in createTable. String srcCatalogProps = FlinkCreateTableOptions.toJson( - getName(), tablePath.getDatabaseName(), tablePath.getObjectName(), catalogProps); + getName(), tablePath.getDatabaseName(), tablePath.getObjectName()); Map tableProps = table.properties(); if (tableProps.containsKey(FlinkCreateTableOptions.CONNECTOR_PROPS_KEY) @@ -427,7 +423,12 @@ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ig + "create table without 'connector'='iceberg' related properties in an iceberg table."); } - Preconditions.checkArgument(table instanceof ResolvedCatalogTable, "table should be resolved"); + Preconditions.checkArgument( + table instanceof ResolvedCatalogTable, + "Expected a ResolvedCatalogTable but got: %s. " + + "Iceberg Flink catalog only supports resolved catalog tables " + + "(Materialized tables and other table kinds are not supported).", + table == null ? "null" : table.getClass().getName()); createIcebergTable(tablePath, (ResolvedCatalogTable) table, ignoreIfExists); } diff --git a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalogFactory.java b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalogFactory.java index 33cbc92ddeec..c1889dc6bd03 100644 --- a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalogFactory.java +++ b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalogFactory.java @@ -170,7 +170,6 @@ protected Catalog createCatalog( defaultDatabase, baseNamespace, catalogLoader, - properties, cacheEnabled, cacheExpirationIntervalMs); } diff --git a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/FlinkCreateTableOptions.java b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/FlinkCreateTableOptions.java index 0612260bfe7d..067b42bba954 100644 --- a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/FlinkCreateTableOptions.java +++ b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/FlinkCreateTableOptions.java @@ -18,7 +18,6 @@ */ package org.apache.iceberg.flink; -import java.util.Map; import org.apache.flink.configuration.ConfigOption; import org.apache.flink.configuration.ConfigOptions; import org.apache.iceberg.util.JsonUtil; @@ -27,14 +26,11 @@ class FlinkCreateTableOptions { private final String catalogName; private final String catalogDb; private final String catalogTable; - private final Map catalogProps; - private FlinkCreateTableOptions( - String catalogName, String catalogDb, String catalogTable, Map props) { + private FlinkCreateTableOptions(String catalogName, String catalogDb, String catalogTable) { this.catalogName = catalogName; this.catalogDb = catalogDb; this.catalogTable = catalogTable; - this.catalogProps = props; } public static final ConfigOption CATALOG_NAME = @@ -61,12 +57,6 @@ private FlinkCreateTableOptions( .noDefaultValue() .withDescription("Table name managed in the underlying iceberg catalog and database."); - public static final ConfigOption> CATALOG_PROPS = - ConfigOptions.key("catalog-props") - .mapType() - .noDefaultValue() - .withDescription("Properties for the underlying catalog for iceberg table."); - public static final ConfigOption USE_DYNAMIC_ICEBERG_SINK = ConfigOptions.key("use-dynamic-iceberg-sink") .booleanType() @@ -89,15 +79,13 @@ private FlinkCreateTableOptions( public static final String CONNECTOR_PROPS_KEY = "connector"; public static final String LOCATION_KEY = "location"; - static String toJson( - String catalogName, String catalogDb, String catalogTable, Map catalogProps) { + static String toJson(String catalogName, String catalogDb, String catalogTable) { return JsonUtil.generate( gen -> { gen.writeStartObject(); gen.writeStringField(CATALOG_NAME.key(), catalogName); gen.writeStringField(CATALOG_DATABASE.key(), catalogDb); gen.writeStringField(CATALOG_TABLE.key(), catalogTable); - JsonUtil.writeStringMap(CATALOG_PROPS.key(), catalogProps, gen); gen.writeEndObject(); }, false); @@ -110,9 +98,8 @@ static FlinkCreateTableOptions fromJson(String createTableOptions) { String catalogName = JsonUtil.getString(CATALOG_NAME.key(), node); String catalogDb = JsonUtil.getString(CATALOG_DATABASE.key(), node); String catalogTable = JsonUtil.getString(CATALOG_TABLE.key(), node); - Map catalogProps = JsonUtil.getStringMap(CATALOG_PROPS.key(), node); - return new FlinkCreateTableOptions(catalogName, catalogDb, catalogTable, catalogProps); + return new FlinkCreateTableOptions(catalogName, catalogDb, catalogTable); }); } @@ -127,8 +114,4 @@ String catalogDb() { String catalogTable() { return catalogTable; } - - Map catalogProps() { - return catalogProps; - } } diff --git a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/FlinkDynamicTableFactory.java b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/FlinkDynamicTableFactory.java index b2e2e33b9291..6306ee7a0a1c 100644 --- a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/FlinkDynamicTableFactory.java +++ b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/FlinkDynamicTableFactory.java @@ -237,13 +237,13 @@ private static TableLoader createTableLoader( } /** - * Merges source catalog properties with connector properties. Iceberg Catalog properties are - * serialized as json in FlinkCatalog#getTable to be able to isolate catalog props from iceberg - * table props, Here, we flatten and merge them back to use to create catalog. + * Merges source catalog properties (catalog name, database, table) with connector properties. + * Source catalog name, database, table are serialized as json in FlinkCatalog#getTable to be able + * to isolate them from iceberg table props, Here, we flatten and merge them back. * * @param tableProps the existing table properties - * @return a map of merged properties, with source catalog properties taking precedence when keys - * conflict + * @return a map of merged properties, defaulting to source catalog name, database and table + * unless overridden. */ private static Map mergeSrcCatalogProps(Map tableProps) { String srcCatalogProps = tableProps.get(FlinkCreateTableOptions.SRC_CATALOG_PROPS_KEY); @@ -257,7 +257,6 @@ private static Map mergeSrcCatalogProps(Map tabl FlinkCreateTableOptions.CATALOG_DATABASE.key(), createTableOptions.catalogDb()); mergedProps.put( FlinkCreateTableOptions.CATALOG_TABLE.key(), createTableOptions.catalogTable()); - mergedProps.putAll(createTableOptions.catalogProps()); tableProps.forEach( (k, v) -> { diff --git a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/ConvertEqualityDeletes.java b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/ConvertEqualityDeletes.java new file mode 100644 index 000000000000..f51df3338d02 --- /dev/null +++ b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/ConvertEqualityDeletes.java @@ -0,0 +1,286 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.api; + +import java.util.Collections; +import java.util.List; +import java.util.Set; +import org.apache.flink.annotation.Experimental; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.streaming.api.datastream.BroadcastStream; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; +import org.apache.iceberg.Schema; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableUtil; +import org.apache.iceberg.flink.maintenance.operator.DVPosition; +import org.apache.iceberg.flink.maintenance.operator.DVWriteResult; +import org.apache.iceberg.flink.maintenance.operator.EqualityConvertCommitter; +import org.apache.iceberg.flink.maintenance.operator.EqualityConvertDVWriter; +import org.apache.iceberg.flink.maintenance.operator.EqualityConvertPKIndex; +import org.apache.iceberg.flink.maintenance.operator.EqualityConvertPlan; +import org.apache.iceberg.flink.maintenance.operator.EqualityConvertPlanner; +import org.apache.iceberg.flink.maintenance.operator.EqualityConvertReader; +import org.apache.iceberg.flink.maintenance.operator.IndexCommand; +import org.apache.iceberg.flink.maintenance.operator.ReadCommand; +import org.apache.iceberg.flink.maintenance.operator.SerializedEqualityValues; +import org.apache.iceberg.flink.maintenance.operator.TaskResultAggregator; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.types.Types.NestedField; + +/** + * Creates the equality delete to DV conversion data stream. Runs a single iteration of the + * conversion for every {@link Trigger} event. + * + *

The pipeline reads equality delete files from a staging branch, converts them to deletion + * vectors (DVs) using a primary key index stored in Flink state, and commits the data files and DVs + * to the target branch. + * + *

The conversion is split into parallel stages: + * + *

    + *
  1. Planner (p=1): scans staging branch, emits file-level ReadCommands with phase timestamps + *
  2. Reader (p=N): reads files, emits row-level IndexCommands + *
  3. PKIndex (p=N): maintains PK index shards, resolves equality deletes to DV positions + *
  4. DVWriter (p=N, keyed by data file path): buffers positions per file, writes Puffin DVs + * inline + *
  5. Committer (p=1): commits data files and DVs to the target branch + *
+ * + *

Mutual exclusion with concurrent maintenance tasks (e.g. compaction) is enforced by the Flink + * maintenance framework lock. + */ +@Experimental +public class ConvertEqualityDeletes { + static final String PLANNER_TASK_NAME = "EqConvert Planner"; + static final String READER_TASK_NAME = "EqConvert Reader"; + static final String PK_INDEX_TASK_NAME = "EqConvert PKIndex"; + static final String DV_WRITER_TASK_NAME = "EqConvert DVWriter"; + static final String UPSTREAM_ABORT_TASK_NAME = "EqConvert UpstreamAbort"; + static final String COMMIT_TASK_NAME = "EqConvert Commit"; + static final String AGGREGATOR_TASK_NAME = "EqConvert Aggregator"; + + private ConvertEqualityDeletes() {} + + public static Builder builder() { + return new Builder(); + } + + public static class Builder extends MaintenanceTaskBuilder { + private String stagingBranch; + private String targetBranch = SnapshotRef.MAIN_BRANCH; + private List equalityFieldColumns = Collections.emptyList(); + + @Override + String maintenanceTaskName() { + return "ConvertEqualityDeletes"; + } + + /** Sets the staging branch name that holds the equality delete files and data files. */ + public Builder stagingBranch(String newStagingBranch) { + this.stagingBranch = newStagingBranch; + return this; + } + + /** + * Sets the target branch where converted data files and DVs are committed. Defaults to the main + * branch. + */ + public Builder targetBranch(String newTargetBranch) { + this.targetBranch = newTargetBranch; + return this; + } + + /** + * Sets the equality field columns used by the worker index. Required. Must match the equality + * field columns the writer uses for staging eq-delete files. Mirrors {@link + * org.apache.iceberg.flink.sink.IcebergSink.Builder#equalityFieldColumns}. + * + *

The partition source columns of an equality delete's spec must be a subset of these + * columns. Writes via Flink's IcebergSink already ensure this. + */ + public Builder equalityFieldColumns(List columns) { + Preconditions.checkNotNull(columns, "equalityFieldColumns must not be null"); + Preconditions.checkArgument(!columns.isEmpty(), "equalityFieldColumns must not be empty"); + this.equalityFieldColumns = ImmutableList.copyOf(columns); + return this; + } + + @Override + DataStream append(DataStream trigger) { + Preconditions.checkNotNull(stagingBranch, "stagingBranch must be set"); + Preconditions.checkArgument( + !equalityFieldColumns.isEmpty(), "equalityFieldColumns must be set on the builder"); + Set eqFieldIds = resolveEqualityFieldIds(); + + // Planner (p=1): emits ReadCommands with phase timestamps and watermarks + SingleOutputStreamOperator planned = + setSlotSharingGroup( + trigger + .transform( + operatorName(PLANNER_TASK_NAME), + TypeInformation.of(ReadCommand.class), + new EqualityConvertPlanner( + tableName(), + taskName(), + tableLoader(), + stagingBranch, + targetBranch, + eqFieldIds)) + .uid(PLANNER_TASK_NAME + uidSuffix()) + .forceNonParallel()); + + // Reader (p=N): reads files, emits IndexCommands + SingleOutputStreamOperator index = + setSlotSharingGroup( + planned + .rebalance() + .process( + new EqualityConvertReader( + tableLoader(), eqFieldIds, stagingBranch.equals(targetBranch))) + .name(operatorName(READER_TASK_NAME)) + .uid(READER_TASK_NAME + uidSuffix()) + .setParallelism(parallelism())); + + // Broadcast from the planner to the PKIndex to clear the entire index + BroadcastStream clearIndexBroadcast = + planned + .getSideOutput(EqualityConvertPlanner.CLEAR_BROADCAST_STREAM) + .broadcast(EqualityConvertPKIndex.CLEAR_BROADCAST_DESCRIPTOR); + + // PKIndex (p=N): keyed by full PK, phase-aware buffering. + SingleOutputStreamOperator dvPositions = + setSlotSharingGroup( + index + .keyBy(IndexCommand::key, TypeInformation.of(SerializedEqualityValues.class)) + .connect(clearIndexBroadcast) + .process(new EqualityConvertPKIndex(stagingBranch.equals(targetBranch))) + .name(operatorName(PK_INDEX_TASK_NAME)) + .uid(PK_INDEX_TASK_NAME + uidSuffix()) + .setParallelism(parallelism())); + + // Reader-side abort signals bypass the PKIndex and feed the DVWriter directly, so a reader + // failure can short-circuit the cycle without waiting on a keyed shuffle. This is not a full + // short-circuit: the abort is keyed by data file path (empty for ABORT), so only one resolver + // subtask observes it; the others still write their buffered DVs, which the committer then + // drops. + DataStream readerAborts = + index.getSideOutput(EqualityConvertReader.READER_ABORT_STREAM); + DataStream dvPositionsWithAborts = dvPositions.union(readerAborts); + + // Metadata side output from planner + DataStream metadata = + planned.getSideOutput(EqualityConvertPlanner.METADATA_STREAM); + + // DVWriter (p=N, keyed by data file path): groups positions per file, writes Puffin DV + // files inline, emits a DVWriteResult per cycle. Plan metadata broadcast so every subtask + // sees it. + SingleOutputStreamOperator resolved = + setSlotSharingGroup( + dvPositionsWithAborts + .keyBy(DVPosition::dataFilePath) + .connect(metadata.broadcast()) + .transform( + operatorName(DV_WRITER_TASK_NAME), + TypeInformation.of(DVWriteResult.class), + new EqualityConvertDVWriter( + tableName(), taskName(), tableLoader(), targetBranch)) + .uid(DV_WRITER_TASK_NAME + uidSuffix()) + .setParallelism(parallelism())); + + // Upstream errors become abort signals so a partial read never commits. The same error side + // outputs also feed the aggregator below to surface the exception in TaskResult; the two + // consumers serve different purposes and must both exist. + DataStream upstreamAborts = + setSlotSharingGroup( + index + .getSideOutput(TaskResultAggregator.ERROR_STREAM) + .union(dvPositions.getSideOutput(TaskResultAggregator.ERROR_STREAM)) + .map(e -> DVWriteResult.ABORT) + .returns(TypeInformation.of(DVWriteResult.class)) + .name(operatorName(UPSTREAM_ABORT_TASK_NAME)) + .uid(UPSTREAM_ABORT_TASK_NAME + uidSuffix()) + .forceNonParallel()); + + // Committer (p=1): commits data files + DVs to main. + SingleOutputStreamOperator committed = + setSlotSharingGroup( + resolved + .union(upstreamAborts) + .connect(metadata) + .transform( + operatorName(COMMIT_TASK_NAME), + TypeInformation.of(Trigger.class), + new EqualityConvertCommitter( + tableName(), taskName(), tableLoader(), stagingBranch, targetBranch)) + .uid(COMMIT_TASK_NAME + uidSuffix()) + .forceNonParallel()); + + // Aggregator (p=1): collects errors and emits TaskResult. + return setSlotSharingGroup( + committed + .connect( + planned + .getSideOutput(TaskResultAggregator.ERROR_STREAM) + .union(index.getSideOutput(TaskResultAggregator.ERROR_STREAM)) + .union(dvPositions.getSideOutput(TaskResultAggregator.ERROR_STREAM)) + .union(resolved.getSideOutput(TaskResultAggregator.ERROR_STREAM)) + .union(committed.getSideOutput(TaskResultAggregator.ERROR_STREAM))) + .transform( + operatorName(AGGREGATOR_TASK_NAME), + TypeInformation.of(TaskResult.class), + new TaskResultAggregator(tableName(), taskName(), index())) + .uid(AGGREGATOR_TASK_NAME + uidSuffix()) + .forceNonParallel()); + } + + private Set resolveEqualityFieldIds() { + if (!tableLoader().isOpen()) { + tableLoader().open(); + } + + Table table = tableLoader().loadTable(); + int formatVersion = TableUtil.formatVersion(table); + Preconditions.checkArgument( + formatVersion >= 3, + "ConvertEqualityDeletes requires table format version >= 3 (DVs), " + + "but table '%s' is version %s", + tableName(), + formatVersion); + + Schema schema = table.schema(); + List fieldIds = Lists.newArrayListWithCapacity(equalityFieldColumns.size()); + for (String column : equalityFieldColumns) { + NestedField field = schema.findField(column); + Preconditions.checkArgument( + field != null, + "Equality field column '%s' not found in table schema %s", + column, + schema); + fieldIds.add(field.fieldId()); + } + + return ImmutableSet.copyOf(fieldIds); + } + } +} diff --git a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/TableMaintenance.java b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/TableMaintenance.java index 025a6d17c023..46b38858b8aa 100644 --- a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/TableMaintenance.java +++ b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/TableMaintenance.java @@ -26,6 +26,7 @@ import java.util.UUID; import javax.annotation.Nullable; import org.apache.flink.annotation.Internal; +import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.eventtime.TimestampAssigner; import org.apache.flink.api.common.eventtime.TimestampAssignerSupplier; @@ -378,7 +379,9 @@ private DataStream changeStream(String tableName, TableLoader loade // Create a monitor source to provide the TableChange stream MonitorSource source = new MonitorSource( - loader, RateLimiterStrategy.perSecond(1.0 / rateLimit.getSeconds()), maxReadBack); + loader, + RateLimiterStrategy.perSecond(monitorRatePerSecond(rateLimit.toMillis())), + maxReadBack); return setSlotSharingGroup( env.fromSource( source, @@ -395,6 +398,16 @@ private static String nameFor(MaintenanceTaskBuilder streamBuilder, int taskI return String.format(Locale.ROOT, "%s [%d]", streamBuilder.maintenanceTaskName(), taskIndex); } + /** + * Monitor poll rate per rate-limit interval, in checks/second. We compute from millis instead + * of seconds, otherwise sub-second intervals could be truncated to 0, yielding an infinite rate + * which busy-loops the source. + */ + @VisibleForTesting + static double monitorRatePerSecond(long rateLimitMillis) { + return 1000.0 / rateLimitMillis; + } + private SingleOutputStreamOperator setSlotSharingGroup( SingleOutputStreamOperator operator) { return slotSharingGroup == null ? operator : operator.slotSharingGroup(slotSharingGroup); diff --git a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java new file mode 100644 index 000000000000..95c9ccd5de1c --- /dev/null +++ b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertCommitter.java @@ -0,0 +1,372 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.operator; + +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import org.apache.flink.annotation.Internal; +import org.apache.flink.metrics.Counter; +import org.apache.flink.metrics.MetricGroup; +import org.apache.flink.streaming.api.operators.AbstractStreamOperator; +import org.apache.flink.streaming.api.operators.TwoInputStreamOperator; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.RowDelta; +import org.apache.iceberg.Table; +import org.apache.iceberg.exceptions.CommitStateUnknownException; +import org.apache.iceberg.flink.TableLoader; +import org.apache.iceberg.flink.maintenance.api.Trigger; +import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.apache.iceberg.util.ContentFileUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Commits data files and DVs to the target branch. Receives {@link DVWriteResult}s from parallel + * {@link EqualityConvertDVWriter} instances (input 1) and an {@link EqualityConvertPlan} from the + * planner (input 2). Assembles the final file lists and commits using a {@link RowDelta} operation + * once the plan result and done-timestamp watermark have both arrived. + * + *

The commit is gated on the plan's done-timestamp watermark. + * + *

Watermarks are forwarded only after the cycle commits, never mid-cycle. The {@link + * LockRemover} releases the maintenance lock once a watermark past the trigger's start epoch + * reaches it. The planner emits phase watermarks in the middle of a cycle; forwarding those would + * release the lock before this commit, letting the TriggerManager start a concurrent cycle that + * re-processes the same uncommitted staging snapshot. + * + *

Emits a {@link Trigger} after each cycle (commit, no-op, or error) so the downstream {@link + * TaskResultAggregator} can track task completion. This is the sole source of Trigger records for + * the Aggregator. + * + *

No-op vs error: a no-op cycle (empty plan result from {@code + * EqualityConvertPlanner.emitNoOpResult}) returns early in {@code commitIfNeeded} without writing + * anything; the Trigger emit in {@code processWatermark} still happens, so the maintenance task + * completes cleanly. Errors are reported via {@link TaskResultAggregator#ERROR_STREAM} side output; + * the Aggregator collects them and surfaces failure on its own watermark. + * + *

The committer is intentionally stateless: {@code bufferedResults} and {@code planResult} are + * not checkpointed. The maintenance framework ensures mutual exclusive tasks, and on restart the + * planner re-derives its position from {@link #COMMITTED_STAGING_SNAPSHOT_PROPERTY} on main, so the + * committer never receives a plan for an already-committed staging snapshot. + */ +@Internal +public class EqualityConvertCommitter extends AbstractStreamOperator + implements TwoInputStreamOperator { + + private static final Logger LOG = LoggerFactory.getLogger(EqualityConvertCommitter.class); + + public static final String COMMITTED_STAGING_SNAPSHOT_PROPERTY = + "equality-convert-staging-snapshot"; + + private static final String ADDED_DV_NUM_METRIC = "addedDvNum"; + private static final String COMMIT_DURATION_MS_METRIC = "commitDurationMs"; + + private final String tableName; + private final String taskName; + private final TableLoader tableLoader; + private final String targetBranch; + private final boolean stagingOnTargetBranch; + + private transient Table table; + private transient List bufferedResults; + private transient EqualityConvertPlan planResult; + + private transient Counter errorCounter; + private transient Counter addedDataFileNumCounter; + private transient Counter addedDataFileSizeCounter; + private transient Counter addedDvNumCounter; + private transient Counter commitDurationMsCounter; + + public EqualityConvertCommitter( + String tableName, + String taskName, + TableLoader tableLoader, + String stagingBranch, + String targetBranch) { + this.tableName = tableName; + this.taskName = taskName; + this.tableLoader = tableLoader; + this.targetBranch = targetBranch; + this.stagingOnTargetBranch = stagingBranch.equals(targetBranch); + } + + @Override + public void open() throws Exception { + super.open(); + if (!tableLoader.isOpen()) { + tableLoader.open(); + } + + this.table = tableLoader.loadTable(); + this.bufferedResults = Lists.newArrayList(); + + MetricGroup taskMetricGroup = + TableMaintenanceMetrics.groupFor(getRuntimeContext(), tableName, taskName, 0); + this.errorCounter = taskMetricGroup.counter(TableMaintenanceMetrics.ERROR_COUNTER); + this.addedDataFileNumCounter = + taskMetricGroup.counter(TableMaintenanceMetrics.ADDED_DATA_FILE_NUM_METRIC); + this.addedDataFileSizeCounter = + taskMetricGroup.counter(TableMaintenanceMetrics.ADDED_DATA_FILE_SIZE_METRIC); + this.addedDvNumCounter = taskMetricGroup.counter(ADDED_DV_NUM_METRIC); + this.commitDurationMsCounter = taskMetricGroup.counter(COMMIT_DURATION_MS_METRIC); + } + + @Override + public void processElement1(StreamRecord record) { + bufferedResults.add(record.getValue()); + } + + @Override + public void processElement2(StreamRecord record) { + planResult = record.getValue(); + } + + @Override + public void processWatermark(Watermark mark) throws Exception { + if (planResult == null || mark.getTimestamp() < planResult.doneTimestamp()) { + // Hold back watermarks until the cycle commits so the LockRemover keeps the maintenance lock + // for the whole cycle. Forwarding the planner's mid-cycle phase watermarks will release the + // lock early and could let the next trigger run a concurrent cycle on the same staging + // snapshot. + return; + } + + try { + commitIfNeeded(); + } catch (Exception e) { + LOG.error( + "Failed to commit equality convert result for table {} task {}", tableName, taskName, e); + output.collect(TaskResultAggregator.ERROR_STREAM, new StreamRecord<>(e)); + errorCounter.inc(); + } + + // Emit Trigger for the Aggregator (even on error or no-op). + output.collect(new StreamRecord<>(Trigger.create(planResult.triggerTimestamp(), 0))); + + bufferedResults.clear(); + planResult = null; + + super.processWatermark(mark); + } + + @Override + public void close() throws Exception { + super.close(); + tableLoader.close(); + } + + private void commitIfNeeded() { + for (DVWriteResult result : bufferedResults) { + if (result.isAbort()) { + LOG.warn( + "Skipping commit for table {} task {}: a DV writer reported an error.", + tableName, + taskName); + deleteUncommittedDVs(); + return; + } + } + + // No-op cycle: the planner emitted an empty plan result (see + // EqualityConvertPlanner.emitNoOpResult) because the next staging snapshot was filtered by + // shouldSkip or there was nothing new on staging. processWatermark still forwards a Trigger to + // TaskResultAggregator, so the maintenance task completes cleanly. + if (planResult.noOp()) { + return; + } + + table.refresh(); + + List allDvFiles = Lists.newArrayList(); + List allRewrittenDvFiles = Lists.newArrayList(); + for (DVWriteResult result : bufferedResults) { + allDvFiles.addAll(result.dvFiles()); + allRewrittenDvFiles.addAll(result.rewrittenDvFiles()); + } + + RowDelta rowDelta = buildRowDelta(planResult.dataFiles(), allDvFiles, allRewrittenDvFiles); + + long startNano = System.nanoTime(); + try { + commit(rowDelta); + } catch (CommitStateUnknownException e) { + // Commit outcome unknown: the DVs may already be referenced by a committed snapshot. Leave + // them for Remove Orphan Files rather than risk deleting live data. + throw e; + } catch (Exception e) { + // Commit definitively failed: the DVs this cycle wrote are unreferenced. Delete them so a + // failed cycle does not leak Puffin files. Rewritten DVs stay (still live on the target). + deleteUncommittedDVs(); + throw e; + } + + long durationMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNano); + commitDurationMsCounter.inc(durationMs); + + LOG.info( + "Committed {} data files and {} DV files to branch '{}' for table {} in {} ms. " + + "Processed staging snapshot {}.", + planResult.dataFiles().size(), + allDvFiles.size(), + targetBranch, + tableName, + durationMs, + planResult.stagingSnapshotId()); + + // Only count files actually added by this commit. When sameBranch, the writer already + // committed the data files to target and buildRowDelta does not re-add them. + if (!stagingOnTargetBranch) { + for (DataFile dataFile : planResult.dataFiles()) { + addedDataFileNumCounter.inc(); + addedDataFileSizeCounter.inc(dataFile.fileSizeInBytes()); + } + } + + addedDvNumCounter.inc(allDvFiles.size()); + } + + @VisibleForTesting + void commit(RowDelta rowDelta) { + rowDelta.commit(); + } + + /** + * Deletes the DVs this cycle wrote but did not commit (abort or definite commit failure). Only + * the newly written DVs are removed; rewritten DVs remain referenced on the target branch. Best + * effort: a delete failure is logged, not propagated, so it never masks the original error. + */ + private void deleteUncommittedDVs() { + for (DVWriteResult result : bufferedResults) { + if (result.isAbort()) { + continue; + } + + for (DeleteFile dvFile : result.dvFiles()) { + try { + table.io().deleteFile(dvFile.location()); + } catch (RuntimeException e) { + LOG.warn( + "Failed to delete uncommitted DV {} for table {} task {}", + dvFile.location(), + tableName, + taskName, + e); + } + } + } + } + + private RowDelta buildRowDelta( + List dataFiles, List allDvFiles, List allRewrittenDvFiles) { + RowDelta rowDelta = table.newRowDelta(); + + // Fail the commit on external target-branch activity since the planner's snapshot. The next + // trigger detects the change and reindexes. + if (planResult.mainSnapshotId() != null) { + rowDelta.validateFromSnapshot(planResult.mainSnapshotId()); + } + + rowDelta.validateNoConflictingDataFiles(); + rowDelta.validateNoConflictingDeleteFiles(); + + Set referencedDataFiles = Sets.newHashSet(); + for (DeleteFile dvFile : allDvFiles) { + if (dvFile.referencedDataFile() != null) { + referencedDataFiles.add(dvFile.referencedDataFile()); + } + } + + if (!referencedDataFiles.isEmpty()) { + rowDelta.validateDataFilesExist(referencedDataFiles); + } + + // When stagingBranch == targetBranch, the writer already committed data files and DVs to the + // target branch. Re-adding them here would produce duplicate manifest entries. Skip those + // paths; only the new DVs from the writer need to be added. + if (!stagingOnTargetBranch) { + for (DataFile dataFile : dataFiles) { + rowDelta.addRows(dataFile); + } + } + + for (DeleteFile dvFile : allDvFiles) { + rowDelta.addDeletes(dvFile); + } + + if (!stagingOnTargetBranch) { + addStagingDeletes(rowDelta, referencedDataFiles, planResult.stagingDVFiles()); + } + + removeRewrittenDVs( + rowDelta, allRewrittenDvFiles, planResult.stagingDVFiles(), stagingOnTargetBranch); + + rowDelta.toBranch(targetBranch); + rowDelta.set( + COMMITTED_STAGING_SNAPSHOT_PROPERTY, String.valueOf(planResult.stagingSnapshotId())); + return rowDelta; + } + + /** Adds staging delete files, skipping DVs that overlap with conversion DVs (V3 rule). */ + private static void addStagingDeletes( + RowDelta rowDelta, Set dvCoveredDataFiles, List stagingDVFiles) { + for (DeleteFile stagingDelete : stagingDVFiles) { + // V3 allows one DV per data file. When a staging snapshot contains both a writer-committed + // DV and an eq-delete that resolves to additional positions in the same data file, the + // writer folds the staging DV into a new merged DV (via + // EqualityConvertDVWriter.collectExistingDVs). Skip the superseded staging DV; adding both + // would commit two DVs for the same data file. + if (ContentFileUtil.isDV(stagingDelete) + && stagingDelete.referencedDataFile() != null + && dvCoveredDataFiles.contains(stagingDelete.referencedDataFile())) { + continue; + } + + rowDelta.addDeletes(stagingDelete); + } + } + + /** + * Removes rewritten DVs. On a separate target branch, staging DVs are not yet on target, so they + * are skipped: removeDeletes would fail. On a shared branch they are already on target and must + * be removed like any other rewritten DV; the merged DV that supersedes them would otherwise be a + * second DV for the same data file (V3 allows one). + */ + private static void removeRewrittenDVs( + RowDelta rowDelta, + List allRewrittenDvFiles, + List stagingDVFiles, + boolean stagingOnTargetBranch) { + Set stagingDeleteLocations = Sets.newHashSet(); + for (DeleteFile sd : stagingDVFiles) { + stagingDeleteLocations.add(sd.location()); + } + + for (DeleteFile rewrittenDv : allRewrittenDvFiles) { + if (stagingOnTargetBranch || !stagingDeleteLocations.contains(rewrittenDv.location())) { + rowDelta.removeDeletes(rewrittenDv); + } + } + } +} diff --git a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertPlanner.java b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertPlanner.java new file mode 100644 index 000000000000..89e5510dd848 --- /dev/null +++ b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/operator/EqualityConvertPlanner.java @@ -0,0 +1,782 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.operator; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import org.apache.flink.annotation.Internal; +import org.apache.flink.annotation.VisibleForTesting; +import org.apache.flink.api.common.state.ListState; +import org.apache.flink.api.common.state.ListStateDescriptor; +import org.apache.flink.api.common.typeinfo.Types; +import org.apache.flink.metrics.Counter; +import org.apache.flink.metrics.MetricGroup; +import org.apache.flink.runtime.state.StateInitializationContext; +import org.apache.flink.runtime.state.StateSnapshotContext; +import org.apache.flink.streaming.api.operators.AbstractStreamOperator; +import org.apache.flink.streaming.api.operators.OneInputStreamOperator; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.util.OutputTag; +import org.apache.iceberg.ContentFile; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileContent; +import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.PartitionField; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SnapshotChanges; +import org.apache.iceberg.SnapshotSummary; +import org.apache.iceberg.Table; +import org.apache.iceberg.flink.TableLoader; +import org.apache.iceberg.flink.maintenance.api.Trigger; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.apache.iceberg.util.ContentFileUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Planner for the equality delete conversion pipeline. For each trigger, it picks the oldest + * staging snapshot that hasn't been converted yet and emits {@link ReadCommand}s describing the + * files its downstream readers and workers must process. + * + *

Each trigger runs two steps in order: + * + *

    + *
  1. {@link #ensureIndexCurrent}: updates {@link #lastStagingSnapshotId} from main's history, + * bootstraps the worker index from main on first run, and reindexes when external commits + * (e.g. compaction) have advanced main past the currently-indexed snapshot. + *
  2. {@link #processStagingSnapshot}: resolve the chosen staging snapshot's eq deletes against + * the (now-current) index, pass through any DV files, and index the snapshot's new data files + * for the next cycle. + *
+ * + * Watermarks separate phases that gate the worker's keyed state. The contract is documented on + * {@link #advancePhase()}. + * + *

An {@link EqualityConvertPlan} with the current cycle's metadata is emitted via the {@link + * #METADATA_STREAM} side output after the read commands. + * + *

Assumes a single equality-field set supplied via the builder; staging eq-deletes with a + * different {@code equalityFieldIds} fail fast in {@link #retrieveStagingFiles}. Concurrent writes + * on the target branch are handled by {@link #ensureIndexCurrent} reindexing from the new main + * snapshot; commit-time conflicts are caught by {@code RowDelta.validateFromSnapshot}. + */ +@Internal +public class EqualityConvertPlanner extends AbstractStreamOperator + implements OneInputStreamOperator { + + private static final Logger LOG = LoggerFactory.getLogger(EqualityConvertPlanner.class); + + public static final OutputTag METADATA_STREAM = + new OutputTag<>("metadata-stream") {}; + + public static final OutputTag CLEAR_BROADCAST_STREAM = + new OutputTag<>("clear-broadcast-stream") {}; + + private static final String PROCESSED_EQ_DELETE_FILE_NUM_METRIC = "processedEqDeleteFileNum"; + private static final String PROCESSED_STAGING_SNAPSHOT_NUM_METRIC = "processedStagingSnapshotNum"; + private static final String SKIPPED_NO_OP_CYCLES_METRIC = "skippedNoOpCycles"; + private static final String REINDEX_COUNT_METRIC = "reindexCount"; + + private final String tableName; + private final String taskName; + private final TableLoader tableLoader; + private final String stagingBranch; + private final String targetBranch; + private final boolean stagingOnTargetBranch; + // Equality-field-id set the worker keys on. Supplied via the builder; every staging + // eq-delete's equalityFieldIds() must match exactly. + private final Set eqFieldIds; + + // Main snapshot id the worker's index reflects. + private transient ListState indexSnapshotState; + // Main sequence number the worker's index reflects. + private transient ListState indexedSequenceNumberState; + // Equality field IDs the index was built with, allows to detect reconfiguration. + private transient ListState eqFieldIdsState; + + private transient Table table; + + private transient Long lastMainSnapshotId; + private transient Long lastStagingSnapshotId; + private transient Long indexSnapshotId; + private transient Long indexedSequenceNumber; + + private transient long nextPhaseTs; + + private transient Counter processedEqDeleteFileNumCounter; + private transient Counter processedStagingSnapshotNumCounter; + private transient Counter skippedNoOpCyclesCounter; + private transient Counter reindexCounter; + private transient Counter errorCounter; + + public EqualityConvertPlanner( + String tableName, + String taskName, + TableLoader tableLoader, + String stagingBranch, + String targetBranch, + Set eqFieldIds) { + this.tableName = tableName; + this.taskName = taskName; + this.tableLoader = tableLoader; + this.stagingBranch = stagingBranch; + this.targetBranch = targetBranch; + this.stagingOnTargetBranch = stagingBranch.equals(targetBranch); + Preconditions.checkArgument( + eqFieldIds != null && !eqFieldIds.isEmpty(), "eqFieldIds must not be null or empty"); + this.eqFieldIds = ImmutableSet.copyOf(eqFieldIds); + } + + @Override + public void open() throws Exception { + super.open(); + if (!tableLoader.isOpen()) { + tableLoader.open(); + } + + table = tableLoader.loadTable(); + + MetricGroup taskMetricGroup = + TableMaintenanceMetrics.groupFor(getRuntimeContext(), tableName, taskName, 0); + this.processedEqDeleteFileNumCounter = + taskMetricGroup.counter(PROCESSED_EQ_DELETE_FILE_NUM_METRIC); + this.processedStagingSnapshotNumCounter = + taskMetricGroup.counter(PROCESSED_STAGING_SNAPSHOT_NUM_METRIC); + this.skippedNoOpCyclesCounter = taskMetricGroup.counter(SKIPPED_NO_OP_CYCLES_METRIC); + this.reindexCounter = taskMetricGroup.counter(REINDEX_COUNT_METRIC); + this.errorCounter = taskMetricGroup.counter(TableMaintenanceMetrics.ERROR_COUNTER); + } + + @Override + public void initializeState(StateInitializationContext context) throws Exception { + super.initializeState(context); + indexSnapshotState = + context + .getOperatorStateStore() + .getListState(new ListStateDescriptor<>("indexSnapshotId", Types.LONG)); + + indexSnapshotId = null; + for (Long stateValue : indexSnapshotState.get()) { + Preconditions.checkState( + indexSnapshotId == null, "indexSnapshotId state should hold at most one value"); + indexSnapshotId = stateValue; + } + + indexedSequenceNumberState = + context + .getOperatorStateStore() + .getListState(new ListStateDescriptor<>("indexedSequenceNumber", Types.LONG)); + + indexedSequenceNumber = null; + for (Long stateValue : indexedSequenceNumberState.get()) { + Preconditions.checkState( + indexedSequenceNumber == null, + "indexedSequenceNumber state should hold at most one value"); + indexedSequenceNumber = stateValue; + } + + eqFieldIdsState = + context + .getOperatorStateStore() + .getListState(new ListStateDescriptor<>("eqFieldIds", Types.INT)); + Set restoredEqFieldIds = Sets.newHashSet(eqFieldIdsState.get()); + Preconditions.checkState( + restoredEqFieldIds.isEmpty() || restoredEqFieldIds.equals(eqFieldIds), + "Equality field IDs changed across restart: restored=%s, configured=%s. " + + "Reconfiguring equality-field columns is not supported; " + + "restart from a clean state (no savepoint).", + restoredEqFieldIds, + eqFieldIds); + } + + @Override + public void snapshotState(StateSnapshotContext context) throws Exception { + super.snapshotState(context); + indexSnapshotState.clear(); + if (indexSnapshotId != null) { + indexSnapshotState.add(indexSnapshotId); + } + + indexedSequenceNumberState.clear(); + if (indexedSequenceNumber != null) { + indexedSequenceNumberState.add(indexedSequenceNumber); + } + + eqFieldIdsState.clear(); + for (int id : eqFieldIds) { + eqFieldIdsState.add(id); + } + } + + @Override + public void processElement(StreamRecord element) throws Exception { + long triggerTs = element.getTimestamp(); + nextPhaseTs = Math.max(triggerTs, nextPhaseTs + 1); + + Long currentMainSnapshotId = lastMainSnapshotId; + try { + table.refresh(); + Snapshot mainSnapshot = table.snapshot(targetBranch); + currentMainSnapshotId = mainSnapshot != null ? mainSnapshot.snapshotId() : null; + + ensureIndexCurrent(mainSnapshot); + + Snapshot nextToProcess = + nextUnprocessedStagingSnapshot(table.snapshot(stagingBranch), mainSnapshot); + + if (nextToProcess == null) { + LOG.info("Nothing new to convert on staging branch '{}'.", stagingBranch); + emitNoOpResult(triggerTs, currentMainSnapshotId); + return; + } + + processStagingSnapshot(nextToProcess, triggerTs, currentMainSnapshotId); + } catch (Exception e) { + LOG.error("Error processing equality deletes for table {} task {}", tableName, taskName, e); + output.collect(TaskResultAggregator.ERROR_STREAM, new StreamRecord<>(e)); + errorCounter.inc(); + emitDrainResult(triggerTs, currentMainSnapshotId); + } + } + + /** + * Brings the worker's index up to date with the current state of the target branch: + * + *

    + *
  • Updates {@link #lastStagingSnapshotId} from the most recent committer marker on main. + *
  • Bootstraps the index from main on the first trigger with a non-null main snapshot. + *
  • Reindexes from main when external commits (e.g. compaction or direct writes) have + * advanced main past the currently-indexed snapshot. + *
+ * + *

No-op when main hasn't moved since the last trigger. Otherwise the history walk is bounded + * to commits added since {@link #lastMainSnapshotId}. + */ + private void ensureIndexCurrent(Snapshot mainSnapshot) { + Long currentMainSnapshotId = mainSnapshot != null ? mainSnapshot.snapshotId() : null; + + if (Objects.equals(lastMainSnapshotId, currentMainSnapshotId)) { + return; + } + + LastCommittedWork info = discoverLastCommittedWork(mainSnapshot); + updateLastStagingSnapshotId(info); + + boolean bootstrap = mainSnapshot != null && indexSnapshotId == null; + boolean reindex = indexSnapshotId != null && info.externalCommitCount() > 0; + if (bootstrap || reindex) { + LOG.info( + "{} worker index from main snapshot {} for field IDs {}.", + bootstrap ? "Bootstrapping" : "Reindexing", + currentMainSnapshotId, + eqFieldIds); + if (reindex) { + // Evict keyed entries the reindex will not re-add (e.g. data file removed by CoW). + output.collect( + CLEAR_BROADCAST_STREAM, + new StreamRecord<>( + IndexCommand.clearBeforeReindex( + currentMainSnapshotId, mainSnapshot.sequenceNumber()))); + reindexCounter.inc(); + } + + indexSnapshotId = currentMainSnapshotId; + indexedSequenceNumber = mainSnapshot.sequenceNumber(); + emitMainDataReadCommands(mainSnapshot); + } + + lastMainSnapshotId = currentMainSnapshotId; + } + + private void updateLastStagingSnapshotId(LastCommittedWork info) { + if (info.lastCommittedStaging() != null) { + lastStagingSnapshotId = info.lastCommittedStaging(); + return; + } + + Preconditions.checkState( + lastMainSnapshotId == null || lastStagingSnapshotId == null || info.reachedLastInspected(), + "No COMMITTED_STAGING_SNAPSHOT marker reachable on target branch '%s' for table %s, " + + "but a prior marker was seen (lastStagingSnapshotId=%s). Target may have been " + + "rewritten (rollback, replace_main, or snapshot expiration). " + + "Manual intervention required.", + targetBranch, + tableName, + lastStagingSnapshotId); + } + + /** + * Walks main back from head looking for the most recent snapshot tagged with {@link + * EqualityConvertCommitter#COMMITTED_STAGING_SNAPSHOT_PROPERTY}. Returns the staging snapshot id + * recorded there (or {@code null} if not reached) and the count of intervening external commits. + * + *

The walk stops at whichever comes first: + * + *

    + *
  • The first snapshot carrying the committer marker. + *
  • {@link #lastMainSnapshotId} — anything older was inspected on a previous trigger. + *
+ */ + private LastCommittedWork discoverLastCommittedWork(Snapshot mainSnapshot) { + Long lastCommittedStaging = null; + int externalCount = 0; + boolean reachedLastInspected = false; + Snapshot current = mainSnapshot; + while (current != null) { + if (lastMainSnapshotId != null && current.snapshotId() == lastMainSnapshotId) { + reachedLastInspected = true; + break; + } + + String prop = + current.summary().get(EqualityConvertCommitter.COMMITTED_STAGING_SNAPSHOT_PROPERTY); + if (prop != null) { + lastCommittedStaging = Long.parseLong(prop); + break; + } + + externalCount++; + current = parentOf(current); + } + + return new LastCommittedWork(lastCommittedStaging, externalCount, reachedLastInspected); + } + + private record LastCommittedWork( + Long lastCommittedStaging, int externalCommitCount, boolean reachedLastInspected) {} + + /** + * Walks staging history from head back to the stop point and returns the oldest unprocessed + * snapshot to convert this cycle, or {@code null} if there's nothing new. + * + *

The stop point is: + * + *

    + *
  • the last-processed staging snapshot, if known; + *
  • otherwise, on cold start with {@code stagingBranch != targetBranch}, the common ancestor + * with target; + *
  • otherwise, on cold start with {@code stagingBranch == targetBranch}, the root of history + * ({@code null}). {@link #shouldSkip} filters already-converted snapshots in O(1) via the + * {@code COMMITTED_STAGING_SNAPSHOT_PROPERTY} marker, and pure-insert snapshots via the + * same-branch eq-delete predicate. + *
+ * + *

When {@code stagingBranch == targetBranch}, the writer commits eq-deletes and data files + * directly to target; the converter performs in-place eq-delete-to-DV compaction. On cold start + * we walk the full history so eq-deletes committed before the converter started are still picked + * up. + */ + private Snapshot nextUnprocessedStagingSnapshot(Snapshot stagingHead, Snapshot mainSnapshot) { + if (stagingHead == null) { + return null; + } + + Long stopAt; + if (lastStagingSnapshotId != null) { + stopAt = lastStagingSnapshotId; + } else if (stagingOnTargetBranch) { + stopAt = null; + } else { + stopAt = findCommonAncestor(stagingHead, mainSnapshot); + } + + Snapshot current = stagingHead; + Snapshot oldestUnprocessed = null; + while (current != null) { + if (stopAt != null && current.snapshotId() == stopAt) { + break; + } + + if (!shouldSkip(current)) { + oldestUnprocessed = current; + } + + current = parentOf(current); + } + + return oldestUnprocessed; + } + + /** + * Resolves the eq deletes in {@code stagingSnapshot} against the current index and emits the + * cycle's metadata. Phase ordering (separated by watermarks): + * + *

    + *
  1. Eq delete read commands. Eq deletes resolve in the worker. + *
  2. Staging data files. Indexed for the NEXT cycle's eq-delete resolution. + *
+ * + * Cold-start bootstrap of the index from main data is handled separately in {@link + * #processElement}, which runs once before the first cycle. + */ + private void processStagingSnapshot( + Snapshot stagingSnapshot, long triggerTs, Long currentMainSnapshotId) { + + StagingInputs inputs = retrieveStagingFiles(stagingSnapshot); + Preconditions.checkState( + !inputs.isEmpty(), + "Staging snapshot %s has no convertible inputs; shouldSkip should have filtered it.", + stagingSnapshot.snapshotId()); + + emitDeletePhase(inputs.eqDeleteFiles()); + emitSnapshotDataPhase(inputs.newDataFiles()); + + LOG.info( + "Emitted read commands for {} new data files from staging branch '{}'.", + inputs.newDataFiles().size(), + stagingBranch); + + processedStagingSnapshotNumCounter.inc(); + + output.collect( + METADATA_STREAM, + new StreamRecord<>( + new EqualityConvertPlan( + inputs.newDataFiles(), + inputs.stagingDVFiles(), + stagingSnapshot.snapshotId(), + currentMainSnapshotId, + triggerTs, + nextPhaseTs))); + + advancePhase(); + } + + /** + * Classifies the files added by {@code stagingSnapshot} into data files, eq delete files, and DV + * files. Throws if the snapshot: + * + *
    + *
  • Removes data files (rewrites on the staging branch aren't supported). + *
  • Contains V2 positional delete files (the converter expects a V3 staging branch written by + * Flink, which produces only deletion vectors for deletes). + *
  • Contains an eq-delete file whose {@code equalityFieldIds()} doesn't match the + * builder-configured set (silent wrong-key serialization otherwise). + *
+ */ + private StagingInputs retrieveStagingFiles(Snapshot stagingSnapshot) { + SnapshotChanges changes = SnapshotChanges.builderFor(table).snapshot(stagingSnapshot).build(); + + // Rewrites on the staging branch would require rewriting the corresponding DVs against new + // data files on target. Not implemented; fail fast instead of silently dropping work. + if (changes.removedDataFiles().iterator().hasNext()) { + throw new IllegalStateException( + String.format( + "Staging snapshot %s on branch '%s' removes data files; " + + "equality delete conversion does not support rewrites on the staging branch. " + + "Run compaction on the target branch instead.", + stagingSnapshot.snapshotId(), stagingBranch)); + } + + List newDataFiles = Lists.newArrayList(); + List stagingDVFiles = Lists.newArrayList(); + List eqDeleteFiles = Lists.newArrayList(); + + for (DataFile dataFile : changes.addedDataFiles()) { + newDataFiles.add(dataFile); + } + + for (DeleteFile deleteFile : changes.addedDeleteFiles()) { + if (deleteFile.content() == FileContent.EQUALITY_DELETES) { + Set deleteFieldIds = Sets.newHashSet(deleteFile.equalityFieldIds()); + Preconditions.checkState( + deleteFieldIds.equals(eqFieldIds), + "Staging snapshot %s on branch '%s' contains an equality delete file %s with " + + "equalityFieldIds=%s, which does not match the configured eqFieldIds=%s. " + + "The writer must use the same equality field IDs as the converter.", + stagingSnapshot.snapshotId(), + stagingBranch, + deleteFile.location(), + deleteFieldIds, + eqFieldIds); + validateDeleteSpecPartitionColumns(stagingSnapshot, deleteFile); + eqDeleteFiles.add(deleteFile); + } else if (ContentFileUtil.isDV(deleteFile)) { + stagingDVFiles.add(deleteFile); + } else { + throw new IllegalStateException( + String.format( + "Staging snapshot %s on branch '%s' contains a V2 positional delete file (%s); " + + "equality delete conversion expects a V3 staging branch written by Flink, " + + "which produces only deletion vectors for deletes.", + stagingSnapshot.snapshotId(), stagingBranch, deleteFile.location())); + } + } + + return new StagingInputs(newDataFiles, stagingDVFiles, eqDeleteFiles); + } + + private void validateDeleteSpecPartitionColumns(Snapshot stagingSnapshot, DeleteFile deleteFile) { + PartitionSpec spec = table.specs().get(deleteFile.specId()); + for (PartitionField field : spec.fields()) { + Preconditions.checkState( + eqFieldIds.contains(field.sourceId()), + "Staging snapshot %s on branch '%s' contains an equality delete file %s under spec %s, " + + "which partitions by field '%s' (source id %s) that is not an equality field %s. " + + "Partition columns must be a subset of the equality fields.", + stagingSnapshot.snapshotId(), + stagingBranch, + deleteFile.location(), + spec.specId(), + field.name(), + field.sourceId(), + eqFieldIds); + } + } + + /** Files added by one staging snapshot, classified for cycle emission. */ + private record StagingInputs( + List newDataFiles, + List stagingDVFiles, + List eqDeleteFiles) { + + boolean isEmpty() { + return newDataFiles.isEmpty() && eqDeleteFiles.isEmpty() && stagingDVFiles.isEmpty(); + } + } + + private void emitDeletePhase(List eqDeleteFiles) { + for (DeleteFile deleteFile : eqDeleteFiles) { + PartitionSpec spec = table.specs().get(deleteFile.specId()); + output.collect( + new StreamRecord<>( + ReadCommand.eqDeleteFile( + deleteFile, + spec, + indexSnapshotId, + indexedSequenceNumber, + dataSequenceNumber(deleteFile)), + nextPhaseTs)); + processedEqDeleteFileNumCounter.inc(); + } + + advancePhase(); + } + + private void emitSnapshotDataPhase(List snapshotDataFiles) { + // Shared-branch skip: when stagingBranch == targetBranch, these files are already on target + // and were indexed by bootstrap/reindex. Re-emitting would duplicate entries in + // dataRowPositions. + if (!stagingOnTargetBranch) { + for (DataFile dataFile : snapshotDataFiles) { + PartitionSpec spec = table.specs().get(dataFile.specId()); + output.collect( + new StreamRecord<>( + ReadCommand.stagingDataFile( + new FlinkAddedRowsScanTask(dataFile, spec), + indexSnapshotId, + indexedSequenceNumber, + dataSequenceNumber(dataFile)), + nextPhaseTs)); + } + } + + advancePhase(); + } + + private void emitNoOpResult(long triggerTimestamp, Long currentMainSnapshotId) { + skippedNoOpCyclesCounter.inc(); + emitDrainResult(triggerTimestamp, currentMainSnapshotId); + } + + /** Emits an empty plan result and advances the phase so the pipeline drains. */ + private void emitDrainResult(long triggerTimestamp, Long currentMainSnapshotId) { + output.collect( + METADATA_STREAM, + new StreamRecord<>( + EqualityConvertPlan.noOp(currentMainSnapshotId, triggerTimestamp, nextPhaseTs))); + advancePhase(); + } + + /** + * Emits {@link ReadCommand}s for every data file on {@code mainSnapshot} so the worker indexes + * them for the configured equality-field set. Existing DVs attached to a data file are loaded by + * the reader and their positions are skipped. V2 positional deletes are not expected on main; the + * reader throws if it encounters one. Equality deletes attached to the scan task are skipped + * during indexing (they are processed via the planner's eq-delete read commands). + */ + private void emitMainDataReadCommands(Snapshot mainSnapshot) { + long commitSnapshotId = mainSnapshot.snapshotId(); + + try (CloseableIterable tasks = + table.newScan().useSnapshot(commitSnapshotId).planFiles()) { + for (FileScanTask task : tasks) { + output.collect( + new StreamRecord<>( + ReadCommand.dataFile( + task, indexSnapshotId, indexedSequenceNumber, dataSequenceNumber(task.file())), + nextPhaseTs)); + } + } catch (IOException e) { + throw new UncheckedIOException("Failed to plan files for main index", e); + } + + LOG.info( + "Emitted main data read commands for field IDs {} from snapshot {}.", + eqFieldIds, + commitSnapshotId); + + advancePhase(); + } + + /** + * Emits a phase-end watermark and bumps the phase timestamp. Every phase-emitting method must + * call this exactly once after its records; the worker uses these watermarks to gate keyed-state + * transitions. Missing or extra calls silently break ordering. + */ + private void advancePhase() { + output.emitWatermark(new Watermark(nextPhaseTs)); + nextPhaseTs++; + } + + /** + * Returns {@code true} if {@code snapshot} can be skipped: + * + *
    + *
  • It was already committed by us (carries {@link + * EqualityConvertCommitter#COMMITTED_STAGING_SNAPSHOT_PROPERTY}), OR + *
  • it adds no data or delete files (e.g. delete-file-only removals), OR + *
  • {@code stagingBranch == targetBranch} and the snapshot adds no equality-delete files. + * Pure-insert (and data-file-only) commits on the shared branch don't need a conversion + * cycle: their data is already on target, and the worker's index stays fresh via {@link + * #ensureIndexCurrent} when the next eq-delete arrives. + *
+ * + *

Filter checks read per-snapshot counts from {@link Snapshot#summary()}; we don't parse + * manifests here. Manifest parsing happens later in {@link #retrieveStagingFiles} only for the + * chosen snapshot. + */ + private boolean shouldSkip(Snapshot snapshot) { + Map summary = snapshot.summary(); + if (summary.containsKey(EqualityConvertCommitter.COMMITTED_STAGING_SNAPSHOT_PROPERTY)) { + return true; + } + + long addedDataFiles = summaryCount(summary, SnapshotSummary.ADDED_FILES_PROP); + long addedDeleteFiles = summaryCount(summary, SnapshotSummary.ADDED_DELETE_FILES_PROP); + if (addedDataFiles == 0 && addedDeleteFiles == 0) { + LOG.info( + "Skipping staging snapshot {}: no added data or delete files.", snapshot.snapshotId()); + return true; + } + + if (stagingOnTargetBranch + && summaryCount(summary, SnapshotSummary.ADD_EQ_DELETE_FILES_PROP) == 0) { + return true; + } + + return false; + } + + private static long summaryCount(Map summary, String key) { + String value = summary.get(key); + return value == null ? 0L : Long.parseLong(value); + } + + /** + * Returns the id of the newest snapshot that is reachable from both the staging and target branch + * heads (i.e. the most recent common ancestor where the two branches last matched), or {@code + * null} if they share no history. Used on cold start to know where staging diverged from target + * so we can skip converting snapshots that already exist on target. + */ + private Long findCommonAncestor(Snapshot stagingHead, Snapshot mainHead) { + if (mainHead == null) { + return null; + } + + Set stagingSeen = Sets.newHashSet(); + Set mainSeen = Sets.newHashSet(); + Snapshot stagingCurrent = stagingHead; + Snapshot mainCurrent = mainHead; + + while (stagingCurrent != null || mainCurrent != null) { + if (stagingCurrent != null) { + long id = stagingCurrent.snapshotId(); + if (mainSeen.contains(id)) { + return id; + } + + stagingSeen.add(id); + stagingCurrent = parentOf(stagingCurrent); + } + + if (mainCurrent != null) { + long id = mainCurrent.snapshotId(); + if (stagingSeen.contains(id)) { + return id; + } + + mainSeen.add(id); + mainCurrent = parentOf(mainCurrent); + } + } + + return null; + } + + /** Returns the parent snapshot, or {@code null} if {@code snapshot} has no parent. */ + private Snapshot parentOf(Snapshot snapshot) { + Long parentId = snapshot.parentId(); + return parentId != null ? table.snapshot(parentId) : null; + } + + private static long dataSequenceNumber(ContentFile file) { + Long sequenceNumber = file.dataSequenceNumber(); + Preconditions.checkNotNull( + sequenceNumber, "Missing data sequence number for committed file %s", file.location()); + return sequenceNumber; + } + + @Override + public void close() throws Exception { + super.close(); + tableLoader.close(); + } + + @VisibleForTesting + long reindexCount() { + return reindexCounter.getCount(); + } + + @VisibleForTesting + long skippedNoOpCycles() { + return skippedNoOpCyclesCounter.getCount(); + } + + @VisibleForTesting + long processedStagingSnapshotNum() { + return processedStagingSnapshotNumCounter.getCount(); + } + + @VisibleForTesting + long processedEqDeleteFileNum() { + return processedEqDeleteFileNumCounter.getCount(); + } +} diff --git a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/source/DataIterator.java b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/source/DataIterator.java index 3beda960cec8..2cb7667a489a 100644 --- a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/source/DataIterator.java +++ b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/source/DataIterator.java @@ -86,6 +86,7 @@ public void seek(int startingFileOffset, long startingRecordOffset) { combinedTask); for (long i = 0L; i < startingFileOffset; ++i) { tasks.next(); + fileOffset += 1; } updateCurrentIterator(); @@ -103,9 +104,6 @@ public void seek(int startingFileOffset, long startingRecordOffset) { combinedTask)); } } - - fileOffset = startingFileOffset; - recordOffset = startingRecordOffset; } @Override diff --git a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/CatalogTestBase.java b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/CatalogTestBase.java index 062ff68d5d85..e45abe25f919 100644 --- a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/CatalogTestBase.java +++ b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/CatalogTestBase.java @@ -89,6 +89,7 @@ public void before() { config.put(CatalogProperties.URI, getURI(hiveConf)); } config.put(CatalogProperties.WAREHOUSE_LOCATION, String.format("file://%s", warehouseRoot())); + config.put("extra-catalog-prop", "extra-value"); this.flinkDatabase = catalogName + "." + DATABASE; this.icebergNamespace = diff --git a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java index 020663a7de02..30daf6d55ef2 100644 --- a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java +++ b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogTable.java @@ -33,8 +33,9 @@ import org.apache.flink.table.api.Schema.UnresolvedPrimaryKey; import org.apache.flink.table.api.TableException; import org.apache.flink.table.api.ValidationException; +import org.apache.flink.table.catalog.CatalogMaterializedTable; import org.apache.flink.table.catalog.CatalogTable; -import org.apache.flink.table.catalog.CommonCatalogOptions; +import org.apache.flink.table.catalog.IntervalFreshness; import org.apache.flink.table.catalog.ObjectPath; import org.apache.flink.table.catalog.exceptions.TableNotExistException; import org.apache.iceberg.BaseTable; @@ -229,19 +230,15 @@ public void testCreateTableLikeInFlinkCatalog() throws TableNotExistException { .column("id", DataTypes.BIGINT()) .build()); - // `type` option is filtered out by Flink - // https://github.com/apache/flink/blob/edc3d68736de73665440f4313ddcfd9142d8d42b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/factories/FactoryUtil.java#L378 - Map filteredOptions = Maps.newHashMap(config); - filteredOptions.remove(CommonCatalogOptions.CATALOG_TYPE.key()); - - String srcCatalogProps = - FlinkCreateTableOptions.toJson(catalogName, DATABASE, "tl", filteredOptions); + String srcCatalogProps = FlinkCreateTableOptions.toJson(catalogName, DATABASE, "tl"); Map options = catalogTable.getOptions(); assertThat(options) .containsEntry( FlinkCreateTableOptions.CONNECTOR_PROPS_KEY, FlinkDynamicTableFactory.FACTORY_IDENTIFIER) .containsEntry(FlinkCreateTableOptions.SRC_CATALOG_PROPS_KEY, srcCatalogProps); + assertThat(options.get(FlinkCreateTableOptions.SRC_CATALOG_PROPS_KEY)) + .doesNotContain("extra-catalog-prop", "extra-value"); } @TestTemplate @@ -750,6 +747,31 @@ private void validateTableFiles(Table tbl, DataFile... expectedFiles) { assertThat(actualFilePaths).as("Files should match").isEqualTo(expectedFilePaths); } + @TestTemplate + public void testCreateMaterializedTableIsUnsupported() { + CatalogMaterializedTable materializedTable = + CatalogMaterializedTable.newBuilder() + .schema( + org.apache.flink.table.api.Schema.newBuilder() + .column("id", DataTypes.BIGINT()) + .build()) + .definitionQuery("SELECT id FROM tl") + .freshness(IntervalFreshness.ofMinute("5")) + .logicalRefreshMode(CatalogMaterializedTable.LogicalRefreshMode.AUTOMATIC) + .refreshMode(CatalogMaterializedTable.RefreshMode.CONTINUOUS) + .refreshStatus(CatalogMaterializedTable.RefreshStatus.INITIALIZING) + .build(); + + assertThatThrownBy( + () -> + getTableEnv() + .getCatalog(catalogName) + .get() + .createTable(new ObjectPath(DATABASE, "mt_table"), materializedTable, false)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Materialized tables and other table kinds are not supported"); + } + private Table table(String name) { return validationCatalog.loadTable(TableIdentifier.of(icebergNamespace, name)); } diff --git a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestConvertEqualityDeletes.java b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestConvertEqualityDeletes.java new file mode 100644 index 000000000000..1c2f230315be --- /dev/null +++ b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestConvertEqualityDeletes.java @@ -0,0 +1,1333 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.api; + +import static org.apache.iceberg.flink.SimpleDataUtil.createRecord; +import static org.apache.iceberg.flink.maintenance.api.ConvertEqualityDeletes.COMMIT_TASK_NAME; +import static org.apache.iceberg.flink.maintenance.api.ConvertEqualityDeletes.PLANNER_TASK_NAME; +import static org.apache.iceberg.flink.maintenance.operator.EqualityConvertCommitter.COMMITTED_STAGING_SNAPSHOT_PROPERTY; +import static org.apache.iceberg.flink.maintenance.operator.TableMaintenanceMetrics.ADDED_DATA_FILE_NUM_METRIC; +import static org.apache.iceberg.flink.maintenance.operator.TableMaintenanceMetrics.ADDED_DATA_FILE_SIZE_METRIC; +import static org.apache.iceberg.flink.maintenance.operator.TableMaintenanceMetrics.ERROR_COUNTER; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Path; +import java.time.Duration; +import java.util.List; +import java.util.Set; +import java.util.stream.StreamSupport; +import org.apache.flink.core.execution.JobClient; +import org.apache.flink.streaming.api.graph.StreamGraphGenerator; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileContent; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.Files; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; +import org.apache.iceberg.PartitionData; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.RewriteFiles; +import org.apache.iceberg.Schema; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.Table; +import org.apache.iceberg.data.FileHelpers; +import org.apache.iceberg.data.GenericAppenderHelper; +import org.apache.iceberg.data.IcebergGenerics; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.deletes.PositionDelete; +import org.apache.iceberg.flink.SimpleDataUtil; +import org.apache.iceberg.flink.maintenance.operator.MetricsReporterFactoryForTests; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.StructLikeSet; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class TestConvertEqualityDeletes extends MaintenanceTaskTestBase { + + private static final String STAGING_BRANCH = "__flink_staging_test"; + + @TempDir private Path tempDir; + + @Test + void testRejectsFormatVersion2() { + createTableWithDelete(2); + + assertThatThrownBy( + () -> + ConvertEqualityDeletes.builder() + .stagingBranch(STAGING_BRANCH) + .equalityFieldColumns(ImmutableList.of("id", "data")) + .append( + infra.triggerStream(), + DUMMY_TABLE_NAME, + DUMMY_TASK_NAME, + 0, + tableLoader(), + UID_SUFFIX, + StreamGraphGenerator.DEFAULT_SLOT_SHARING_GROUP, + 1)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("format version >= 3"); + } + + @Test + void testRejectsUnknownEqualityFieldColumns() { + createTableWithDelete(3); + + assertThatThrownBy( + () -> + ConvertEqualityDeletes.builder() + .stagingBranch(STAGING_BRANCH) + .equalityFieldColumns(ImmutableList.of("nonexistent")) + .append( + infra.triggerStream(), + DUMMY_TABLE_NAME, + DUMMY_TASK_NAME, + 0, + tableLoader(), + UID_SUFFIX, + StreamGraphGenerator.DEFAULT_SLOT_SHARING_GROUP, + 1)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Equality field column 'nonexistent' not found in table schema"); + } + + @Test + void testConvertEqualityDeletesToDVs() throws Exception { + Table table = createTableWithDelete(3); + + // Insert initial data to main + insert(table, 1, "a"); + insert(table, 2, "b"); + insert(table, 3, "c"); + + assertThat(dataFileCount(table)).isEqualTo(3); + + // Create staging branch from current main state + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Write a new data file (simulating insert of id=4) + DataFile newDataFile = writeDataFile(table, createRecord(4, "d")); + + // Write an equality delete for id=2 (simulating delete of row "b") + DeleteFile eqDelete = writeEqualityDelete(table, 2, "b"); + + // Commit both to the staging branch + table.newRowDelta().addRows(newDataFile).addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Verify staging branch has the eq delete + long stagingEqDeleteCount = + table.snapshot(STAGING_BRANCH).deleteManifests(table.io()).stream() + .flatMap( + m -> + StreamSupport.stream( + ManifestFiles.readDeleteManifest(m, table.io(), table.specs()) + .spliterator(), + false)) + .filter(f -> f.content() == FileContent.EQUALITY_DELETES) + .count(); + assertThat(stagingEqDeleteCount).isEqualTo(1); + + // Wire the ConvertEqualityDeletes maintenance task + appendConvertTask(); + + // Run the maintenance task + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + // Verify main branch now has 4 data files (3 original + 1 new) + table.refresh(); + assertThat(dataFileCount(table)).isEqualTo(4); + + // Verify main branch has exactly one DV (id=2 deleted from its single-row data file) + long mainDvCount = + table.currentSnapshot().deleteManifests(table.io()).stream() + .flatMap( + m -> + StreamSupport.stream( + ManifestFiles.readDeleteManifest(m, table.io(), table.specs()) + .spliterator(), + false)) + .filter(f -> f.content() == FileContent.POSITION_DELETES) + .count(); + assertThat(mainDvCount).isEqualTo(1); + + // Verify no equality deletes on main + long mainEqDeleteCount = + table.currentSnapshot().deleteManifests(table.io()).stream() + .flatMap( + m -> + StreamSupport.stream( + ManifestFiles.readDeleteManifest(m, table.io(), table.specs()) + .spliterator(), + false)) + .filter(f -> f.content() == FileContent.EQUALITY_DELETES) + .count(); + assertThat(mainEqDeleteCount).isEqualTo(0); + + // Verify data correctness: id=2 should be deleted, id=4 should be added + SimpleDataUtil.assertTableRecords( + table, ImmutableList.of(createRecord(1, "a"), createRecord(3, "c"), createRecord(4, "d"))); + } + + @Test + void testMetrics() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // One staging snapshot: a new data file (id=3) plus an eq-delete (id=1). The conversion commits + // exactly one data file and one DV to main. + DataFile newDataFile = writeDataFile(table, createRecord(3, "c")); + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(newDataFile).addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + // Only metrics named on TableMaintenanceMetrics flow through the test reporter. Among the + // converter operators only the planner and committer own an ERROR_COUNTER; the parallel reader, + // PK index, and DV writer report failures through ERROR_STREAM instead. The committer also + // counts the data files it adds. Operator-specific counters (reindexCount, addedDvNum, ...) are + // asserted by the operator unit tests. A -1 expected value means "present, value not checked". + MetricsReporterFactoryForTests.assertCounters( + new ImmutableMap.Builder, Long>() + .put(errorKey(PLANNER_TASK_NAME), 0L) + .put(errorKey(COMMIT_TASK_NAME), 0L) + .put(metricKey(COMMIT_TASK_NAME, ADDED_DATA_FILE_NUM_METRIC), 1L) + .put(metricKey(COMMIT_TASK_NAME, ADDED_DATA_FILE_SIZE_METRIC), -1L) + .build()); + } + + private static List errorKey(String taskName) { + return metricKey(taskName, ERROR_COUNTER); + } + + private static List metricKey(String taskName, String metric) { + return ImmutableList.of(taskName + "[0]", DUMMY_TABLE_NAME, DUMMY_TASK_NAME, "0", metric); + } + + @Test + void testNoOpWhenStagingBranchEmpty() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + // Create staging branch with no new files + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + // Should complete successfully with no changes + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + assertThat(dataFileCount(table)).isEqualTo(1); + SimpleDataUtil.assertTableRecords(table, ImmutableList.of(createRecord(1, "a"))); + } + + @Test + void testMultipleEqualityDeletes() throws Exception { + Table table = createTableWithDelete(3); + + insert(table, 1, "a"); + insert(table, 2, "b"); + insert(table, 3, "c"); + insert(table, 4, "d"); + insert(table, 5, "e"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Delete id=1 and id=4 via equality deletes on staging + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + DeleteFile eqDelete2 = writeEqualityDelete(table, 4, "d"); + + table + .newRowDelta() + .addDeletes(eqDelete1) + .addDeletes(eqDelete2) + .toBranch(STAGING_BRANCH) + .commit(); + table.refresh(); + + appendConvertTask(); + + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + SimpleDataUtil.assertTableRecords( + table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"), createRecord(5, "e"))); + } + + @Test + void testDuplicateKeyAcrossDataFiles() throws Exception { + Table table = createTableWithDelete(3); + + // Two data files with the same key (id=1, data="a") + insert(table, 1, "a"); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Eq delete for id=1 should produce DVs for both data files containing id=1 + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + // Only id=2 should remain + SimpleDataUtil.assertTableRecords(table, ImmutableList.of(createRecord(2, "b"))); + } + + @Test + void testMultiSnapshotStagingWithPerSnapshotScoping() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging snapshot 1: delete id=1 from main + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging snapshot 2: re-insert id=1 (new data file) + DataFile newDataFile = writeDataFile(table, createRecord(1, "a")); + table.newAppend().appendFile(newDataFile).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + // Cycle 1: processes S1 (delete id=1) + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.success()).isTrue(); + + // Cycle 2: processes S2 (re-insert id=1) + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.success()).isTrue(); + + table.refresh(); + // id=1 should still exist: the delete from S1 removed the original, + // but S2 re-inserted it. Per-snapshot scoping ensures S1's delete + // doesn't affect S2's data. + assertRecords(table, ImmutableList.of(createRecord(1, "a"))); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testInsertThenDeleteAcrossCycles() throws Exception { + Table table = createTableWithDelete(3); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging snapshot 1: insert id=1 (data-only, no eq deletes) + DataFile insertS1 = writeDataFile(table, createRecord(1, "a")); + table.newAppend().appendFile(insertS1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging snapshot 2: eq delete the row written in S1 + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + // Cycle 1: processes S1 (insert id=1), commits data file to main + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.exceptions()).isEmpty(); + assertThat(result1.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "a"))); + + // Cycle 2: processes S2 (eq delete id=1), must find id=1 on main + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of()); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testInsertUpdateDeleteInsertUpdateChain() throws Exception { + Table table = createTableWithDelete(3); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S1: insert K=1, V=A + DataFile s1 = writeDataFile(table, createRecord(1, "a")); + table.newAppend().appendFile(s1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S2: update K=1 to V=B (eq delete + insert in same commit) + DataFile s2 = writeDataFile(table, createRecord(1, "b")); + DeleteFile e2 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(s2).addDeletes(e2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S3: delete K=1 + DeleteFile e3 = writeEqualityDelete(table, 1, "b"); + table.newRowDelta().addDeletes(e3).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S4: insert K=1, V=C + DataFile s4 = writeDataFile(table, createRecord(1, "c")); + table.newAppend().appendFile(s4).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S5: update K=1 to V=D (eq delete + insert in same commit) + DataFile s5 = writeDataFile(table, createRecord(1, "d")); + DeleteFile e5 = writeEqualityDelete(table, 1, "c"); + table.newRowDelta().addRows(s5).addDeletes(e5).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + long time = System.currentTimeMillis(); + + // Cycle 1: S1 inserts K=1, V=A + long time1 = time; + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.exceptions()).isEmpty(); + assertThat(result1.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "a"))); + + // Cycle 2: S2 updates K=1 from V=A to V=B (eq delete + insert) + long time2 = time + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "b"))); + + // Cycle 3: S3 deletes K=1 + long time3 = time + 2; + infra.source().sendRecord(Trigger.create(time3, 0), time3); + TaskResult result3 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result3.exceptions()).isEmpty(); + assertThat(result3.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of()); + + // Cycle 4: S4 inserts K=1, V=C + long time4 = time + 3; + infra.source().sendRecord(Trigger.create(time4, 0), time4); + TaskResult result4 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result4.exceptions()).isEmpty(); + assertThat(result4.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "c"))); + + // Cycle 5: S5 updates K=1 from V=C to V=D (eq delete + insert) + long time5 = time + 4; + infra.source().sendRecord(Trigger.create(time5, 0), time5); + TaskResult result5 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result5.exceptions()).isEmpty(); + assertThat(result5.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "d"))); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testParallelInsertOfToBeDeletedKeySurvives() throws Exception { + Table table = createTableWithDelete(3); + + // Main holds the original (1, "a"); the staging eq-delete below removes this copy. + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // One staging snapshot updates id=1 in place: re-insert (1, "a") plus an eq-delete (1, "a") in + // the same commit. The re-insert shares the equality key with the delete and carries the + // delete's sequence, so it must survive: the delete only removes the lower-sequence main copy. + // At parallelism > 1 the staging-data ADD can reach the index before the eq-delete resolves. + // Event-time phase ordering is what keeps the re-insert from being accidentally deleted. + DataFile reinsert = writeDataFile(table, createRecord(1, "a")); + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(reinsert).addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(STAGING_BRANCH); + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "a"))); + } + + @Test + void testDVMergeAcrossConversionCycles() throws Exception { + Table table = createTableWithDelete(3); + + // Single data file with 3 rows so DV merge applies to the same file + insert( + table, ImmutableList.of(createRecord(1, "a"), createRecord(2, "b"), createRecord(3, "c"))); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Cycle 1 setup: eq delete for id=1 + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + // Cycle 1: convert eq delete for id=1 + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"))); + + // Cycle 2 setup: eq delete for id=2 (committed while job is running) + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Cycle 2: convert eq delete for id=2, should merge DV with existing + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(3, "c"))); + + // Verify: main has DVs, no equality deletes + assertNoEqualityDeletesOnMain(table, 0); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testConversionCorrectAfterCompaction() throws Exception { + Table table = createTableWithDelete(3); + + // Three separate data files + insert(table, 1, "a"); + insert(table, 2, "b"); + insert(table, 3, "c"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Cycle 1: delete id=1 + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"))); + + // Compact file2 and file3 on main into one file (leave file1 + its DV untouched) + Set allDataFiles = Sets.newHashSet(); + for (ManifestFile manifest : table.currentSnapshot().dataManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.read(manifest, table.io(), table.specs())) { + for (DataFile df : reader) { + allDataFiles.add(df.copy()); + } + } + } + + // Find file1 (contains id=1) by checking which file has a DV against it + Set dvReferencedFiles = Sets.newHashSet(); + for (ManifestFile manifest : table.currentSnapshot().deleteManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { + for (DeleteFile df : reader) { + if (df.referencedDataFile() != null) { + dvReferencedFiles.add(df.referencedDataFile()); + } + } + } + } + + Set filesToCompact = Sets.newHashSet(); + for (DataFile df : allDataFiles) { + if (!dvReferencedFiles.contains(df.location())) { + filesToCompact.add(df); + } + } + + assertThat(filesToCompact).hasSize(2); + + DataFile compactedFile = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"))); + RewriteFiles rewrite = table.newRewrite(); + for (DataFile old : filesToCompact) { + rewrite.deleteFile(old); + } + + rewrite.addFile(compactedFile); + rewrite.commit(); + table.refresh(); + + assertRecords(table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"))); + + // Cycle 2: delete id=2 (should target the compacted file) + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(3, "c"))); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testConvertEqualityDeletesPartitionedTable() throws Exception { + Table table = createPartitionedTableWithDelete(3); + + // Insert data into two partitions + insertPartitioned(table, 1, "a"); + insertPartitioned(table, 2, "b"); + insertPartitioned(table, 3, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Partition-scoped equality delete for id=1 in partition data="a" + DeleteFile eqDelete = writePartitionedEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + // id=1 deleted from partition "a", id=2 in partition "b" and id=3 in partition "a" remain + assertRecords(table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "a"))); + } + + @Test + void testEqualityDeleteIsScopedToItsPartition() throws Exception { + Table table = createPartitionedTableWithDelete(3); + + // Same PK (id=1) exists in two partitions. An eq delete in one partition must not delete + // rows in the other. + insertPartitioned(table, 1, "a"); + insertPartitioned(table, 1, "b"); + insertPartitioned(table, 2, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete = writePartitionedEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + // Only (1, "a") is deleted; (1, "b") remains because the eq delete was scoped to partition + // "a", and (2, "a") remains because its equality field values don't match. + assertRecords(table, ImmutableList.of(createRecord(1, "b"), createRecord(2, "a"))); + } + + @Test + void testStagingPositionDeleteMergedIntoConversionDV() throws Exception { + Table table = createTableWithDelete(3); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S1: write a data file with two rows (id=1 at pos 0, id=2 at pos 1). + DataFile data = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(1, "a"), createRecord(2, "b"))); + table.newAppend().appendFile(data).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S2: eq delete matches row 0 (will produce a conversion DV at pos 0) + a position + // delete DV referencing the same data file at pos 1. Both DVs target the same data + // file and must be merged into a single DV (V3 invariant). + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + DeleteFile stagingDV = writeStagingDV(table, data.location(), 1L); + table + .newRowDelta() + .addDeletes(eqDelete) + .addDeletes(stagingDV) + .toBranch(STAGING_BRANCH) + .commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + // Cycle 1: commits S1's data file to main + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.success()).isTrue(); + + // Cycle 2: converts S2's eq delete to DV, merges with staging DV + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + table.refresh(); + // Both rows from S1 must be masked: pos 0 by the conversion DV, pos 1 by the staging DV. + assertRecords(table, ImmutableList.of()); + + // Exactly one DV per data file (V3 invariant): the resolver must have folded the + // staging DV's positions into the conversion DV. + long dvCount = + table.currentSnapshot().deleteManifests(table.io()).stream() + .flatMap( + m -> + StreamSupport.stream( + ManifestFiles.readDeleteManifest(m, table.io(), table.specs()) + .spliterator(), + false)) + .filter(f -> f.content() == FileContent.POSITION_DELETES) + .filter(f -> data.location().equals(f.referencedDataFile())) + .count(); + assertThat(dvCount).isEqualTo(1); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testStagingDataFilesOnlyNoEqDeletes() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging has only a new data file, no equality deletes + DataFile newDataFile = writeDataFile(table, createRecord(2, "b")); + table.newAppend().appendFile(newDataFile).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + assertThat(dataFileCount(table)).isEqualTo(2); + SimpleDataUtil.assertTableRecords( + table, ImmutableList.of(createRecord(1, "a"), createRecord(2, "b"))); + } + + @Test + void testReindexAfterExternalCommit() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Cycle 1: delete id=1 + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(2, "b"))); + + // External commit: insert id=3 directly to main (not via staging) + insert(table, 3, "c"); + + // Cycle 2: delete id=2 (should reindex because of external commit) + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(3, "c"))); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testReindexEvictsGhostKeyAfterExternalDataFileRemoval() throws Exception { + // CoW removal case: an external commit removes a data file, leaving a stale + // PK in the worker's index. A later staging eq-delete for that PK must NOT produce a DV + // referencing the removed file. The external commit advances main, so the next cycle reindexes + // and the worker clears the ghost key before resolving the delete. + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + table.refresh(); + DataFile file1 = table.currentSnapshot().addedDataFiles(table.io()).iterator().next().copy(); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Cycle 1: delete id=2. Bootstraps the worker index from main (id=1 -> file1, id=2 -> file2). + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "a"))); + + // External CoW-style removal: drop file1 (id=1) from main without re-adding the row. The + // worker's index still holds the ghost id=1 -> file1 until the next reindex clears it. + table.newDelete().deleteFile(file1).commit(); + table.refresh(); + assertRecords(table, ImmutableList.of()); + + // Cycle 2: stage an eq-delete for the removed key id=1. Without ghost eviction the worker + // would emit a DV position against the now-absent file1. + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + // No deletion vector may reference the removed file1. + table.refresh(); + assertThat(dvReferencedDataFiles(table)).doesNotContain(file1.location()); + assertRecords(table, ImmutableList.of()); + } finally { + closeJobClient(jobClient); + } + } + + private static Set dvReferencedDataFiles(Table table) { + Set referenced = Sets.newHashSet(); + for (ManifestFile manifest : table.currentSnapshot().deleteManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { + for (DeleteFile df : reader) { + if (df.referencedDataFile() != null) { + referenced.add(df.referencedDataFile()); + } + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + return referenced; + } + + private DataFile writeDataFile(Table table, Record record) throws IOException { + return new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(record)); + } + + private DeleteFile writeStagingDV(Table table, String dataFilePath, long position) + throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + + PositionDelete delete = PositionDelete.create(); + delete.set(dataFilePath, position, null); + return FileHelpers.writePosDeleteFile( + table, + Files.localOutput(file), + new PartitionData(PartitionSpec.unpartitioned().partitionType()), + Lists.newArrayList(delete), + 3); + } + + private DeleteFile writeEqualityDelete(Table table, Integer id, String data) throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + + Schema eqDeleteSchema = table.schema(); + return FileHelpers.writeDeleteFile( + table, + Files.localOutput(file), + new PartitionData(PartitionSpec.unpartitioned().partitionType()), + Lists.newArrayList(createRecord(id, data)), + eqDeleteSchema); + } + + private DeleteFile writePartitionedEqualityDelete(Table table, Integer id, String data) + throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + + PartitionData partition = new PartitionData(table.spec().partitionType()); + partition.set(0, data); + return FileHelpers.writeDeleteFile( + table, + Files.localOutput(file), + partition, + Lists.newArrayList(createRecord(id, data)), + table.schema()); + } + + private static long dataFileCount(Table table) { + table.refresh(); + long count = 0; + for (ManifestFile manifest : table.currentSnapshot().dataManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.read(manifest, table.io(), table.specs())) { + for (DataFile ignored : reader) { + count++; + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + return count; + } + + @Test + void testStagingEqualsTargetBranch() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + // Write eq delete directly to main (no separate staging branch) + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + DataFile newData = writeDataFile(table, createRecord(3, "c")); + table.newRowDelta().addRows(newData).addDeletes(eqDelete).commit(); + table.refresh(); + + appendConvertTask(SnapshotRef.MAIN_BRANCH); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + // Cycle 1: process the eq delete for id=1 + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result1.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"))); + long dataFilesAfterCycle1 = dataFileCount(table); + // Expect 3: two from insert() + one from the writer's rowDelta.addRows(newData). + // When stagingBranch == targetBranch, the committer must NOT re-add newData via + // rowDelta.addRows(...) — that would duplicate (count=4). + assertThat(dataFilesAfterCycle1).isEqualTo(3); + + // Cycle 2: no-op (converter's own commit must be skipped) + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result2.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"))); + assertThat(dataFileCount(table)).isEqualTo(dataFilesAfterCycle1); + + // New eq delete for id=2 committed directly to main between cycles + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + DataFile newData2 = writeDataFile(table, createRecord(4, "d")); + table.newRowDelta().addRows(newData2).addDeletes(eqDelete2).commit(); + table.refresh(); + + // Cycle 3: process the new eq delete + long time3 = time2 + 1; + infra.source().sendRecord(Trigger.create(time3, 0), time3); + TaskResult result3 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result3.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(3, "c"), createRecord(4, "d"))); + long dataFilesAfterCycle3 = dataFileCount(table); + + // Cycle 4: no-op again + long time4 = time3 + 1; + infra.source().sendRecord(Trigger.create(time4, 0), time4); + TaskResult result4 = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result4.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(3, "c"), createRecord(4, "d"))); + assertThat(dataFileCount(table)).isEqualTo(dataFilesAfterCycle3); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testStagingEqualsTargetBranchColdStartCatchUp() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + insert(table, 3, "c"); + + // Writer commits three eq-deletes to main BEFORE the converter starts. + // Cold start must pick up the unconverted history, not just the head snapshot. + table.newRowDelta().addDeletes(writeEqualityDelete(table, 1, "a")).commit(); + table.newRowDelta().addDeletes(writeEqualityDelete(table, 2, "b")).commit(); + table.newRowDelta().addDeletes(writeEqualityDelete(table, 3, "c")).commit(); + table.refresh(); + + appendConvertTask(SnapshotRef.MAIN_BRANCH); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + // One unconverted snapshot per cycle, oldest first. After three cycles every eq-delete + // commit has its own committer commit carrying the marker. + for (int cycle = 1; cycle <= 3; cycle++) { + long ts = System.currentTimeMillis() + cycle; + infra.source().sendRecord(Trigger.create(ts, 0), ts); + TaskResult result = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result.success()).isTrue(); + } + + table.refresh(); + long convertedCount = + StreamSupport.stream(table.snapshots().spliterator(), false) + .filter(s -> s.summary().containsKey(COMMITTED_STAGING_SNAPSHOT_PROPERTY)) + .count(); + assertThat(convertedCount).isEqualTo(3); + assertRecords(table, ImmutableList.of()); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testStagingEqualsTargetBranchReinsertAfterDeleteSurvives() throws Exception { + Table table = createTableWithDelete(3); + + // Shared branch: insert id=1, eq-delete id=1, then re-insert id=1. The re-insert has a higher + // sequence than the delete and must survive the conversion (sequence-aware resolution). + insert(table, 1, "a"); + table.newRowDelta().addDeletes(writeEqualityDelete(table, 1, "a")).commit(); + table.newAppend().appendFile(writeDataFile(table, createRecord(1, "a"))).commit(); + table.refresh(); + + appendConvertTask(SnapshotRef.MAIN_BRANCH); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + // One cycle converts the eq-delete: the original row is deleted, the newer re-insert is + // not (its sequence is at or above the delete's). + long ts = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(ts, 0), ts); + TaskResult result = infra.sink().poll(Duration.ofSeconds(10)); + assertThat(result.exceptions()).isEmpty(); + assertThat(result.success()).isTrue(); + + table.refresh(); + assertRecords(table, ImmutableList.of(createRecord(1, "a"))); + } finally { + closeJobClient(jobClient); + } + } + + @Test + void testStagingEqualsTargetBranchMergesStagingDvIntoSingleDv() throws Exception { + Table table = createTableWithDelete(3); + + // One data file with two rows: id=1 at pos 0, id=2 at pos 1, committed to main. + DataFile data = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(1, "a"), createRecord(2, "b"))); + table.newAppend().appendFile(data).commit(); + table.refresh(); + + // Same-branch commit: an eq-delete for id=1 (resolves to a conversion DV at pos 0) plus a + // writer DV at pos 1 on the same data file. The resolver folds the staging DV into the + // conversion DV; on a shared branch the committer must remove the superseded staging DV. + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + DeleteFile stagingDV = writeStagingDV(table, data.location(), 1L); + table.newRowDelta().addDeletes(eqDelete).addDeletes(stagingDV).commit(); + table.refresh(); + + appendConvertTask(SnapshotRef.MAIN_BRANCH); + runAndWaitForSuccess(infra.env(), infra.source(), infra.sink()); + + table.refresh(); + // Both rows masked: pos 0 by the conversion DV, pos 1 by the merged-in staging DV. + assertRecords(table, ImmutableList.of()); + + // Exactly one DV per data file (V3 invariant). Without removing the rewritten staging DV on a + // shared branch, the data file would carry two DVs. + long dvCount = + table.currentSnapshot().deleteManifests(table.io()).stream() + .flatMap( + m -> + StreamSupport.stream( + ManifestFiles.readDeleteManifest(m, table.io(), table.specs()) + .spliterator(), + false)) + .filter(f -> f.content() == FileContent.POSITION_DELETES) + .filter(f -> data.location().equals(f.referencedDataFile())) + .count(); + assertThat(dvCount).isEqualTo(1); + } + + @Test + void testReaderErrorSkipsCommit() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + long mainSnapshotBeforeStaging = table.currentSnapshot().snapshotId(); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging data file + eq delete file, both referenced by the staging commit. + DataFile newDataFile = writeDataFile(table, createRecord(2, "b")); + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(newDataFile).addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + File eqDeleteLocalFile = new File(eqDelete.location().replace("file:", "")); + + // Delete the eq delete file; the committer must abort rather than committing data without its + // DV. + assertThat(eqDeleteLocalFile.delete()).isTrue(); + + appendConvertTask(); + + JobClient jobClient = null; + try { + jobClient = infra.env().executeAsync(); + + long time1 = System.currentTimeMillis(); + infra.source().sendRecord(Trigger.create(time1, 0), time1); + TaskResult result1 = infra.sink().poll(Duration.ofSeconds(10)); + + assertThat(result1.success()).isFalse(); + assertThat(result1.exceptions()).isNotEmpty(); + + table.refresh(); + // Main must not have advanced (no commit happened). + assertThat(table.currentSnapshot().snapshotId()).isEqualTo(mainSnapshotBeforeStaging); + SimpleDataUtil.assertTableRecords(table, ImmutableList.of(createRecord(1, "a"))); + + // Restore the eq delete file content by rewriting an identical delete, and retry: + // the planner must re-process the same staging snapshot (cursor didn't advance on failure). + DeleteFile recreated = + FileHelpers.writeDeleteFile( + table, + Files.localOutput(eqDeleteLocalFile), + new PartitionData(PartitionSpec.unpartitioned().partitionType()), + Lists.newArrayList(createRecord(1, "a")), + table.schema()); + assertThat(recreated.location()).isEqualTo(eqDelete.location()); + + long time2 = time1 + 1; + infra.source().sendRecord(Trigger.create(time2, 0), time2); + TaskResult result2 = infra.sink().poll(Duration.ofSeconds(10)); + + assertThat(result2.exceptions()).isEmpty(); + assertThat(result2.success()).isTrue(); + + table.refresh(); + // Staging data file committed with DV for id=1: should see id=2 only. + SimpleDataUtil.assertTableRecords(table, ImmutableList.of(createRecord(2, "b"))); + } finally { + closeJobClient(jobClient); + } + } + + private void appendConvertTask() { + appendConvertTask(STAGING_BRANCH); + } + + private void appendConvertTask(String stagingBranch) { + ConvertEqualityDeletes.builder() + .stagingBranch(stagingBranch) + .equalityFieldColumns(ImmutableList.of("id", "data")) + .parallelism(2) + .append( + infra.triggerStream(), + DUMMY_TABLE_NAME, + DUMMY_TASK_NAME, + 0, + tableLoader(), + UID_SUFFIX, + StreamGraphGenerator.DEFAULT_SLOT_SHARING_GROUP, + 1) + .sinkTo(infra.sink()); + } + + private static void assertRecords(Table table, List expected) throws IOException { + table.refresh(); + Types.StructType type = SimpleDataUtil.SCHEMA.asStruct(); + + StructLikeSet expectedSet = StructLikeSet.create(type); + expectedSet.addAll(expected); + + try (CloseableIterable iterable = + IcebergGenerics.read(table) + .useSnapshot(table.currentSnapshot().snapshotId()) + .project(SimpleDataUtil.SCHEMA) + .build()) { + StructLikeSet actualSet = StructLikeSet.create(type); + for (Record record : iterable) { + actualSet.add(record); + } + + assertThat(actualSet).isEqualTo(expectedSet); + } + } + + private static void assertNoEqualityDeletesOnMain(Table table, long expectedEqDeleteCount) { + long mainEqDeleteCount = 0; + for (ManifestFile manifest : table.currentSnapshot().deleteManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { + for (DeleteFile f : reader) { + if (f.content() == FileContent.EQUALITY_DELETES) { + mainEqDeleteCount++; + } + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + assertThat(mainEqDeleteCount).isEqualTo(expectedEqDeleteCount); + } +} diff --git a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestConvertEqualityDeletesE2E.java b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestConvertEqualityDeletesE2E.java new file mode 100644 index 000000000000..f6d1fd9464e2 --- /dev/null +++ b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestConvertEqualityDeletesE2E.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.api; + +import static org.apache.iceberg.flink.SimpleDataUtil.createRecord; +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.time.Duration; +import org.apache.flink.core.execution.JobClient; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.Files; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; +import org.apache.iceberg.PartitionData; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.Table; +import org.apache.iceberg.data.FileHelpers; +import org.apache.iceberg.data.GenericAppenderHelper; +import org.apache.iceberg.flink.SimpleDataUtil; +import org.apache.iceberg.flink.maintenance.operator.OperatorTestBase; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.util.ContentFileUtil; +import org.awaitility.Awaitility; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +/** + * End-to-end test for {@link ConvertEqualityDeletes} wired through the {@link TableMaintenance} + * framework. Verifies that the converter actually runs and commits a DV when the framework triggers + * it, exercising the full operator graph including the framework's monitor source, trigger manager, + * and lock remover. + */ +class TestConvertEqualityDeletesE2E extends OperatorTestBase { + private static final String STAGING_BRANCH = "staging"; + + @TempDir private Path tempDir; + private StreamExecutionEnvironment env; + + @BeforeEach + public void beforeEach() { + this.env = StreamExecutionEnvironment.getExecutionEnvironment(); + } + + @ParameterizedTest + @ValueSource(strings = {STAGING_BRANCH, SnapshotRef.MAIN_BRANCH}) + void testConvertEqualityDeletesE2E(String stagingBranch) throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + // When staging is a separate branch, fork it from main first. + if (!stagingBranch.equals(SnapshotRef.MAIN_BRANCH)) { + table.manageSnapshots().createBranch(stagingBranch).commit(); + table.refresh(); + } + + // Commit a new data file + eq delete to staging. This pre-job snapshot exercises both the + // "new data file" and "eq delete" paths in one cycle. + DataFile newData = writeDataFile(table, 3, "c"); + DeleteFile firstDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(newData).addDeletes(firstDelete).toBranch(stagingBranch).commit(); + table.refresh(); + assertThat(dvCountOnMain(table)).isZero(); + + TableMaintenance.forTable(env, tableLoader(), LOCK_FACTORY) + .uidSuffix("ConvertEqualityDeletesE2EUID-" + stagingBranch) + .rateLimit(Duration.ofMillis(50)) + .lockCheckDelay(Duration.ofMillis(50)) + .add( + ConvertEqualityDeletes.builder() + .scheduleOnInterval(Duration.ofMillis(100)) + .stagingBranch(stagingBranch) + .targetBranch(SnapshotRef.MAIN_BRANCH) + .equalityFieldColumns(ImmutableList.of("id", "data")) + .parallelism(2)) + .append(); + + JobClient jobClient = null; + try { + jobClient = env.executeAsync(); + + // Cycle 1: row 1 deleted by the converted DV; row 3 added on staging and committed to main. + Awaitility.await() + .atMost(Duration.ofMinutes(5)) + .pollInterval(Duration.ofMillis(200)) + .untilAsserted(() -> assertThat(dvCountOnMain(table)).isEqualTo(1)); + SimpleDataUtil.assertTableRecords( + table, ImmutableList.of(createRecord(2, "b"), createRecord(3, "c"))); + + // Cycle 2: commit a second staging snapshot while the job is still running. The framework's + // next interval-trigger should pick it up and produce a second DV against the data file + // holding id=2. + table.refresh(); + DeleteFile secondDelete = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(secondDelete).toBranch(stagingBranch).commit(); + + Awaitility.await() + .atMost(Duration.ofMinutes(5)) + .pollInterval(Duration.ofMillis(200)) + .untilAsserted(() -> assertThat(dvCountOnMain(table)).isEqualTo(2)); + SimpleDataUtil.assertTableRecords(table, ImmutableList.of(createRecord(3, "c"))); + } finally { + closeJobClient(jobClient); + } + } + + private DataFile writeDataFile(Table table, Integer id, String data) throws IOException { + return new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(SimpleDataUtil.createRecord(id, data))); + } + + private DeleteFile writeEqualityDelete(Table table, Integer id, String data) throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + return FileHelpers.writeDeleteFile( + table, + Files.localOutput(file), + new PartitionData(PartitionSpec.unpartitioned().partitionType()), + Lists.newArrayList(SimpleDataUtil.createRecord(id, data)), + table.schema()); + } + + private static long dvCountOnMain(Table table) throws IOException { + table.refresh(); + if (table.currentSnapshot() == null) { + return 0; + } + + long count = 0; + for (ManifestFile manifest : table.currentSnapshot().deleteManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { + for (DeleteFile file : reader) { + if (ContentFileUtil.isDV(file)) { + count++; + } + } + } + } + + return count; + } +} diff --git a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestMaintenanceE2E.java b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestMaintenanceE2E.java index fe8457167a1f..f786f1cdb29d 100644 --- a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestMaintenanceE2E.java +++ b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestMaintenanceE2E.java @@ -24,8 +24,11 @@ import java.time.Duration; import org.apache.flink.core.execution.JobClient; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.SnapshotRef; import org.apache.iceberg.Table; import org.apache.iceberg.flink.maintenance.operator.OperatorTestBase; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -116,4 +119,34 @@ void testE2eUseCoordinator() throws Exception { closeJobClient(jobClient); } } + + @Test + void testE2eConvertEqualityDeletes() throws Exception { + // Converter requires V3 (DV support); replace the V2 table created in @BeforeEach. + dropTable(); + createTable(3, FileFormat.PARQUET); + + TableMaintenance.forTable(env, tableLoader(), LOCK_FACTORY) + .uidSuffix("E2eConvertEqualityDeletesUID") + .rateLimit(Duration.ofMinutes(10)) + .lockCheckDelay(Duration.ofSeconds(10)) + .add( + ConvertEqualityDeletes.builder() + .scheduleOnEqDeleteFileCount(1) + .stagingBranch("staging") + .targetBranch(SnapshotRef.MAIN_BRANCH) + .equalityFieldColumns(ImmutableList.of("id")) + .parallelism(2)) + .append(); + + JobClient jobClient = null; + try { + jobClient = env.executeAsync(); + + // Just make sure that we are able to instantiate the flow + assertThat(jobClient).isNotNull(); + } finally { + closeJobClient(jobClient); + } + } } diff --git a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestTableMaintenance.java b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestTableMaintenance.java index 49219d5b4698..96b2fe04d66e 100644 --- a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestTableMaintenance.java +++ b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestTableMaintenance.java @@ -350,6 +350,14 @@ void testUidAndSlotSharingGroupForMonitorSource() throws IOException { checkSlotSharingGroupsAreSet(env, SLOT_SHARING_GROUP); } + @Test + void testMonitorRatePerSecond() { + // Sub-second rate limits must not yield infinite (unthrottled) monitor rates. + assertThat(TableMaintenance.Builder.monitorRatePerSecond(50)).isEqualTo(20.0); + assertThat(TableMaintenance.Builder.monitorRatePerSecond(100)).isEqualTo(10.0); + assertThat(TableMaintenance.Builder.monitorRatePerSecond(60_000)).isEqualTo(1.0 / 60); + } + /** * Sends the events though the {@link ManualSource} provided, and waits until the given number of * records are processed. diff --git a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertCommitter.java b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertCommitter.java new file mode 100644 index 000000000000..54f0b1758d90 --- /dev/null +++ b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertCommitter.java @@ -0,0 +1,569 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.operator; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.IOException; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.util.TwoInputStreamOperatorTestHarness; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; +import org.apache.iceberg.RowDelta; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.Table; +import org.apache.iceberg.data.FileHelpers; +import org.apache.iceberg.data.GenericRecord; +import org.apache.iceberg.deletes.PositionDelete; +import org.apache.iceberg.exceptions.CommitStateUnknownException; +import org.apache.iceberg.exceptions.ValidationException; +import org.apache.iceberg.flink.maintenance.api.Trigger; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.junit.jupiter.api.Test; + +class TestEqualityConvertCommitter extends OperatorTestBase { + + @Test + void commitsDataFilesToMainBranch() throws Exception { + Table table = createTable(3, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long snapshotIdBefore = table.currentSnapshot().snapshotId(); + long stagingSnapshotId = 42L; + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + stagingSnapshotId, + snapshotIdBefore, + doneTs - 1, + doneTs); + + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + harness.processBothWatermarks(new Watermark(doneTs)); + + assertThat(harness.extractOutputValues()).hasSize(1); + table.refresh(); + assertThat(table.currentSnapshot().snapshotId()).isNotEqualTo(snapshotIdBefore); + assertThat( + table + .currentSnapshot() + .summary() + .get(EqualityConvertCommitter.COMMITTED_STAGING_SNAPSHOT_PROPERTY)) + .isEqualTo(String.valueOf(stagingSnapshotId)); + } + } + + @Test + void holdsBackWatermarkUntilCommit() throws Exception { + Table table = createTable(3, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long snapshotIdBefore = table.currentSnapshot().snapshotId(); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + 42L, + snapshotIdBefore, + doneTs - 2, + doneTs); + + harness.processElement2(new StreamRecord<>(planResult, doneTs - 2)); + + // A phase watermark before the plan's done timestamp must not be forwarded: it would let the + // LockRemover release the maintenance lock before this cycle commits. + harness.processBothWatermarks(new Watermark(doneTs - 1)); + assertThat(harness.extractOutputValues()).isEmpty(); + assertThat(watermarks(harness)).isEmpty(); + + // The done-timestamp watermark commits the cycle; the watermark is forwarded only now. + harness.processBothWatermarks(new Watermark(doneTs)); + assertThat(harness.extractOutputValues()).hasSize(1); + assertThat(watermarks(harness)).containsExactly(new Watermark(doneTs)); + } + } + + @Test + void skipsCommitForEmptyCycle() throws Exception { + Table table = createTable(3, FileFormat.PARQUET); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = EqualityConvertPlan.noOp(null, doneTs - 1, doneTs); + + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + harness.processBothWatermarks(new Watermark(doneTs)); + + assertThat(harness.extractOutputValues()).hasSize(1); + table.refresh(); + assertThat(table.currentSnapshot()).isNull(); + } + } + + @Test + void abortsCommitWhenDVWriterFailed() throws Exception { + Table table = createTable(3, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long snapshotIdBefore = table.currentSnapshot().snapshotId(); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + 42L, + snapshotIdBefore, + doneTs - 1, + doneTs); + + // Receive an abort signal from the DVWriter followed by the planning planResult. + harness.processElement1(new StreamRecord<>(DVWriteResult.ABORT, doneTs)); + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + + harness.processBothWatermarks(new Watermark(doneTs)); + + // The cycle must complete (Trigger emitted) but nothing must be committed to the table. + assertThat(harness.extractOutputValues()).hasSize(1); + table.refresh(); + assertThat(table.currentSnapshot().snapshotId()).isEqualTo(snapshotIdBefore); + } + } + + @Test + void failsCommitWhenExternalCommitLandsAfterPlan() throws Exception { + Table table = createTable(3, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long mainSnapshotIdAtPlan = table.currentSnapshot().snapshotId(); + long stagingSnapshotId = 555L; + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + stagingSnapshotId, + mainSnapshotIdAtPlan, + doneTs - 1, + doneTs); + + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + + // External writer appends to main between plan and commit. + insert(table, 2, "b"); + table.refresh(); + long mainSnapshotIdAfterExternal = table.currentSnapshot().snapshotId(); + assertThat(mainSnapshotIdAfterExternal).isNotEqualTo(mainSnapshotIdAtPlan); + + harness.processBothWatermarks(new Watermark(doneTs)); + + // Trigger still fires (so the aggregator records the cycle), but the commit + // must have been rejected by validateNoConflictingDataFiles. + assertThat(harness.extractOutputValues()).hasSize(1); + + List> errors = + Lists.newArrayList(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)); + assertThat(errors).hasSize(1); + Exception failure = errors.get(0).getValue(); + assertThat(failure).isInstanceOf(ValidationException.class); + assertThat(failure) + .hasMessageContaining("Found conflicting files that can contain records matching"); + + // Target head is still the external commit, not our cycle's commit. + table.refresh(); + assertThat(table.currentSnapshot().snapshotId()).isEqualTo(mainSnapshotIdAfterExternal); + assertThat( + table + .currentSnapshot() + .summary() + .get(EqualityConvertCommitter.COMMITTED_STAGING_SNAPSHOT_PROPERTY)) + .isNull(); + } + } + + @Test + void failsReplayOfCommittedPlanFromOlderState() throws Exception { + Table table = createTable(3, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long mainSnapshotIdAtPlan = table.currentSnapshot().snapshotId(); + long stagingSnapshotId = 909L; + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + stagingSnapshotId, + mainSnapshotIdAtPlan, + doneTs - 1, + doneTs); + + // Cycle 1 commits, advancing the target past mainSnapshotIdAtPlan and writing the marker. + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + harness.processBothWatermarks(new Watermark(doneTs)); + assertThat(harness.extractOutputValues()).hasSize(1); + + table.refresh(); + long committedSnapshotId = table.currentSnapshot().snapshotId(); + assertThat(committedSnapshotId).isNotEqualTo(mainSnapshotIdAtPlan); + + // Restart from older state: the identical plan with its stale mainSnapshotId is replayed. + // The committer is stateless, so validateFromSnapshot is the only guard against a duplicate + // commit. It must reject the replay. + long doneTs2 = doneTs + 1; + EqualityConvertPlan replay = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + stagingSnapshotId, + mainSnapshotIdAtPlan, + doneTs2 - 1, + doneTs2); + + harness.processElement2(new StreamRecord<>(replay, doneTs2 - 1)); + harness.processBothWatermarks(new Watermark(doneTs2)); + + // Trigger still fires (so the aggregator records the cycle), but the commit was rejected. + assertThat(harness.extractOutputValues()).hasSize(2); + + List> errors = + Lists.newArrayList(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)); + assertThat(errors).hasSize(1); + Exception failure = errors.get(0).getValue(); + assertThat(failure).isInstanceOf(ValidationException.class); + assertThat(failure) + .hasMessageContaining("Found conflicting files that can contain records matching"); + + // No duplicate commit: target head is still cycle 1's commit. + table.refresh(); + assertThat(table.currentSnapshot().snapshotId()).isEqualTo(committedSnapshotId); + } + } + + @Test + void deletesWrittenDvsWhenCommitFails() throws Exception { + Table table = createTable(2, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long mainSnapshotIdAtPlan = table.currentSnapshot().snapshotId(); + + DeleteFile writtenDv = writePosDeleteFile(table, stagingDataFile.location(), 0L); + assertThat(table.io().newInputFile(writtenDv.location()).exists()).isTrue(); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + 777L, + mainSnapshotIdAtPlan, + doneTs - 1, + doneTs); + + harness.processElement1( + new StreamRecord<>( + new DVWriteResult(Lists.newArrayList(writtenDv), Lists.newArrayList()), doneTs)); + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + + // External writer appends to main between plan and commit, so the commit is rejected. + insert(table, 2, "b"); + table.refresh(); + + harness.processBothWatermarks(new Watermark(doneTs)); + + assertThat(harness.extractOutputValues()).hasSize(1); + + List> errors = + Lists.newArrayList(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)); + assertThat(errors).hasSize(1); + assertThat(errors.get(0).getValue()).isInstanceOf(ValidationException.class); + + // The DV written this cycle is unreferenced after the failed commit and must be deleted. + assertThat(table.io().newInputFile(writtenDv.location()).exists()).isFalse(); + } + } + + @Test + void deletesSiblingDvsOnAbortButKeepsRewrittenDvs() throws Exception { + Table table = createTable(2, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long snapshotIdBefore = table.currentSnapshot().snapshotId(); + + DeleteFile writtenDv = writePosDeleteFile(table, stagingDataFile.location(), 0L); + DeleteFile rewrittenDv = writePosDeleteFile(table, stagingDataFile.location(), 1L); + assertThat(table.io().newInputFile(writtenDv.location()).exists()).isTrue(); + assertThat(table.io().newInputFile(rewrittenDv.location()).exists()).isTrue(); + + try (TwoInputStreamOperatorTestHarness harness = + createHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + 42L, + snapshotIdBefore, + doneTs - 1, + doneTs); + + // One writer task wrote a DV (rewriting an existing one); a sibling task aborted. + harness.processElement1( + new StreamRecord<>( + new DVWriteResult(Lists.newArrayList(writtenDv), Lists.newArrayList(rewrittenDv)), + doneTs)); + harness.processElement1(new StreamRecord<>(DVWriteResult.ABORT, doneTs)); + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + harness.processBothWatermarks(new Watermark(doneTs)); + + assertThat(harness.extractOutputValues()).hasSize(1); + table.refresh(); + assertThat(table.currentSnapshot().snapshotId()).isEqualTo(snapshotIdBefore); + + // The freshly written DV is removed; the rewritten DV is still live on target and stays. + assertThat(table.io().newInputFile(writtenDv.location()).exists()).isFalse(); + assertThat(table.io().newInputFile(rewrittenDv.location()).exists()).isTrue(); + } + } + + @Test + void retainsWrittenDvsWhenCommitStateUnknown() throws Exception { + Table table = createTable(2, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile stagingDataFile = getFirstDataFile(table); + long mainSnapshotIdAtPlan = table.currentSnapshot().snapshotId(); + + DeleteFile writtenDv = writePosDeleteFile(table, stagingDataFile.location(), 0L); + assertThat(table.io().newInputFile(writtenDv.location()).exists()).isTrue(); + + EqualityConvertCommitter committer = + new EqualityConvertCommitter( + DUMMY_TABLE_NAME, DUMMY_TASK_NAME, tableLoader(), "staging", SnapshotRef.MAIN_BRANCH) { + @Override + void commit(RowDelta rowDelta) { + throw new CommitStateUnknownException( + new RuntimeException("simulated unknown commit state")); + } + }; + + try (TwoInputStreamOperatorTestHarness harness = + new TwoInputStreamOperatorTestHarness<>(committer)) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(stagingDataFile), + Lists.newArrayList(), + 888L, + mainSnapshotIdAtPlan, + doneTs - 1, + doneTs); + + harness.processElement1( + new StreamRecord<>( + new DVWriteResult(Lists.newArrayList(writtenDv), Lists.newArrayList()), doneTs)); + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + harness.processBothWatermarks(new Watermark(doneTs)); + + assertThat(harness.extractOutputValues()).hasSize(1); + + List> errors = + Lists.newArrayList(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)); + assertThat(errors).hasSize(1); + assertThat(errors.get(0).getValue()).isInstanceOf(CommitStateUnknownException.class); + + // Commit outcome unknown: the DV may be live, so it must not be deleted. + assertThat(table.io().newInputFile(writtenDv.location()).exists()).isTrue(); + } + } + + @Test + void removesRewrittenStagingDvOnSharedBranch() throws Exception { + // Shared branch: stagingBranch == targetBranch == main, so the writer's DVs are already on + // target. A staging DV folded into a merged conversion DV must be removed, else the data file + // would carry two DVs (V3 allows one per data file). + Table table = createTable(3, FileFormat.PARQUET); + insert(table, 1, "a"); + + DataFile dataFile = getFirstDataFile(table); + + // Staging DV already committed on the shared target branch; the writer rewrites it by folding + // it into a merged DV for the same data file. + DeleteFile stagingDv = writeDV(table, dataFile.location(), 0L); + table.newRowDelta().addDeletes(stagingDv).commit(); + table.refresh(); + long mainSnapshotId = table.currentSnapshot().snapshotId(); + + DeleteFile mergedDv = writeDV(table, dataFile.location(), 0L); + + try (TwoInputStreamOperatorTestHarness harness = + sharedBranchHarness()) { + harness.open(); + + long doneTs = System.currentTimeMillis(); + EqualityConvertPlan planResult = + new EqualityConvertPlan( + Lists.newArrayList(), + Lists.newArrayList(stagingDv), + 123L, + mainSnapshotId, + doneTs - 1, + doneTs); + + harness.processElement1( + new StreamRecord<>( + new DVWriteResult(Lists.newArrayList(mergedDv), Lists.newArrayList(stagingDv)), + doneTs)); + harness.processElement2(new StreamRecord<>(planResult, doneTs - 1)); + harness.processBothWatermarks(new Watermark(doneTs)); + + assertThat(harness.extractOutputValues()).hasSize(1); + + // Exactly one DV references the data file: the merged DV, with the rewritten staging DV + // removed rather than left as a second DV on the same data file. + table.refresh(); + List dvs = deletesForDataFile(table, dataFile.location()); + assertThat(dvs).hasSize(1); + assertThat(dvs.get(0).location()).isEqualTo(mergedDv.location()); + } + } + + private TwoInputStreamOperatorTestHarness + createHarness() throws Exception { + return new TwoInputStreamOperatorTestHarness<>( + new EqualityConvertCommitter( + DUMMY_TABLE_NAME, DUMMY_TASK_NAME, tableLoader(), "staging", SnapshotRef.MAIN_BRANCH)); + } + + private TwoInputStreamOperatorTestHarness + sharedBranchHarness() throws Exception { + return new TwoInputStreamOperatorTestHarness<>( + new EqualityConvertCommitter( + DUMMY_TABLE_NAME, + DUMMY_TASK_NAME, + tableLoader(), + SnapshotRef.MAIN_BRANCH, + SnapshotRef.MAIN_BRANCH)); + } + + private static List watermarks(TwoInputStreamOperatorTestHarness harness) { + return harness.getOutput().stream() + .filter(Watermark.class::isInstance) + .map(Watermark.class::cast) + .collect(Collectors.toList()); + } + + private static List deletesForDataFile(Table table, String dataFilePath) { + List deletes = Lists.newArrayList(); + for (ManifestFile manifest : table.currentSnapshot().deleteManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { + for (DeleteFile file : reader) { + if (dataFilePath.equals(file.referencedDataFile())) { + deletes.add(file.copy()); + } + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + return deletes; + } + + private static DeleteFile writeDV(Table table, String dataFilePath, long... positions) + throws IOException { + List> deletes = Lists.newArrayList(); + GenericRecord nested = GenericRecord.create(table.schema()); + for (long pos : positions) { + PositionDelete delete = PositionDelete.create(); + delete.set(dataFilePath, pos, nested); + deletes.add(delete); + } + + return FileHelpers.writePosDeleteFile(table, null, null, deletes, 3); + } + + private static DataFile getFirstDataFile(Table table) { + for (ManifestFile manifest : table.currentSnapshot().dataManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.read(manifest, table.io(), table.specs())) { + for (DataFile file : reader) { + return file.copy(); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + throw new IllegalStateException("No data files found"); + } +} diff --git a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertPlanner.java b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertPlanner.java new file mode 100644 index 000000000000..1ee5c5d0d5fd --- /dev/null +++ b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/operator/TestEqualityConvertPlanner.java @@ -0,0 +1,1155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.flink.maintenance.operator; + +import static org.apache.iceberg.flink.SimpleDataUtil.createRecord; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.flink.runtime.checkpoint.OperatorSubtaskState; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.Files; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; +import org.apache.iceberg.PartitionData; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.RewriteFiles; +import org.apache.iceberg.Schema; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.Table; +import org.apache.iceberg.data.FileHelpers; +import org.apache.iceberg.data.GenericAppenderHelper; +import org.apache.iceberg.data.GenericRecord; +import org.apache.iceberg.data.Record; +import org.apache.iceberg.deletes.PositionDelete; +import org.apache.iceberg.flink.maintenance.api.Trigger; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class TestEqualityConvertPlanner extends OperatorTestBase { + + private static final String STAGING_BRANCH = "__flink_staging_test"; + + @TempDir private Path tempDir; + + @Test + void bootstrapsIndexFromBuilderEqualityFieldIds() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + // Staging branch exists but contains no eq-delete files yet. The planner still populates the + // worker index from main using the builder-configured equality field set so the index is ready + // when the first delete arrives. + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH, Lists.newArrayList(1, 2))) { + harness.open(); + sendTrigger(harness); + + List commands = harness.extractOutputValues(); + assertThat(countDataFileTasks(commands)).isEqualTo(2); + assertThat(countEqDeleteTasks(commands)).isZero(); + + assertThat(harness.getSideOutput(EqualityConvertPlanner.METADATA_STREAM)).hasSize(1); + } + } + + @Test + void failsWhenStagingEqDeleteFieldIdsMismatchBuilder() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Builder configured for [1, 2], but writer produces an eq delete with [1] only. + DeleteFile mismatched = writeIdOnlyEqualityDelete(table, 1); + table.newRowDelta().addDeletes(mismatched).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH, Lists.newArrayList(1, 2))) { + harness.open(); + sendTrigger(harness); + + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).hasSize(1); + } + } + + @Test + void failsWhenStagingEqDeleteSpecPartitionsByNonEqualityColumn() throws Exception { + Table table = createPartitionedTableWithDelete(3); + insertPartitioned(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete = writeIdOnlyPartitionedEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH, Lists.newArrayList(1))) { + harness.open(); + sendTrigger(harness); + + List> errOutput = + Lists.newArrayList(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)); + + assertThat(errOutput).hasSize(1); + assertThat(errOutput.get(0).getValue()) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Partition columns must be a subset of the equality fields."); + } + } + + @Test + void doesNotDuplicateNewDataFilesWhenStagingEqualsTarget() throws Exception { + // When stagingBranch == targetBranch, the writer commits new data files directly to main. + // Bootstrap scans the main snapshot (which already includes those files) and indexes them. + // The planner must NOT also emit the staging-data phase for the same files — that would + // duplicate ADD_DATA_ROW commands for the same (PK, file, position) in the worker's index. + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + // Writer commits (new-data id=3) + (eq-delete id=1) in one RowDelta directly to main. + DataFile newDataFile = writeDataFile(table, createRecord(3, "c")); + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(newDataFile).addDeletes(eqDelete).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(SnapshotRef.MAIN_BRANCH)) { + harness.open(); + sendTrigger(harness); + + List commands = harness.extractOutputValues(); + + // Bootstrap from main emits one data-file command per file (id=1, id=2, id=3) = 3. + // Without the same-branch guard, emitSnapshotDataPhase would also emit newDataFile → 4. + assertThat(countDataFileTasks(commands)).isEqualTo(3); + assertThat(countEqDeleteTasks(commands)).isEqualTo(1); + + long newDataFileCount = + commands.stream() + .filter(c -> c.task() instanceof FileScanTask) + .filter(c -> c.task().file().location().equals(newDataFile.location())) + .count(); + assertThat(newDataFileCount).isEqualTo(1); + } + } + + @Test + void emitsReadCommandsForEqualityDeletes() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + insert(table, 3, "c"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + List commands = harness.extractOutputValues(); + // 3 DATA_FILE (from main) + 1 EQ_DELETE_FILE + assertThat(countDataFileTasks(commands)).isEqualTo(3); + assertThat(countEqDeleteTasks(commands)).isEqualTo(1); + assertThat(planner(harness).processedStagingSnapshotNum()).isEqualTo(1); + assertThat(planner(harness).processedEqDeleteFileNum()).isEqualTo(1); + + List> metadata = + Lists.newArrayList(harness.getSideOutput(EqualityConvertPlanner.METADATA_STREAM)); + assertThat(metadata).hasSize(1); + assertThat(metadata.get(0).getValue().dataFiles()).isEmpty(); + } + } + + @Test + void emitsDataFileFromInsertOnlyStagingSnapshot() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // S1: eq delete targeting main data + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + long s1SnapshotId = table.snapshot(STAGING_BRANCH).snapshotId(); + + // S2: insert-only (no eq deletes), but its data must still be indexed for the configured + // equality fields + DataFile insertS2 = writeDataFile(table, createRecord(2, "b")); + table.newAppend().appendFile(insertS2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + // Trigger 1: processes S1 (eq delete) + sendTrigger(harness); + int afterFirst = harness.extractOutputValues().size(); + assertThat(countEqDeleteTasks(harness.extractOutputValues())).isEqualTo(1); + + // Record S1's conversion on main, so the planner walks past S1 before processing S2. + simulateConvertCommit(table, s1SnapshotId); + + // Trigger 2: processes S2 (insert-only). Must emit its data file so the configured equality + // fields stay indexed. + sendTrigger(harness); + List allCommands = harness.extractOutputValues(); + List trigger2Commands = allCommands.subList(afterFirst, allCommands.size()); + + Set dataFilePaths = + trigger2Commands.stream() + .filter(c -> c.task() instanceof FileScanTask) + .map(TestEqualityConvertPlanner::filePath) + .collect(Collectors.toSet()); + + assertThat(dataFilePaths).contains(insertS2.location()); + } + } + + @Test + void includesNewDataFilesFromStaging() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DataFile newDataFile = writeDataFile(table, createRecord(2, "b")); + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(newDataFile).addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + List commands = harness.extractOutputValues(); + // 1 main data file + 1 eq delete + 1 staging data file + assertThat(countDataFileTasks(commands)).isEqualTo(2); + assertThat(countEqDeleteTasks(commands)).isEqualTo(1); + + List> metadata = + Lists.newArrayList(harness.getSideOutput(EqualityConvertPlanner.METADATA_STREAM)); + assertThat(metadata).hasSize(1); + assertThat(metadata.get(0).getValue().dataFiles()).hasSize(1); + } + } + + @Test + void findsIntersectionWhenMainAdvancedAfterStagingFork() throws Exception { + Table table = createTableWithDelete(3); + // Pre-fork main: two snapshots. + insert(table, 1, "a"); + insert(table, 2, "b"); + + // Fork staging from current main head. + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Main advances with two more snapshots after the fork. These are on main's + // lineage but not on staging's. + insert(table, 3, "c"); + insert(table, 4, "d"); + + // Staging gets one new snapshot containing an equality delete. + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + // Planner identifies the fork point and processes only the staging-only + // commit (the eq delete), and emits all four current main data files + // for the index refresh. + List commands = harness.extractOutputValues(); + assertThat(countEqDeleteTasks(commands)).isEqualTo(1); + assertThat(countDataFileTasks(commands)).isEqualTo(4); + } + } + + @Test + void skipsAlreadyProcessedStagingSnapshot() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + long stagingSnapshotId = table.snapshot(STAGING_BRANCH).snapshotId(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + sendTrigger(harness); + int firstTriggerCount = harness.extractOutputValues().size(); + assertThat(firstTriggerCount).isGreaterThan(0); + assertThat(planner(harness).skippedNoOpCycles()).isZero(); + + // Simulate the committer committing to main with the staging snapshot property + // (the planner promotes pending only after confirming the commit landed on main). + simulateConvertCommit(table, stagingSnapshotId); + + sendTrigger(harness); + assertThat(harness.extractOutputValues()).hasSize(firstTriggerCount); + assertThat(planner(harness).skippedNoOpCycles()).isEqualTo(1); + } + } + + @Test + void noOutputWhenStagingBranchEmpty() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + // Bootstrap from main for the configured field set: 1 main DATA_FILE; no-op metadata still + // emitted because there's nothing on staging to convert. + assertThat(countDataFileTasks(harness.extractOutputValues())).isEqualTo(1); + assertThat(countEqDeleteTasks(harness.extractOutputValues())).isZero(); + assertThat(harness.getSideOutput(EqualityConvertPlanner.METADATA_STREAM)).hasSize(1); + assertThat(planner(harness).skippedNoOpCycles()).isEqualTo(1); + assertThat(planner(harness).reindexCount()).isZero(); + } + } + + @Test + void propagatesStagingOnlyPositionalDeletes() throws Exception { + Table table = createTableWithDelete(3); + DataFile mainData = writeDataFile(table, createRecord(1, "a")); + table.newAppend().appendFile(mainData).commit(); + table.refresh(); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging snapshot has only a DV (no eq deletes, no new data files). Without the + // planner forwarding stagingDVFiles in this case, the committer would never + // see the DV and it would be lost. + DeleteFile stagingDV = writeStagingDV(table, mainData.location(), 0L); + table.newRowDelta().addDeletes(stagingDV).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + List commands = harness.extractOutputValues(); + assertThat(countDataFileTasks(commands)).isEqualTo(1); + assertThat(countEqDeleteTasks(commands)).isZero(); + + List> metadata = + Lists.newArrayList(harness.getSideOutput(EqualityConvertPlanner.METADATA_STREAM)); + assertThat(metadata).hasSize(1); + EqualityConvertPlan result = metadata.get(0).getValue(); + assertThat(result.dataFiles()).isEmpty(); + assertThat(result.stagingDVFiles()).hasSize(1); + assertThat(result.stagingDVFiles().get(0).location()).isEqualTo(stagingDV.location()); + } + } + + @Test + void phaseTimestampsAreMonotonicallyIncreasing() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Staging: eq delete + new data file (triggers 3 phases: main data, eq delete, staging data) + DataFile newDataFile = writeDataFile(table, createRecord(2, "b")); + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addRows(newDataFile).addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + long triggerTs = 100L; + sendTrigger(harness, triggerTs); + + List output = Lists.newArrayList(harness.getOutput()); + assertThat(((StreamRecord) output.get(0)).getValue().task()) + .isInstanceOf(FileScanTask.class); + assertThat(output.get(1)).isEqualTo(new Watermark(triggerTs)); + + assertThat(((StreamRecord) output.get(2)).getValue().task()) + .isInstanceOf(EqualityDeleteFileScanTask.class); + assertThat(output.get(3)).isEqualTo(new Watermark(triggerTs + 1)); + + assertThat(((StreamRecord) output.get(4)).getValue().task()) + .isInstanceOf(FileScanTask.class); + assertThat(output.get(5)).isEqualTo(new Watermark(triggerTs + 2)); + } + } + + @Test + void routesExceptionToErrorStream() throws Exception { + createTableWithDelete(3); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + dropTable(); + + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).isNull(); + sendTrigger(harness); + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).hasSize(1); + } + } + + @Test + void failsOnRemovedDataFilesOnStagingBranch() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + Set oldDataFiles = Sets.newHashSet(); + for (ManifestFile manifest : table.currentSnapshot().dataManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.read(manifest, table.io(), table.specs())) { + for (DataFile df : reader) { + oldDataFiles.add(df.copy()); + } + } + } + + assertThat(oldDataFiles).hasSize(2); + + // Rewrite on the staging branch (not main). Equality delete conversion does not support + // rewrites on staging; the planner must fail the cycle instead of silently dropping the + // removed files. + DataFile compactedFile = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(1, "a"), createRecord(2, "b"))); + RewriteFiles rewrite = table.newRewrite(); + for (DataFile old : oldDataFiles) { + rewrite.deleteFile(old); + } + + rewrite.addFile(compactedFile); + rewrite.toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).hasSize(1); + // Bootstrap runs before processCycle's failure, so the main data commands are already on + // the wire. Only the cycle itself fails. + assertThat(countDataFileTasks(harness.extractOutputValues())).isEqualTo(2); + assertThat(countEqDeleteTasks(harness.extractOutputValues())).isZero(); + } + } + + @Test + void reEmitsMainDataAfterCompaction() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + sendTrigger(harness); + int firstTriggerCount = harness.extractOutputValues().size(); + // 2 DATA_FILE (main) + 1 EQ_DELETE_FILE + assertThat(firstTriggerCount).isEqualTo(3); + + // Compact data files on main: rewrite 2 files into 1 + Set oldDataFiles = Sets.newHashSet(); + for (ManifestFile manifest : table.currentSnapshot().dataManifests(table.io())) { + try (ManifestReader reader = + ManifestFiles.read(manifest, table.io(), table.specs())) { + for (DataFile df : reader) { + oldDataFiles.add(df.copy()); + } + } + } + + assertThat(oldDataFiles).hasSize(2); + + DataFile compactedFile = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(1, "a"), createRecord(2, "b"))); + RewriteFiles rewrite = table.newRewrite(); + for (DataFile old : oldDataFiles) { + rewrite.deleteFile(old); + } + + rewrite.addFile(compactedFile); + rewrite.commit(); + table.refresh(); + + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + sendTrigger(harness); + List allCommands = harness.extractOutputValues(); + List trigger2Commands = + allCommands.subList(firstTriggerCount, allCommands.size()); + + // 1 DATA_FILE (compacted main) + 1 EQ_DELETE_FILE + assertThat(countDataFileTasks(trigger2Commands)).isEqualTo(1); + assertThat(countEqDeleteTasks(trigger2Commands)).isEqualTo(1); + + // DATA_FILE should reference the compacted file + List dataCmds = + trigger2Commands.stream() + .filter(c -> c.task() instanceof FileScanTask) + .collect(Collectors.toList()); + for (ReadCommand cmd : dataCmds) { + assertThat(cmd.task().file().location()).isEqualTo(compactedFile.location()); + } + } + } + + @Test + void refreshesIndexBeforeFirstCycleProcessesEqDeletes() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Stage MULTIPLE eq-delete snapshots before the first trigger arrives. The first trigger + // bootstraps the index from main for the configured equality fields before processing any eq + // deletes, then processes one snapshot per trigger (oldest first). + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + long s2SnapshotId = table.snapshot(STAGING_BRANCH).snapshotId(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + sendTrigger(harness); + int afterFirst = harness.extractOutputValues().size(); + + // First trigger processes the OLDER staging snapshot (eqDelete1). Bootstrap built the index + // from main once: 2 main DATA_FILE + 1 EQ_DELETE_FILE = 3 commands. + assertThat(afterFirst).isEqualTo(3); + assertThat(countDataFileTasks(harness.extractOutputValues())).isEqualTo(2); + assertThat(countEqDeleteTasks(harness.extractOutputValues())).isEqualTo(1); + + simulateConvertCommit(table, table.snapshot(STAGING_BRANCH).parentId()); + + // Second trigger processes the newer staging snapshot. The index is already up-to-date. + sendTrigger(harness); + List allCommands = harness.extractOutputValues(); + List trigger2Commands = allCommands.subList(afterFirst, allCommands.size()); + assertThat(countDataFileTasks(trigger2Commands)).isZero(); + assertThat(countEqDeleteTasks(trigger2Commands)).isEqualTo(1); + + simulateConvertCommit(table, s2SnapshotId); + } + } + + @Test + void noMainReEmitWhenUnchanged() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + sendTrigger(harness); + int firstTriggerCount = harness.extractOutputValues().size(); + // 2 DATA_FILE (main) + 1 EQ_DELETE_FILE + assertThat(firstTriggerCount).isEqualTo(3); + assertThat(harness.getSideOutput(EqualityConvertPlanner.METADATA_STREAM)).hasSize(1); + + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + sendTrigger(harness); + List allCommands = harness.extractOutputValues(); + List trigger2Commands = + allCommands.subList(firstTriggerCount, allCommands.size()); + + // Only 1 EQ_DELETE_FILE, no main re-emission + assertThat(countDataFileTasks(trigger2Commands)).isEqualTo(0); + assertThat(countEqDeleteTasks(trigger2Commands)).isEqualTo(1); + + assertThat(harness.getSideOutput(EqualityConvertPlanner.METADATA_STREAM)).hasSize(2); + } + } + + @Test + void noMainReEmitAfterOwnCommit() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + insert(table, 2, "b"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + long indexedStagingSnapshotId = table.snapshot(STAGING_BRANCH).snapshotId(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + sendTrigger(harness); + int firstTriggerCount = harness.extractOutputValues().size(); + + // Record the converter's own commit (COMMITTED_STAGING_SNAPSHOT property) for the staging + // snapshot the first trigger processed. The next trigger reads the property off main and + // advances lastStagingSnapshotId. No reindex should be triggered. + simulateConvertCommit(table, indexedStagingSnapshotId); + + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + sendTrigger(harness); + List allCommands = harness.extractOutputValues(); + List trigger2Commands = + allCommands.subList(firstTriggerCount, allCommands.size()); + + // Own commits don't trigger index rebuild. + assertThat(countDataFileTasks(trigger2Commands)).isEqualTo(0); + assertThat(countEqDeleteTasks(trigger2Commands)).isEqualTo(1); + } + } + + @Test + void reIndexesAfterExternalCommit() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + sendTrigger(harness); + int firstTriggerCount = harness.extractOutputValues().size(); + // 1 DATA_FILE (main) + 1 EQ_DELETE_FILE + assertThat(firstTriggerCount).isEqualTo(2); + assertThat(planner(harness).reindexCount()).isZero(); + assertThat(planner(harness).processedStagingSnapshotNum()).isEqualTo(1); + assertThat(planner(harness).processedEqDeleteFileNum()).isEqualTo(1); + + // External commit: no COMMITTED_STAGING_SNAPSHOT_PROPERTY + DataFile externalFile = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(10, "z"))); + table.newAppend().appendFile(externalFile).commit(); + table.refresh(); + + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + sendTrigger(harness); + List allCommands = harness.extractOutputValues(); + List trigger2Commands = + allCommands.subList(firstTriggerCount, allCommands.size()); + + // External commit triggers re-index: main data files are re-emitted + assertThat(countDataFileTasks(trigger2Commands)).isGreaterThan(0); + assertThat(countEqDeleteTasks(trigger2Commands)).isEqualTo(1); + assertThat(planner(harness).reindexCount()).isEqualTo(1); + assertThat(planner(harness).processedStagingSnapshotNum()).isEqualTo(2); + assertThat(planner(harness).processedEqDeleteFileNum()).isEqualTo(2); + } + } + + @Test + void emitsClearIndexBroadcastOnReindex() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + // Trigger 1 bootstraps the index. Bootstrap does not broadcast CLEAR_INDEX. + sendTrigger(harness); + assertThat(harness.getSideOutput(EqualityConvertPlanner.CLEAR_BROADCAST_STREAM)) + .isNullOrEmpty(); + + // External commit advances main. The next trigger reindexes and must broadcast CLEAR_INDEX + // so workers evict ghost keys (e.g. a PK whose data file was removed by external CoW). + DataFile externalFile = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(10, "z"))); + table.newAppend().appendFile(externalFile).commit(); + table.refresh(); + long mainAfterExternal = table.snapshot(SnapshotRef.MAIN_BRANCH).snapshotId(); + long mainSeqAfterExternal = table.snapshot(SnapshotRef.MAIN_BRANCH).sequenceNumber(); + + DeleteFile eqDelete2 = writeEqualityDelete(table, 2, "b"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + sendTrigger(harness); + + List clears = + harness.getSideOutput(EqualityConvertPlanner.CLEAR_BROADCAST_STREAM).stream() + .map(StreamRecord::getValue) + .collect(Collectors.toList()); + assertThat(clears).hasSize(1); + assertThat(clears.get(0).type()).isEqualTo(IndexCommand.Type.CLEAR_INDEX); + assertThat(clears.get(0).mainSnapshotId()).isEqualTo(mainAfterExternal); + assertThat(clears.get(0).mainSequenceNumber()).isEqualTo(mainSeqAfterExternal); + } + } + + @Test + void detectsMainBranchChangeWithoutNewStagingSnapshots() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete1 = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete1).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + // Trigger 1: build the index and process the existing eq delete. + sendTrigger(harness); + int afterFirst = harness.extractOutputValues().size(); + assertThat(afterFirst).isGreaterThan(0); + + // External commit on main (no COMMITTED_STAGING_SNAPSHOT_PROPERTY). + DataFile externalFile = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(10, "z"))); + table.newAppend().appendFile(externalFile).commit(); + table.refresh(); + + // Trigger 2: nothing new on staging, but the planner must still notice the + // external main change and immediately re-emit main data for index rebuild. + sendTrigger(harness); + List allAfterTrigger2 = harness.extractOutputValues(); + List trigger2Commands = + allAfterTrigger2.subList(afterFirst, allAfterTrigger2.size()); + assertThat(countDataFileTasks(trigger2Commands)).isGreaterThan(0); + + // A subsequent staging eq delete should NOT re-emit main data because the index + // was already rebuilt in trigger 2. + int afterSecond = allAfterTrigger2.size(); + DeleteFile eqDelete2 = writeEqualityDelete(table, 10, "z"); + table.newRowDelta().addDeletes(eqDelete2).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + sendTrigger(harness); + List all = harness.extractOutputValues(); + List trigger3 = all.subList(afterSecond, all.size()); + assertThat(countDataFileTasks(trigger3)).isEqualTo(0); + assertThat(countEqDeleteTasks(trigger3)).isEqualTo(1); + } + } + + @Test + void stateRestoredAfterCheckpoint() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + long stagingSnapshotId = table.snapshot(STAGING_BRANCH).snapshotId(); + + OperatorSubtaskState state; + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + + // First trigger bootstraps the index and processes the staging snapshot. + sendTrigger(harness); + int commandCount = harness.extractOutputValues().size(); + assertThat(commandCount).isGreaterThan(0); + + // Record S1's conversion on main (committer marker) so the planner advances + // lastStagingSnapshotId on the second trigger. + simulateConvertCommit(table, stagingSnapshotId); + + // Second trigger reads the marker and advances lastStagingSnapshotId. + sendTrigger(harness); + assertThat(harness.extractOutputValues()).hasSize(commandCount); + + state = harness.snapshot(1, System.currentTimeMillis()); + } + + // Restore from checkpoint: the restored index is not re-indexed because its state matches the + // snapshot COMMITTED_STAGING_SNAPSHOT_PROPERTY property. + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.initializeState(state); + harness.open(); + + sendTrigger(harness); + assertThat(harness.extractOutputValues()).isEmpty(); + } + } + + @Test + void failsOnV2PosDeleteOnStagingBranch() throws Exception { + Table table = createTableWithDelete(2); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + // Flink-written staging branches are V3 and only produce DVs for deletes. A V2 positional + // delete on staging is a writer-side misconfiguration and should fail the cycle rather than + // be silently absorbed. + DataFile dataFile = writeDataFile(table, createRecord(2, "b")); + DeleteFile posDelete = writePosDeleteFile(table, dataFile.location(), 0); + table.newRowDelta().addRows(dataFile).addDeletes(posDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).hasSize(1); + } + } + + @Test + void skipsCommittedSnapshotAfterCommitLandsButCheckpointDoesNot() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + long stagingSnapshotId = table.snapshot(STAGING_BRANCH).snapshotId(); + + // Capture the planner's checkpoint before any cycle has run. + OperatorSubtaskState state; + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + state = harness.snapshot(1, System.currentTimeMillis()); + } + + // The staging commit landed on main but lastStagingSnapshotId was not checkpointed. + simulateConvertCommit(table, stagingSnapshotId); + + // On restore, the planner walks main, finds the COMMITTED_STAGING_SNAPSHOT_PROPERTY, + // advances lastStagingSnapshotId, and skips re-emitting the already-committed snapshot. + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.initializeState(state); + harness.open(); + + sendTrigger(harness); + + List commands = harness.extractOutputValues(); + assertThat(countEqDeleteTasks(commands)).isEqualTo(0); + // Bootstrap from main creates the index from the main data files on the first trigger even + // though the staging snapshot itself is already committed and skipped. Main now has the + // original insert plus simulateConvertCommit's marker file = 2 data files. + assertThat(countDataFileTasks(commands)).isEqualTo(2); + } + } + + @Test + void failsWhenCommitMarkerDisappears() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + long startSnapshotId = table.currentSnapshot().snapshotId(); + + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + DeleteFile eqDelete = writeEqualityDelete(table, 1, "a"); + table.newRowDelta().addDeletes(eqDelete).toBranch(STAGING_BRANCH).commit(); + table.refresh(); + long stagingSnapshotId = table.snapshot(STAGING_BRANCH).snapshotId(); + + simulateConvertCommit(table, stagingSnapshotId); + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH)) { + harness.open(); + sendTrigger(harness); + + table.manageSnapshots().rollbackTo(startSnapshotId).commit(); + table.refresh(); + + sendTrigger(harness); + + assertThat(harness.getSideOutput(TaskResultAggregator.ERROR_STREAM)).hasSize(1); + assertThat( + harness + .getSideOutput(TaskResultAggregator.ERROR_STREAM) + .poll() + .getValue() + .getMessage()) + .contains("No COMMITTED_STAGING_SNAPSHOT marker reachable"); + } + } + + @Test + void failsOnEqFieldIdsChangeAcrossRestart() throws Exception { + Table table = createTableWithDelete(3); + insert(table, 1, "a"); + table.manageSnapshots().createBranch(STAGING_BRANCH).commit(); + table.refresh(); + + OperatorSubtaskState state; + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH, Lists.newArrayList(1, 2))) { + harness.open(); + state = harness.snapshot(1, System.currentTimeMillis()); + } + + try (OneInputStreamOperatorTestHarness harness = + createHarness(STAGING_BRANCH, Lists.newArrayList(1))) { + assertThatThrownBy(() -> harness.initializeState(state)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Equality field IDs changed across restart"); + } + } + + private OneInputStreamOperatorTestHarness createHarness( + String stagingBranch) throws Exception { + // Default matches the [1, 2] equality-field-id list produced by writeEqualityDelete. + return createHarness(stagingBranch, Lists.newArrayList(1, 2)); + } + + private OneInputStreamOperatorTestHarness createHarness( + String stagingBranch, List eqFieldIds) throws Exception { + return new OneInputStreamOperatorTestHarness<>( + new EqualityConvertPlanner( + DUMMY_TABLE_NAME, + DUMMY_TASK_NAME, + tableLoader(), + stagingBranch, + SnapshotRef.MAIN_BRANCH, + Sets.newHashSet(eqFieldIds))); + } + + private static void sendTrigger(OneInputStreamOperatorTestHarness harness) + throws Exception { + sendTrigger(harness, System.currentTimeMillis()); + } + + private static void sendTrigger(OneInputStreamOperatorTestHarness harness, long time) + throws Exception { + harness.processElement(new StreamRecord<>(Trigger.create(time, 0), time)); + } + + private DataFile writeDataFile(Table table, Record record) throws IOException { + return new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(record)); + } + + private DeleteFile writeEqualityDelete(Table table, Integer id, String data) throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + return FileHelpers.writeDeleteFile( + table, + Files.localOutput(file), + new PartitionData(PartitionSpec.unpartitioned().partitionType()), + Lists.newArrayList(createRecord(id, data)), + table.schema()); + } + + private DeleteFile writeIdOnlyEqualityDelete(Table table, int id) throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + Schema idOnly = table.schema().select("id"); + Record record = GenericRecord.create(idOnly); + record.setField("id", id); + return FileHelpers.writeDeleteFile( + table, + Files.localOutput(file), + new PartitionData(PartitionSpec.unpartitioned().partitionType()), + Lists.newArrayList(record), + idOnly); + } + + private DeleteFile writeIdOnlyPartitionedEqualityDelete(Table table, int id, String data) + throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + Schema idOnly = table.schema().select("id"); + Record record = GenericRecord.create(idOnly); + record.setField("id", id); + PartitionData partition = new PartitionData(table.spec().partitionType()); + partition.set(0, data); + return FileHelpers.writeDeleteFile( + table, Files.localOutput(file), partition, Lists.newArrayList(record), idOnly); + } + + private DeleteFile writeStagingDV(Table table, String dataFilePath, long position) + throws IOException { + File file = File.createTempFile("junit", null, tempDir.toFile()); + assertThat(file.delete()).isTrue(); + + PositionDelete delete = PositionDelete.create(); + delete.set(dataFilePath, position, null); + return FileHelpers.writePosDeleteFile( + table, + Files.localOutput(file), + new PartitionData(PartitionSpec.unpartitioned().partitionType()), + Lists.newArrayList(delete), + 3); + } + + private static long countDataFileTasks(List commands) { + return commands.stream().filter(c -> c.task() instanceof FileScanTask).count(); + } + + private static long countEqDeleteTasks(List commands) { + return commands.stream().filter(TestEqualityConvertPlanner::isEqDelete).count(); + } + + private static EqualityConvertPlanner planner( + OneInputStreamOperatorTestHarness harness) { + return (EqualityConvertPlanner) harness.getOperator(); + } + + private static boolean isEqDelete(ReadCommand cmd) { + return cmd.task() instanceof EqualityDeleteFileScanTask; + } + + private static String filePath(ReadCommand cmd) { + return cmd.task().file().location(); + } + + private void simulateConvertCommit(Table table, long stagingSnapshotId) throws IOException { + DataFile dummy = + new GenericAppenderHelper(table, FileFormat.PARQUET, tempDir) + .writeFile(Lists.newArrayList(createRecord(-1, "marker" + stagingSnapshotId))); + table + .newRowDelta() + .addRows(dummy) + .set( + EqualityConvertCommitter.COMMITTED_STAGING_SNAPSHOT_PROPERTY, + String.valueOf(stagingSnapshotId)) + .commit(); + table.refresh(); + } +} diff --git a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceSql.java b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceSql.java index 0cdaf8371cbd..6cfc18868af0 100644 --- a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceSql.java +++ b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/source/TestIcebergSourceSql.java @@ -179,7 +179,10 @@ public void testReadFlinkDynamicTable() throws Exception { List expected = generateExpectedRecords(false); SqlHelpers.sql( getTableEnv(), - "create table `default_catalog`.`default_database`.flink_table LIKE iceberg_catalog.`default`.%s", + "create table `default_catalog`.`default_database`.flink_table " + + "WITH ('catalog-type'='hadoop', 'warehouse'='%s') " + + "LIKE iceberg_catalog.`default`.%s", + CATALOG_EXTENSION.warehouse(), TestFixtures.TABLE); // Read from table in flink catalog @@ -199,8 +202,11 @@ public void testWatermarkInvalidConfig() { getStreamingTableEnv(), "CREATE TABLE %s " + "(eventTS AS CAST(t1 AS TIMESTAMP(3)), " - + "WATERMARK FOR eventTS AS SOURCE_WATERMARK()) LIKE iceberg_catalog.`default`.%s", + + "WATERMARK FOR eventTS AS SOURCE_WATERMARK()) " + + "WITH ('catalog-type'='hadoop', 'warehouse'='%s') " + + "LIKE iceberg_catalog.`default`.%s", flinkTable, + CATALOG_EXTENSION.warehouse(), TestFixtures.TABLE); assertThatThrownBy(() -> SqlHelpers.sql(getStreamingTableEnv(), "SELECT * FROM %s", flinkTable)) @@ -218,8 +224,11 @@ public void testWatermarkValidConfig() throws Exception { getStreamingTableEnv(), "CREATE TABLE %s " + "(eventTS AS CAST(t1 AS TIMESTAMP(3)), " - + "WATERMARK FOR eventTS AS SOURCE_WATERMARK()) WITH ('watermark-column'='t1') LIKE iceberg_catalog.`default`.%s", + + "WATERMARK FOR eventTS AS SOURCE_WATERMARK()) " + + "WITH ('catalog-type'='hadoop', 'warehouse'='%s', 'watermark-column'='t1') " + + "LIKE iceberg_catalog.`default`.%s", flinkTable, + CATALOG_EXTENSION.warehouse(), TestFixtures.TABLE); TestHelpers.assertRecordsWithOrder( diff --git a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/source/reader/ReaderUtil.java b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/source/reader/ReaderUtil.java index 3b094ba02298..5a6787001170 100644 --- a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/source/reader/ReaderUtil.java +++ b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/source/reader/ReaderUtil.java @@ -57,6 +57,21 @@ private ReaderUtil() {} public static FileScanTask createFileTask( List records, File file, FileFormat fileFormat, Schema schema) throws IOException { + return createFileTask( + records, + file, + fileFormat, + schema, + ResidualEvaluator.unpartitioned(Expressions.alwaysTrue())); + } + + public static FileScanTask createFileTask( + List records, + File file, + FileFormat fileFormat, + Schema schema, + ResidualEvaluator residuals) + throws IOException { DataWriter writer = new GenericFileWriterFactory.Builder() .dataSchema(schema) @@ -69,7 +84,6 @@ public static FileScanTask createFileTask( DataFile dataFile = writer.toDataFile(); - ResidualEvaluator residuals = ResidualEvaluator.unpartitioned(Expressions.alwaysTrue()); return new BaseFileScanTask( dataFile, null, diff --git a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/source/reader/TestArrayPoolDataIteratorBatcherRowData.java b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/source/reader/TestArrayPoolDataIteratorBatcherRowData.java index 4f6d182bb3c1..67f049dbf0b5 100644 --- a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/source/reader/TestArrayPoolDataIteratorBatcherRowData.java +++ b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/source/reader/TestArrayPoolDataIteratorBatcherRowData.java @@ -21,6 +21,7 @@ import static org.assertj.core.api.Assertions.assertThat; import java.io.File; +import java.io.IOException; import java.nio.file.Path; import java.util.Arrays; import java.util.List; @@ -32,13 +33,18 @@ import org.apache.iceberg.CombinedScanTask; import org.apache.iceberg.FileFormat; import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.data.RandomGenericData; import org.apache.iceberg.data.Record; +import org.apache.iceberg.expressions.Expression; +import org.apache.iceberg.expressions.Expressions; +import org.apache.iceberg.expressions.ResidualEvaluator; import org.apache.iceberg.flink.FlinkConfigOptions; import org.apache.iceberg.flink.TestFixtures; import org.apache.iceberg.flink.TestHelpers; import org.apache.iceberg.flink.source.DataIterator; import org.apache.iceberg.io.CloseableIterator; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -354,4 +360,182 @@ public void testMultipleFilesWithSeekPosition() throws Exception { assertThat(recordBatchIterator).isExhausted(); } + + @Test + public void testDataIteratorWithResidualFilter() throws IOException { + GenericRecord record0 = GenericRecord.create(TestFixtures.SCHEMA); + record0.setField("data", "file-1"); + record0.setField("id", 0L); + record0.setField("dt", "-"); + + GenericRecord record1 = GenericRecord.create(TestFixtures.SCHEMA); + record1.setField("data", "file-2"); + record1.setField("id", 1L); + record1.setField("dt", "-"); + + List fileRecords0 = ImmutableList.of(record0); + List fileRecords1 = ImmutableList.of(record1); + + validateFileOffsetWithResidualFilter(fileRecords0, fileRecords1, Expressions.alwaysTrue()); + + validateFileOffsetWithResidualFilter( + fileRecords0, fileRecords1, Expressions.greaterThan("id", 0)); + } + + private void validateFileOffsetWithResidualFilter( + List fileRecords0, List fileRecords1, Expression residualFilter) + throws IOException { + ResidualEvaluator residualEvaluator = ResidualEvaluator.unpartitioned(residualFilter); + + FileScanTask fileTask0 = + ReaderUtil.createFileTask( + fileRecords0, + File.createTempFile("junit", null, temporaryFolder.toFile()), + FILE_FORMAT, + TestFixtures.SCHEMA, + residualEvaluator); + + FileScanTask fileTask1 = + ReaderUtil.createFileTask( + fileRecords1, + File.createTempFile("junit", null, temporaryFolder.toFile()), + FILE_FORMAT, + TestFixtures.SCHEMA, + residualEvaluator); + + CombinedScanTask combinedTask = new BaseCombinedScanTask(Arrays.asList(fileTask0, fileTask1)); + + DataIterator dataIterator = ReaderUtil.createDataIterator(combinedTask); + + dataIterator.seek(0, 0); + + while (dataIterator.hasNext()) { + assertThat(dataIterator.fileOffset()).isEqualTo(dataIterator.next().getLong(1)); + } + } + + @Test + public void testInitializationWithHeadFilesSkipped() throws IOException { + GenericRecord record0 = GenericRecord.create(TestFixtures.SCHEMA); + record0.setField("data", "a"); + record0.setField("id", 1L); + record0.setField("dt", "-"); + + GenericRecord record1 = GenericRecord.create(TestFixtures.SCHEMA); + record1.setField("data", "a"); + record1.setField("id", 10L); + record1.setField("dt", "-"); + + ResidualEvaluator residuals = ResidualEvaluator.unpartitioned(Expressions.greaterThan("id", 5)); + + FileScanTask fileTask0 = + ReaderUtil.createFileTask( + ImmutableList.of(record0), + File.createTempFile("junit", null, temporaryFolder.toFile()), + FILE_FORMAT, + TestFixtures.SCHEMA, + residuals); + + FileScanTask fileTask1 = + ReaderUtil.createFileTask( + ImmutableList.of(record1), + File.createTempFile("junit", null, temporaryFolder.toFile()), + FILE_FORMAT, + TestFixtures.SCHEMA, + residuals); + + CombinedScanTask combinedTask = new BaseCombinedScanTask(Arrays.asList(fileTask0, fileTask1)); + + DataIterator dataIterator = ReaderUtil.createDataIterator(combinedTask); + + dataIterator.seek(0, 0); + + assertThat(dataIterator.fileOffset()) + .as("File offset should be 1 because file 0 should be skipped") + .isEqualTo(1); + + TestHelpers.assertRowData(TestFixtures.SCHEMA, record1, dataIterator.next()); + } + + @Test + void testSeekResumesCorrectlyAfterHeadFilesSkipped() throws IOException { + // Setup: [F0(filtered), F1(1 record), F2(2 records)] + // F1 has only 1 record so that after reading F1, the checkpoint position (fileOffset=1, + // recordOffset=1) falls at a file boundary. Without the fix, fileOffset is incorrectly + // reported as 0 instead of 1 after seek(0,0) skips F0. This causes F2's first record to + // be reported at (fileOffset=1, recordOffset=1). On restore with seek(1, 1), the iterator + // skips only F0 + 1 record from F1, landing at the start of F2 instead of after F2 record0, + // resulting in F2 record0 being read again (duplicate). + GenericRecord filteredRecord = GenericRecord.create(TestFixtures.SCHEMA); + filteredRecord.setField("data", "a"); + filteredRecord.setField("id", 1L); + filteredRecord.setField("dt", "-"); + + GenericRecord f1Record0 = GenericRecord.create(TestFixtures.SCHEMA); + f1Record0.setField("data", "b"); + f1Record0.setField("id", 10L); + f1Record0.setField("dt", "-"); + + GenericRecord f2Record0 = GenericRecord.create(TestFixtures.SCHEMA); + f2Record0.setField("data", "d"); + f2Record0.setField("id", 20L); + f2Record0.setField("dt", "-"); + + GenericRecord f2Record1 = GenericRecord.create(TestFixtures.SCHEMA); + f2Record1.setField("data", "e"); + f2Record1.setField("id", 21L); + f2Record1.setField("dt", "-"); + + ResidualEvaluator residuals = ResidualEvaluator.unpartitioned(Expressions.greaterThan("id", 5)); + + FileScanTask fileTask0 = + ReaderUtil.createFileTask( + ImmutableList.of(filteredRecord), + File.createTempFile("junit", null, temporaryFolder.toFile()), + FILE_FORMAT, + TestFixtures.SCHEMA, + residuals); + + FileScanTask fileTask1 = + ReaderUtil.createFileTask( + ImmutableList.of(f1Record0), + File.createTempFile("junit", null, temporaryFolder.toFile()), + FILE_FORMAT, + TestFixtures.SCHEMA, + residuals); + + FileScanTask fileTask2 = + ReaderUtil.createFileTask( + ImmutableList.of(f2Record0, f2Record1), + File.createTempFile("junit", null, temporaryFolder.toFile()), + FILE_FORMAT, + TestFixtures.SCHEMA, + residuals); + + CombinedScanTask combinedTask = + new BaseCombinedScanTask(Arrays.asList(fileTask0, fileTask1, fileTask2)); + + // First read: consume file 1's record and file 2's first record + DataIterator dataIterator = ReaderUtil.createDataIterator(combinedTask); + dataIterator.seek(0, 0); + + TestHelpers.assertRowData(TestFixtures.SCHEMA, f1Record0, dataIterator.next()); + // F1 exhausted, updateCurrentIterator moves to F2 + TestHelpers.assertRowData(TestFixtures.SCHEMA, f2Record0, dataIterator.next()); + + // Capture checkpoint position after reading F2 record0. + // With the fix: fileOffset=2, recordOffset=1 (correctly points into F2) + // Without fix: fileOffset=1, recordOffset=1 (incorrectly points to "F1 record 1") + int checkpointFileOffset = dataIterator.fileOffset(); + long checkpointRecordOffset = dataIterator.recordOffset(); + + // Simulate restore: seek to the checkpoint position + DataIterator restoredIterator = ReaderUtil.createDataIterator(combinedTask); + restoredIterator.seek(checkpointFileOffset, checkpointRecordOffset); + + // After restore, should read only F2 record1 (the unread remainder) + assertThat(restoredIterator.hasNext()).isTrue(); + TestHelpers.assertRowData(TestFixtures.SCHEMA, f2Record1, restoredIterator.next()); + assertThat(restoredIterator.hasNext()).isFalse(); + } } diff --git a/format/expressions-spec.md b/format/expressions-spec.md new file mode 100644 index 000000000000..01aa7d8b353d --- /dev/null +++ b/format/expressions-spec.md @@ -0,0 +1,311 @@ +--- +title: "Expressions Spec" +--- + + +# Iceberg Expressions + +This document defines the structure and behavior of expressions for use in Iceberg specifications. The purpose is to define a common structure that enables simple expressions to be stored and exchanged. + +Stored expressions are needed for use cases like data validations (`CHECK` constraints) and default values (for instance, `current_timestamp()`). Expressions are exchanged in use cases like server-side scan planning in the catalog protocol. + + +## Overview + +The goal of this specification is to define a simple expression structure and avoid complexity. + +To remain simple, the expressions that can be represented are deliberately constrained to value expressions (constants, references, and function calls) and predicates (comparisons that produce true or false). + +This approach is intended to keep focus on the logical structure of expressions. Complexity is pushed to the functions that are called, which are a limited set of well-defined and portable functions (like Iceberg partition transforms) or [user-defined functions][udf-spec] that can use the full range of SQL capabilities. Multi-dialect UDFs are responsible for any SQL constructs that are specific to an engine, rather than importing and duplicating dialects in Iceberg expressions. + +This is consistent with Iceberg's conservative approach in other specs. Expressions and predicates are an important part of Iceberg implementation APIs, but have been deliberately limited in specifications. For example, sort orders and partition fields are strictly limited to a small set of transforms over well-defined inputs (source field IDs). This spec is widening what can be expressed, but depends on function calls for complex tasks. + +This specification covers the structure of Iceberg expressions and includes appendices that specify serialization as JSON and a set of portable functions defined by Iceberg specifications. + +[udf-spec]: https://iceberg.apache.org/udf-spec + + +## Structure + +Iceberg expressions have two types: + +* **Value expressions** represent data values and transformations of values (function calls) that produce any Iceberg type +* **Predicates** represent comparisons of value expressions as well as combinations of predicates with boolean logic (and, or, not) + + +### Value expressions + +A value expression is an expression that produces a typed value. + +Value expressions can be one of three types: a constant value, a field reference, or a function applied to zero or more value expressions. + + +#### Constant values + +A constant or literal is the simplest type of value expression that represents a specific typed value. + + +#### Field reference + +A field reference represents the value of a specific field in a row. When an expression is evaluated on a row, it returns the value of the field. + +Field references may be named references (unbound) or ID references (bound). ID references identify a field by field ID from a schema. Named references identify a field by name that must be resolved to an ID (bound to a schema) to access the field. + +ID references are used for stored expressions, where the identity of the column is determined when the stored expression is created. For example, column constraints are tied to field IDs so that renaming a column does not invalidate the reference in its stored constraint. + +Named references are used when the identity of the column is determined when the expression is evaluated. For example, query filters are resolved each time a query runs so server-side planning uses unbound named references. + +The context in which an expression is used determines the type of references that are valid. Iceberg specifications should document whether ID references, named references, or both are allowed. + + +#### Apply function + +An apply expression represents the result of a function applied to (or called on) zero or more values produced by child value expressions or predicates. + +Functions are referenced using a catalog and a function identifier. + +* The function identifier consists of 0 or more namespace names followed by the function name. At least one part, the function name, is required. +* Catalog is optional and is assumed to be the catalog in which the referencing object is stored if it is not present or is null + +The catalog name identifies the catalog where the function definition can be loaded or is a reserved name that identifies a set of functions. As in the view and UDF specs, catalog names represent connection configurations that may differ across environments. Omitting catalog names is recommended to avoid depending on consistent environments. For example, if a table has a CHECK constraint that references a UDF without a catalog name (missing or null), the UDF should be loaded from the table's catalog. + +The reserved names used to identify sets are: + +* `sql_functions` is used for functions defined by the SQL standard +* `iceberg_functions` is used for functions defined in this specification + +Engines may document and use a catalog name to identify their built-in functions that are not part of the SQL spec, like `spark_builtin_functions.to_utc_timestamp`. + +Function references are unambiguous and are not interpreted using session context. Producers are responsible for resolving catalog, namespace, and name if the session is relevant. For example, if a SQL engine uses its current catalog and namespace to find a function, the resolved catalog and namespace must be used to produce an unambiguous function reference. + + +#### Value expression types + +The type produced by a value expression may change. For example, an ID reference may produce a widened type after the underlying column's type is promoted. + +A value expression's result type is determined when it is bound to a specific input schema. + +Function calls may produce different types when function definitions change, and type changes may change the definition that is resolved for a function name. For example, if the input field passed to `identity(int) -> int` is promoted from `int` to `long`, the resolved `identity` function can change to `identity(long) -> long` if it is defined. + +If types are incompatible at runtime, implementations binding or evaluating expressions may apply type promotion to align types for predicates and to resolve functions. Implementations may choose when to promote values to accommodate engines that differ in casting behavior. However, implementations must fail rather than insert unsafe casts. + + +### Predicates + +A predicate is a boolean expression that produces true or false. + +Predicates can be constants (true or false), tests of a value expression, comparisons of value expressions, or logical combinations of predicates (AND, OR, NOT). + +Value expressions are not valid predicates, even when the expression is expected to return a boolean value. Value expressions must be compared or tested to produce a predicate. For example, `is_empty(str_col)` is not a valid predicate because it may produce `null`, but `is_empty(str_col) = true` is a valid predicate. + + +#### Tests + +Tests are predicates that test a single value expression, optionally using a constant or set of constants. Constants must all have the same type and must be non-null and non-NaN. Tests are: + +| Test | Allowed types | Constant type | Description | +|-------------------------|---------------|---------------|-------------| +| `IS NULL` | any | | true iff the value is null | +| `IS NOT NULL` | any | | true iff the value is not null | +| `IS NaN` | float, double | | true iff the value is an IEEE 754 NaN | +| `IS NOT NaN` | float, double | | true iff the value is not an IEEE 754 NaN | +| `STARTS WITH const` | string | string | true iff the constant is a prefix of the value | +| `NOT STARTS WITH const` | string | string | true iff the constant is not a prefix of the value | +| `IN (constant set)` | any primitive | same as value | true iff the value is equal to any constant | +| `NOT IN (constant set)` | any primitive | same as value | true iff the value is not equal to any constant | + + +#### Comparisons + +Comparisons are predicates that compare two value expressions with the same primitive type. + +If value expression types in a comparison are incompatible, implementations should align types using type promotion. For instance, `int_col > 5.0` should promote int values to float. If the types cannot be aligned according to type promotion rules (for instance, `"goats" > -Infinity`), the predicate cannot be evaluated and implementations must fail. + +Comparisons are: + +| Comparison | Description | +|-------------|-------------| +| `=` | Is equal (is not distinct from) | +| `!=` | Is not equal | +| `<` | Less than | +| `<=` | Less than or equal | +| `>` | Greater than | +| `>=` | Greater than or equal | + +Comparisons must be null-safe. For any two operands a and b: + +* `a = b` is true if both are null, or both are non-null and equal; otherwise false +* `a != b` is the boolean negation of `a = b` +* `a < b` and `a > b` are false when either operand is null; otherwise they use the order defined above +* `a <= b` is `(a = b) OR (a < b)`; `a >= b` is `(a = b) OR (a > b)`; both are true when both operands are null and false when only one operand is null + +This table shows examples of these rules after evaluating value expressions to constants: + +| Comparison | Result | +|----------------|---------| +| `null = null` | `true` | +| `34 = null` | `false` | +| `null != null` | `false` | +| `34 != null` | `true` | +| `null < null` | `false` | +| `null <= null` | `true` | +| `34 < null` | `false` | + +Value expressions that are the direct child of a comparison must not be either a null or NaN constant. However, comparisons must handle null and NaN values that are the result of evaluating a value expression. For example, `x = get_item(map, "key")` is valid although `get_item` may return a null value, but `x = null` must be rejected because `x IS NULL` is the correct unambiguous predicate. Similarly, `multiply(a, b)` may produce NaN for `a=0.0` and `b=Infinity` and is valid, but `x = NaN` must be rejected because `x IS NaN` is the correct test. + +Primitive types are compared using signed comparison, except for the following types: + +* `false` is less than `true` for `boolean` +* `fixed` and `binary` use unsigned byte-wise comparison +* `string` uses unsigned byte-wise comparison of the UTF-8 representation; it is not the Unicode Collation Algorithm +* `uuid` uses unsigned byte-wise comparison of the UUID bytes +* `decimal` uses signed comparison independent of scale; this is equivalent to comparison of unscaled values because type alignment produces values with the same scale +* `float` and `double` use IEEE 754 order for all non-NaN values; see below for NaN comparison rules + +For floating point values, comparison with NaN behaves similarly to comparison of values with null. NaN should be specifically handled using `IS NaN` and `IS NOT NaN` tests. However, when value expressions produce a NaN value, the following rules must be applied: + +* `a = b` is true if both are NaN, or both are non-NaN and equal; false otherwise +* `a != b` is the boolean negation of `a = b` +* `a < b` and `a > b` are false when either operand is NaN; otherwise the IEEE 754 order is used +* `a <= b` is `(a = b) OR (a < b)`; `a >= b` is `(a = b) OR (a > b)`; both are true when both operands are NaN and false when only one operand is NaN + + +#### Boolean logic + +Predicates must use 2-valued boolean logic. Evaluation of all predicates must produce `true` or `false`. + +Engines that implement SQL 3-valued boolean logic must add `IS NULL` and `IS NOT NULL` to produce the 2-valued equivalent. This avoids bugs in engines and languages that do not natively implement 3-valued logic. For example, the SQL predicate `x < 10` should be passed as `x < 10 AND x IS NOT NULL` for a SQL `WHERE` condition (or `x < 10`; see null-safe comparisons below). For a `CHECK` constraint, the expression is passed as `x < 10 OR x IS NULL`. This ensures that implementations will make the correct determination, rather than depending on context to interpret a null result (`WHERE` vs `CHECK`). + +Logical combinations are boolean operators applied to predicates. `AND` and `OR` are binary operations and `NOT` is a unary operation. `AND`, `OR`, and `NOT` do not accept null values because predicates cannot produce them. + + +### Compatibility with REST catalog expressions + +Prior to this spec, REST APIs used a more restrictive, term-based form of predicates and references. Those forms are now deprecated, but should be supported for backward compatibility to allow older clients to interact with newer REST catalog services. + +The deprecated expressions were passed in 3 places: + +* As `filter` passed to server-side scan planning +* As `filter` passed to the service in `ScanReport` +* As `residual` passed to the client with a scan task + +Both server-side scan planning and the report endpoint should continue to accept filters from older clients by parsing term-based expressions (see [Appendix B: JSON serialization](#backward-compatibility)). + +Residuals passed from services back to clients that do not use the new syntax would cause clients to fail. Services are allowed to omit the residual so that it is calculated on the client side (intended to avoid duplicating large IN filters). For compatibility, REST services should omit residuals from tasks, but may include them if the service detects support for newer predicates (for example, via client version). + + +## Appendix A: Iceberg functions + +This section defines the functions in the `iceberg_functions` reserved catalog name. + +* `if_else(condition: predicate, when_true: T, when_false: T) -> T`: returns the value of `when_true` when `condition` is true and `when_false` otherwise + +### Partition transforms + +Iceberg partition transforms are also defined as functions (other than `void`). + +All partition transforms produce `null` for a `null` input value. + +| Function name | Description | Source types | Result type | +|-------------------|--------------------------------------------------------------|----------------------------------------------------------------------|-------------| +| `identity(value)` | Source value, unmodified | Any primitive except for `geometry`, `geography`, and `variant` | Source type | +| `year(value)` | Extract a date or timestamp year, as years from 1970 | `date`, `timestamp`, `timestamptz`, `timestamp_ns`, `timestamptz_ns` | `int` | +| `month(value)` | Extract a date or timestamp month, as months from 1970-01-01 | `date`, `timestamp`, `timestamptz`, `timestamp_ns`, `timestamptz_ns` | `int` | +| `day(value)` | Extract a date or timestamp day, as days from 1970-01-01 | `date`, `timestamp`, `timestamptz`, `timestamp_ns`, `timestamptz_ns` | `date` | +| `hour(value)` | Extract a timestamp hour, as hours from 1970-01-01 00:00:00 | `timestamp`, `timestamptz`, `timestamp_ns`, `timestamptz_ns` | `int` | + +Note that `year`, `month`, and `hour` transforms produce ordinal values and not human-readable values. For example, `year(2018-05-13)` produces `48`, not `2018`. + +`bucket` and `truncate` are called as 2-argument functions. The first argument is an `int` parameter (`N` or `W` from the table spec) and the second argument is the value to transform. For example, `bucket(256, id)` calls `bucket[256]`. + +| Parameterized function name | Description | Source types | Result type | +|-----------------------------|-----------------------------------------------------------------------|----------------------------------------------------------------------------------------------|-------------| +| `bucket(N, value)` | Hash of value, mod `N` (see [table spec details][bucket-ref]) | Any primitive except for `geometry`, `geography`, `variant`, `boolean`, `float`, or `double` | `int` | +| `truncate(W, value)` | Value truncated to width `W` (see [table spec details][truncate-ref]) | `int`, `long`, `decimal`, `string`, `binary` | Source type | + +[bucket-ref]: spec/#bucket-transform-details +[truncate-ref]: spec/#truncate-transform-details + + +## Appendix B: JSON serialization + +Iceberg expressions are serialized as JSON objects in table, view, and UDF metadata, and in the REST protocol for catalogs. + +### Value expressions + +``` +EXPR: LITERAL | REFERENCE | APPLY + +LITERAL: VALUE + | { "type": "literal", "value": VALUE } + | { "type": "literal", "value": VALUE, "data-type": DATA_TYPE } +LITERALS: [ LITERAL* ] + | { "type": "literals", "values": [ VALUE* ], "data-type": DATA_TYPE } + +REFERENCE: BOUND_REF | UNBOUND_REF +BOUND_REF: { "type": "reference", "id": ID } +UNBOUND_REF: { "type": "reference", "name": NAME } + +APPLY: { "type": "apply", "function": FUNC_REF, "arguments": [ FUNC_ARG* ] } +FUNC_ARG: EXPR | PREDICATE +FUNC_REF: NAME + | [ NAME* ] + | { "identifier": [ NAME* ] } + | { "catalog": NAME, "identifier": [ NAME* ] } + +ID: integer +NAME: string + +VALUE: single value JSON from the table spec +DATA_TYPE: Iceberg type from the spec +``` + +If a function reference is a string, that string is the one-part identifier and the catalog is missing/null. +If a function reference is a list of strings, it is the function identifier and the catalog is missing/null. + +### Predicates + +``` +PREDICATE: true | false + | { "type": "not", "child": PREDICATE } + | { "type": BINARY_OP, "left": PREDICATE, "right": PREDICATE } + | { "type": UNARY_OP, "child": EXPR } + | { "type": CMP_OP, "left": EXPR, "right": EXPR } + | { "type": SET_OP, "child": EXPR, "values": LITERALS } + | DEPRECATED_PREDICATE + +BINARY_OP: "and" | "or" +UNARY_OP: "is-null" | "not-null" | "is-nan" | "not-nan" +CMP_OP: "lt" | "lt-eq" | "gt" | "gt-eq" | "eq" | "not-eq" + | "starts-with" | "not-starts-with" +SET_OP: "in" | "not-in" +``` + +### Backward compatibility + +``` +DEPRECATED_PREDICATE: + | { "type": UNARY_OP, "term": TERM } + | { "type": CMP_OP, "term": TERM, "value": LITERAL } + | { "type": SET_OP, "term": TERM, "values": LITERALS } + +DEPRECATED_REF: { "type": "reference", "term": NAME } + +TERM: NAME | DEPRECATED_REF | TRANSFORM +TRANSFORM: { "type": "transform", "transform": NAME, "term": TERM } +``` diff --git a/format/spec.md b/format/spec.md index b88f2af1edcc..be28a6c5f000 100644 --- a/format/spec.md +++ b/format/spec.md @@ -1685,7 +1685,7 @@ Types are serialized according to this table: |**`map`**|`JSON object: {`
  `"type": "map",`
  `"key-id": ,`
  `"key": ,`
  `"value-id": ,`
  `"value-required": `
  `"value": `
`}`|`{`
  `"type": "map",`
  `"key-id": 4,`
  `"key": "string",`
  `"value-id": 5,`
  `"value-required": false,`
  `"value": "double"`
`}`| | **`variant`**| `JSON string: "variant"`|`"variant"`| | **`geometry(C)`** |`JSON string: "geometry()"`|`"geometry(srid:4326)"`| -| **`geography(C, A)`** |`JSON string: "geography(,)"`|`"geography(srid:4326,spherical)"`| +| **`geography(C, A)`** |`JSON string: "geography(, )"`|`"geography(srid:4326, spherical)"`| Note that default values are serialized using the JSON single-value serialization in [Appendix D](#appendix-d-single-value-serialization). diff --git a/format/udf-spec.md b/format/udf-spec.md index a11f10ffd2fd..7a5e89010cac 100644 --- a/format/udf-spec.md +++ b/format/udf-spec.md @@ -80,6 +80,7 @@ must produce values of the declared `return-type`. | Requirement | Field name | Type | Description | |-------------|----------------------|--------------------------------|---------------------------------------------------------------------------------------------------| | *required* | `definition-id` | `string` | An identifier derived from canonical parameter-type tuple (see [Definition ID](#definition-id)). | +| *optional* | `specific-name` | `string` | A user-assignable name for this definition; must be unique (see [Specific Name](#specific-name)). | | *required* | `parameters` | `list` | Ordered list of [function parameters](#parameter). Invocation order **must** match this list. | | *required* | `return-type` | `string` or `object` | Declared return type using [Types](#types). | | *optional* | `return-nullable` | `boolean` | A hint to indicate whether the return value is nullable or not. Default: `true`. | @@ -131,6 +132,13 @@ Examples of complete definition-id signatures: * `int,string` – two parameters: int and string * `int,list,struct` – three parameters: an int, a list and a struct +#### Specific Name + +The `specific-name` is an optional, user-assignable name for a single definition, analogous to the SQL standard's +routine *specific name*. It provides a stable, human-readable handle for a definition that is independent of its +signature (e.g., for SQL statements such as `DROP SPECIFIC FUNCTION`). +When present, `specific-name` **must** be unique among all definitions within the UDF metadata. + ### Definition Version Each definition can evolve over time by introducing new versions. diff --git a/gcp-bundle/build.gradle b/gcp-bundle/build.gradle index 9c4907bcdaa5..39b230745b9b 100644 --- a/gcp-bundle/build.gradle +++ b/gcp-bundle/build.gradle @@ -32,6 +32,9 @@ project(":iceberg-gcp-bundle") { dependencies { implementation platform(libs.google.libraries.bom) + // Align the shaded Jackson to the project version; the Google libraries BOM + // otherwise pulls in an older jackson-databind with known CVEs. + implementation platform(libs.jackson.bom) implementation "com.google.cloud:google-cloud-storage" implementation "com.google.cloud:google-cloud-bigquery" implementation "com.google.cloud:google-cloud-core" diff --git a/gcp-bundle/runtime-deps.txt b/gcp-bundle/runtime-deps.txt index 0ff407294f5d..84cc4fe3c0f3 100644 --- a/gcp-bundle/runtime-deps.txt +++ b/gcp-bundle/runtime-deps.txt @@ -1,35 +1,35 @@ -com.fasterxml.jackson.core:jackson-annotations:2.18 -com.fasterxml.jackson.core:jackson-core:2.18 -com.fasterxml.jackson.core:jackson-databind:2.18 -com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.18 -com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.18 +com.fasterxml.jackson.core:jackson-annotations:2.22 +com.fasterxml.jackson.core:jackson-core:2.22 +com.fasterxml.jackson.core:jackson-databind:2.22 +com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.22 +com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.22 com.fasterxml.woodstox:woodstox-core:7.0 -com.github.ben-manes.caffeine:caffeine:3.1 +com.github.ben-manes.caffeine:caffeine:3.2 com.google.android:annotations:4.1 com.google.api-client:google-api-client:2.7 -com.google.api.grpc:gapic-google-cloud-storage-v2:2.68 -com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:3.28 -com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.200 -com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.200 -com.google.api.grpc:grpc-google-cloud-storage-v2:2.68 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:3.28 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha:3.28 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.200 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.200 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta:3.28 -com.google.api.grpc:proto-google-cloud-kms-v1:0.186 -com.google.api.grpc:proto-google-cloud-monitoring-v3:3.93 -com.google.api.grpc:proto-google-cloud-storage-v2:2.68 -com.google.api.grpc:proto-google-common-protos:2.71 -com.google.api.grpc:proto-google-iam-v1:1.66 -com.google.api:api-common:2.63 -com.google.api:gax-grpc:2.80 -com.google.api:gax-httpjson:2.80 -com.google.api:gax:2.80 +com.google.api.grpc:gapic-google-cloud-storage-v2:2.69 +com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:3.29 +com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:3.29 +com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:3.29 +com.google.api.grpc:grpc-google-cloud-storage-v2:2.69 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:3.29 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha:3.29 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:3.29 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:3.29 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta:3.29 +com.google.api.grpc:proto-google-cloud-kms-v1:2.96 +com.google.api.grpc:proto-google-cloud-monitoring-v3:3.94 +com.google.api.grpc:proto-google-cloud-storage-v2:2.69 +com.google.api.grpc:proto-google-common-protos:2.72 +com.google.api.grpc:proto-google-iam-v1:1.67 +com.google.api:api-common:2.64 +com.google.api:gax-grpc:2.81 +com.google.api:gax-httpjson:2.81 +com.google.api:gax:2.81 com.google.apis:google-api-services-bigquery:v2-rev20251012-2.0 com.google.apis:google-api-services-storage:v1-rev20260204-2.0 -com.google.auth:google-auth-library-credentials:1.47 -com.google.auth:google-auth-library-oauth2-http:1.47 +com.google.auth:google-auth-library-credentials:1.48 +com.google.auth:google-auth-library-oauth2-http:1.48 com.google.auto.value:auto-value-annotations:1.11 com.google.cloud.gcs.analytics:client:1.3 com.google.cloud.gcs.analytics:common:1.3 @@ -37,19 +37,19 @@ com.google.cloud.gcs.analytics:gcs-analytics-core:1.3 com.google.cloud.opentelemetry:detector-resources-support:0.33 com.google.cloud.opentelemetry:exporter-metrics:0.33 com.google.cloud.opentelemetry:shared-resourcemapping:0.33 -com.google.cloud:google-cloud-bigquery:2.66 -com.google.cloud:google-cloud-bigquerystorage:3.28 -com.google.cloud:google-cloud-core-grpc:2.70 -com.google.cloud:google-cloud-core-http:2.70 -com.google.cloud:google-cloud-core:2.70 -com.google.cloud:google-cloud-kms:2.95 -com.google.cloud:google-cloud-monitoring:3.93 -com.google.cloud:google-cloud-storage:2.68 +com.google.cloud:google-cloud-bigquery:2.67 +com.google.cloud:google-cloud-bigquerystorage:3.29 +com.google.cloud:google-cloud-core-grpc:2.71 +com.google.cloud:google-cloud-core-http:2.71 +com.google.cloud:google-cloud-core:2.71 +com.google.cloud:google-cloud-kms:2.96 +com.google.cloud:google-cloud-monitoring:3.94 +com.google.cloud:google-cloud-storage:2.69 com.google.code.gson:gson:2.13 -com.google.errorprone:error_prone_annotations:2.48 +com.google.errorprone:error_prone_annotations:2.49 com.google.flatbuffers:flatbuffers-java:24.3 com.google.guava:failureaccess:1.0 -com.google.guava:guava:33.5 +com.google.guava:guava:33.6 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava com.google.http-client:google-http-client-apache-v2:2.1 com.google.http-client:google-http-client-appengine:2.1 @@ -85,16 +85,16 @@ io.opencensus:opencensus-api:0.31 io.opencensus:opencensus-contrib-http-util:0.31 io.opentelemetry.contrib:opentelemetry-gcp-resources:1.37 io.opentelemetry.semconv:opentelemetry-semconv:1.29 -io.opentelemetry:opentelemetry-api:1.57 -io.opentelemetry:opentelemetry-common:1.57 -io.opentelemetry:opentelemetry-context:1.57 -io.opentelemetry:opentelemetry-exporter-logging:1.52 -io.opentelemetry:opentelemetry-sdk-common:1.57 -io.opentelemetry:opentelemetry-sdk-extension-autoconfigure-spi:1.57 -io.opentelemetry:opentelemetry-sdk-logs:1.57 -io.opentelemetry:opentelemetry-sdk-metrics:1.57 -io.opentelemetry:opentelemetry-sdk-trace:1.57 -io.opentelemetry:opentelemetry-sdk:1.57 +io.opentelemetry:opentelemetry-api:1.63 +io.opentelemetry:opentelemetry-common:1.63 +io.opentelemetry:opentelemetry-context:1.63 +io.opentelemetry:opentelemetry-exporter-logging:1.63 +io.opentelemetry:opentelemetry-sdk-common:1.63 +io.opentelemetry:opentelemetry-sdk-extension-autoconfigure-spi:1.62 +io.opentelemetry:opentelemetry-sdk-logs:1.63 +io.opentelemetry:opentelemetry-sdk-metrics:1.63 +io.opentelemetry:opentelemetry-sdk-trace:1.63 +io.opentelemetry:opentelemetry-sdk:1.63 io.perfmark:perfmark-api:0.27 javax.annotation:javax.annotation-api:1.3 org.apache.arrow:arrow-format:17.0 diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 1a31b6f103fd..a30280fcaff0 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -33,7 +33,7 @@ arrow = "15.0.2" avro = "1.12.1" assertj-core = "3.27.7" awaitility = "4.3.0" -awssdk-bom = "2.46.5" +awssdk-bom = "2.46.15" azuresdk-bom = "1.3.7" awssdk-s3accessgrants = "2.4.1" bouncycastle = "1.84" @@ -45,14 +45,14 @@ delta-standalone = "3.3.2" delta-spark = "3.3.2" derby = "10.15.2.0" esotericsoftware-kryo = "4.0.3" -errorprone-annotations = "2.49.0" +errorprone-annotations = "2.50.0" failsafe = "3.3.2" findbugs-jsr305 = "3.0.2" flink120 = { strictly = "1.20.1"} flink20 = { strictly = "2.0.0"} flink21 = { strictly = "2.1.0"} -google-libraries-bom = "26.83.0" -gcs-analytics-core = "1.3.0" +google-libraries-bom = "26.84.0" +gcs-analytics-core = "1.3.1" guava = "33.6.0-jre" hadoop3 = "3.4.3" httpcomponents-httpclient5 = "5.6.1" @@ -60,8 +60,9 @@ hive2 = { strictly = "2.3.10"} # see rich version usage explanation above immutables-value = "2.12.2" jackson-annotations = "2.22" jackson-bom = "2.22.0" -jackson214 = { strictly = "2.14.2"} jackson215 = { strictly = "2.15.2"} # see rich version usage explanation above +jackson218 = { strictly = "2.18.8"} +jackson221 = { strictly = "2.21.4"} jakarta-el-api = "3.0.3" jakarta-servlet-api = "6.1.0" jaxb-api = "2.3.1" @@ -77,7 +78,7 @@ lz4Java = "1.11.0" microprofile-openapi-api = "3.1.2" mockito = "4.11.0" mockserver = "5.15.0" -nessie = "0.107.9" +nessie = "0.108.0" netty-buffer = "4.2.15.Final" object-client-bundle = "3.3.2" orc = "1.9.8" @@ -157,8 +158,6 @@ jackson-bom = { module = "com.fasterxml.jackson:jackson-bom", version.ref = "jac jackson-core = { module = "com.fasterxml.jackson.core:jackson-core", version.ref = "jackson-bom" } jackson-databind = { module = "com.fasterxml.jackson.core:jackson-databind", version.ref = "jackson-bom" } jackson-annotations = { module = "com.fasterxml.jackson.core:jackson-annotations", version.ref = "jackson-annotations" } -jackson214-bom = { module = "com.fasterxml.jackson:jackson-bom", version.ref = "jackson214" } -jackson215-bom = { module = "com.fasterxml.jackson:jackson-bom", version.ref = "jackson215" } jaxb-api = { module = "javax.xml.bind:jaxb-api", version.ref = "jaxb-api" } jaxb-runtime = { module = "org.glassfish.jaxb:jaxb-runtime", version.ref = "jaxb-runtime" } kafka-clients = { module = "org.apache.kafka:kafka-clients", version.ref = "kafka" } diff --git a/kafka-connect/build.gradle b/kafka-connect/build.gradle index 43eb245d93a3..61b5ed53fcfb 100644 --- a/kafka-connect/build.gradle +++ b/kafka-connect/build.gradle @@ -75,13 +75,13 @@ project(':iceberg-kafka-connect:iceberg-kafka-connect-runtime') { exclude group: 'org.jspecify', module: 'jspecify' // force upgrades for dependencies with known vulnerabilities... resolutionStrategy { - force 'org.codehaus.jettison:jettison:1.5.5' + force 'org.codehaus.jettison:jettison:1.5.6' force 'org.xerial.snappy:snappy-java:1.1.10.8' force 'org.apache.commons:commons-compress:1.28.0' force 'org.apache.hadoop.thirdparty:hadoop-shaded-guava:1.5.0' force 'com.fasterxml.woodstox:woodstox-core:6.7.0' force 'commons-beanutils:commons-beanutils:1.11.0' - force 'io.grpc:grpc-netty-shaded:1.81.0' + force 'io.grpc:grpc-netty-shaded:1.82.0' } } } @@ -179,7 +179,6 @@ project(':iceberg-kafka-connect:iceberg-kafka-connect-runtime') { integrationImplementation platform(libs.jackson.bom) integrationImplementation libs.jackson.core integrationImplementation libs.jackson.databind - integrationImplementation libs.jackson.databind integrationImplementation libs.kafka.clients integrationImplementation libs.kafka.connect.api integrationImplementation libs.kafka.connect.json diff --git a/kafka-connect/kafka-connect-runtime/runtime-deps.txt b/kafka-connect/kafka-connect-runtime/runtime-deps.txt index 48d11bdf6f0f..1cb307041212 100644 --- a/kafka-connect/kafka-connect-runtime/runtime-deps.txt +++ b/kafka-connect/kafka-connect-runtime/runtime-deps.txt @@ -18,39 +18,39 @@ com.github.luben:zstd-jni:1.5 com.github.pjfanning:jersey-json:1.22 com.google.android:annotations:4.1 com.google.api-client:google-api-client:2.7 -com.google.api.grpc:gapic-google-cloud-storage-v2:2.68 -com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:3.28 -com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:0.200 -com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:0.200 -com.google.api.grpc:grpc-google-cloud-storage-v2:2.68 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:3.28 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha:3.28 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:0.200 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:0.200 -com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta:3.28 -com.google.api.grpc:proto-google-cloud-monitoring-v3:3.93 -com.google.api.grpc:proto-google-cloud-storage-v2:2.68 -com.google.api.grpc:proto-google-common-protos:2.71 -com.google.api.grpc:proto-google-iam-v1:1.66 -com.google.api:api-common:2.63 -com.google.api:gax-grpc:2.80 -com.google.api:gax-httpjson:2.80 -com.google.api:gax:2.80 +com.google.api.grpc:gapic-google-cloud-storage-v2:2.69 +com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1:3.29 +com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1:3.29 +com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2:3.29 +com.google.api.grpc:grpc-google-cloud-storage-v2:2.69 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1:3.29 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1alpha:3.29 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta1:3.29 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta2:3.29 +com.google.api.grpc:proto-google-cloud-bigquerystorage-v1beta:3.29 +com.google.api.grpc:proto-google-cloud-monitoring-v3:3.94 +com.google.api.grpc:proto-google-cloud-storage-v2:2.69 +com.google.api.grpc:proto-google-common-protos:2.72 +com.google.api.grpc:proto-google-iam-v1:1.67 +com.google.api:api-common:2.64 +com.google.api:gax-grpc:2.81 +com.google.api:gax-httpjson:2.81 +com.google.api:gax:2.81 com.google.apis:google-api-services-bigquery:v2-rev20251012-2.0 com.google.apis:google-api-services-storage:v1-rev20260204-2.0 -com.google.auth:google-auth-library-credentials:1.47 -com.google.auth:google-auth-library-oauth2-http:1.47 +com.google.auth:google-auth-library-credentials:1.48 +com.google.auth:google-auth-library-oauth2-http:1.48 com.google.auto.value:auto-value-annotations:1.11 com.google.cloud.opentelemetry:detector-resources-support:0.33 com.google.cloud.opentelemetry:exporter-metrics:0.33 com.google.cloud.opentelemetry:shared-resourcemapping:0.33 -com.google.cloud:google-cloud-bigquery:2.66 -com.google.cloud:google-cloud-bigquerystorage:3.28 -com.google.cloud:google-cloud-core-grpc:2.70 -com.google.cloud:google-cloud-core-http:2.70 -com.google.cloud:google-cloud-core:2.70 -com.google.cloud:google-cloud-monitoring:3.93 -com.google.cloud:google-cloud-storage:2.68 +com.google.cloud:google-cloud-bigquery:2.67 +com.google.cloud:google-cloud-bigquerystorage:3.29 +com.google.cloud:google-cloud-core-grpc:2.71 +com.google.cloud:google-cloud-core-http:2.71 +com.google.cloud:google-cloud-core:2.71 +com.google.cloud:google-cloud-monitoring:3.94 +com.google.cloud:google-cloud-storage:2.69 com.google.code.findbugs:jsr305:3.0 com.google.code.gson:gson:2.13 com.google.errorprone:error_prone_annotations:2.48 @@ -90,7 +90,7 @@ io.grpc:grpc-core:1.81 io.grpc:grpc-googleapis:1.81 io.grpc:grpc-grpclb:1.81 io.grpc:grpc-inprocess:1.81 -io.grpc:grpc-netty-shaded:1.81 +io.grpc:grpc-netty-shaded:1.82 io.grpc:grpc-opentelemetry:1.81 io.grpc:grpc-protobuf-lite:1.81 io.grpc:grpc-protobuf:1.81 @@ -128,15 +128,15 @@ io.opencensus:opencensus-api:0.31 io.opencensus:opencensus-contrib-http-util:0.31 io.opentelemetry.contrib:opentelemetry-gcp-resources:1.37 io.opentelemetry.semconv:opentelemetry-semconv:1.29 -io.opentelemetry:opentelemetry-api:1.57 -io.opentelemetry:opentelemetry-common:1.57 -io.opentelemetry:opentelemetry-context:1.57 -io.opentelemetry:opentelemetry-sdk-common:1.57 -io.opentelemetry:opentelemetry-sdk-extension-autoconfigure-spi:1.57 -io.opentelemetry:opentelemetry-sdk-logs:1.57 -io.opentelemetry:opentelemetry-sdk-metrics:1.57 -io.opentelemetry:opentelemetry-sdk-trace:1.57 -io.opentelemetry:opentelemetry-sdk:1.57 +io.opentelemetry:opentelemetry-api:1.62 +io.opentelemetry:opentelemetry-common:1.62 +io.opentelemetry:opentelemetry-context:1.62 +io.opentelemetry:opentelemetry-sdk-common:1.62 +io.opentelemetry:opentelemetry-sdk-extension-autoconfigure-spi:1.62 +io.opentelemetry:opentelemetry-sdk-logs:1.62 +io.opentelemetry:opentelemetry-sdk-metrics:1.62 +io.opentelemetry:opentelemetry-sdk-trace:1.62 +io.opentelemetry:opentelemetry-sdk:1.62 io.perfmark:perfmark-api:0.27 io.projectreactor.netty:reactor-netty-core:1.2 io.projectreactor.netty:reactor-netty-http:1.2 diff --git a/kafka-connect/kafka-connect/src/main/java/org/apache/iceberg/connect/data/RecordConverter.java b/kafka-connect/kafka-connect/src/main/java/org/apache/iceberg/connect/data/RecordConverter.java index a152af2fd02a..41e70d67555a 100644 --- a/kafka-connect/kafka-connect/src/main/java/org/apache/iceberg/connect/data/RecordConverter.java +++ b/kafka-connect/kafka-connect/src/main/java/org/apache/iceberg/connect/data/RecordConverter.java @@ -81,9 +81,13 @@ import org.apache.kafka.connect.data.Field; import org.apache.kafka.connect.data.Struct; import org.apache.kafka.connect.errors.ConnectException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; class RecordConverter { + private static final Logger LOG = LoggerFactory.getLogger(RecordConverter.class); + private static final ObjectMapper MAPPER = new ObjectMapper(); private static final DateTimeFormatter OFFSET_TIMESTAMP_FORMAT = @@ -255,11 +259,19 @@ private GenericRecord convertToStruct( hasSchemaUpdates = true; } } + Object recordFieldValue = struct.get(recordField); + if (recordFieldValue == null && schemaUpdateConsumer != null && !hasSchemaUpdates) { + evolveSchemaFromConnectSchema( + recordField.schema(), + tableField.type(), + tableField.fieldId(), + schemaUpdateConsumer); + } if (!hasSchemaUpdates) { result.setField( tableField.name(), convertValue( - struct.get(recordField), + recordFieldValue, tableField.type(), tableField.fieldId(), schemaUpdateConsumer)); @@ -269,6 +281,87 @@ private GenericRecord convertToStruct( return result; } + /** + * Recursively traverses the Connect schema and emits all evolution events (addColumn, updateType, + * and makeOptional) at every nested level. + * + *

Unlike {@link #convertToStruct(Struct, StructType, int, SchemaUpdate.Consumer)} which skips + * a field's children once an update is detected (deferring nested discovery to re-conversion), + * this method always recurses through the full schema. + */ + private void evolveSchemaFromConnectSchema( + org.apache.kafka.connect.data.Schema recordSchema, + Type tableType, + int tableFieldId, + SchemaUpdate.Consumer schemaUpdateConsumer) { + if (recordSchema == null) { + return; + } + switch (recordSchema.type()) { + case STRUCT: + if (tableType.isStructType()) { + StructType structType = tableType.asStructType(); + for (Field field : recordSchema.fields()) { + NestedField nestedField = lookupStructField(field.name(), structType, tableFieldId); + if (nestedField == null) { + String parentFieldName = + tableFieldId < 0 ? null : tableSchema.findColumnName(tableFieldId); + Type type = SchemaUtils.toIcebergType(field.schema(), config); + schemaUpdateConsumer.addColumn(parentFieldName, field.name(), type); + } else { + PrimitiveType evolveDataType = + SchemaUtils.needsDataTypeUpdate(nestedField.type(), field.schema()); + if (evolveDataType != null) { + String fieldName = tableSchema.findColumnName(nestedField.fieldId()); + schemaUpdateConsumer.updateType(fieldName, evolveDataType); + } + if (nestedField.isRequired() && field.schema().isOptional()) { + String fieldName = tableSchema.findColumnName(nestedField.fieldId()); + schemaUpdateConsumer.makeOptional(fieldName); + } + evolveSchemaFromConnectSchema( + field.schema(), nestedField.type(), nestedField.fieldId(), schemaUpdateConsumer); + } + } + } else { + logMismatchedType(recordSchema.type(), tableType); + } + break; + case ARRAY: + if (tableType.isListType()) { + ListType listType = tableType.asListType(); + evolveSchemaFromConnectSchema( + recordSchema.valueSchema(), + listType.elementType(), + listType.elementId(), + schemaUpdateConsumer); + } else { + logMismatchedType(recordSchema.type(), tableType); + } + break; + case MAP: + if (tableType.isMapType()) { + MapType mapType = tableType.asMapType(); + evolveSchemaFromConnectSchema( + recordSchema.valueSchema(), + mapType.valueType(), + mapType.valueId(), + schemaUpdateConsumer); + } else { + logMismatchedType(recordSchema.type(), tableType); + } + break; + default: + break; + } + } + + private void logMismatchedType( + org.apache.kafka.connect.data.Schema.Type recordSchemaType, Type tableType) { + LOG.warn( + "Record schema of type {} does not match table of type {}", recordSchemaType, tableType); + } + private NestedField lookupStructField(String fieldName, StructType schema, int structFieldId) { if (nameMapping == null) { return config.schemaCaseInsensitive() diff --git a/kafka-connect/kafka-connect/src/main/java/org/apache/iceberg/connect/data/SchemaUtils.java b/kafka-connect/kafka-connect/src/main/java/org/apache/iceberg/connect/data/SchemaUtils.java index b0dd56b45d67..89d7878172cb 100644 --- a/kafka-connect/kafka-connect/src/main/java/org/apache/iceberg/connect/data/SchemaUtils.java +++ b/kafka-connect/kafka-connect/src/main/java/org/apache/iceberg/connect/data/SchemaUtils.java @@ -55,6 +55,7 @@ import org.apache.iceberg.types.Types.StructType; import org.apache.iceberg.types.Types.TimeType; import org.apache.iceberg.types.Types.TimestampType; +import org.apache.iceberg.types.Types.UUIDType; import org.apache.iceberg.util.Pair; import org.apache.iceberg.util.Tasks; import org.apache.kafka.connect.data.Date; @@ -286,6 +287,10 @@ Type toIcebergType(Schema valueSchema) { .collect(Collectors.toList()); return StructType.of(structFields); case STRING: + if ("uuid".equals(valueSchema.name())) { + return UUIDType.get(); + } + return StringType.get(); default: return StringType.get(); } diff --git a/kafka-connect/kafka-connect/src/test/java/org/apache/iceberg/connect/data/TestRecordConverter.java b/kafka-connect/kafka-connect/src/test/java/org/apache/iceberg/connect/data/TestRecordConverter.java index f2abb9d17da7..50a1ae6ad1c4 100644 --- a/kafka-connect/kafka-connect/src/test/java/org/apache/iceberg/connect/data/TestRecordConverter.java +++ b/kafka-connect/kafka-connect/src/test/java/org/apache/iceberg/connect/data/TestRecordConverter.java @@ -233,6 +233,22 @@ public void testMapConvert() { assertRecordValues(record); } + @Test + public void testUUIDStringConversion() { + Table table = mock(Table.class); + when(table.schema()) + .thenReturn(new org.apache.iceberg.Schema(NestedField.required(1, "uuid", UUIDType.get()))); + + Schema connectSchema = + SchemaBuilder.struct().field("uuid", SchemaBuilder.string().name("uuid").build()).build(); + Struct data = new Struct(connectSchema).put("uuid", UUID_VAL.toString()); + + RecordConverter converter = new RecordConverter(table, config); + Record record = converter.convert(data); + + assertThat(record.getField("uuid")).isEqualTo(UUID_VAL); + } + @Test public void testEmptyListAndMapConvert() { Table table = mock(Table.class); @@ -874,6 +890,426 @@ private void assertTypesAddedFromStruct(Function fn) { assertThat(fn.apply("ma")).isInstanceOf(MapType.class); } + @Test + public void testNestedSchemaEvolutionStructWithNullValue() { + org.apache.iceberg.Schema nestedStructSchema = + new org.apache.iceberg.Schema( + NestedField.required(1, "id", IntegerType.get()), + NestedField.optional( + 2, "nested", StructType.of(NestedField.required(3, "a", IntegerType.get())))); + + Table table = mock(Table.class); + when(table.schema()).thenReturn(nestedStructSchema); + RecordConverter converter = new RecordConverter(table, config); + + Schema connectNestedSchema = + SchemaBuilder.struct() + .optional() + .field("a", Schema.INT32_SCHEMA) + .field("b", Schema.OPTIONAL_STRING_SCHEMA) + .build(); + Schema connectSchema = + SchemaBuilder.struct() + .field("id", Schema.INT32_SCHEMA) + .field("nested", connectNestedSchema) + .build(); + Struct data = new Struct(connectSchema).put("id", 1).put("nested", null); + + SchemaUpdate.Consumer consumer = new SchemaUpdate.Consumer(); + Record result = converter.convert(data, consumer); + + assertThat(result.getField("id")).isEqualTo(1); + assertThat(result.getField("nested")).isNull(); + + Collection addCols = consumer.addColumns(); + assertThat(addCols).hasSize(1); + AddColumn addCol = addCols.iterator().next(); + assertThat(addCol.parentName()).isEqualTo("nested"); + assertThat(addCol.name()).isEqualTo("b"); + assertThat(addCol.type()).isInstanceOf(StringType.class); + } + + @Test + public void testNoSchemaEvolutionStructWithNullValue() { + org.apache.iceberg.Schema nestedStructSchema = + new org.apache.iceberg.Schema( + NestedField.required(1, "id", IntegerType.get()), + NestedField.optional( + 2, "nested", StructType.of(NestedField.required(3, "a", IntegerType.get())))); + + Table table = mock(Table.class); + when(table.schema()).thenReturn(nestedStructSchema); + RecordConverter converter = new RecordConverter(table, config); + + Schema connectNestedSchema = + SchemaBuilder.struct().optional().field("a", Schema.INT32_SCHEMA).build(); + Schema connectSchema = + SchemaBuilder.struct() + .field("id", Schema.INT32_SCHEMA) + .field("nested", connectNestedSchema) + .build(); + Struct data = new Struct(connectSchema).put("id", 1).put("nested", null); + + SchemaUpdate.Consumer consumer = new SchemaUpdate.Consumer(); + Record result = converter.convert(data, consumer); + + assertThat(result.getField("id")).isEqualTo(1); + assertThat(result.getField("nested")).isNull(); + + assertThat(consumer.addColumns()).isEmpty(); + assertThat(consumer.makeOptionals()).isEmpty(); + assertThat(consumer.updateTypes()).isEmpty(); + assertThat(consumer.empty()).isTrue(); + } + + @Test + @SuppressWarnings("unchecked") + public void testNestedSchemaEvolutionListOfStructsWithNullValue() { + org.apache.iceberg.Schema tableSchema = + new org.apache.iceberg.Schema( + NestedField.required( + 1, + "items", + ListType.ofRequired( + 2, + StructType.of( + NestedField.required(3, "product_id", IntegerType.get()), + NestedField.optional( + 4, + "details", + StructType.of(NestedField.required(5, "name", StringType.get()))))))); + + Table table = mock(Table.class); + when(table.schema()).thenReturn(tableSchema); + RecordConverter converter = new RecordConverter(table, config); + + Schema detailsSchema = + SchemaBuilder.struct() + .optional() + .field("name", Schema.OPTIONAL_STRING_SCHEMA) + .field("price", Schema.OPTIONAL_FLOAT64_SCHEMA) + .build(); + Schema itemSchema = + SchemaBuilder.struct() + .field("product_id", Schema.INT32_SCHEMA) + .field("details", detailsSchema) + .build(); + Schema connectSchema = + SchemaBuilder.struct().field("items", SchemaBuilder.array(itemSchema).build()).build(); + + Struct item = new Struct(itemSchema).put("product_id", 101).put("details", null); + Struct data = new Struct(connectSchema).put("items", ImmutableList.of(item)); + + SchemaUpdate.Consumer consumer = new SchemaUpdate.Consumer(); + Record result = converter.convert(data, consumer); + + List items = (List) result.getField("items"); + assertThat(items).hasSize(1); + assertThat(items.get(0).getField("product_id")).isEqualTo(101); + assertThat(items.get(0).getField("details")).isNull(); + + Collection addCols = consumer.addColumns(); + assertThat(addCols).hasSize(1); + AddColumn addCol = addCols.iterator().next(); + assertThat(addCol.parentName()).isEqualTo("items.element.details"); + assertThat(addCol.name()).isEqualTo("price"); + assertThat(addCol.type()).isInstanceOf(DoubleType.class); + } + + @Test + @SuppressWarnings("unchecked") + public void testNestedSchemaEvolutionMapOfStructsWithNullValue() { + org.apache.iceberg.Schema tableSchema = + new org.apache.iceberg.Schema( + NestedField.optional( + 1, + "metadata", + MapType.ofRequired( + 2, + 3, + StringType.get(), + StructType.of( + NestedField.required(4, "key", StringType.get()), + NestedField.optional( + 5, + "info", + StructType.of(NestedField.required(6, "name", StringType.get()))))))); + + Table table = mock(Table.class); + when(table.schema()).thenReturn(tableSchema); + RecordConverter converter = new RecordConverter(table, config); + + Schema infoSchema = + SchemaBuilder.struct() + .optional() + .field("name", Schema.OPTIONAL_STRING_SCHEMA) + .field("value", Schema.OPTIONAL_STRING_SCHEMA) + .build(); + Schema mapValueSchema = + SchemaBuilder.struct().field("key", Schema.STRING_SCHEMA).field("info", infoSchema).build(); + Schema connectSchema = + SchemaBuilder.struct() + .field("metadata", SchemaBuilder.map(Schema.STRING_SCHEMA, mapValueSchema).build()) + .build(); + + Struct mapValue = new Struct(mapValueSchema).put("key", "source_system").put("info", null); + Struct data = new Struct(connectSchema).put("metadata", ImmutableMap.of("source", mapValue)); + + SchemaUpdate.Consumer consumer = new SchemaUpdate.Consumer(); + Record result = converter.convert(data, consumer); + + Map metadata = (Map) result.getField("metadata"); + assertThat(metadata).containsKey("source"); + assertThat(metadata.get("source").getField("key")).isEqualTo("source_system"); + assertThat(metadata.get("source").getField("info")).isNull(); + + Collection addCols = consumer.addColumns(); + assertThat(addCols).hasSize(1); + AddColumn addCol = addCols.iterator().next(); + assertThat(addCol.parentName()).isEqualTo("metadata.value.info"); + assertThat(addCol.name()).isEqualTo("value"); + assertThat(addCol.type()).isInstanceOf(StringType.class); + } + + @Test + public void testNestedSchemaEvolutionStructInStructWithNullParent() { + org.apache.iceberg.Schema tableSchema = + new org.apache.iceberg.Schema( + NestedField.required(1, "id", IntegerType.get()), + NestedField.optional( + 2, + "customer", + StructType.of( + NestedField.required(3, "customer_id", IntegerType.get()), + NestedField.optional( + 4, + "details", + StructType.of(NestedField.required(5, "name", StringType.get())))))); + + Table table = mock(Table.class); + when(table.schema()).thenReturn(tableSchema); + RecordConverter converter = new RecordConverter(table, config); + + Schema detailsSchema = + SchemaBuilder.struct() + .optional() + .field("name", Schema.OPTIONAL_STRING_SCHEMA) + .field("email", Schema.OPTIONAL_STRING_SCHEMA) + .build(); + Schema customerSchema = + SchemaBuilder.struct() + .optional() + .field("customer_id", Schema.INT32_SCHEMA) + .field("details", detailsSchema) + .build(); + Schema connectSchema = + SchemaBuilder.struct() + .field("id", Schema.INT32_SCHEMA) + .field("customer", customerSchema) + .build(); + Struct data = new Struct(connectSchema).put("id", 1).put("customer", null); + + SchemaUpdate.Consumer consumer = new SchemaUpdate.Consumer(); + Record result = converter.convert(data, consumer); + + assertThat(result.getField("id")).isEqualTo(1); + assertThat(result.getField("customer")).isNull(); + + Collection addCols = consumer.addColumns(); + assertThat(addCols).hasSize(1); + AddColumn addCol = addCols.iterator().next(); + assertThat(addCol.parentName()).isEqualTo("customer.details"); + assertThat(addCol.name()).isEqualTo("email"); + assertThat(addCol.type()).isInstanceOf(StringType.class); + } + + @Test + public void testNestedSchemaEvolutionTypePromotionWithNullValue() { + org.apache.iceberg.Schema tableSchema = + new org.apache.iceberg.Schema( + NestedField.required(1, "id", IntegerType.get()), + NestedField.optional( + 2, + "nested", + StructType.of( + NestedField.required(3, "a", IntegerType.get()), + NestedField.required(4, "b", FloatType.get())))); + + Table table = mock(Table.class); + when(table.schema()).thenReturn(tableSchema); + RecordConverter converter = new RecordConverter(table, config); + + Schema connectNestedSchema = + SchemaBuilder.struct() + .optional() + .field("a", Schema.INT64_SCHEMA) + .field("b", Schema.FLOAT64_SCHEMA) + .build(); + Schema connectSchema = + SchemaBuilder.struct() + .field("id", Schema.INT32_SCHEMA) + .field("nested", connectNestedSchema) + .build(); + Struct data = new Struct(connectSchema).put("id", 1).put("nested", null); + + SchemaUpdate.Consumer consumer = new SchemaUpdate.Consumer(); + Record result = converter.convert(data, consumer); + + assertThat(result.getField("id")).isEqualTo(1); + assertThat(result.getField("nested")).isNull(); + + Collection updates = consumer.updateTypes(); + assertThat(updates).hasSize(2); + Map updateMap = Maps.newHashMap(); + updates.forEach(update -> updateMap.put(update.name(), update)); + assertThat(updateMap.get("nested.a").type()).isInstanceOf(LongType.class); + assertThat(updateMap.get("nested.b").type()).isInstanceOf(DoubleType.class); + } + + @Test + public void testNestedSchemaEvolutionMakeOptionalWithNullValue() { + org.apache.iceberg.Schema tableSchema = + new org.apache.iceberg.Schema( + NestedField.required(1, "id", IntegerType.get()), + NestedField.optional( + 2, + "nested", + StructType.of( + NestedField.required(3, "a", IntegerType.get()), + NestedField.required(4, "b", StringType.get())))); + + Table table = mock(Table.class); + when(table.schema()).thenReturn(tableSchema); + RecordConverter converter = new RecordConverter(table, config); + + Schema connectNestedSchema = + SchemaBuilder.struct() + .optional() + .field("a", Schema.OPTIONAL_INT32_SCHEMA) + .field("b", Schema.OPTIONAL_STRING_SCHEMA) + .build(); + Schema connectSchema = + SchemaBuilder.struct() + .field("id", Schema.INT32_SCHEMA) + .field("nested", connectNestedSchema) + .build(); + Struct data = new Struct(connectSchema).put("id", 1).put("nested", null); + + SchemaUpdate.Consumer consumer = new SchemaUpdate.Consumer(); + Record result = converter.convert(data, consumer); + + assertThat(result.getField("id")).isEqualTo(1); + assertThat(result.getField("nested")).isNull(); + + Collection makeOptionals = consumer.makeOptionals(); + assertThat(makeOptionals).hasSize(2); + Map optionalMap = Maps.newHashMap(); + makeOptionals.forEach(mo -> optionalMap.put(mo.name(), mo)); + assertThat(optionalMap).containsKey("nested.a"); + assertThat(optionalMap).containsKey("nested.b"); + } + + @Test + public void testSchemaEvolutionForFieldAndNestedFieldsAcrossTwoRecords() { + org.apache.iceberg.Schema tableSchema = + new org.apache.iceberg.Schema( + NestedField.required(1, "id", IntegerType.get()), + NestedField.required( + 2, "nested", StructType.of(NestedField.required(3, "x", IntegerType.get())))); + + Table table = mock(Table.class); + when(table.schema()).thenReturn(tableSchema); + RecordConverter converter = new RecordConverter(table, config); + + Schema nestedSchema = + SchemaBuilder.struct() + .optional() + .field("x", Schema.INT32_SCHEMA) + .field("y", Schema.OPTIONAL_STRING_SCHEMA) + .build(); + Schema connectSchema = + SchemaBuilder.struct() + .field("id", Schema.INT32_SCHEMA) + .field("nested", nestedSchema) + .build(); + + Struct data = new Struct(connectSchema).put("id", 1).put("nested", null); + + SchemaUpdate.Consumer consumer = new SchemaUpdate.Consumer(); + Record result = converter.convert(data, consumer); + + assertThat(result.getField("id")).isEqualTo(1); + assertThat(result.getField("nested")).isNull(); + + Collection makeOptionals = consumer.makeOptionals(); + assertThat(makeOptionals).hasSize(1); + assertThat(makeOptionals.iterator().next().name()).isEqualTo("nested"); + + Collection addCols = consumer.addColumns(); + assertThat(addCols).hasSize(0); + + org.apache.iceberg.Schema updatedSchema = + new org.apache.iceberg.Schema( + NestedField.required(1, "id", IntegerType.get()), + NestedField.optional( + 2, "nested", StructType.of(NestedField.required(3, "x", IntegerType.get())))); + when(table.schema()).thenReturn(updatedSchema); + RecordConverter converter2 = new RecordConverter(table, config); + + SchemaUpdate.Consumer consumer2 = new SchemaUpdate.Consumer(); + Record result2 = converter2.convert(data, consumer2); + + assertThat(result2.getField("id")).isEqualTo(1); + assertThat(result2.getField("nested")).isNull(); + + Collection addCols2 = consumer2.addColumns(); + assertThat(addCols2).hasSize(1); + AddColumn addCol2 = addCols2.iterator().next(); + assertThat(addCol2.parentName()).isEqualTo("nested"); + assertThat(addCol2.name()).isEqualTo("y"); + assertThat(addCol2.type()).isInstanceOf(StringType.class); + } + + @Test + public void testNoNestedSchemaEvolutionMapKeyWithNullValue() { + org.apache.iceberg.Schema tableSchema = + new org.apache.iceberg.Schema( + NestedField.required(1, "id", IntegerType.get()), + NestedField.optional( + 2, + "data", + MapType.ofRequired( + 3, + 4, + StructType.of(NestedField.required(5, "k1", StringType.get())), + StringType.get()))); + + Table table = mock(Table.class); + when(table.schema()).thenReturn(tableSchema); + RecordConverter converter = new RecordConverter(table, config); + + Schema keySchema = + SchemaBuilder.struct() + .field("k1", Schema.STRING_SCHEMA) + .field("k2", Schema.OPTIONAL_STRING_SCHEMA) + .build(); + Schema mapSchema = + SchemaBuilder.map(keySchema, Schema.OPTIONAL_STRING_SCHEMA).optional().build(); + Schema connectSchema = + SchemaBuilder.struct().field("id", Schema.INT32_SCHEMA).field("data", mapSchema).build(); + + Struct data = new Struct(connectSchema).put("id", 1).put("data", null); + + SchemaUpdate.Consumer consumer = new SchemaUpdate.Consumer(); + Record result = converter.convert(data, consumer); + + assertThat(result.getField("id")).isEqualTo(1); + assertThat(result.getField("data")).isNull(); + + Collection addCols = consumer.addColumns(); + assertThat(addCols).hasSize(0); + } + @Test public void testEvolveTypeDetectionStruct() { org.apache.iceberg.Schema tableSchema = diff --git a/kafka-connect/kafka-connect/src/test/java/org/apache/iceberg/connect/data/TestSchemaUtils.java b/kafka-connect/kafka-connect/src/test/java/org/apache/iceberg/connect/data/TestSchemaUtils.java index bde2452128b9..9443ed467696 100644 --- a/kafka-connect/kafka-connect/src/test/java/org/apache/iceberg/connect/data/TestSchemaUtils.java +++ b/kafka-connect/kafka-connect/src/test/java/org/apache/iceberg/connect/data/TestSchemaUtils.java @@ -63,6 +63,7 @@ import org.apache.iceberg.types.Types.StructType; import org.apache.iceberg.types.Types.TimeType; import org.apache.iceberg.types.Types.TimestampType; +import org.apache.iceberg.types.Types.UUIDType; import org.apache.kafka.connect.data.Date; import org.apache.kafka.connect.data.Decimal; import org.apache.kafka.connect.data.Schema; @@ -331,4 +332,12 @@ public void testInferIcebergTypeEmpty() { assertThat(SchemaUtils.inferIcebergType(ImmutableMap.of("nested", ImmutableMap.of()), config)) .isNull(); } + + @Test + public void testToIcebergTypeUUIDLogicalTypeOnString() { + IcebergSinkConfig config = mock(IcebergSinkConfig.class); + + Schema uuidSchema = SchemaBuilder.string().name("uuid").build(); + assertThat(SchemaUtils.toIcebergType(uuidSchema, config)).isInstanceOf(UUIDType.class); + } } diff --git a/open-api/requirements.txt b/open-api/requirements.txt index 6cab5500c6ef..3e3b59c33bb4 100644 --- a/open-api/requirements.txt +++ b/open-api/requirements.txt @@ -16,5 +16,5 @@ # under the License. openapi-spec-validator==0.9.0 -datamodel-code-generator==0.60.0 +datamodel-code-generator==0.64.1 yamllint==1.38.0 diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 4e05d10e9cdd..99fe855147c8 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -18,7 +18,7 @@ from __future__ import annotations from datetime import date, timedelta -from typing import Literal +from typing import Dict, Literal from uuid import UUID from pydantic import Base64Str, BaseModel, ConfigDict, Field, RootModel @@ -269,10 +269,13 @@ class Summary(BaseModel): model_config = ConfigDict( extra='allow', ) - __pydantic_extra__: dict[str, str] operation: Literal['append', 'replace', 'overwrite', 'delete'] +Summary.__annotations__['__pydantic_extra__'] = Dict[str, str] +Summary.model_rebuild(force=True) + + class Snapshot(BaseModel): snapshot_id: int = Field(..., alias='snapshot-id') parent_snapshot_id: int | None = Field(None, alias='parent-snapshot-id') @@ -365,17 +368,17 @@ class AssignUUIDUpdate(BaseUpdate): Assigning a UUID to a table/view should only be done when creating the table/view. It is not safe to re-assign the UUID if a table/view already has a UUID assigned """ - action: Literal['assign-uuid'] = 'assign-uuid' + action: Literal['assign-uuid'] uuid: str class UpgradeFormatVersionUpdate(BaseUpdate): - action: Literal['upgrade-format-version'] = 'upgrade-format-version' + action: Literal['upgrade-format-version'] format_version: int = Field(..., alias='format-version') class SetCurrentSchemaUpdate(BaseUpdate): - action: Literal['set-current-schema'] = 'set-current-schema' + action: Literal['set-current-schema'] schema_id: int = Field( ..., alias='schema-id', @@ -384,12 +387,12 @@ class SetCurrentSchemaUpdate(BaseUpdate): class AddPartitionSpecUpdate(BaseUpdate): - action: Literal['add-spec'] = 'add-spec' + action: Literal['add-spec'] spec: PartitionSpec class SetDefaultSpecUpdate(BaseUpdate): - action: Literal['set-default-spec'] = 'set-default-spec' + action: Literal['set-default-spec'] spec_id: int = Field( ..., alias='spec-id', @@ -398,12 +401,12 @@ class SetDefaultSpecUpdate(BaseUpdate): class AddSortOrderUpdate(BaseUpdate): - action: Literal['add-sort-order'] = 'add-sort-order' + action: Literal['add-sort-order'] sort_order: SortOrder = Field(..., alias='sort-order') class SetDefaultSortOrderUpdate(BaseUpdate): - action: Literal['set-default-sort-order'] = 'set-default-sort-order' + action: Literal['set-default-sort-order'] sort_order_id: int = Field( ..., alias='sort-order-id', @@ -412,47 +415,47 @@ class SetDefaultSortOrderUpdate(BaseUpdate): class AddSnapshotUpdate(BaseUpdate): - action: Literal['add-snapshot'] = 'add-snapshot' + action: Literal['add-snapshot'] snapshot: Snapshot class SetSnapshotRefUpdate(BaseUpdate, SnapshotReference): - action: Literal['set-snapshot-ref'] = 'set-snapshot-ref' + action: Literal['set-snapshot-ref'] ref_name: str = Field(..., alias='ref-name') class RemoveSnapshotsUpdate(BaseUpdate): - action: Literal['remove-snapshots'] = 'remove-snapshots' + action: Literal['remove-snapshots'] snapshot_ids: list[int] = Field(..., alias='snapshot-ids') class RemoveSnapshotRefUpdate(BaseUpdate): - action: Literal['remove-snapshot-ref'] = 'remove-snapshot-ref' + action: Literal['remove-snapshot-ref'] ref_name: str = Field(..., alias='ref-name') class SetLocationUpdate(BaseUpdate): - action: Literal['set-location'] = 'set-location' + action: Literal['set-location'] location: str class SetPropertiesUpdate(BaseUpdate): - action: Literal['set-properties'] = 'set-properties' + action: Literal['set-properties'] updates: dict[str, str] class RemovePropertiesUpdate(BaseUpdate): - action: Literal['remove-properties'] = 'remove-properties' + action: Literal['remove-properties'] removals: list[str] class AddViewVersionUpdate(BaseUpdate): - action: Literal['add-view-version'] = 'add-view-version' + action: Literal['add-view-version'] view_version: ViewVersion = Field(..., alias='view-version') class SetCurrentViewVersionUpdate(BaseUpdate): - action: Literal['set-current-view-version'] = 'set-current-view-version' + action: Literal['set-current-view-version'] view_version_id: int = Field( ..., alias='view-version-id', @@ -461,32 +464,32 @@ class SetCurrentViewVersionUpdate(BaseUpdate): class RemoveStatisticsUpdate(BaseUpdate): - action: Literal['remove-statistics'] = 'remove-statistics' + action: Literal['remove-statistics'] snapshot_id: int = Field(..., alias='snapshot-id') class RemovePartitionStatisticsUpdate(BaseUpdate): - action: Literal['remove-partition-statistics'] = 'remove-partition-statistics' + action: Literal['remove-partition-statistics'] snapshot_id: int = Field(..., alias='snapshot-id') class RemovePartitionSpecsUpdate(BaseUpdate): - action: Literal['remove-partition-specs'] = 'remove-partition-specs' + action: Literal['remove-partition-specs'] spec_ids: list[int] = Field(..., alias='spec-ids') class RemoveSchemasUpdate(BaseUpdate): - action: Literal['remove-schemas'] = 'remove-schemas' + action: Literal['remove-schemas'] schema_ids: list[int] = Field(..., alias='schema-ids') class AddEncryptionKeyUpdate(BaseUpdate): - action: Literal['add-encryption-key'] = 'add-encryption-key' + action: Literal['add-encryption-key'] encryption_key: EncryptedKey = Field(..., alias='encryption-key') class RemoveEncryptionKeyUpdate(BaseUpdate): - action: Literal['remove-encryption-key'] = 'remove-encryption-key' + action: Literal['remove-encryption-key'] key_id: str = Field(..., alias='key-id') @@ -519,7 +522,7 @@ class AssertRefSnapshotId(TableRequirement): """ - type: Literal['assert-ref-snapshot-id'] = 'assert-ref-snapshot-id' + type: Literal['assert-ref-snapshot-id'] ref: str snapshot_id: int = Field(..., alias='snapshot-id') @@ -529,7 +532,7 @@ class AssertLastAssignedFieldId(TableRequirement): The table's last assigned column id must match the requirement's `last-assigned-field-id` """ - type: Literal['assert-last-assigned-field-id'] = 'assert-last-assigned-field-id' + type: Literal['assert-last-assigned-field-id'] last_assigned_field_id: int = Field(..., alias='last-assigned-field-id') @@ -538,7 +541,7 @@ class AssertCurrentSchemaId(TableRequirement): The table's current schema id must match the requirement's `current-schema-id` """ - type: Literal['assert-current-schema-id'] = 'assert-current-schema-id' + type: Literal['assert-current-schema-id'] current_schema_id: int = Field(..., alias='current-schema-id') @@ -547,9 +550,7 @@ class AssertLastAssignedPartitionId(TableRequirement): The table's last assigned partition id must match the requirement's `last-assigned-partition-id` """ - type: Literal['assert-last-assigned-partition-id'] = ( - 'assert-last-assigned-partition-id' - ) + type: Literal['assert-last-assigned-partition-id'] last_assigned_partition_id: int = Field(..., alias='last-assigned-partition-id') @@ -558,7 +559,7 @@ class AssertDefaultSpecId(TableRequirement): The table's default spec id must match the requirement's `default-spec-id` """ - type: Literal['assert-default-spec-id'] = 'assert-default-spec-id' + type: Literal['assert-default-spec-id'] default_spec_id: int = Field(..., alias='default-spec-id') @@ -567,7 +568,7 @@ class AssertDefaultSortOrderId(TableRequirement): The table's default sort order id must match the requirement's `default-sort-order-id` """ - type: Literal['assert-default-sort-order-id'] = 'assert-default-sort-order-id' + type: Literal['assert-default-sort-order-id'] default_sort_order_id: int = Field(..., alias='default-sort-order-id') @@ -1141,7 +1142,7 @@ class TransformTerm(BaseModel): class SetPartitionStatisticsUpdate(BaseUpdate): - action: Literal['set-partition-statistics'] = 'set-partition-statistics' + action: Literal['set-partition-statistics'] partition_statistics: PartitionStatisticsFile = Field( ..., alias='partition-statistics' ) @@ -1251,7 +1252,7 @@ class Term(RootModel[Reference | TransformTerm]): class SetStatisticsUpdate(BaseUpdate): - action: Literal['set-statistics'] = 'set-statistics' + action: Literal['set-statistics'] snapshot_id: int | None = Field( None, alias='snapshot-id', @@ -1515,7 +1516,7 @@ class ViewMetadata(BaseModel): class AddSchemaUpdate(BaseUpdate): - action: Literal['add-schema'] = 'add-schema' + action: Literal['add-schema'] schema_: Schema = Field(..., alias='schema') last_column_id: int | None = Field( None, diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 990f2285e53b..419041d12a87 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -2152,10 +2152,12 @@ components: required: false schema: - type: string - enum: - - vended-credentials - - remote-signing + type: array + items: + type: string + enum: + - vended-credentials + - remote-signing style: simple explode: false example: "vended-credentials,remote-signing" @@ -3093,6 +3095,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - uuid properties: action: @@ -3105,6 +3108,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - format-version properties: action: @@ -3117,6 +3121,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - schema properties: action: @@ -3138,6 +3143,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - schema-id properties: action: @@ -3151,6 +3157,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - spec properties: action: @@ -3163,6 +3170,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - spec-id properties: action: @@ -3176,6 +3184,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - sort-order properties: action: @@ -3188,6 +3197,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - sort-order-id properties: action: @@ -3201,6 +3211,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - snapshot properties: action: @@ -3214,6 +3225,7 @@ components: - $ref: '#/components/schemas/BaseUpdate' - $ref: '#/components/schemas/SnapshotReference' required: + - action - ref-name properties: action: @@ -3226,6 +3238,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - snapshot-ids properties: action: @@ -3241,6 +3254,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - ref-name properties: action: @@ -3253,6 +3267,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - location properties: action: @@ -3265,6 +3280,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - updates properties: action: @@ -3279,6 +3295,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - removals properties: action: @@ -3293,6 +3310,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - view-version properties: action: @@ -3305,6 +3323,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - view-version-id properties: action: @@ -3318,6 +3337,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - statistics properties: action: @@ -3337,6 +3357,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - snapshot-id properties: action: @@ -3350,6 +3371,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - partition-statistics properties: action: @@ -3362,6 +3384,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - snapshot-id properties: action: @@ -3375,6 +3398,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - spec-ids properties: action: @@ -3389,6 +3413,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - schema-ids properties: action: @@ -3403,6 +3428,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - encryption-key properties: action: @@ -3415,6 +3441,7 @@ components: allOf: - $ref: '#/components/schemas/BaseUpdate' required: + - action - key-id properties: action: @@ -3513,6 +3540,7 @@ components: allOf: - $ref: '#/components/schemas/TableRequirement' required: + - type - ref - snapshot-id properties: @@ -3532,6 +3560,7 @@ components: allOf: - $ref: '#/components/schemas/TableRequirement' required: + - type - last-assigned-field-id properties: type: @@ -3546,6 +3575,7 @@ components: allOf: - $ref: '#/components/schemas/TableRequirement' required: + - type - current-schema-id properties: type: @@ -3560,6 +3590,7 @@ components: allOf: - $ref: '#/components/schemas/TableRequirement' required: + - type - last-assigned-partition-id properties: type: @@ -3574,6 +3605,7 @@ components: allOf: - $ref: '#/components/schemas/TableRequirement' required: + - type - default-spec-id properties: type: @@ -3588,6 +3620,7 @@ components: allOf: - $ref: '#/components/schemas/TableRequirement' required: + - type - default-sort-order-id properties: type: diff --git a/orc/src/main/java/org/apache/iceberg/orc/OrcFileAppender.java b/orc/src/main/java/org/apache/iceberg/orc/OrcFileAppender.java index 48b0e837f1a9..aef43c053142 100644 --- a/orc/src/main/java/org/apache/iceberg/orc/OrcFileAppender.java +++ b/orc/src/main/java/org/apache/iceberg/orc/OrcFileAppender.java @@ -54,9 +54,6 @@ class OrcFileAppender implements FileAppender { private final OrcRowWriter valueWriter; private boolean isClosed = false; - @SuppressWarnings("unused") // Currently used in tests TODO remove this redundant field - private final Configuration conf; - private final MetricsConfig metricsConfig; OrcFileAppender( @@ -67,7 +64,6 @@ class OrcFileAppender implements FileAppender { Map metadata, int batchSize, MetricsConfig metricsConfig) { - this.conf = conf; this.file = file; this.batchSize = batchSize; this.metricsConfig = metricsConfig; diff --git a/orc/src/main/java/org/apache/iceberg/orc/OrcMetrics.java b/orc/src/main/java/org/apache/iceberg/orc/OrcMetrics.java index 867828820d2b..7498d6e1558f 100644 --- a/orc/src/main/java/org/apache/iceberg/orc/OrcMetrics.java +++ b/orc/src/main/java/org/apache/iceberg/orc/OrcMetrics.java @@ -21,6 +21,9 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.sql.Timestamp; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.temporal.ChronoUnit; import java.util.List; import java.util.Map; import java.util.Objects; @@ -267,9 +270,7 @@ private static Optional fromOrcMin( TimestampColumnStatistics tColStats = (TimestampColumnStatistics) columnStats; Timestamp minValue = tColStats.getMinimumUTC(); min = - Optional.ofNullable(minValue) - .map(v -> DateTimeUtil.microsFromInstant(v.toInstant())) - .orElse(null); + Optional.ofNullable(minValue).map(v -> timestampBound(type, v.toInstant())).orElse(null); } else if (columnStats instanceof BooleanColumnStatistics) { BooleanColumnStatistics booleanStats = (BooleanColumnStatistics) columnStats; min = booleanStats.getFalseCount() <= 0; @@ -322,9 +323,7 @@ private static Optional fromOrcMax( TimestampColumnStatistics tColStats = (TimestampColumnStatistics) columnStats; Timestamp maxValue = tColStats.getMaximumUTC(); max = - Optional.ofNullable(maxValue) - .map(v -> DateTimeUtil.microsFromInstant(v.toInstant())) - .orElse(null); + Optional.ofNullable(maxValue).map(v -> timestampBound(type, v.toInstant())).orElse(null); } else if (columnStats instanceof BooleanColumnStatistics) { BooleanColumnStatistics booleanStats = (BooleanColumnStatistics) columnStats; max = booleanStats.getTrueCount() > 0; @@ -333,6 +332,15 @@ private static Optional fromOrcMax( Conversions.toByteBuffer(type, truncateIfNeeded(Bound.UPPER, type, max, metricsMode))); } + private static long timestampBound(Type type, Instant instant) { + // timestamp_ns columns store bounds as nanoseconds from epoch, while timestamp columns store + // microseconds (see Conversions for TIMESTAMP and TIMESTAMP_NANO). + if (type.typeId() == Type.TypeID.TIMESTAMP_NANO) { + return ChronoUnit.NANOS.between(DateTimeUtil.EPOCH, instant.atOffset(ZoneOffset.UTC)); + } + return DateTimeUtil.microsFromInstant(instant); + } + private static Object replaceNaN(double value, double replacement) { return Double.isNaN(value) ? replacement : value; } diff --git a/orc/src/test/java/org/apache/iceberg/orc/TestTableProperties.java b/orc/src/test/java/org/apache/iceberg/orc/TestTableProperties.java index ce3985597ed0..f045dafeafee 100644 --- a/orc/src/test/java/org/apache/iceberg/orc/TestTableProperties.java +++ b/orc/src/test/java/org/apache/iceberg/orc/TestTableProperties.java @@ -21,15 +21,15 @@ import static org.assertj.core.api.Assertions.assertThat; import java.io.File; -import java.util.Random; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; import org.apache.iceberg.FileFormat; import org.apache.iceberg.Files; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; import org.apache.iceberg.TableProperties; -import org.apache.iceberg.common.DynFields; +import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.data.Record; import org.apache.iceberg.data.orc.GenericOrcWriter; import org.apache.iceberg.deletes.EqualityDeleteWriter; @@ -38,8 +38,8 @@ import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.types.Types; import org.apache.orc.CompressionKind; -import org.apache.orc.OrcConf; -import org.apache.orc.OrcFile.CompressionStrategy; +import org.apache.orc.OrcFile; +import org.apache.orc.Reader; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -54,23 +54,15 @@ public class TestTableProperties { @TempDir private File testFile; @Test - public void testOrcTableProperties() throws Exception { - Random random = new Random(); - int numOfCodecs = CompressionKind.values().length; - int numOfStrategies = CompressionStrategy.values().length; - - long stripeSizeBytes = 32L * 1024 * 1024; - long blockSizeBytes = 128L * 1024 * 1024; - String codecAsString = CompressionKind.values()[random.nextInt(numOfCodecs)].name(); - String strategyAsString = CompressionStrategy.values()[random.nextInt(numOfStrategies)].name(); + public void testOrcTablePropertiesForDataFile() throws Exception { + String codecAsString = CompressionKind.SNAPPY.name(); ImmutableMap properties = ImmutableMap.of( - TableProperties.ORC_STRIPE_SIZE_BYTES, String.valueOf(stripeSizeBytes), - TableProperties.ORC_BLOCK_SIZE_BYTES, String.valueOf(blockSizeBytes), - TableProperties.ORC_COMPRESSION, codecAsString, - TableProperties.ORC_COMPRESSION_STRATEGY, strategyAsString, - TableProperties.DEFAULT_FILE_FORMAT, FileFormat.ORC.name()); + TableProperties.ORC_COMPRESSION, + codecAsString, + TableProperties.DEFAULT_FILE_FORMAT, + FileFormat.ORC.name()); String warehouse = folder.getAbsolutePath(); String tablePath = warehouse.concat("/test"); @@ -81,42 +73,30 @@ public void testOrcTableProperties() throws Exception { assertThat(testFile.delete()).isTrue(); - FileAppender writer = + try (FileAppender writer = ORC.write(Files.localOutput(testFile)) .forTable(table) .createWriterFunc(GenericOrcWriter::buildWriter) - .build(); - - DynFields.BoundField confField = - DynFields.builder().hiddenImpl(writer.getClass(), "conf").build(writer); - - Configuration configuration = confField.get(); - assertThat(OrcConf.BLOCK_SIZE.getLong(configuration)).isEqualTo(blockSizeBytes); - assertThat(OrcConf.STRIPE_SIZE.getLong(configuration)).isEqualTo(stripeSizeBytes); - assertThat(OrcConf.COMPRESS.getString(configuration)).isEqualTo(codecAsString); - assertThat(OrcConf.COMPRESSION_STRATEGY.getString(configuration)).isEqualTo(strategyAsString); - assertThat(configuration.get(TableProperties.DEFAULT_FILE_FORMAT)) - .isEqualTo(FileFormat.ORC.name()); + .build()) { + writer.add(GenericRecord.create(SCHEMA).copy(ImmutableMap.of("id", 1, "data", "a"))); + } + + Reader reader = + OrcFile.createReader( + new Path(testFile.toURI()), OrcFile.readerOptions(new Configuration())); + assertThat(reader.getCompressionKind()).isEqualTo(CompressionKind.SNAPPY); } @Test - public void testOrcTableDeleteProperties() throws Exception { - Random random = new Random(); - int numOfCodecs = CompressionKind.values().length; - int numOfStrategies = CompressionStrategy.values().length; - - long stripeSizeBytes = 32L * 1024 * 1024; - long blockSizeBytes = 128L * 1024 * 1024; - String codecAsString = CompressionKind.values()[random.nextInt(numOfCodecs)].name(); - String strategyAsString = CompressionStrategy.values()[random.nextInt(numOfStrategies)].name(); + public void testOrcTablePropertiesForDeleteFile() throws Exception { + String codecAsString = CompressionKind.SNAPPY.name(); ImmutableMap properties = ImmutableMap.of( - TableProperties.DELETE_ORC_STRIPE_SIZE_BYTES, String.valueOf(stripeSizeBytes), - TableProperties.DELETE_ORC_BLOCK_SIZE_BYTES, String.valueOf(blockSizeBytes), - TableProperties.DELETE_ORC_COMPRESSION, codecAsString, - TableProperties.DELETE_ORC_COMPRESSION_STRATEGY, strategyAsString, - TableProperties.DEFAULT_FILE_FORMAT, FileFormat.ORC.name()); + TableProperties.DELETE_ORC_COMPRESSION, + codecAsString, + TableProperties.DEFAULT_FILE_FORMAT, + FileFormat.ORC.name()); String warehouse = folder.getAbsolutePath(); String tablePath = warehouse.concat("/test"); @@ -127,26 +107,18 @@ public void testOrcTableDeleteProperties() throws Exception { assertThat(testFile.delete()).isTrue(); - EqualityDeleteWriter deleteWriter = + try (EqualityDeleteWriter deleteWriter = ORC.writeDeletes(Files.localOutput(testFile)) .forTable(table) .equalityFieldIds(1) .createWriterFunc(GenericOrcWriter::buildWriter) - .buildEqualityWriter(); - - DynFields.BoundField> writer = - DynFields.builder().hiddenImpl(deleteWriter.getClass(), "appender").build(deleteWriter); - - OrcFileAppender orcFileAppender = writer.get(); - DynFields.BoundField confField = - DynFields.builder().hiddenImpl(orcFileAppender.getClass(), "conf").build(orcFileAppender); - - Configuration configuration = confField.get(); - assertThat(OrcConf.BLOCK_SIZE.getLong(configuration)).isEqualTo(blockSizeBytes); - assertThat(OrcConf.STRIPE_SIZE.getLong(configuration)).isEqualTo(stripeSizeBytes); - assertThat(OrcConf.COMPRESS.getString(configuration)).isEqualTo(codecAsString); - assertThat(OrcConf.COMPRESSION_STRATEGY.getString(configuration)).isEqualTo(strategyAsString); - assertThat(configuration.get(TableProperties.DEFAULT_FILE_FORMAT)) - .isEqualTo(FileFormat.ORC.name()); + .buildEqualityWriter()) { + deleteWriter.write(GenericRecord.create(SCHEMA).copy(ImmutableMap.of("id", 1, "data", "a"))); + } + + Reader reader = + OrcFile.createReader( + new Path(testFile.toURI()), OrcFile.readerOptions(new Configuration())); + assertThat(reader.getCompressionKind()).isEqualTo(CompressionKind.SNAPPY); } } diff --git a/parquet/src/main/java/org/apache/iceberg/data/parquet/BaseParquetReaders.java b/parquet/src/main/java/org/apache/iceberg/data/parquet/BaseParquetReaders.java index b1fd8f43a578..89024951ccaa 100644 --- a/parquet/src/main/java/org/apache/iceberg/data/parquet/BaseParquetReaders.java +++ b/parquet/src/main/java/org/apache/iceberg/data/parquet/BaseParquetReaders.java @@ -224,6 +224,20 @@ public Optional> visit( return Optional.of(ParquetValueReaders.byteBuffers(desc)); } + @Override + public Optional> visit( + LogicalTypeAnnotation.GeometryLogicalTypeAnnotation geometryLogicalType) { + // geometry values are pure WKB stored in a BINARY column + return Optional.of(ParquetValueReaders.byteBuffers(desc)); + } + + @Override + public Optional> visit( + LogicalTypeAnnotation.GeographyLogicalTypeAnnotation geographyLogicalType) { + // geography values are pure WKB stored in a BINARY column + return Optional.of(ParquetValueReaders.byteBuffers(desc)); + } + @Override public Optional> visit( LogicalTypeAnnotation.UUIDLogicalTypeAnnotation uuidLogicalType) { diff --git a/parquet/src/main/java/org/apache/iceberg/data/parquet/BaseParquetWriter.java b/parquet/src/main/java/org/apache/iceberg/data/parquet/BaseParquetWriter.java index 2a986fc62d00..dcc93f939d8e 100644 --- a/parquet/src/main/java/org/apache/iceberg/data/parquet/BaseParquetWriter.java +++ b/parquet/src/main/java/org/apache/iceberg/data/parquet/BaseParquetWriter.java @@ -265,6 +265,20 @@ public Optional> visit( return Optional.of(ParquetValueWriters.byteBuffers(desc)); } + @Override + public Optional> visit( + LogicalTypeAnnotation.GeometryLogicalTypeAnnotation geometryType) { + // geometry values are pure WKB stored in a BINARY column + return Optional.of(ParquetValueWriters.byteBuffers(desc)); + } + + @Override + public Optional> visit( + LogicalTypeAnnotation.GeographyLogicalTypeAnnotation geographyType) { + // geography values are pure WKB stored in a BINARY column + return Optional.of(ParquetValueWriters.byteBuffers(desc)); + } + @Override public Optional> visit( LogicalTypeAnnotation.UUIDLogicalTypeAnnotation uuidLogicalType) { diff --git a/parquet/src/main/java/org/apache/iceberg/parquet/MessageTypeToType.java b/parquet/src/main/java/org/apache/iceberg/parquet/MessageTypeToType.java index 98023bafcb8f..2b01bf882e75 100644 --- a/parquet/src/main/java/org/apache/iceberg/parquet/MessageTypeToType.java +++ b/parquet/src/main/java/org/apache/iceberg/parquet/MessageTypeToType.java @@ -29,9 +29,11 @@ import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.types.EdgeAlgorithm; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import org.apache.iceberg.types.Types.TimestampType; +import org.apache.parquet.column.schema.EdgeInterpolationAlgorithm; import org.apache.parquet.schema.GroupType; import org.apache.parquet.schema.LogicalTypeAnnotation; import org.apache.parquet.schema.MessageType; @@ -254,6 +256,23 @@ public Optional visit(LogicalTypeAnnotation.JsonLogicalTypeAnnotation json public Optional visit(LogicalTypeAnnotation.BsonLogicalTypeAnnotation bsonType) { return Optional.of(Types.BinaryType.get()); } + + @Override + public Optional visit(LogicalTypeAnnotation.GeometryLogicalTypeAnnotation geometryType) { + // a null crs resolves to the Iceberg default in GeometryType.of + return Optional.of(Types.GeometryType.of(geometryType.getCrs())); + } + + @Override + public Optional visit( + LogicalTypeAnnotation.GeographyLogicalTypeAnnotation geographyType) { + // a null crs / algorithm resolves to the Iceberg default in GeographyType.of + EdgeInterpolationAlgorithm algorithm = geographyType.getAlgorithm(); + return Optional.of( + Types.GeographyType.of( + geographyType.getCrs(), + algorithm != null ? EdgeAlgorithm.fromName(algorithm.name()) : null)); + } } private void addAlias(String name, int fieldId) { diff --git a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetMetrics.java b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetMetrics.java index af6566e747b2..42950bc073ec 100644 --- a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetMetrics.java +++ b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetMetrics.java @@ -43,6 +43,7 @@ import org.apache.iceberg.relocated.com.google.common.collect.Streams; import org.apache.iceberg.types.Comparators; import org.apache.iceberg.types.Conversions; +import org.apache.iceberg.types.Type.TypeID; import org.apache.iceberg.types.Types; import org.apache.iceberg.util.BinaryUtil; import org.apache.iceberg.util.NaNUtil; @@ -264,13 +265,19 @@ private FieldMetrics metricsFromFooter( int truncateLength) { if (primitive.getPrimitiveTypeName() == PrimitiveType.PrimitiveTypeName.INT96) { return null; - } else if (truncateLength <= 0) { + } else if (truncateLength <= 0 + || (icebergType != null && isGeospatial(icebergType.typeId()))) { + // Parquet lexicographic min/max is not meaningful for spatial WKB. return counts(fieldId); } else { return bounds(fieldId, icebergType, primitive, truncateLength); } } + private static boolean isGeospatial(TypeID typeId) { + return typeId == TypeID.GEOMETRY || typeId == TypeID.GEOGRAPHY; + } + private FieldMetrics counts(int fieldId) { ColumnPath path = ColumnPath.get(currentPath()); long valueCount = 0; @@ -477,6 +484,11 @@ public Iterable value( return typedResult; } + if (Iterables.isEmpty(valueResult)) { + // missing value stats invalidate typed bounds + return ImmutableList.of(); + } + ParquetVariantUtil.VariantMetrics valueMetrics = Iterables.getOnlyElement(valueResult); if (typedResult != null && valueMetrics.valueCount() == valueMetrics.nullCount()) { // all the variant-encoded values are null, so the typed stats can be used diff --git a/parquet/src/main/java/org/apache/iceberg/parquet/TypeToMessageType.java b/parquet/src/main/java/org/apache/iceberg/parquet/TypeToMessageType.java index d648cbf0694b..f05001f5f43d 100644 --- a/parquet/src/main/java/org/apache/iceberg/parquet/TypeToMessageType.java +++ b/parquet/src/main/java/org/apache/iceberg/parquet/TypeToMessageType.java @@ -30,12 +30,15 @@ import org.apache.iceberg.Schema; import org.apache.iceberg.avro.AvroSchemaUtil; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.types.EdgeAlgorithm; import org.apache.iceberg.types.Type.NestedType; import org.apache.iceberg.types.Type.PrimitiveType; import org.apache.iceberg.types.Type.TypeID; import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types.DecimalType; import org.apache.iceberg.types.Types.FixedType; +import org.apache.iceberg.types.Types.GeographyType; +import org.apache.iceberg.types.Types.GeometryType; import org.apache.iceberg.types.Types.ListType; import org.apache.iceberg.types.Types.MapType; import org.apache.iceberg.types.Types.NestedField; @@ -43,6 +46,7 @@ import org.apache.iceberg.types.Types.TimestampNanoType; import org.apache.iceberg.types.Types.TimestampType; import org.apache.iceberg.variants.Variant; +import org.apache.parquet.column.schema.EdgeInterpolationAlgorithm; import org.apache.parquet.schema.GroupType; import org.apache.parquet.schema.LogicalTypeAnnotation; import org.apache.parquet.schema.LogicalTypeAnnotation.TimeUnit; @@ -280,11 +284,33 @@ public Type primitive( .id(id) .named(name); + case GEOMETRY: + GeometryType geometry = (GeometryType) primitive; + return Types.primitive(BINARY, repetition) + .as(LogicalTypeAnnotation.geometryType(geometry.crs())) + .id(id) + .named(name); + + case GEOGRAPHY: + GeographyType geography = (GeographyType) primitive; + return Types.primitive(BINARY, repetition) + .as( + LogicalTypeAnnotation.geographyType( + geography.crs(), toParquet(geography.algorithm()))) + .id(id) + .named(name); + default: throw new UnsupportedOperationException("Unsupported type for Parquet: " + primitive); } } + private static EdgeInterpolationAlgorithm toParquet(EdgeAlgorithm algorithm) { + // Iceberg and Parquet use the same algorithm names (SPHERICAL, VINCENTY, THOMAS, ANDOYER, + // KARNEY) so the algorithm is mapped by name + return EdgeInterpolationAlgorithm.valueOf(algorithm.name()); + } + private static LogicalTypeAnnotation decimalAnnotation(int precision, int scale) { return LogicalTypeAnnotation.decimalType(scale, precision); } diff --git a/parquet/src/test/java/org/apache/iceberg/parquet/TestInternalParquet.java b/parquet/src/test/java/org/apache/iceberg/parquet/TestInternalParquet.java index a53ac8972528..f193cab5c552 100644 --- a/parquet/src/test/java/org/apache/iceberg/parquet/TestInternalParquet.java +++ b/parquet/src/test/java/org/apache/iceberg/parquet/TestInternalParquet.java @@ -56,6 +56,11 @@ protected boolean supportsVariant() { return true; } + @Override + protected boolean supportsGeospatial() { + return true; + } + @Override protected void writeAndValidate(Schema schema) throws IOException { List expected = RandomInternalData.generate(schema, 100, 1376L); diff --git a/parquet/src/test/java/org/apache/iceberg/parquet/TestParquet.java b/parquet/src/test/java/org/apache/iceberg/parquet/TestParquet.java index afabd2c6b1bd..c1aabab0b251 100644 --- a/parquet/src/test/java/org/apache/iceberg/parquet/TestParquet.java +++ b/parquet/src/test/java/org/apache/iceberg/parquet/TestParquet.java @@ -284,6 +284,52 @@ public void testColumnStatisticsEnabled() throws Exception { } } + @Test + public void testGeospatialFooterMetricsSkipParquetBounds() throws IOException { + Schema binarySchema = new Schema(optional(1, "geom", Types.BinaryType.get())); + Schema geometrySchema = new Schema(optional(1, "geom", Types.GeometryType.crs84())); + Schema geographySchema = new Schema(optional(1, "geom", Types.GeographyType.crs84())); + + File file = createTempFile(temp); + org.apache.avro.Schema avroSchema = AvroSchemaUtil.convert(binarySchema.asStruct()); + GenericData.Record first = new GenericData.Record(avroSchema); + first.put("geom", ByteBuffer.wrap(new byte[] {0x01, 0x02, 0x03})); + GenericData.Record second = new GenericData.Record(avroSchema); + second.put("geom", ByteBuffer.wrap(new byte[] {0x04, 0x05, 0x06})); + + write( + file, binarySchema, Collections.emptyMap(), ParquetAvroWriter::buildWriter, first, second); + + InputFile inputFile = Files.localInput(file); + try (ParquetFileReader reader = ParquetFileReader.open(ParquetIO.file(inputFile))) { + Metrics geometryMetrics = + ParquetMetrics.metrics( + geometrySchema, + reader.getFooter().getFileMetaData().getSchema(), + MetricsConfig.getDefault(), + reader.getFooter(), + Stream.empty()); + + Metrics geographyMetrics = + ParquetMetrics.metrics( + geographySchema, + reader.getFooter().getFileMetaData().getSchema(), + MetricsConfig.getDefault(), + reader.getFooter(), + Stream.empty()); + + assertThat(geometryMetrics.valueCounts()).containsEntry(1, 2L); + assertThat(geometryMetrics.nullValueCounts()).containsEntry(1, 0L); + assertThat(geometryMetrics.lowerBounds()).doesNotContainKey(1); + assertThat(geometryMetrics.upperBounds()).doesNotContainKey(1); + + assertThat(geographyMetrics.valueCounts()).containsEntry(1, 2L); + assertThat(geographyMetrics.nullValueCounts()).containsEntry(1, 0L); + assertThat(geographyMetrics.lowerBounds()).doesNotContainKey(1); + assertThat(geographyMetrics.upperBounds()).doesNotContainKey(1); + } + } + @Test public void testPerColumnDictionaryEncoding() throws Exception { Schema schema = diff --git a/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetDataWriter.java b/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetDataWriter.java index 19f0b9129857..00891b507eef 100644 --- a/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetDataWriter.java +++ b/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetDataWriter.java @@ -55,6 +55,7 @@ import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.RandomUtil; import org.apache.iceberg.variants.Variant; import org.apache.iceberg.variants.VariantMetadata; import org.apache.iceberg.variants.VariantTestUtil; @@ -102,6 +103,65 @@ public void testDataWriter() throws IOException { testDataWriter(SCHEMA, (id, name) -> null); } + @Test + public void testGeospatialRoundTrip() throws IOException { + Schema schema = + new Schema( + Types.NestedField.required(1, "id", Types.LongType.get()), + Types.NestedField.optional(2, "geom", Types.GeometryType.crs84()), + Types.NestedField.optional(3, "geog", Types.GeographyType.crs84())); + + GenericRecord record = GenericRecord.create(schema); + List geoRecords = + ImmutableList.of( + record.copy( + ImmutableMap.of("id", 1L, "geom", wkbPoint(30, 10), "geog", wkbPoint(-5, 40))), + // geog is left null + record.copy(ImmutableMap.of("id", 2L, "geom", wkbPoint(0, 0))), + // both geo columns are left null + record.copy(ImmutableMap.of("id", 3L))); + + OutputFile file = Files.localOutput(createTempFile(temp)); + DataWriter dataWriter = + Parquet.writeData(file) + .schema(schema) + .createWriterFunc(GenericParquetWriter::create) + .overwrite() + .withSpec(PartitionSpec.unpartitioned()) + .build(); + try (dataWriter) { + for (Record geoRecord : geoRecords) { + dataWriter.write(geoRecord); + } + } + + assertThat(dataWriter.toDataFile().recordCount()).isEqualTo(geoRecords.size()); + + List writtenRecords; + try (CloseableIterable reader = + Parquet.read(file.toInputFile()) + .project(schema) + .createReaderFunc(fileSchema -> GenericParquetReaders.buildReader(schema, fileSchema)) + .build()) { + writtenRecords = Lists.newArrayList(reader); + } + + assertThat(writtenRecords).hasSameSizeAs(geoRecords); + for (int i = 0; i < geoRecords.size(); i++) { + assertThat(writtenRecords.get(i).getField("id")).isEqualTo(geoRecords.get(i).getField("id")); + assertThat(writtenRecords.get(i).getField("geom")) + .as("geometry WKB should round-trip unchanged") + .isEqualTo(geoRecords.get(i).getField("geom")); + assertThat(writtenRecords.get(i).getField("geog")) + .as("geography WKB should round-trip unchanged") + .isEqualTo(geoRecords.get(i).getField("geog")); + } + } + + private static ByteBuffer wkbPoint(double xCoord, double yCoord) { + return ByteBuffer.wrap(RandomUtil.wkbPoint(xCoord, yCoord)); + } + private void testDataWriter(Schema schema, VariantShreddingFunction variantShreddingFunc) throws IOException { OutputFile file = Files.localOutput(createTempFile(temp)); diff --git a/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetSchemaUtil.java b/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetSchemaUtil.java index 51bfc1e811f4..6acf5eda0e86 100644 --- a/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetSchemaUtil.java +++ b/parquet/src/test/java/org/apache/iceberg/parquet/TestParquetSchemaUtil.java @@ -22,13 +22,17 @@ import static org.apache.iceberg.types.Types.NestedField.required; import static org.assertj.core.api.Assertions.assertThat; +import java.util.List; import java.util.concurrent.atomic.AtomicInteger; import org.apache.iceberg.Schema; import org.apache.iceberg.mapping.MappingUtil; import org.apache.iceberg.mapping.NameMapping; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.types.EdgeAlgorithm; import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types; import org.apache.iceberg.variants.Variant; +import org.apache.parquet.column.schema.EdgeInterpolationAlgorithm; import org.apache.parquet.schema.GroupType; import org.apache.parquet.schema.LogicalTypeAnnotation; import org.apache.parquet.schema.MessageType; @@ -338,6 +342,90 @@ public void testVariantTypesWithoutAssigningIds() { .isEqualTo(expectedSchema.asStruct()); } + @Test + public void testGeospatialTypeRoundTrip() { + List fields = + Lists.newArrayList( + required(1, "geom_default", Types.GeometryType.crs84()), + optional(2, "geom_3857", Types.GeometryType.of("EPSG:3857")), + required(3, "geog_default", Types.GeographyType.crs84())); + // cover the by-name algorithm mapping for every edge algorithm in both directions + int nextId = fields.size() + 1; + for (EdgeAlgorithm algorithm : EdgeAlgorithm.values()) { + fields.add( + optional(nextId++, "geog_" + algorithm, Types.GeographyType.of("EPSG:4326", algorithm))); + } + + Schema schema = new Schema(fields); + MessageType messageType = ParquetSchemaUtil.convert(schema, "geo_table"); + Schema actualSchema = ParquetSchemaUtil.convert(messageType); + assertThat(actualSchema.asStruct()) + .as("Schema must round-trip through Parquet geometry/geography logical types") + .isEqualTo(schema.asStruct()); + } + + @Test + public void testGeospatialAnnotationsWithOmittedParameters() { + // unset CRS and algorithm parameters must map to Iceberg's defaults, and explicit values + // (including explicit default values) must be preserved + MessageType messageType = + org.apache.parquet.schema.Types.buildMessage() + .required(PrimitiveTypeName.BINARY) + .as(LogicalTypeAnnotation.geometryType(null)) + .id(1) + .named("geom_bare") + .optional(PrimitiveTypeName.BINARY) + .as(LogicalTypeAnnotation.geometryType("OGC:CRS84")) + .id(2) + .named("geom_explicit_default") + .required(PrimitiveTypeName.BINARY) + .as(LogicalTypeAnnotation.geographyType(null, null)) + .id(3) + .named("geog_bare") + .optional(PrimitiveTypeName.BINARY) + .as(LogicalTypeAnnotation.geographyType("EPSG:4326", null)) + .id(4) + .named("geog_crs_only") + .optional(PrimitiveTypeName.BINARY) + .as(LogicalTypeAnnotation.geographyType(null, EdgeInterpolationAlgorithm.ANDOYER)) + .id(5) + .named("geog_algorithm_only") + .optional(PrimitiveTypeName.BINARY) + .as( + LogicalTypeAnnotation.geographyType( + "EPSG:4326", EdgeInterpolationAlgorithm.SPHERICAL)) + .id(6) + .named("geog_explicit_spherical") + .named("geo_table"); + + Schema expectedSchema = + new Schema( + required(1, "geom_bare", Types.GeometryType.crs84()), + optional(2, "geom_explicit_default", Types.GeometryType.crs84()), + required(3, "geog_bare", Types.GeographyType.crs84()), + optional(4, "geog_crs_only", Types.GeographyType.of("EPSG:4326")), + optional( + 5, + "geog_algorithm_only", + Types.GeographyType.of(Types.GeographyType.DEFAULT_CRS, EdgeAlgorithm.ANDOYER)), + optional( + 6, + "geog_explicit_spherical", + Types.GeographyType.of("EPSG:4326", EdgeAlgorithm.SPHERICAL))); + + Schema actualSchema = ParquetSchemaUtil.convert(messageType); + assertThat(actualSchema.asStruct()) + .as("Geometry and geography annotations must convert to the expected Iceberg types") + .isEqualTo(expectedSchema.asStruct()); + assertThat(actualSchema.findType("geom_bare").toString()).isEqualTo("geometry(OGC:CRS84)"); + assertThat(actualSchema.findType("geom_explicit_default").toString()) + .isEqualTo("geometry(OGC:CRS84)"); + assertThat(actualSchema.findType("geog_bare").toString()) + .isEqualTo("geography(OGC:CRS84, spherical)"); + assertThat(actualSchema.findType("geog_explicit_spherical").toString()) + .isEqualTo("geography(EPSG:4326, spherical)"); + } + @Test public void testSchemaConversionForHiveStyleLists() { String parquetSchemaString = diff --git a/parquet/src/test/java/org/apache/iceberg/parquet/TestVariantMetrics.java b/parquet/src/test/java/org/apache/iceberg/parquet/TestVariantMetrics.java index b451558a39e8..b7b63d6f6b80 100644 --- a/parquet/src/test/java/org/apache/iceberg/parquet/TestVariantMetrics.java +++ b/parquet/src/test/java/org/apache/iceberg/parquet/TestVariantMetrics.java @@ -26,7 +26,9 @@ import java.util.Map; import java.util.Set; import java.util.UUID; +import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.Metrics; +import org.apache.iceberg.MetricsConfig; import org.apache.iceberg.Schema; import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.data.Record; @@ -34,6 +36,7 @@ import org.apache.iceberg.inmemory.InMemoryOutputFile; import org.apache.iceberg.io.FileAppender; import org.apache.iceberg.io.OutputFile; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.types.Conversions; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; @@ -48,6 +51,10 @@ import org.apache.iceberg.variants.VariantTestUtil; import org.apache.iceberg.variants.VariantValue; import org.apache.iceberg.variants.Variants; +import org.apache.parquet.column.ParquetProperties; +import org.apache.parquet.hadoop.ParquetFileWriter; +import org.apache.parquet.hadoop.metadata.CompressionCodecName; +import org.apache.parquet.schema.MessageType; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.FieldSource; @@ -479,6 +486,46 @@ public void testShreddedObjectFieldTypeMismatch() throws IOException { .isEqualTo(Map.of(1, Types.LongType.get(), 2, Types.VariantType.get())); } + @Test + public void testShreddedValueColumnWithEmptyStats() throws IOException { + // typed bounds are dropped when value-column stats are missing on a shredded variant + OutputFile out = new InMemoryOutputFile(); + GenericRecord record = GenericRecord.create(SCHEMA); + + VariantShreddingFunction shredding = + (id, name) -> ParquetVariantUtil.toParquetSchema(Variants.of((byte) 0)); + MessageType parquetSchema = ParquetSchemaUtil.convert(SCHEMA, "table", shredding); + ParquetProperties props = ParquetProperties.builder().withStatisticsEnabled(false).build(); + + // Parquet.write() cannot disable stats on variant sub-columns (no field IDs) + ParquetWriter writer = + new ParquetWriter<>( + new Configuration(), + out, + SCHEMA, + parquetSchema, + 1024, + ImmutableMap.of(), + (s, m) -> InternalWriter.create(s.asStruct(), m), + CompressionCodecName.SNAPPY, + props, + MetricsConfig.getDefault(), + ParquetFileWriter.Mode.CREATE, + null, + false); + + try (writer) { + record.setField("id", 1L); + record.setField("var", Variant.of(EMPTY, Variants.of((byte) 5))); + writer.add(record); + } + + Metrics metrics = writer.metrics(); + assertThat(metrics.recordCount()).isEqualTo(1L); + assertThat(metrics.lowerBounds()).doesNotContainKey(2); + assertThat(metrics.upperBounds()).doesNotContainKey(2); + } + private Metrics writeParquet(VariantShreddingFunction shredding, Variant... variants) throws IOException { OutputFile out = new InMemoryOutputFile(); diff --git a/site/docs/developer-snapshot-testing.md b/site/docs/developer-snapshot-testing.md new file mode 100644 index 000000000000..6365e66e590b --- /dev/null +++ b/site/docs/developer-snapshot-testing.md @@ -0,0 +1,112 @@ +--- +title: "Developer Snapshot Testing" +--- + + +{# Mirrors getProjectVersion() in build.gradle for released versions: bump minor, reset patch to 0, and append -SNAPSHOT. #} +{% set icebergVersionParts = icebergVersion.split('.') %} +{% set snapshotMinorVersion = (icebergVersionParts[1] | int) + 1 %} +{% set snapshotVersion = icebergVersionParts[0] ~ '.' ~ snapshotMinorVersion ~ '.0-SNAPSHOT' %} + +# Developer Snapshot Testing + +!!! warning "For Iceberg developers only" + + Nightly snapshots are **unreleased** development artifacts. They are **not** + official Apache releases, are not intended for general use, may change or + break at any time, and **must not be used in production**. If you are not + actively participating in Iceberg development or following dev list + discussions, use an official [release](releases.md) instead. + +Every night, Apache Iceberg publishes snapshots of every module from unreleased +changes on `main` to support active Iceberg developers, engine maintainers, and +automated testing. +Per the ASF [release policy](https://apache.org/legal/release-policy#publication), +unreleased artifacts are developer resources for testing ongoing development and +are not a substitute for official releases. + +## Snapshot version + +Snapshots are published daily at 00:00 UTC under the version `{{ snapshotVersion }}`, +which tracks the next unreleased development version by incrementing the latest +release minor version and resetting the patch version to 0. + +## Development-only usage + +Active developers who need to validate unreleased Iceberg changes can use the +[Apache snapshot repository](https://repository.apache.org/content/repositories/snapshots/org/apache/iceberg/) +when a development task or [dev list](community.md#mailing-lists) discussion +calls for testing snapshots. Add snapshot dependencies only to temporary local +test builds; do not commit them to production applications, user documentation, +or release validation workflows. + +The examples below use `iceberg-core` as the module under test. Replace it only +with the Iceberg module needed for the development task you are validating. + +=== "Gradle (development only)" + + ```gradle + repositories { + mavenCentral() + maven { + url = uri("https://repository.apache.org/content/repositories/snapshots") + } + } + + dependencies { + implementation "org.apache.iceberg:iceberg-core:{{ snapshotVersion }}" + } + ``` + +=== "Maven (development only)" + + ```xml + + + apache-snapshots + https://repository.apache.org/content/repositories/snapshots + + true + + + + + + + org.apache.iceberg + iceberg-core + {{ snapshotVersion }} + + + ``` + +=== "sbt (development only)" + + ```scala + resolvers += "Apache Snapshots" at "https://repository.apache.org/content/repositories/snapshots" + + libraryDependencies += "org.apache.iceberg" % "iceberg-core" % "{{ snapshotVersion }}" + ``` + +=== "Spark (development only)" + + ```sh + spark-shell \ + --repositories https://repository.apache.org/content/repositories/snapshots \ + --packages org.apache.iceberg:iceberg-spark-runtime-{{ sparkVersionMajor }}:{{ snapshotVersion }} + ``` diff --git a/site/docs/multi-engine-support.md b/site/docs/multi-engine-support.md index bdfe87123aa3..d153c6104bba 100644 --- a/site/docs/multi-engine-support.md +++ b/site/docs/multi-engine-support.md @@ -135,6 +135,6 @@ This allows the Iceberg support to evolve with the engine. Projects such as [Trino](https://trino.io/docs/current/connector/iceberg.html) and [Presto](https://prestodb.io/docs/current/connector/iceberg.html) are good examples of such support strategy. In this approach, an Iceberg version upgrade is needed for an engine to consume new Iceberg features. -To facilitate engine development against unreleased Iceberg features, a daily snapshot is published in the [Apache snapshot repository](https://repository.apache.org/content/repositories/snapshots/org/apache/iceberg/). +To facilitate engine development against unreleased Iceberg features, [developer snapshot testing](developer-snapshot-testing.md) is available for active developers and engine maintainers testing ongoing development. If bringing an engine directly to the Iceberg main repository is needed, please raise a discussion thread in the [Iceberg community](community.md). diff --git a/site/mkdocs-dev.yml b/site/mkdocs-dev.yml index f8d4d71bf316..1557a73f041a 100644 --- a/site/mkdocs-dev.yml +++ b/site/mkdocs-dev.yml @@ -82,6 +82,7 @@ nav: - Project: - Contributing: contribute.md - Multi-engine support: multi-engine-support.md + - Developer snapshot testing: developer-snapshot-testing.md - Benchmarks: benchmarks.md - Security: security.md - How to release: how-to-release.md diff --git a/site/nav.yml b/site/nav.yml index c5e9fa5ed0ba..a83f66165acc 100644 --- a/site/nav.yml +++ b/site/nav.yml @@ -98,6 +98,7 @@ nav: - Project: - Contributing: contribute.md - Multi-engine support: multi-engine-support.md + - Developer snapshot testing: developer-snapshot-testing.md - Benchmarks: benchmarks.md - Security: security.md - How to release: how-to-release.md diff --git a/site/requirements.txt b/site/requirements.txt index 8b38df3fc236..b33232b8f104 100644 --- a/site/requirements.txt +++ b/site/requirements.txt @@ -23,4 +23,4 @@ mkdocs-monorepo-plugin @ git+https://github.com/bitsondatadev/mkdocs-monorepo-pl mkdocs-redirects==1.2.3 mkdocs-rss-plugin==1.19.0 mkdocs-exclude-search==0.6.6 -pymarkdownlnt==0.9.37 +pymarkdownlnt==0.9.38 diff --git a/spark/v3.5/build.gradle b/spark/v3.5/build.gradle index 68bdb1c21a98..964b8af9a52b 100644 --- a/spark/v3.5/build.gradle +++ b/spark/v3.5/build.gradle @@ -306,6 +306,40 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio archiveClassifier.set(null) } + tasks.register('shadowSourceJar', Jar) { + archiveClassifier.set('sources') + + from(sourceSets.main.allSource) + + from(project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}").file('src/main/java')) + from(project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}").file('src/main/scala')) + from(project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}").file('src/main/java')) + from(project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}").file('src/main/scala')) + from(project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}").file('src/main/antlr')) + + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + } + + def sparkProject = project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}") + def sparkExtensionsProject = project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}") + + javadoc { + dependsOn sparkProject.tasks.named('classes') + dependsOn sparkExtensionsProject.tasks.named('classes') + + source = files( + sourceSets.main.allJava, + sparkProject.fileTree('src/main/java'), + sparkExtensionsProject.fileTree('src/main/java') + ) + + classpath = files( + sourceSets.main.compileClasspath, + sparkProject.sourceSets.main.compileClasspath, + sparkExtensionsProject.sourceSets.main.compileClasspath + ) + } + task integrationTest(type: Test) { useJUnitPlatform() description = "Test Spark3 Runtime Jar against Spark ${sparkMajorVersion}" @@ -322,6 +356,11 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio enabled = false } + tasks.matching { it.name == 'sourceJar' }.configureEach { + enabled = false + dependsOn shadowSourceJar + } + apply from: "${rootDir}/runtime-deps.gradle" } diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java index 72988ae0ed9e..36f28acd9955 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java @@ -34,11 +34,11 @@ import static org.apache.iceberg.TableProperties.FORMAT_VERSION; import static org.apache.iceberg.TableProperties.ORC_VECTORIZATION_ENABLED; import static org.apache.iceberg.TableProperties.PARQUET_VECTORIZATION_ENABLED; -import static org.apache.iceberg.TableProperties.SPARK_WRITE_PARTITIONED_FANOUT_ENABLED; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_HASH; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_NONE; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_RANGE; +import static org.apache.iceberg.spark.SparkTableProperties.WRITE_PARTITIONED_FANOUT_ENABLED; import static org.assertj.core.api.Assertions.assertThat; import java.io.IOException; @@ -172,7 +172,7 @@ protected void initTable() { fileFormat, WRITE_DISTRIBUTION_MODE, distributionMode, - SPARK_WRITE_PARTITIONED_FANOUT_ENABLED, + WRITE_PARTITIONED_FANOUT_ENABLED, String.valueOf(fanoutEnabled), DATA_PLANNING_MODE, planningMode.modeName(), diff --git a/spark/v3.5/spark-runtime/runtime-deps.txt b/spark/v3.5/spark-runtime/runtime-deps.txt index 1fcb7e284d8c..18c17ad30177 100644 --- a/spark/v3.5/spark-runtime/runtime-deps.txt +++ b/spark/v3.5/spark-runtime/runtime-deps.txt @@ -34,7 +34,7 @@ org.eclipse.collections:eclipse-collections-api:11.1 org.eclipse.collections:eclipse-collections:11.1 org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1 org.locationtech.jts:jts-core:1.20 -org.projectnessie.nessie:nessie-client:0.107 -org.projectnessie.nessie:nessie-model:0.107 +org.projectnessie.nessie:nessie-client:0.108 +org.projectnessie.nessie:nessie-model:0.108 org.roaringbitmap:RoaringBitmap:1.6 org.threeten:threeten-extra:1.7 diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkTableProperties.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkTableProperties.java new file mode 100644 index 000000000000..6ca02220a8d8 --- /dev/null +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkTableProperties.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.spark; + +/** Spark-specific table properties that control Spark integration behavior. */ +public class SparkTableProperties { + + private SparkTableProperties() {} + + public static final String WRITE_PARTITIONED_FANOUT_ENABLED = "write.spark.fanout.enabled"; + public static final boolean WRITE_PARTITIONED_FANOUT_ENABLED_DEFAULT = false; + + public static final String WRITE_ACCEPT_ANY_SCHEMA = "write.spark.accept-any-schema"; + public static final boolean WRITE_ACCEPT_ANY_SCHEMA_DEFAULT = false; + + public static final String WRITE_AUTO_SCHEMA_EVOLUTION = + "write.spark.auto-schema-evolution.enabled"; + public static final boolean WRITE_AUTO_SCHEMA_EVOLUTION_DEFAULT = true; + + public static final String WRITE_ADVISORY_PARTITION_SIZE_BYTES = + "write.spark.advisory-partition-size-bytes"; +} diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkValueConverter.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkValueConverter.java index 28b717ac090e..af405bb04d9d 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkValueConverter.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkValueConverter.java @@ -129,7 +129,7 @@ public static Object convertToSpark(Type type, Object object) { case STRUCT: case LIST: case MAP: - return new UnsupportedOperationException("Complex types currently not supported"); + throw new UnsupportedOperationException("Complex types currently not supported"); case DATE: return DateTimeUtils.daysToLocalDate((int) object); case TIMESTAMP: diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java index 9da48ae51e5c..537a14e4ab52 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java @@ -228,7 +228,7 @@ private boolean fanoutWriterEnabled(boolean defaultValue) { return confParser .booleanConf() .option(SparkWriteOptions.FANOUT_ENABLED) - .tableProperty(TableProperties.SPARK_WRITE_PARTITIONED_FANOUT_ENABLED) + .tableProperty(SparkTableProperties.WRITE_PARTITIONED_FANOUT_ENABLED) .defaultValue(defaultValue) .parse(); } @@ -717,7 +717,7 @@ private long advisoryPartitionSize(long defaultValue) { .longConf() .option(SparkWriteOptions.ADVISORY_PARTITION_SIZE) .sessionConf(SparkSQLProperties.ADVISORY_PARTITION_SIZE) - .tableProperty(TableProperties.SPARK_WRITE_ADVISORY_PARTITION_SIZE_BYTES) + .tableProperty(SparkTableProperties.WRITE_ADVISORY_PARTITION_SIZE_BYTES) .defaultValue(defaultValue) .parse(); } diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java index aedb25e4a4a6..11935e815e76 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.io.OutputStreamWriter; import java.nio.charset.StandardCharsets; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; @@ -31,9 +32,13 @@ import org.apache.iceberg.ContentFile; import org.apache.iceberg.DataFile; import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileContent; import org.apache.iceberg.FileFormat; import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.ManifestContent; import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.PartitionStatisticsFile; import org.apache.iceberg.RewriteTablePathUtil; @@ -69,13 +74,14 @@ import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.relocated.com.google.common.collect.Sets; import org.apache.iceberg.spark.JobGroupInfo; import org.apache.iceberg.spark.source.SerializableTableWithSize; import org.apache.iceberg.util.DeleteFileSet; import org.apache.iceberg.util.Pair; import org.apache.iceberg.util.Tasks; -import org.apache.spark.api.java.function.ForeachFunction; +import org.apache.spark.api.java.function.FlatMapFunction; import org.apache.spark.api.java.function.MapFunction; import org.apache.spark.api.java.function.ReduceFunction; import org.apache.spark.broadcast.Broadcast; @@ -87,6 +93,7 @@ import org.apache.spark.sql.functions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import scala.Tuple2; public class RewriteTablePathSparkAction extends BaseSparkAction implements RewriteTablePath { @@ -272,7 +279,8 @@ private String jobDesc() { *
    *
  • Rebuild version files to staging *
  • Rebuild manifest list files to staging - *
  • Rebuild manifest to staging + *
  • Rewrite referenced position delete files to staging + *
  • Rebuild manifests to staging *
  • Get all files needed to move *
*/ @@ -308,24 +316,29 @@ private Result rebuildMetadata() { RewriteResult rewriteManifestListResult = new RewriteResult<>(); manifestListResults.forEach(rewriteManifestListResult::append); - // rebuild manifest files - Set metaFiles = rewriteManifestListResult.toRewrite(); - RewriteContentFileResult rewriteManifestResult = - rewriteManifests(deltaSnapshots, endMetadata, metaFiles); + Set manifestFiles = rewriteManifestListResult.toRewrite(); // rebuild position delete files - Set deleteFiles = - rewriteManifestResult.toRewrite().stream() - .filter(e -> e instanceof DeleteFile) - .map(e -> (DeleteFile) e) - .collect(Collectors.toCollection(DeleteFileSet::create)); - rewritePositionDeletes(deleteFiles); + Set deleteManifests = + manifestFiles.stream() + .filter(manifest -> manifest.content() == ManifestContent.DELETES) + .collect(Collectors.toSet()); + Set deleteFilesToRewrite = positionDeletesToRewrite(deleteManifests); + Map rewrittenDeleteFileSizes = rewritePositionDeletes(deleteFilesToRewrite); + + // rebuild manifest files + RewriteContentFileResult rewriteManifestResult = + rewriteManifests( + deltaSnapshots, + endMetadata, + manifestFiles, + sparkContext().broadcast(rewrittenDeleteFileSizes)); ImmutableRewriteTablePath.Result.Builder builder = ImmutableRewriteTablePath.Result.builder() .stagingLocation(stagingDir) - .rewrittenDeleteFilePathsCount(deleteFiles.size()) - .rewrittenManifestFilePathsCount(metaFiles.size()) + .rewrittenDeleteFilePathsCount(deleteFilesToRewrite.size()) + .rewrittenManifestFilePathsCount(manifestFiles.size()) .latestVersion(RewriteTablePathUtil.fileName(endVersionName)); if (!createFileList) { @@ -561,7 +574,10 @@ public RewriteContentFileResult appendDeleteFile(RewriteResult r1) { /** Rewrite manifest files in a distributed manner and return rewritten data files path pairs. */ private RewriteContentFileResult rewriteManifests( - Set deltaSnapshots, TableMetadata tableMetadata, Set toRewrite) { + Set deltaSnapshots, + TableMetadata tableMetadata, + Set toRewrite, + Broadcast> rewrittenDeleteFileSizes) { if (toRewrite.isEmpty()) { return new RewriteContentFileResult(); } @@ -581,7 +597,8 @@ private RewriteContentFileResult rewriteManifests( stagingDir, tableMetadata.formatVersion(), sourcePrefix, - targetPrefix), + targetPrefix, + rewrittenDeleteFileSizes), Encoders.bean(RewriteContentFileResult.class)) // duplicates are expected here as the same data file can have different statuses // (e.g. added and deleted) @@ -594,7 +611,8 @@ private static MapFunction toManifests( String stagingLocation, int format, String sourcePrefix, - String targetPrefix) { + String targetPrefix, + Broadcast> rewrittenDeleteFileSizes) { return manifestFile -> { RewriteContentFileResult result = new RewriteContentFileResult(); @@ -619,7 +637,8 @@ private static MapFunction toManifests( stagingLocation, format, sourcePrefix, - targetPrefix)); + targetPrefix, + rewrittenDeleteFileSizes)); break; default: throw new UnsupportedOperationException( @@ -665,7 +684,8 @@ private static RewriteResult writeDeleteManifest( String stagingLocation, int format, String sourcePrefix, - String targetPrefix) { + String targetPrefix, + Broadcast> rewrittenDeleteFileSizes) { try { String stagingPath = RewriteTablePathUtil.stagingPath(manifestFile.path(), sourcePrefix, stagingLocation); @@ -682,27 +702,120 @@ private static RewriteResult writeDeleteManifest( specsById, sourcePrefix, targetPrefix, - stagingLocation); + stagingLocation, + rewrittenDeleteFileSizes.value()); } catch (IOException e) { throw new RuntimeIOException(e); } } - private void rewritePositionDeletes(Set toRewrite) { + /** + * Enumerate the distinct position delete files referenced by the given delete manifests. Deduped + * by identity (location, offset, size) so a file shared across manifests is counted once; the + * physical rewrite is further deduped by location in {@link #rewritePositionDeletes}. + */ + private Set positionDeletesToRewrite(Set deleteManifests) { + if (deleteManifests.isEmpty()) { + return Collections.emptySet(); + } + + Encoder manifestFileEncoder = Encoders.javaSerialization(ManifestFile.class); + Dataset manifestDS = + spark().createDataset(Lists.newArrayList(deleteManifests), manifestFileEncoder); + Encoder deleteFileEncoder = Encoders.javaSerialization(DeleteFile.class); + + List referencedDeleteFiles = + manifestDS + .repartition(deleteManifests.size()) + .flatMap(positionDeletesInManifest(tableBroadcast()), deleteFileEncoder) + .collectAsList(); + + return DeleteFileSet.of(referencedDeleteFiles); + } + + private static FlatMapFunction positionDeletesInManifest( + Broadcast tableArg) { + return manifestFile -> { + Table table = tableArg.getValue(); + List deleteFiles = Lists.newArrayList(); + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifestFile, table.io(), table.specs())) { + for (DeleteFile deleteFile : reader) { + if (deleteFile.content() == FileContent.POSITION_DELETES) { + deleteFiles.add(deleteFile.copy()); + } + } + } + return deleteFiles.iterator(); + }; + } + + /** + * Rewrite the given position delete files in parallel, returning a map from each source delete + * file path to the size of its rewritten file. Physical files are deduped by location, so a + * Puffin file holding multiple DVs is rewritten once and its size keyed once. + */ + private Map rewritePositionDeletes(Set toRewrite) { if (toRewrite.isEmpty()) { - return; + return Collections.emptyMap(); + } + + // Multiple DVs can share one Puffin file at different blob offsets; rewrite each physical file + // once. The measured size is keyed by location and applied to every referencing manifest entry. + Map byLocation = Maps.newHashMapWithExpectedSize(toRewrite.size()); + for (DeleteFile deleteFile : toRewrite) { + byLocation.putIfAbsent(deleteFile.location(), deleteFile); } + List physicalFiles = Lists.newArrayList(byLocation.values()); Encoder deleteFileEncoder = Encoders.javaSerialization(DeleteFile.class); - Dataset deleteFileDs = - spark().createDataset(Lists.newArrayList(toRewrite), deleteFileEncoder); + Dataset deleteFileDS = spark().createDataset(physicalFiles, deleteFileEncoder); PositionDeleteReaderWriter posDeleteReaderWriter = new SparkPositionDeleteReaderWriter(); - deleteFileDs - .repartition(toRewrite.size()) - .foreach( - rewritePositionDelete( - tableBroadcast(), sourcePrefix, targetPrefix, stagingDir, posDeleteReaderWriter)); + List> rewrittenSizes = + deleteFileDS + .repartition(physicalFiles.size()) + .map( + rewritePositionDelete( + tableBroadcast(), + sourcePrefix, + targetPrefix, + stagingDir, + posDeleteReaderWriter), + Encoders.tuple(Encoders.STRING(), Encoders.LONG())) + .collectAsList(); + + Map sizesBySourcePath = Maps.newHashMapWithExpectedSize(rewrittenSizes.size()); + for (Tuple2 entry : rewrittenSizes) { + sizesBySourcePath.put(entry._1(), entry._2()); + } + return sizesBySourcePath; + } + + private static MapFunction> rewritePositionDelete( + Broadcast
tableArg, + String sourcePrefixArg, + String targetPrefixArg, + String stagingLocationArg, + PositionDeleteReaderWriter posDeleteReaderWriter) { + return deleteFile -> { + FileIO io = tableArg.getValue().io(); + String newPath = + RewriteTablePathUtil.stagingPath( + deleteFile.location(), sourcePrefixArg, stagingLocationArg); + OutputFile outputFile = io.newOutputFile(newPath); + PartitionSpec spec = tableArg.getValue().specs().get(deleteFile.specId()); + long rewrittenLength = + RewriteTablePathUtil.rewritePositionDelete( + deleteFile, + outputFile, + io, + spec, + sourcePrefixArg, + targetPrefixArg, + posDeleteReaderWriter); + return new Tuple2<>(deleteFile.location(), rewrittenLength); + }; } private static class SparkPositionDeleteReaderWriter implements PositionDeleteReaderWriter { @@ -724,30 +837,6 @@ public PositionDeleteWriter writer( } } - private ForeachFunction rewritePositionDelete( - Broadcast
tableArg, - String sourcePrefixArg, - String targetPrefixArg, - String stagingLocationArg, - PositionDeleteReaderWriter posDeleteReaderWriter) { - return deleteFile -> { - FileIO io = tableArg.getValue().io(); - String newPath = - RewriteTablePathUtil.stagingPath( - deleteFile.location(), sourcePrefixArg, stagingLocationArg); - OutputFile outputFile = io.newOutputFile(newPath); - PartitionSpec spec = tableArg.getValue().specs().get(deleteFile.specId()); - RewriteTablePathUtil.rewritePositionDeleteFile( - deleteFile, - outputFile, - io, - spec, - sourcePrefixArg, - targetPrefixArg, - posDeleteReaderWriter); - }; - } - private static CloseableIterable positionDeletesReader( InputFile inputFile, FileFormat format, PartitionSpec spec) { return FormatModelRegistry.readBuilder(format, Record.class, inputFile) diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java index 1348afff6475..3cb5d1809f95 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java @@ -59,6 +59,7 @@ import org.apache.iceberg.spark.SparkReadOptions; import org.apache.iceberg.spark.SparkSQLProperties; import org.apache.iceberg.spark.SparkSchemaUtil; +import org.apache.iceberg.spark.SparkTableProperties; import org.apache.iceberg.spark.SparkTableUtil; import org.apache.iceberg.spark.SparkUtil; import org.apache.iceberg.spark.SparkV2Filters; @@ -156,8 +157,8 @@ public SparkTable( boolean acceptAnySchema = PropertyUtil.propertyAsBoolean( icebergTable.properties(), - TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA, - TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA_DEFAULT); + SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA, + SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA_DEFAULT); this.capabilities = acceptAnySchema ? CAPABILITIES_WITH_ACCEPT_ANY_SCHEMA : CAPABILITIES; this.isTableRewrite = isTableRewrite; } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestSparkDistributionAndOrderingUtil.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestSparkDistributionAndOrderingUtil.java index ca86350346cd..e1592525f31e 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestSparkDistributionAndOrderingUtil.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestSparkDistributionAndOrderingUtil.java @@ -20,12 +20,12 @@ import static org.apache.iceberg.TableProperties.DELETE_DISTRIBUTION_MODE; import static org.apache.iceberg.TableProperties.MERGE_DISTRIBUTION_MODE; -import static org.apache.iceberg.TableProperties.SPARK_WRITE_PARTITIONED_FANOUT_ENABLED; import static org.apache.iceberg.TableProperties.UPDATE_DISTRIBUTION_MODE; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_HASH; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_NONE; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_RANGE; +import static org.apache.iceberg.spark.SparkTableProperties.WRITE_PARTITIONED_FANOUT_ENABLED; import static org.apache.spark.sql.connector.write.RowLevelOperation.Command.DELETE; import static org.apache.spark.sql.connector.write.RowLevelOperation.Command.MERGE; import static org.apache.spark.sql.connector.write.RowLevelOperation.Command.UPDATE; @@ -3020,10 +3020,10 @@ private void checkPositionDeltaDistributionAndOrdering( } private void disableFanoutWriters(Table table) { - table.updateProperties().set(SPARK_WRITE_PARTITIONED_FANOUT_ENABLED, "false").commit(); + table.updateProperties().set(WRITE_PARTITIONED_FANOUT_ENABLED, "false").commit(); } private void enableFanoutWriters(Table table) { - table.updateProperties().set(SPARK_WRITE_PARTITIONED_FANOUT_ENABLED, "true").commit(); + table.updateProperties().set(WRITE_PARTITIONED_FANOUT_ENABLED, "true").commit(); } } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestSparkValueConverter.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestSparkValueConverter.java index c7a2e6c18fca..3ffc303759db 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestSparkValueConverter.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestSparkValueConverter.java @@ -19,10 +19,13 @@ package org.apache.iceberg.spark; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import java.util.List; import org.apache.iceberg.Schema; import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.data.Record; +import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import org.apache.spark.sql.Row; import org.apache.spark.sql.RowFactory; @@ -83,6 +86,21 @@ public void testSparkNullPrimitiveConvert() { assertCorrectNullConversion(schema); } + @Test + public void testConvertToSparkComplexTypesThrow() { + Types.StructType struct = + Types.StructType.of(Types.NestedField.required(1, "lat", Types.FloatType.get())); + Types.ListType list = Types.ListType.ofOptional(1, Types.StringType.get()); + Types.MapType map = + Types.MapType.ofOptional(1, 2, Types.StringType.get(), Types.StringType.get()); + + for (Type type : List.of(struct, list, map)) { + assertThatThrownBy(() -> SparkValueConverter.convertToSpark(type, "unused")) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining("Complex types currently not supported"); + } + } + private void assertCorrectNullConversion(Schema schema) { Row sparkRow = RowFactory.create(1, null); Record record = GenericRecord.create(schema); diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java index 130078a344a5..d74d8a29f994 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java @@ -41,23 +41,18 @@ import java.io.File; import java.io.IOException; import java.io.UncheckedIOException; -import java.nio.file.Path; import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Random; import java.util.Set; -import java.util.TreeMap; import java.util.UUID; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.LongStream; import java.util.stream.Stream; import org.apache.hadoop.conf.Configuration; -import org.apache.iceberg.AppendFiles; import org.apache.iceberg.BaseTable; import org.apache.iceberg.ContentFile; import org.apache.iceberg.DataFile; @@ -107,12 +102,10 @@ import org.apache.iceberg.io.OutputFile; import org.apache.iceberg.io.OutputFileFactory; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; import org.apache.iceberg.relocated.com.google.common.collect.Iterables; import org.apache.iceberg.relocated.com.google.common.collect.Lists; -import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.relocated.com.google.common.collect.Sets; import org.apache.iceberg.relocated.com.google.common.collect.Streams; import org.apache.iceberg.spark.FileRewriteCoordinator; @@ -136,7 +129,6 @@ import org.apache.spark.sql.catalyst.analysis.NoSuchTableException; import org.apache.spark.sql.internal.SQLConf; import org.apache.spark.sql.types.DataTypes; -import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.TestTemplate; @@ -150,18 +142,6 @@ public class TestRewriteDataFilesAction extends TestBase { @TempDir private File tableDir; private static final int SCALE = 400000; - // Row group size used by createTable(); part of the unpartitioned cache key so a future - // override of this property can't silently hand back cached files of a different shape. - private static final int INPUT_PARQUET_ROW_GROUP_SIZE_BYTES = 20 * 1024; - - // Cache of pre-written input data files keyed by table shape (schema/spec/props are - // fixed per key), so identical large inputs are materialized via Spark only once per JVM - // fork and reused by every test that asks for the same shape. The Spark write of SCALE - // rows dominates these tests; the rewrite under test still runs per test on a fresh table. - @TempDir private static Path inputCacheDir; - private static final Map> INPUT_FILE_CACHE = Maps.newConcurrentMap(); - private static final Map INPUT_CACHE_LOCKS = Maps.newConcurrentMap(); - private static final AtomicInteger INPUT_CACHE_SEQ = new AtomicInteger(); private static final HadoopTables TABLES = new HadoopTables(new Configuration()); private static final Schema SCHEMA = @@ -189,16 +169,6 @@ public static void setupSpark() { spark.conf().set(SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), "false"); } - @AfterAll - public static void clearInputFileCache() { - // inputCacheDir is a static @TempDir that JUnit recreates if the class runs twice in one JVM - // (IDE re-run, forkCount=0). Clear the cache so a second run can't return DataFiles pointing - // into the deleted first-run directory. - INPUT_FILE_CACHE.clear(); - INPUT_CACHE_LOCKS.clear(); - INPUT_CACHE_SEQ.set(0); - } - @BeforeEach public void setupTableLocation() { this.tableLocation = tableDir.toURI().toString(); @@ -2210,9 +2180,6 @@ protected void shouldHaveSnapshots(Table table, int expectedSnapshots) { } protected void shouldHaveNoOrphans(Table table) { - // Cached input files live under the static inputCacheDir, outside table.location(), so - // deleteOrphanFiles (which only scans the table prefix) never sees them by design. Orphan - // coverage therefore does not extend to the shared cached inputs. assertThat( actions() .deleteOrphanFiles(table) @@ -2325,9 +2292,7 @@ protected Table createTable() { Table table = TABLES.create(SCHEMA, spec, options, tableLocation); table .updateProperties() - .set( - TableProperties.PARQUET_ROW_GROUP_SIZE_BYTES, - Integer.toString(INPUT_PARQUET_ROW_GROUP_SIZE_BYTES)) + .set(TableProperties.PARQUET_ROW_GROUP_SIZE_BYTES, Integer.toString(20 * 1024)) .commit(); assertThat(table.currentSnapshot()).as("Table must be empty").isNull(); return table; @@ -2340,21 +2305,8 @@ protected Table createTable() { * @return the created table */ protected Table createTable(int files) { - String key = - String.format( - "unpartitioned|fv=%d|rowGroup=%d|files=%d|rows=%d", - formatVersion, INPUT_PARQUET_ROW_GROUP_SIZE_BYTES, files, SCALE); - List inputFiles = - cachedInputFiles( - key, - () -> { - Table golden = createTable(); - writeRecords(files, SCALE); - golden.refresh(); - return golden; - }); Table table = createTable(); - appendInputFiles(table, inputFiles); + writeRecords(files, SCALE); table.refresh(); return table; } @@ -2362,85 +2314,14 @@ protected Table createTable(int files) { protected Table createTablePartitioned( int partitions, int files, int numRecords, Map options) { PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).identity("c1").truncate("c2", 2).build(); - String key = - String.format( - "partitioned|fv=%d|spec=%s|opts=%s|files=%d|rows=%d|partitions=%d", - formatVersion, spec, new TreeMap<>(options), files, numRecords, partitions); - List inputFiles = - cachedInputFiles( - key, - () -> { - Table golden = TABLES.create(SCHEMA, spec, options, tableLocation); - assertThat(golden.currentSnapshot()).as("Table must be empty").isNull(); - writeRecords(files, numRecords, partitions); - golden.refresh(); - return golden; - }); Table table = TABLES.create(SCHEMA, spec, options, tableLocation); assertThat(table.currentSnapshot()).as("Table must be empty").isNull(); - appendInputFiles(table, inputFiles); + + writeRecords(files, numRecords, partitions); table.refresh(); return table; } - /** - * Returns the input data files for a given table shape, materializing them with Spark exactly - * once per JVM fork and reusing them afterwards. On a cache miss the {@code goldenBuilder} writes - * the data into a stable cache location (kept alive for the whole class via a static {@link - * TempDir}); on a hit the cached {@link DataFile}s are returned and re-appended to a fresh table - * by {@link #appendInputFiles}. The data is deterministic (fixed RNG seed) so reuse is - * byte-identical to regenerating it. - */ - private List cachedInputFiles(String key, Supplier
goldenBuilder) { - List cached = INPUT_FILE_CACHE.get(key); - if (cached != null) { - return cached; - } - // Serialize builds per key: concurrent callers requesting the same table shape block on the - // first build and then reuse its result, instead of materializing identical input twice. The - // heavy Spark write happens outside any map lock, so distinct shapes can still build in - // parallel. - Object lock = INPUT_CACHE_LOCKS.computeIfAbsent(key, ignored -> new Object()); - synchronized (lock) { - List existing = INPUT_FILE_CACHE.get(key); - if (existing != null) { - return existing; - } - String savedLocation = this.tableLocation; - try { - this.tableLocation = - inputCacheDir.resolve("input-" + INPUT_CACHE_SEQ.incrementAndGet()).toUri().toString(); - Table golden = goldenBuilder.get(); - // includeColumnStats() is required: a plain scan drops lower/upper bounds and - // value counts, and re-appending stat-less files breaks tests that read bounds. - // planFiles() returns a CloseableIterable holding manifest readers open, so close it via - // try-with-resources; otherwise every cache miss leaks file descriptors and can leave - // manifest files locked on Windows. - List built; - try (CloseableIterable tasks = - golden.newScan().includeColumnStats().planFiles()) { - built = - Streams.stream(tasks) - .map(FileScanTask::file) - .map(DataFile::copy) - .collect(ImmutableList.toImmutableList()); - } catch (IOException e) { - throw new UncheckedIOException("Failed to plan cached input files", e); - } - INPUT_FILE_CACHE.put(key, built); - return built; - } finally { - this.tableLocation = savedLocation; - } - } - } - - private static void appendInputFiles(Table table, List inputFiles) { - AppendFiles append = table.newAppend(); - inputFiles.forEach(append::appendFile); - append.commit(); - } - protected Table createTablePartitioned(int partitions, int files) { return createTablePartitioned( partitions, diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteTablePathsAction.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteTablePathsAction.java index c5db04762f21..9660beae187e 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteTablePathsAction.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteTablePathsAction.java @@ -30,6 +30,7 @@ import java.nio.file.Path; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.function.Predicate; @@ -41,15 +42,21 @@ import org.apache.iceberg.BaseTable; import org.apache.iceberg.DataFile; import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileFormat; import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; import org.apache.iceberg.Parameter; import org.apache.iceberg.ParameterizedTestExtension; import org.apache.iceberg.Parameters; import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.RowDelta; import org.apache.iceberg.Schema; import org.apache.iceberg.Snapshot; import org.apache.iceberg.SnapshotChanges; import org.apache.iceberg.StaticTableOperations; +import org.apache.iceberg.StructLike; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableProperties; @@ -62,14 +69,18 @@ import org.apache.iceberg.data.FileHelpers; import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.data.Record; +import org.apache.iceberg.deletes.BaseDVFileWriter; +import org.apache.iceberg.deletes.DVFileWriter; import org.apache.iceberg.deletes.PositionDelete; import org.apache.iceberg.hadoop.HadoopTables; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.OutputFile; +import org.apache.iceberg.io.OutputFileFactory; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.Iterables; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; import org.apache.iceberg.spark.SparkCatalog; import org.apache.iceberg.spark.TestBase; import org.apache.iceberg.spark.source.ThreeColumnRecord; @@ -629,12 +640,7 @@ public void testPositionDeletesDeduplication() throws Exception { // in a new manifest, which will cause duplicate DeleteFile objects when processing tableWithPosDeletes.newRowDelta().addDeletes(positionDeletes).commit(); - // This should NOT throw AlreadyExistsException - the fix uses DeleteFileSet to deduplicate - // Without the fix (using Collectors.toSet()), this would fail because: - // 1. Both manifests contain entries for the same delete file - // 2. Processing returns two different DeleteFile objects for the same file - // 3. HashSet doesn't deduplicate them (DeleteFile doesn't override equals()) - // 4. rewritePositionDeletes tries to write the same file twice -> AlreadyExistsException + // This should NOT throw AlreadyExistsException RewriteTablePath.Result result = actions() .rewriteTablePath(tableWithPosDeletes) @@ -642,13 +648,295 @@ public void testPositionDeletesDeduplication() throws Exception { .rewriteLocationPrefix(tableWithPosDeletes.location(), targetTableLocation()) .execute(); - // Verify the rewrite completed successfully - should have rewritten exactly 1 delete file - // (the duplicate should be deduplicated by DeleteFileSet) assertThat(result.rewrittenDeleteFilePathsCount()) .as("Should have rewritten exactly 1 delete file after deduplication") .isEqualTo(1); } + // Regression test: when the same position delete file is referenced from manifests in different + // snapshots, it must be rewritten once and the resulting size stamped consistently into every + // manifest that references it. The delete file is enumerated and deduped by path before the + // rewrite, so its measured size is shared across all referencing delete manifests. + @TestTemplate + public void testSharedDeleteFileSizeAcrossManifests() throws Exception { + assumeThat(formatVersion) + .as("Format versions 3+ use DVs with different validation rules") + .isEqualTo(2); + + Table tableWithPosDeletes = + createTableWithSnapshots( + tableDir.toFile().toURI().toString().concat("tableWithSharedDelete"), + 1, + Map.of(TableProperties.DELETE_DEFAULT_FILE_FORMAT, "parquet")); + + DataFile dataFile = + tableWithPosDeletes + .currentSnapshot() + .addedDataFiles(tableWithPosDeletes.io()) + .iterator() + .next(); + + List> deletes = Lists.newArrayList(Pair.of(dataFile.location(), 0L)); + File deleteFile = + new File( + removePrefix(tableWithPosDeletes.location() + "/data/deeply/nested/deletes.parquet")); + DeleteFile positionDeletes = + FileHelpers.writeDeleteFile( + tableWithPosDeletes, + tableWithPosDeletes.io().newOutputFile(deleteFile.toURI().toString()), + deletes, + formatVersion) + .first(); + + tableWithPosDeletes.newRowDelta().addDeletes(positionDeletes).commit(); + tableWithPosDeletes.newRowDelta().addDeletes(positionDeletes).commit(); + + RewriteTablePath.Result result = + actions() + .rewriteTablePath(tableWithPosDeletes) + .stagingLocation(stagingLocation()) + .rewriteLocationPrefix(tableWithPosDeletes.location(), targetTableLocation()) + .execute(); + copyTableFiles(result); + + Table targetTable = TABLES.load(targetTableLocation()); + List deleteManifests = + targetTable.currentSnapshot().deleteManifests(targetTable.io()); + assertThat(deleteManifests) + .as("Expected the shared delete file to be referenced by multiple manifests") + .hasSizeGreaterThanOrEqualTo(2); + for (ManifestFile manifest : deleteManifests) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, targetTable.io(), targetTable.specs())) { + for (DeleteFile df : reader) { + long manifestSize = df.fileSizeInBytes(); + long actualSize = targetTable.io().newInputFile(df.location()).getLength(); + assertThat(manifestSize) + .as( + "file_size_in_bytes in rewritten manifest should match actual file size for %s", + df.location()) + .isEqualTo(actualSize); + } + } + } + } + + // Regression test: a single delete manifest can reference multiple distinct position delete + // files, and each entry must be stamped with its own rewritten size. The two delete files carry + // a different number of records so they rewrite to different sizes, which catches a per-path size + // map that collapses entries to a single size or falls back to the stale original size. + @TestTemplate + public void testMultipleDistinctDeleteFileSizesAfterRewrite() throws Exception { + assumeThat(formatVersion) + .as("Format versions 3+ use DVs with different validation rules") + .isEqualTo(2); + + Table tableWithPosDeletes = + createTableWithSnapshots( + tableDir.toFile().toURI().toString().concat("tableWithDistinctDeletes"), + 2, + Map.of(TableProperties.DELETE_DEFAULT_FILE_FORMAT, "parquet")); + + List dataFiles = Lists.newArrayList(); + tableWithPosDeletes + .snapshots() + .forEach( + snapshot -> snapshot.addedDataFiles(tableWithPosDeletes.io()).forEach(dataFiles::add)); + assertThat(dataFiles).as("Expected two data files to reference from deletes").hasSize(2); + + // One delete file holds a single record, the other holds two, so they rewrite to distinct + // on-disk sizes. + File smallDeleteFile = + new File( + removePrefix( + tableWithPosDeletes.location() + "/data/deeply/nested/deletes-small.parquet")); + DeleteFile smallDelete = + FileHelpers.writeDeleteFile( + tableWithPosDeletes, + tableWithPosDeletes.io().newOutputFile(smallDeleteFile.toURI().toString()), + Lists.newArrayList(Pair.of(dataFiles.get(0).location(), 0L)), + formatVersion) + .first(); + + File largeDeleteFile = + new File( + removePrefix( + tableWithPosDeletes.location() + "/data/deeply/nested/deletes-large.parquet")); + DeleteFile largeDelete = + FileHelpers.writeDeleteFile( + tableWithPosDeletes, + tableWithPosDeletes.io().newOutputFile(largeDeleteFile.toURI().toString()), + Lists.newArrayList( + Pair.of(dataFiles.get(0).location(), 0L), + Pair.of(dataFiles.get(1).location(), 0L)), + formatVersion) + .first(); + + tableWithPosDeletes.newRowDelta().addDeletes(smallDelete).addDeletes(largeDelete).commit(); + + RewriteTablePath.Result result = + actions() + .rewriteTablePath(tableWithPosDeletes) + .stagingLocation(stagingLocation()) + .rewriteLocationPrefix(tableWithPosDeletes.location(), targetTableLocation()) + .execute(); + copyTableFiles(result); + + Table targetTable = TABLES.load(targetTableLocation()); + List rewrittenSizes = Lists.newArrayList(); + for (ManifestFile manifest : targetTable.currentSnapshot().deleteManifests(targetTable.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, targetTable.io(), targetTable.specs())) { + for (DeleteFile df : reader) { + long manifestSize = df.fileSizeInBytes(); + long actualSize = targetTable.io().newInputFile(df.location()).getLength(); + assertThat(manifestSize) + .as( + "file_size_in_bytes in rewritten manifest should match actual file size for %s", + df.location()) + .isEqualTo(actualSize); + rewrittenSizes.add(manifestSize); + } + } + } + + assertThat(rewrittenSizes) + .as( + "The two distinct delete files should rewrite to distinct, independently recorded sizes") + .hasSize(2) + .doesNotHaveDuplicates(); + } + + // Regression test: rewriting delete file paths changes the file size (since the + // embedded data file paths may differ in length), but file_size_in_bytes in the rewritten + // manifest was not updated. Readers that use file_size_in_bytes to elide a stat() call may + // fail. + @TestTemplate + public void testDeleteFileSizeInBytesAfterRewrite() throws Exception { + List> deletes = + Lists.newArrayList( + Pair.of( + SnapshotChanges.builderFor(table) + .build() + .addedDataFiles() + .iterator() + .next() + .location(), + 0L)); + + File file = new File(removePrefix(table.location() + "/data/deeply/nested/deletes.parquet")); + DeleteFile positionDeletes = + FileHelpers.writeDeleteFile( + table, table.io().newOutputFile(file.toURI().toString()), deletes, formatVersion) + .first(); + table.newRowDelta().addDeletes(positionDeletes).commit(); + + RewriteTablePath.Result result = + actions() + .rewriteTablePath(table) + .stagingLocation(stagingLocation()) + .rewriteLocationPrefix(table.location(), targetTableLocation()) + .execute(); + copyTableFiles(result); + + Table targetTable = TABLES.load(targetTableLocation()); + for (ManifestFile manifest : targetTable.currentSnapshot().deleteManifests(targetTable.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, targetTable.io(), targetTable.specs())) { + for (DeleteFile df : reader) { + long manifestSize = df.fileSizeInBytes(); + long actualSize = targetTable.io().newInputFile(df.location()).getLength(); + assertThat(manifestSize) + .as( + "file_size_in_bytes in rewritten manifest should match actual file size for %s", + df.location()) + .isEqualTo(actualSize); + } + } + } + } + + // Regression test: a single Puffin file can hold multiple DVs (one blob per data file) + // referenced by distinct DeleteFile entries that share the same location. The rewrite must + // rewrite + // the physical Puffin file once (rather than colliding on the staging path) and stamp the + // rewritten size into every referencing manifest entry. + @TestTemplate + public void testSharedPuffinDeleteFileSizeAfterRewrite() throws Exception { + assumeThat(formatVersion) + .as("DVs are introduced in v3; v4 writes parquet manifests the test setup cannot read") + .isEqualTo(3); + + Table tableWithDVs = + createTableWithSnapshots( + tableDir.toFile().toURI().toString().concat("tableWithSharedPuffin"), 2); + + List dataFilePaths = Lists.newArrayList(); + tableWithDVs + .snapshots() + .forEach( + snapshot -> + snapshot + .addedDataFiles(tableWithDVs.io()) + .forEach(dataFile -> dataFilePaths.add(dataFile.location()))); + assertThat(dataFilePaths).as("Expected two data files to back two DVs").hasSize(2); + + List dvs = writeDVsForDataFiles(tableWithDVs, dataFilePaths); + assertThat(dvs) + .as("Both DVs should live in a single Puffin file") + .hasSize(2) + .allSatisfy(dv -> assertThat(dv.location()).isEqualTo(dvs.get(0).location())); + + RowDelta rowDelta = tableWithDVs.newRowDelta(); + dvs.forEach(rowDelta::addDeletes); + rowDelta.commit(); + + RewriteTablePath.Result result = + actions() + .rewriteTablePath(tableWithDVs) + .stagingLocation(stagingLocation()) + .rewriteLocationPrefix(tableWithDVs.location(), targetTableLocation()) + .execute(); + assertThat(result.rewrittenDeleteFilePathsCount()) + .as("Two DVs are two delete files with rewritten paths") + .isEqualTo(2); + copyTableFiles(result); + + Table targetTable = TABLES.load(targetTableLocation()); + Set rewrittenLocations = Sets.newHashSet(); + for (ManifestFile manifest : targetTable.currentSnapshot().deleteManifests(targetTable.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, targetTable.io(), targetTable.specs())) { + for (DeleteFile df : reader) { + rewrittenLocations.add(df.location()); + long actualSize = targetTable.io().newInputFile(df.location()).getLength(); + assertThat(df.fileSizeInBytes()) + .as("file_size_in_bytes should match the rewritten Puffin size for %s", df.location()) + .isEqualTo(actualSize); + } + } + } + assertThat(rewrittenLocations) + .as("Both DVs should point at the single rewritten Puffin file") + .hasSize(1); + } + + // Writes one DV per data file path into a single Puffin file, returning the resulting DeleteFiles + // (which share a location but carry distinct blob offsets). + private List writeDVsForDataFiles(Table targetTable, List dataFilePaths) + throws IOException { + OutputFileFactory fileFactory = + OutputFileFactory.builderFor(targetTable, 1, 1).format(FileFormat.PUFFIN).build(); + DVFileWriter writer = new BaseDVFileWriter(fileFactory, p -> null); + try (DVFileWriter closeableWriter = writer) { + for (String path : dataFilePaths) { + closeableWriter.delete(path, 0L, targetTable.spec(), (StructLike) null); + } + } + + return writer.result().deleteFiles(); + } + @TestTemplate public void testEqualityDeletes() throws Exception { Table sourceTable = createTableWithSnapshots(newTableLocation(), 1); diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWriterV2.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWriterV2.java index 0a51d5b16f50..240d5bcf5bc6 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWriterV2.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWriterV2.java @@ -25,9 +25,9 @@ import java.math.BigDecimal; import java.util.List; import org.apache.iceberg.ParameterizedTestExtension; -import org.apache.iceberg.TableProperties; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.spark.Spark3Util; +import org.apache.iceberg.spark.SparkTableProperties; import org.apache.iceberg.spark.TestBaseWithCatalog; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; @@ -60,7 +60,7 @@ public void removeTables() { public void testMergeSchemaFailsWithoutWriterOption() throws Exception { sql( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA); + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA); Dataset twoColDF = jsonToDF( @@ -119,7 +119,7 @@ public void testMergeSchemaWithoutAcceptAnySchema() throws Exception { public void testMergeSchemaSparkProperty() throws Exception { sql( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA); + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA); Dataset twoColDF = jsonToDF( @@ -153,7 +153,7 @@ public void testMergeSchemaSparkProperty() throws Exception { public void testMergeSchemaIcebergProperty() throws Exception { sql( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA); + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA); Dataset twoColDF = jsonToDF( @@ -190,7 +190,7 @@ public void testWriteWithCaseSensitiveOption() throws NoSuchTableException, Pars .sql( String.format( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA)) + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA)) .collect(); String schema = "ID bigint, DaTa string"; @@ -220,7 +220,7 @@ public void testWriteWithCaseSensitiveOption() throws NoSuchTableException, Pars public void testMergeSchemaSparkConfiguration() throws Exception { sql( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA); + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA); Dataset twoColDF = jsonToDF( "id bigint, data string", @@ -255,7 +255,7 @@ public void testMergeSchemaSparkConfiguration() throws Exception { public void testMergeSchemaIgnoreCastingLongToInt() throws Exception { sql( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA); + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA); Dataset bigintDF = jsonToDF( @@ -297,7 +297,7 @@ public void testMergeSchemaIgnoreCastingDoubleToFloat() throws Exception { sql("CREATE TABLE %s (id double, data string) USING iceberg", tableName); sql( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA); + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA); Dataset doubleDF = jsonToDF( @@ -339,7 +339,7 @@ public void testMergeSchemaIgnoreCastingDecimalToDecimalWithNarrowerPrecision() sql("CREATE TABLE %s (id decimal(6,2), data string) USING iceberg", tableName); sql( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA); + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA); Dataset decimalPrecision6DF = jsonToDF( diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java index 70f3b986d23b..7f2dc287ff5c 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java @@ -18,7 +18,7 @@ */ package org.apache.iceberg.spark.source; -import static org.apache.iceberg.TableProperties.SPARK_WRITE_PARTITIONED_FANOUT_ENABLED; +import static org.apache.iceberg.spark.SparkTableProperties.WRITE_PARTITIONED_FANOUT_ENABLED; import static org.apache.iceberg.types.Types.NestedField.optional; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -624,7 +624,7 @@ public void partitionedCreateWithTargetFileSizeViaOption(IcebergOptionsType opti .save(location.toString()); break; case TABLE: - table.updateProperties().set(SPARK_WRITE_PARTITIONED_FANOUT_ENABLED, "true").commit(); + table.updateProperties().set(WRITE_PARTITIONED_FANOUT_ENABLED, "true").commit(); df.select("id", "data") .write() .format("iceberg") diff --git a/spark/v4.0/build.gradle b/spark/v4.0/build.gradle index 3707e01e4865..fce8483aeb20 100644 --- a/spark/v4.0/build.gradle +++ b/spark/v4.0/build.gradle @@ -30,9 +30,9 @@ configure(sparkProjects) { configurations { all { resolutionStrategy { - force "com.fasterxml.jackson.module:jackson-module-scala_${scalaVersion}:${libs.versions.jackson215.get()}" - force "com.fasterxml.jackson.core:jackson-databind:${libs.versions.jackson215.get()}" - force "com.fasterxml.jackson.core:jackson-core:${libs.versions.jackson215.get()}" + force "com.fasterxml.jackson.module:jackson-module-scala_${scalaVersion}:${libs.versions.jackson218.get()}" + force "com.fasterxml.jackson.core:jackson-databind:${libs.versions.jackson218.get()}" + force "com.fasterxml.jackson.core:jackson-core:${libs.versions.jackson218.get()}" } } } @@ -306,6 +306,40 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio archiveClassifier.set(null) } + tasks.register('shadowSourceJar', Jar) { + archiveClassifier.set('sources') + + from(sourceSets.main.allSource) + + from(project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}").file('src/main/java')) + from(project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}").file('src/main/scala')) + from(project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}").file('src/main/java')) + from(project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}").file('src/main/scala')) + from(project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}").file('src/main/antlr')) + + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + } + + def sparkProject = project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}") + def sparkExtensionsProject = project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}") + + javadoc { + dependsOn sparkProject.tasks.named('classes') + dependsOn sparkExtensionsProject.tasks.named('classes') + + source = files( + sourceSets.main.allJava, + sparkProject.fileTree('src/main/java'), + sparkExtensionsProject.fileTree('src/main/java') + ) + + classpath = files( + sourceSets.main.compileClasspath, + sparkProject.sourceSets.main.compileClasspath, + sparkExtensionsProject.sourceSets.main.compileClasspath + ) + } + task integrationTest(type: Test) { useJUnitPlatform() description = "Test Spark3 Runtime Jar against Spark ${sparkMajorVersion}" @@ -322,6 +356,11 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio enabled = false } + tasks.matching { it.name == 'sourceJar' }.configureEach { + enabled = false + dependsOn shadowSourceJar + } + apply from: "${rootDir}/runtime-deps.gradle" } diff --git a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java index 72988ae0ed9e..36f28acd9955 100644 --- a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java +++ b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java @@ -34,11 +34,11 @@ import static org.apache.iceberg.TableProperties.FORMAT_VERSION; import static org.apache.iceberg.TableProperties.ORC_VECTORIZATION_ENABLED; import static org.apache.iceberg.TableProperties.PARQUET_VECTORIZATION_ENABLED; -import static org.apache.iceberg.TableProperties.SPARK_WRITE_PARTITIONED_FANOUT_ENABLED; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_HASH; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_NONE; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_RANGE; +import static org.apache.iceberg.spark.SparkTableProperties.WRITE_PARTITIONED_FANOUT_ENABLED; import static org.assertj.core.api.Assertions.assertThat; import java.io.IOException; @@ -172,7 +172,7 @@ protected void initTable() { fileFormat, WRITE_DISTRIBUTION_MODE, distributionMode, - SPARK_WRITE_PARTITIONED_FANOUT_ENABLED, + WRITE_PARTITIONED_FANOUT_ENABLED, String.valueOf(fanoutEnabled), DATA_PLANNING_MODE, planningMode.modeName(), diff --git a/spark/v4.0/spark-runtime/runtime-deps.txt b/spark/v4.0/spark-runtime/runtime-deps.txt index 1fcb7e284d8c..97841db98914 100644 --- a/spark/v4.0/spark-runtime/runtime-deps.txt +++ b/spark/v4.0/spark-runtime/runtime-deps.txt @@ -1,6 +1,6 @@ com.fasterxml.jackson.core:jackson-annotations:2.22 -com.fasterxml.jackson.core:jackson-core:2.15 -com.fasterxml.jackson.core:jackson-databind:2.15 +com.fasterxml.jackson.core:jackson-core:2.18 +com.fasterxml.jackson.core:jackson-databind:2.18 com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.22 com.github.ben-manes.caffeine:caffeine:2.9 com.google.errorprone:error_prone_annotations:2.10 @@ -34,7 +34,7 @@ org.eclipse.collections:eclipse-collections-api:11.1 org.eclipse.collections:eclipse-collections:11.1 org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1 org.locationtech.jts:jts-core:1.20 -org.projectnessie.nessie:nessie-client:0.107 -org.projectnessie.nessie:nessie-model:0.107 +org.projectnessie.nessie:nessie-client:0.108 +org.projectnessie.nessie:nessie-model:0.108 org.roaringbitmap:RoaringBitmap:1.6 org.threeten:threeten-extra:1.7 diff --git a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/SparkTableProperties.java b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/SparkTableProperties.java new file mode 100644 index 000000000000..6ca02220a8d8 --- /dev/null +++ b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/SparkTableProperties.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.spark; + +/** Spark-specific table properties that control Spark integration behavior. */ +public class SparkTableProperties { + + private SparkTableProperties() {} + + public static final String WRITE_PARTITIONED_FANOUT_ENABLED = "write.spark.fanout.enabled"; + public static final boolean WRITE_PARTITIONED_FANOUT_ENABLED_DEFAULT = false; + + public static final String WRITE_ACCEPT_ANY_SCHEMA = "write.spark.accept-any-schema"; + public static final boolean WRITE_ACCEPT_ANY_SCHEMA_DEFAULT = false; + + public static final String WRITE_AUTO_SCHEMA_EVOLUTION = + "write.spark.auto-schema-evolution.enabled"; + public static final boolean WRITE_AUTO_SCHEMA_EVOLUTION_DEFAULT = true; + + public static final String WRITE_ADVISORY_PARTITION_SIZE_BYTES = + "write.spark.advisory-partition-size-bytes"; +} diff --git a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/SparkValueConverter.java b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/SparkValueConverter.java index 28b717ac090e..af405bb04d9d 100644 --- a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/SparkValueConverter.java +++ b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/SparkValueConverter.java @@ -129,7 +129,7 @@ public static Object convertToSpark(Type type, Object object) { case STRUCT: case LIST: case MAP: - return new UnsupportedOperationException("Complex types currently not supported"); + throw new UnsupportedOperationException("Complex types currently not supported"); case DATE: return DateTimeUtils.daysToLocalDate((int) object); case TIMESTAMP: diff --git a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java index add12e6040b0..71a4a7878fef 100644 --- a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java +++ b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java @@ -232,7 +232,7 @@ private boolean fanoutWriterEnabled(boolean defaultValue) { return confParser .booleanConf() .option(SparkWriteOptions.FANOUT_ENABLED) - .tableProperty(TableProperties.SPARK_WRITE_PARTITIONED_FANOUT_ENABLED) + .tableProperty(SparkTableProperties.WRITE_PARTITIONED_FANOUT_ENABLED) .defaultValue(defaultValue) .parse(); } @@ -731,7 +731,7 @@ private long advisoryPartitionSize(long defaultValue) { .longConf() .option(SparkWriteOptions.ADVISORY_PARTITION_SIZE) .sessionConf(SparkSQLProperties.ADVISORY_PARTITION_SIZE) - .tableProperty(TableProperties.SPARK_WRITE_ADVISORY_PARTITION_SIZE_BYTES) + .tableProperty(SparkTableProperties.WRITE_ADVISORY_PARTITION_SIZE_BYTES) .defaultValue(defaultValue) .parse(); } diff --git a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java index aedb25e4a4a6..11935e815e76 100644 --- a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java +++ b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.io.OutputStreamWriter; import java.nio.charset.StandardCharsets; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; @@ -31,9 +32,13 @@ import org.apache.iceberg.ContentFile; import org.apache.iceberg.DataFile; import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileContent; import org.apache.iceberg.FileFormat; import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.ManifestContent; import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.PartitionStatisticsFile; import org.apache.iceberg.RewriteTablePathUtil; @@ -69,13 +74,14 @@ import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.relocated.com.google.common.collect.Sets; import org.apache.iceberg.spark.JobGroupInfo; import org.apache.iceberg.spark.source.SerializableTableWithSize; import org.apache.iceberg.util.DeleteFileSet; import org.apache.iceberg.util.Pair; import org.apache.iceberg.util.Tasks; -import org.apache.spark.api.java.function.ForeachFunction; +import org.apache.spark.api.java.function.FlatMapFunction; import org.apache.spark.api.java.function.MapFunction; import org.apache.spark.api.java.function.ReduceFunction; import org.apache.spark.broadcast.Broadcast; @@ -87,6 +93,7 @@ import org.apache.spark.sql.functions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import scala.Tuple2; public class RewriteTablePathSparkAction extends BaseSparkAction implements RewriteTablePath { @@ -272,7 +279,8 @@ private String jobDesc() { *
    *
  • Rebuild version files to staging *
  • Rebuild manifest list files to staging - *
  • Rebuild manifest to staging + *
  • Rewrite referenced position delete files to staging + *
  • Rebuild manifests to staging *
  • Get all files needed to move *
*/ @@ -308,24 +316,29 @@ private Result rebuildMetadata() { RewriteResult rewriteManifestListResult = new RewriteResult<>(); manifestListResults.forEach(rewriteManifestListResult::append); - // rebuild manifest files - Set metaFiles = rewriteManifestListResult.toRewrite(); - RewriteContentFileResult rewriteManifestResult = - rewriteManifests(deltaSnapshots, endMetadata, metaFiles); + Set manifestFiles = rewriteManifestListResult.toRewrite(); // rebuild position delete files - Set deleteFiles = - rewriteManifestResult.toRewrite().stream() - .filter(e -> e instanceof DeleteFile) - .map(e -> (DeleteFile) e) - .collect(Collectors.toCollection(DeleteFileSet::create)); - rewritePositionDeletes(deleteFiles); + Set deleteManifests = + manifestFiles.stream() + .filter(manifest -> manifest.content() == ManifestContent.DELETES) + .collect(Collectors.toSet()); + Set deleteFilesToRewrite = positionDeletesToRewrite(deleteManifests); + Map rewrittenDeleteFileSizes = rewritePositionDeletes(deleteFilesToRewrite); + + // rebuild manifest files + RewriteContentFileResult rewriteManifestResult = + rewriteManifests( + deltaSnapshots, + endMetadata, + manifestFiles, + sparkContext().broadcast(rewrittenDeleteFileSizes)); ImmutableRewriteTablePath.Result.Builder builder = ImmutableRewriteTablePath.Result.builder() .stagingLocation(stagingDir) - .rewrittenDeleteFilePathsCount(deleteFiles.size()) - .rewrittenManifestFilePathsCount(metaFiles.size()) + .rewrittenDeleteFilePathsCount(deleteFilesToRewrite.size()) + .rewrittenManifestFilePathsCount(manifestFiles.size()) .latestVersion(RewriteTablePathUtil.fileName(endVersionName)); if (!createFileList) { @@ -561,7 +574,10 @@ public RewriteContentFileResult appendDeleteFile(RewriteResult r1) { /** Rewrite manifest files in a distributed manner and return rewritten data files path pairs. */ private RewriteContentFileResult rewriteManifests( - Set deltaSnapshots, TableMetadata tableMetadata, Set toRewrite) { + Set deltaSnapshots, + TableMetadata tableMetadata, + Set toRewrite, + Broadcast> rewrittenDeleteFileSizes) { if (toRewrite.isEmpty()) { return new RewriteContentFileResult(); } @@ -581,7 +597,8 @@ private RewriteContentFileResult rewriteManifests( stagingDir, tableMetadata.formatVersion(), sourcePrefix, - targetPrefix), + targetPrefix, + rewrittenDeleteFileSizes), Encoders.bean(RewriteContentFileResult.class)) // duplicates are expected here as the same data file can have different statuses // (e.g. added and deleted) @@ -594,7 +611,8 @@ private static MapFunction toManifests( String stagingLocation, int format, String sourcePrefix, - String targetPrefix) { + String targetPrefix, + Broadcast> rewrittenDeleteFileSizes) { return manifestFile -> { RewriteContentFileResult result = new RewriteContentFileResult(); @@ -619,7 +637,8 @@ private static MapFunction toManifests( stagingLocation, format, sourcePrefix, - targetPrefix)); + targetPrefix, + rewrittenDeleteFileSizes)); break; default: throw new UnsupportedOperationException( @@ -665,7 +684,8 @@ private static RewriteResult writeDeleteManifest( String stagingLocation, int format, String sourcePrefix, - String targetPrefix) { + String targetPrefix, + Broadcast> rewrittenDeleteFileSizes) { try { String stagingPath = RewriteTablePathUtil.stagingPath(manifestFile.path(), sourcePrefix, stagingLocation); @@ -682,27 +702,120 @@ private static RewriteResult writeDeleteManifest( specsById, sourcePrefix, targetPrefix, - stagingLocation); + stagingLocation, + rewrittenDeleteFileSizes.value()); } catch (IOException e) { throw new RuntimeIOException(e); } } - private void rewritePositionDeletes(Set toRewrite) { + /** + * Enumerate the distinct position delete files referenced by the given delete manifests. Deduped + * by identity (location, offset, size) so a file shared across manifests is counted once; the + * physical rewrite is further deduped by location in {@link #rewritePositionDeletes}. + */ + private Set positionDeletesToRewrite(Set deleteManifests) { + if (deleteManifests.isEmpty()) { + return Collections.emptySet(); + } + + Encoder manifestFileEncoder = Encoders.javaSerialization(ManifestFile.class); + Dataset manifestDS = + spark().createDataset(Lists.newArrayList(deleteManifests), manifestFileEncoder); + Encoder deleteFileEncoder = Encoders.javaSerialization(DeleteFile.class); + + List referencedDeleteFiles = + manifestDS + .repartition(deleteManifests.size()) + .flatMap(positionDeletesInManifest(tableBroadcast()), deleteFileEncoder) + .collectAsList(); + + return DeleteFileSet.of(referencedDeleteFiles); + } + + private static FlatMapFunction positionDeletesInManifest( + Broadcast
tableArg) { + return manifestFile -> { + Table table = tableArg.getValue(); + List deleteFiles = Lists.newArrayList(); + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifestFile, table.io(), table.specs())) { + for (DeleteFile deleteFile : reader) { + if (deleteFile.content() == FileContent.POSITION_DELETES) { + deleteFiles.add(deleteFile.copy()); + } + } + } + return deleteFiles.iterator(); + }; + } + + /** + * Rewrite the given position delete files in parallel, returning a map from each source delete + * file path to the size of its rewritten file. Physical files are deduped by location, so a + * Puffin file holding multiple DVs is rewritten once and its size keyed once. + */ + private Map rewritePositionDeletes(Set toRewrite) { if (toRewrite.isEmpty()) { - return; + return Collections.emptyMap(); + } + + // Multiple DVs can share one Puffin file at different blob offsets; rewrite each physical file + // once. The measured size is keyed by location and applied to every referencing manifest entry. + Map byLocation = Maps.newHashMapWithExpectedSize(toRewrite.size()); + for (DeleteFile deleteFile : toRewrite) { + byLocation.putIfAbsent(deleteFile.location(), deleteFile); } + List physicalFiles = Lists.newArrayList(byLocation.values()); Encoder deleteFileEncoder = Encoders.javaSerialization(DeleteFile.class); - Dataset deleteFileDs = - spark().createDataset(Lists.newArrayList(toRewrite), deleteFileEncoder); + Dataset deleteFileDS = spark().createDataset(physicalFiles, deleteFileEncoder); PositionDeleteReaderWriter posDeleteReaderWriter = new SparkPositionDeleteReaderWriter(); - deleteFileDs - .repartition(toRewrite.size()) - .foreach( - rewritePositionDelete( - tableBroadcast(), sourcePrefix, targetPrefix, stagingDir, posDeleteReaderWriter)); + List> rewrittenSizes = + deleteFileDS + .repartition(physicalFiles.size()) + .map( + rewritePositionDelete( + tableBroadcast(), + sourcePrefix, + targetPrefix, + stagingDir, + posDeleteReaderWriter), + Encoders.tuple(Encoders.STRING(), Encoders.LONG())) + .collectAsList(); + + Map sizesBySourcePath = Maps.newHashMapWithExpectedSize(rewrittenSizes.size()); + for (Tuple2 entry : rewrittenSizes) { + sizesBySourcePath.put(entry._1(), entry._2()); + } + return sizesBySourcePath; + } + + private static MapFunction> rewritePositionDelete( + Broadcast
tableArg, + String sourcePrefixArg, + String targetPrefixArg, + String stagingLocationArg, + PositionDeleteReaderWriter posDeleteReaderWriter) { + return deleteFile -> { + FileIO io = tableArg.getValue().io(); + String newPath = + RewriteTablePathUtil.stagingPath( + deleteFile.location(), sourcePrefixArg, stagingLocationArg); + OutputFile outputFile = io.newOutputFile(newPath); + PartitionSpec spec = tableArg.getValue().specs().get(deleteFile.specId()); + long rewrittenLength = + RewriteTablePathUtil.rewritePositionDelete( + deleteFile, + outputFile, + io, + spec, + sourcePrefixArg, + targetPrefixArg, + posDeleteReaderWriter); + return new Tuple2<>(deleteFile.location(), rewrittenLength); + }; } private static class SparkPositionDeleteReaderWriter implements PositionDeleteReaderWriter { @@ -724,30 +837,6 @@ public PositionDeleteWriter writer( } } - private ForeachFunction rewritePositionDelete( - Broadcast
tableArg, - String sourcePrefixArg, - String targetPrefixArg, - String stagingLocationArg, - PositionDeleteReaderWriter posDeleteReaderWriter) { - return deleteFile -> { - FileIO io = tableArg.getValue().io(); - String newPath = - RewriteTablePathUtil.stagingPath( - deleteFile.location(), sourcePrefixArg, stagingLocationArg); - OutputFile outputFile = io.newOutputFile(newPath); - PartitionSpec spec = tableArg.getValue().specs().get(deleteFile.specId()); - RewriteTablePathUtil.rewritePositionDeleteFile( - deleteFile, - outputFile, - io, - spec, - sourcePrefixArg, - targetPrefixArg, - posDeleteReaderWriter); - }; - } - private static CloseableIterable positionDeletesReader( InputFile inputFile, FileFormat format, PartitionSpec spec) { return FormatModelRegistry.readBuilder(format, Record.class, inputFile) diff --git a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java index 9e3c9a7e69e6..6060a2ee5c21 100644 --- a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java +++ b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java @@ -60,6 +60,7 @@ import org.apache.iceberg.spark.SparkReadOptions; import org.apache.iceberg.spark.SparkSQLProperties; import org.apache.iceberg.spark.SparkSchemaUtil; +import org.apache.iceberg.spark.SparkTableProperties; import org.apache.iceberg.spark.SparkTableUtil; import org.apache.iceberg.spark.SparkUtil; import org.apache.iceberg.spark.SparkV2Filters; @@ -157,8 +158,8 @@ public SparkTable( boolean acceptAnySchema = PropertyUtil.propertyAsBoolean( icebergTable.properties(), - TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA, - TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA_DEFAULT); + SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA, + SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA_DEFAULT); this.capabilities = acceptAnySchema ? CAPABILITIES_WITH_ACCEPT_ANY_SCHEMA : CAPABILITIES; this.isTableRewrite = isTableRewrite; } diff --git a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/TestSparkDistributionAndOrderingUtil.java b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/TestSparkDistributionAndOrderingUtil.java index ca86350346cd..e1592525f31e 100644 --- a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/TestSparkDistributionAndOrderingUtil.java +++ b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/TestSparkDistributionAndOrderingUtil.java @@ -20,12 +20,12 @@ import static org.apache.iceberg.TableProperties.DELETE_DISTRIBUTION_MODE; import static org.apache.iceberg.TableProperties.MERGE_DISTRIBUTION_MODE; -import static org.apache.iceberg.TableProperties.SPARK_WRITE_PARTITIONED_FANOUT_ENABLED; import static org.apache.iceberg.TableProperties.UPDATE_DISTRIBUTION_MODE; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_HASH; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_NONE; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_RANGE; +import static org.apache.iceberg.spark.SparkTableProperties.WRITE_PARTITIONED_FANOUT_ENABLED; import static org.apache.spark.sql.connector.write.RowLevelOperation.Command.DELETE; import static org.apache.spark.sql.connector.write.RowLevelOperation.Command.MERGE; import static org.apache.spark.sql.connector.write.RowLevelOperation.Command.UPDATE; @@ -3020,10 +3020,10 @@ private void checkPositionDeltaDistributionAndOrdering( } private void disableFanoutWriters(Table table) { - table.updateProperties().set(SPARK_WRITE_PARTITIONED_FANOUT_ENABLED, "false").commit(); + table.updateProperties().set(WRITE_PARTITIONED_FANOUT_ENABLED, "false").commit(); } private void enableFanoutWriters(Table table) { - table.updateProperties().set(SPARK_WRITE_PARTITIONED_FANOUT_ENABLED, "true").commit(); + table.updateProperties().set(WRITE_PARTITIONED_FANOUT_ENABLED, "true").commit(); } } diff --git a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/TestSparkValueConverter.java b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/TestSparkValueConverter.java index c7a2e6c18fca..3ffc303759db 100644 --- a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/TestSparkValueConverter.java +++ b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/TestSparkValueConverter.java @@ -19,10 +19,13 @@ package org.apache.iceberg.spark; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import java.util.List; import org.apache.iceberg.Schema; import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.data.Record; +import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import org.apache.spark.sql.Row; import org.apache.spark.sql.RowFactory; @@ -83,6 +86,21 @@ public void testSparkNullPrimitiveConvert() { assertCorrectNullConversion(schema); } + @Test + public void testConvertToSparkComplexTypesThrow() { + Types.StructType struct = + Types.StructType.of(Types.NestedField.required(1, "lat", Types.FloatType.get())); + Types.ListType list = Types.ListType.ofOptional(1, Types.StringType.get()); + Types.MapType map = + Types.MapType.ofOptional(1, 2, Types.StringType.get(), Types.StringType.get()); + + for (Type type : List.of(struct, list, map)) { + assertThatThrownBy(() -> SparkValueConverter.convertToSpark(type, "unused")) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining("Complex types currently not supported"); + } + } + private void assertCorrectNullConversion(Schema schema) { Row sparkRow = RowFactory.create(1, null); Record record = GenericRecord.create(schema); diff --git a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java index 06d45436955e..38ddefd26a45 100644 --- a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java +++ b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java @@ -41,24 +41,19 @@ import java.io.File; import java.io.IOException; import java.io.UncheckedIOException; -import java.nio.file.Path; import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Random; import java.util.Set; -import java.util.TreeMap; import java.util.UUID; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.LongStream; import java.util.stream.Stream; import java.util.stream.StreamSupport; import org.apache.hadoop.conf.Configuration; -import org.apache.iceberg.AppendFiles; import org.apache.iceberg.BaseTable; import org.apache.iceberg.ContentFile; import org.apache.iceberg.DataFile; @@ -108,12 +103,10 @@ import org.apache.iceberg.io.OutputFile; import org.apache.iceberg.io.OutputFileFactory; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; import org.apache.iceberg.relocated.com.google.common.collect.Iterables; import org.apache.iceberg.relocated.com.google.common.collect.Lists; -import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.relocated.com.google.common.collect.Sets; import org.apache.iceberg.relocated.com.google.common.collect.Streams; import org.apache.iceberg.spark.FileRewriteCoordinator; @@ -137,7 +130,6 @@ import org.apache.spark.sql.catalyst.analysis.NoSuchTableException; import org.apache.spark.sql.internal.SQLConf; import org.apache.spark.sql.types.DataTypes; -import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.TestTemplate; @@ -151,18 +143,6 @@ public class TestRewriteDataFilesAction extends TestBase { @TempDir private File tableDir; private static final int SCALE = 400000; - // Row group size used by createTable(); part of the unpartitioned cache key so a future - // override of this property can't silently hand back cached files of a different shape. - private static final int INPUT_PARQUET_ROW_GROUP_SIZE_BYTES = 20 * 1024; - - // Cache of pre-written input data files keyed by table shape (schema/spec/props are - // fixed per key), so identical large inputs are materialized via Spark only once per JVM - // fork and reused by every test that asks for the same shape. The Spark write of SCALE - // rows dominates these tests; the rewrite under test still runs per test on a fresh table. - @TempDir private static Path inputCacheDir; - private static final Map> INPUT_FILE_CACHE = Maps.newConcurrentMap(); - private static final Map INPUT_CACHE_LOCKS = Maps.newConcurrentMap(); - private static final AtomicInteger INPUT_CACHE_SEQ = new AtomicInteger(); private static final HadoopTables TABLES = new HadoopTables(new Configuration()); private static final Schema SCHEMA = @@ -190,16 +170,6 @@ public static void setupSpark() { spark.conf().set(SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), "false"); } - @AfterAll - public static void clearInputFileCache() { - // inputCacheDir is a static @TempDir that JUnit recreates if the class runs twice in one JVM - // (IDE re-run, forkCount=0). Clear the cache so a second run can't return DataFiles pointing - // into the deleted first-run directory. - INPUT_FILE_CACHE.clear(); - INPUT_CACHE_LOCKS.clear(); - INPUT_CACHE_SEQ.set(0); - } - @BeforeEach public void setupTableLocation() { this.tableLocation = tableDir.toURI().toString(); @@ -2260,9 +2230,6 @@ protected void shouldHaveSnapshots(Table table, int expectedSnapshots) { } protected void shouldHaveNoOrphans(Table table) { - // Cached input files live under the static inputCacheDir, outside table.location(), so - // deleteOrphanFiles (which only scans the table prefix) never sees them by design. Orphan - // coverage therefore does not extend to the shared cached inputs. assertThat( actions() .deleteOrphanFiles(table) @@ -2375,9 +2342,7 @@ protected Table createTable() { Table table = TABLES.create(SCHEMA, spec, options, tableLocation); table .updateProperties() - .set( - TableProperties.PARQUET_ROW_GROUP_SIZE_BYTES, - Integer.toString(INPUT_PARQUET_ROW_GROUP_SIZE_BYTES)) + .set(TableProperties.PARQUET_ROW_GROUP_SIZE_BYTES, Integer.toString(20 * 1024)) .commit(); assertThat(table.currentSnapshot()).as("Table must be empty").isNull(); return table; @@ -2390,21 +2355,8 @@ protected Table createTable() { * @return the created table */ protected Table createTable(int files) { - String key = - String.format( - "unpartitioned|fv=%d|rowGroup=%d|files=%d|rows=%d", - formatVersion, INPUT_PARQUET_ROW_GROUP_SIZE_BYTES, files, SCALE); - List inputFiles = - cachedInputFiles( - key, - () -> { - Table golden = createTable(); - writeRecords(files, SCALE); - golden.refresh(); - return golden; - }); Table table = createTable(); - appendInputFiles(table, inputFiles); + writeRecords(files, SCALE); table.refresh(); return table; } @@ -2412,85 +2364,14 @@ protected Table createTable(int files) { protected Table createTablePartitioned( int partitions, int files, int numRecords, Map options) { PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).identity("c1").truncate("c2", 2).build(); - String key = - String.format( - "partitioned|fv=%d|spec=%s|opts=%s|files=%d|rows=%d|partitions=%d", - formatVersion, spec, new TreeMap<>(options), files, numRecords, partitions); - List inputFiles = - cachedInputFiles( - key, - () -> { - Table golden = TABLES.create(SCHEMA, spec, options, tableLocation); - assertThat(golden.currentSnapshot()).as("Table must be empty").isNull(); - writeRecords(files, numRecords, partitions); - golden.refresh(); - return golden; - }); Table table = TABLES.create(SCHEMA, spec, options, tableLocation); assertThat(table.currentSnapshot()).as("Table must be empty").isNull(); - appendInputFiles(table, inputFiles); + + writeRecords(files, numRecords, partitions); table.refresh(); return table; } - /** - * Returns the input data files for a given table shape, materializing them with Spark exactly - * once per JVM fork and reusing them afterwards. On a cache miss the {@code goldenBuilder} writes - * the data into a stable cache location (kept alive for the whole class via a static {@link - * TempDir}); on a hit the cached {@link DataFile}s are returned and re-appended to a fresh table - * by {@link #appendInputFiles}. The data is deterministic (fixed RNG seed) so reuse is - * byte-identical to regenerating it. - */ - private List cachedInputFiles(String key, Supplier
goldenBuilder) { - List cached = INPUT_FILE_CACHE.get(key); - if (cached != null) { - return cached; - } - // Serialize builds per key: concurrent callers requesting the same table shape block on the - // first build and then reuse its result, instead of materializing identical input twice. The - // heavy Spark write happens outside any map lock, so distinct shapes can still build in - // parallel. - Object lock = INPUT_CACHE_LOCKS.computeIfAbsent(key, ignored -> new Object()); - synchronized (lock) { - List existing = INPUT_FILE_CACHE.get(key); - if (existing != null) { - return existing; - } - String savedLocation = this.tableLocation; - try { - this.tableLocation = - inputCacheDir.resolve("input-" + INPUT_CACHE_SEQ.incrementAndGet()).toUri().toString(); - Table golden = goldenBuilder.get(); - // includeColumnStats() is required: a plain scan drops lower/upper bounds and - // value counts, and re-appending stat-less files breaks tests that read bounds. - // planFiles() returns a CloseableIterable holding manifest readers open, so close it via - // try-with-resources; otherwise every cache miss leaks file descriptors and can leave - // manifest files locked on Windows. - List built; - try (CloseableIterable tasks = - golden.newScan().includeColumnStats().planFiles()) { - built = - Streams.stream(tasks) - .map(FileScanTask::file) - .map(DataFile::copy) - .collect(ImmutableList.toImmutableList()); - } catch (IOException e) { - throw new UncheckedIOException("Failed to plan cached input files", e); - } - INPUT_FILE_CACHE.put(key, built); - return built; - } finally { - this.tableLocation = savedLocation; - } - } - } - - private static void appendInputFiles(Table table, List inputFiles) { - AppendFiles append = table.newAppend(); - inputFiles.forEach(append::appendFile); - append.commit(); - } - protected Table createTablePartitioned(int partitions, int files) { return createTablePartitioned( partitions, diff --git a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteTablePathsAction.java b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteTablePathsAction.java index c5db04762f21..9660beae187e 100644 --- a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteTablePathsAction.java +++ b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteTablePathsAction.java @@ -30,6 +30,7 @@ import java.nio.file.Path; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.function.Predicate; @@ -41,15 +42,21 @@ import org.apache.iceberg.BaseTable; import org.apache.iceberg.DataFile; import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileFormat; import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; import org.apache.iceberg.Parameter; import org.apache.iceberg.ParameterizedTestExtension; import org.apache.iceberg.Parameters; import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.RowDelta; import org.apache.iceberg.Schema; import org.apache.iceberg.Snapshot; import org.apache.iceberg.SnapshotChanges; import org.apache.iceberg.StaticTableOperations; +import org.apache.iceberg.StructLike; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableProperties; @@ -62,14 +69,18 @@ import org.apache.iceberg.data.FileHelpers; import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.data.Record; +import org.apache.iceberg.deletes.BaseDVFileWriter; +import org.apache.iceberg.deletes.DVFileWriter; import org.apache.iceberg.deletes.PositionDelete; import org.apache.iceberg.hadoop.HadoopTables; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.OutputFile; +import org.apache.iceberg.io.OutputFileFactory; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.Iterables; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; import org.apache.iceberg.spark.SparkCatalog; import org.apache.iceberg.spark.TestBase; import org.apache.iceberg.spark.source.ThreeColumnRecord; @@ -629,12 +640,7 @@ public void testPositionDeletesDeduplication() throws Exception { // in a new manifest, which will cause duplicate DeleteFile objects when processing tableWithPosDeletes.newRowDelta().addDeletes(positionDeletes).commit(); - // This should NOT throw AlreadyExistsException - the fix uses DeleteFileSet to deduplicate - // Without the fix (using Collectors.toSet()), this would fail because: - // 1. Both manifests contain entries for the same delete file - // 2. Processing returns two different DeleteFile objects for the same file - // 3. HashSet doesn't deduplicate them (DeleteFile doesn't override equals()) - // 4. rewritePositionDeletes tries to write the same file twice -> AlreadyExistsException + // This should NOT throw AlreadyExistsException RewriteTablePath.Result result = actions() .rewriteTablePath(tableWithPosDeletes) @@ -642,13 +648,295 @@ public void testPositionDeletesDeduplication() throws Exception { .rewriteLocationPrefix(tableWithPosDeletes.location(), targetTableLocation()) .execute(); - // Verify the rewrite completed successfully - should have rewritten exactly 1 delete file - // (the duplicate should be deduplicated by DeleteFileSet) assertThat(result.rewrittenDeleteFilePathsCount()) .as("Should have rewritten exactly 1 delete file after deduplication") .isEqualTo(1); } + // Regression test: when the same position delete file is referenced from manifests in different + // snapshots, it must be rewritten once and the resulting size stamped consistently into every + // manifest that references it. The delete file is enumerated and deduped by path before the + // rewrite, so its measured size is shared across all referencing delete manifests. + @TestTemplate + public void testSharedDeleteFileSizeAcrossManifests() throws Exception { + assumeThat(formatVersion) + .as("Format versions 3+ use DVs with different validation rules") + .isEqualTo(2); + + Table tableWithPosDeletes = + createTableWithSnapshots( + tableDir.toFile().toURI().toString().concat("tableWithSharedDelete"), + 1, + Map.of(TableProperties.DELETE_DEFAULT_FILE_FORMAT, "parquet")); + + DataFile dataFile = + tableWithPosDeletes + .currentSnapshot() + .addedDataFiles(tableWithPosDeletes.io()) + .iterator() + .next(); + + List> deletes = Lists.newArrayList(Pair.of(dataFile.location(), 0L)); + File deleteFile = + new File( + removePrefix(tableWithPosDeletes.location() + "/data/deeply/nested/deletes.parquet")); + DeleteFile positionDeletes = + FileHelpers.writeDeleteFile( + tableWithPosDeletes, + tableWithPosDeletes.io().newOutputFile(deleteFile.toURI().toString()), + deletes, + formatVersion) + .first(); + + tableWithPosDeletes.newRowDelta().addDeletes(positionDeletes).commit(); + tableWithPosDeletes.newRowDelta().addDeletes(positionDeletes).commit(); + + RewriteTablePath.Result result = + actions() + .rewriteTablePath(tableWithPosDeletes) + .stagingLocation(stagingLocation()) + .rewriteLocationPrefix(tableWithPosDeletes.location(), targetTableLocation()) + .execute(); + copyTableFiles(result); + + Table targetTable = TABLES.load(targetTableLocation()); + List deleteManifests = + targetTable.currentSnapshot().deleteManifests(targetTable.io()); + assertThat(deleteManifests) + .as("Expected the shared delete file to be referenced by multiple manifests") + .hasSizeGreaterThanOrEqualTo(2); + for (ManifestFile manifest : deleteManifests) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, targetTable.io(), targetTable.specs())) { + for (DeleteFile df : reader) { + long manifestSize = df.fileSizeInBytes(); + long actualSize = targetTable.io().newInputFile(df.location()).getLength(); + assertThat(manifestSize) + .as( + "file_size_in_bytes in rewritten manifest should match actual file size for %s", + df.location()) + .isEqualTo(actualSize); + } + } + } + } + + // Regression test: a single delete manifest can reference multiple distinct position delete + // files, and each entry must be stamped with its own rewritten size. The two delete files carry + // a different number of records so they rewrite to different sizes, which catches a per-path size + // map that collapses entries to a single size or falls back to the stale original size. + @TestTemplate + public void testMultipleDistinctDeleteFileSizesAfterRewrite() throws Exception { + assumeThat(formatVersion) + .as("Format versions 3+ use DVs with different validation rules") + .isEqualTo(2); + + Table tableWithPosDeletes = + createTableWithSnapshots( + tableDir.toFile().toURI().toString().concat("tableWithDistinctDeletes"), + 2, + Map.of(TableProperties.DELETE_DEFAULT_FILE_FORMAT, "parquet")); + + List dataFiles = Lists.newArrayList(); + tableWithPosDeletes + .snapshots() + .forEach( + snapshot -> snapshot.addedDataFiles(tableWithPosDeletes.io()).forEach(dataFiles::add)); + assertThat(dataFiles).as("Expected two data files to reference from deletes").hasSize(2); + + // One delete file holds a single record, the other holds two, so they rewrite to distinct + // on-disk sizes. + File smallDeleteFile = + new File( + removePrefix( + tableWithPosDeletes.location() + "/data/deeply/nested/deletes-small.parquet")); + DeleteFile smallDelete = + FileHelpers.writeDeleteFile( + tableWithPosDeletes, + tableWithPosDeletes.io().newOutputFile(smallDeleteFile.toURI().toString()), + Lists.newArrayList(Pair.of(dataFiles.get(0).location(), 0L)), + formatVersion) + .first(); + + File largeDeleteFile = + new File( + removePrefix( + tableWithPosDeletes.location() + "/data/deeply/nested/deletes-large.parquet")); + DeleteFile largeDelete = + FileHelpers.writeDeleteFile( + tableWithPosDeletes, + tableWithPosDeletes.io().newOutputFile(largeDeleteFile.toURI().toString()), + Lists.newArrayList( + Pair.of(dataFiles.get(0).location(), 0L), + Pair.of(dataFiles.get(1).location(), 0L)), + formatVersion) + .first(); + + tableWithPosDeletes.newRowDelta().addDeletes(smallDelete).addDeletes(largeDelete).commit(); + + RewriteTablePath.Result result = + actions() + .rewriteTablePath(tableWithPosDeletes) + .stagingLocation(stagingLocation()) + .rewriteLocationPrefix(tableWithPosDeletes.location(), targetTableLocation()) + .execute(); + copyTableFiles(result); + + Table targetTable = TABLES.load(targetTableLocation()); + List rewrittenSizes = Lists.newArrayList(); + for (ManifestFile manifest : targetTable.currentSnapshot().deleteManifests(targetTable.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, targetTable.io(), targetTable.specs())) { + for (DeleteFile df : reader) { + long manifestSize = df.fileSizeInBytes(); + long actualSize = targetTable.io().newInputFile(df.location()).getLength(); + assertThat(manifestSize) + .as( + "file_size_in_bytes in rewritten manifest should match actual file size for %s", + df.location()) + .isEqualTo(actualSize); + rewrittenSizes.add(manifestSize); + } + } + } + + assertThat(rewrittenSizes) + .as( + "The two distinct delete files should rewrite to distinct, independently recorded sizes") + .hasSize(2) + .doesNotHaveDuplicates(); + } + + // Regression test: rewriting delete file paths changes the file size (since the + // embedded data file paths may differ in length), but file_size_in_bytes in the rewritten + // manifest was not updated. Readers that use file_size_in_bytes to elide a stat() call may + // fail. + @TestTemplate + public void testDeleteFileSizeInBytesAfterRewrite() throws Exception { + List> deletes = + Lists.newArrayList( + Pair.of( + SnapshotChanges.builderFor(table) + .build() + .addedDataFiles() + .iterator() + .next() + .location(), + 0L)); + + File file = new File(removePrefix(table.location() + "/data/deeply/nested/deletes.parquet")); + DeleteFile positionDeletes = + FileHelpers.writeDeleteFile( + table, table.io().newOutputFile(file.toURI().toString()), deletes, formatVersion) + .first(); + table.newRowDelta().addDeletes(positionDeletes).commit(); + + RewriteTablePath.Result result = + actions() + .rewriteTablePath(table) + .stagingLocation(stagingLocation()) + .rewriteLocationPrefix(table.location(), targetTableLocation()) + .execute(); + copyTableFiles(result); + + Table targetTable = TABLES.load(targetTableLocation()); + for (ManifestFile manifest : targetTable.currentSnapshot().deleteManifests(targetTable.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, targetTable.io(), targetTable.specs())) { + for (DeleteFile df : reader) { + long manifestSize = df.fileSizeInBytes(); + long actualSize = targetTable.io().newInputFile(df.location()).getLength(); + assertThat(manifestSize) + .as( + "file_size_in_bytes in rewritten manifest should match actual file size for %s", + df.location()) + .isEqualTo(actualSize); + } + } + } + } + + // Regression test: a single Puffin file can hold multiple DVs (one blob per data file) + // referenced by distinct DeleteFile entries that share the same location. The rewrite must + // rewrite + // the physical Puffin file once (rather than colliding on the staging path) and stamp the + // rewritten size into every referencing manifest entry. + @TestTemplate + public void testSharedPuffinDeleteFileSizeAfterRewrite() throws Exception { + assumeThat(formatVersion) + .as("DVs are introduced in v3; v4 writes parquet manifests the test setup cannot read") + .isEqualTo(3); + + Table tableWithDVs = + createTableWithSnapshots( + tableDir.toFile().toURI().toString().concat("tableWithSharedPuffin"), 2); + + List dataFilePaths = Lists.newArrayList(); + tableWithDVs + .snapshots() + .forEach( + snapshot -> + snapshot + .addedDataFiles(tableWithDVs.io()) + .forEach(dataFile -> dataFilePaths.add(dataFile.location()))); + assertThat(dataFilePaths).as("Expected two data files to back two DVs").hasSize(2); + + List dvs = writeDVsForDataFiles(tableWithDVs, dataFilePaths); + assertThat(dvs) + .as("Both DVs should live in a single Puffin file") + .hasSize(2) + .allSatisfy(dv -> assertThat(dv.location()).isEqualTo(dvs.get(0).location())); + + RowDelta rowDelta = tableWithDVs.newRowDelta(); + dvs.forEach(rowDelta::addDeletes); + rowDelta.commit(); + + RewriteTablePath.Result result = + actions() + .rewriteTablePath(tableWithDVs) + .stagingLocation(stagingLocation()) + .rewriteLocationPrefix(tableWithDVs.location(), targetTableLocation()) + .execute(); + assertThat(result.rewrittenDeleteFilePathsCount()) + .as("Two DVs are two delete files with rewritten paths") + .isEqualTo(2); + copyTableFiles(result); + + Table targetTable = TABLES.load(targetTableLocation()); + Set rewrittenLocations = Sets.newHashSet(); + for (ManifestFile manifest : targetTable.currentSnapshot().deleteManifests(targetTable.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, targetTable.io(), targetTable.specs())) { + for (DeleteFile df : reader) { + rewrittenLocations.add(df.location()); + long actualSize = targetTable.io().newInputFile(df.location()).getLength(); + assertThat(df.fileSizeInBytes()) + .as("file_size_in_bytes should match the rewritten Puffin size for %s", df.location()) + .isEqualTo(actualSize); + } + } + } + assertThat(rewrittenLocations) + .as("Both DVs should point at the single rewritten Puffin file") + .hasSize(1); + } + + // Writes one DV per data file path into a single Puffin file, returning the resulting DeleteFiles + // (which share a location but carry distinct blob offsets). + private List writeDVsForDataFiles(Table targetTable, List dataFilePaths) + throws IOException { + OutputFileFactory fileFactory = + OutputFileFactory.builderFor(targetTable, 1, 1).format(FileFormat.PUFFIN).build(); + DVFileWriter writer = new BaseDVFileWriter(fileFactory, p -> null); + try (DVFileWriter closeableWriter = writer) { + for (String path : dataFilePaths) { + closeableWriter.delete(path, 0L, targetTable.spec(), (StructLike) null); + } + } + + return writer.result().deleteFiles(); + } + @TestTemplate public void testEqualityDeletes() throws Exception { Table sourceTable = createTableWithSnapshots(newTableLocation(), 1); diff --git a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWriterV2.java b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWriterV2.java index 186d0b2b5204..c1adc405b89e 100644 --- a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWriterV2.java +++ b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWriterV2.java @@ -25,10 +25,10 @@ import java.math.BigDecimal; import java.util.List; import org.apache.iceberg.ParameterizedTestExtension; -import org.apache.iceberg.TableProperties; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.spark.Spark3Util; +import org.apache.iceberg.spark.SparkTableProperties; import org.apache.iceberg.spark.TestBaseWithCatalog; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; @@ -61,7 +61,7 @@ public void removeTables() { public void testMergeSchemaFailsWithoutWriterOption() throws Exception { sql( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA); + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA); Dataset twoColDF = jsonToDF( @@ -120,7 +120,7 @@ public void testMergeSchemaWithoutAcceptAnySchema() throws Exception { public void testMergeSchemaSparkProperty() throws Exception { sql( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA); + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA); Dataset twoColDF = jsonToDF( @@ -154,7 +154,7 @@ public void testMergeSchemaSparkProperty() throws Exception { public void testMergeSchemaIcebergProperty() throws Exception { sql( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA); + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA); Dataset twoColDF = jsonToDF( @@ -196,7 +196,7 @@ public void testWriteWithCaseSensitiveOption() throws NoSuchTableException, Pars .sql( String.format( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA)) + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA)) .collect(); String schema = "ID bigint, DaTa string"; @@ -226,7 +226,7 @@ public void testWriteWithCaseSensitiveOption() throws NoSuchTableException, Pars public void testMergeSchemaSparkConfiguration() throws Exception { sql( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA); + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA); Dataset twoColDF = jsonToDF( "id bigint, data string", @@ -261,7 +261,7 @@ public void testMergeSchemaSparkConfiguration() throws Exception { public void testMergeSchemaIgnoreCastingLongToInt() throws Exception { sql( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA); + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA); Dataset bigintDF = jsonToDF( @@ -303,7 +303,7 @@ public void testMergeSchemaIgnoreCastingDoubleToFloat() throws Exception { sql("CREATE TABLE %s (id double, data string) USING iceberg", tableName); sql( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA); + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA); Dataset doubleDF = jsonToDF( @@ -345,7 +345,7 @@ public void testMergeSchemaIgnoreCastingDecimalToDecimalWithNarrowerPrecision() sql("CREATE TABLE %s (id decimal(6,2), data string) USING iceberg", tableName); sql( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA); + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA); Dataset decimalPrecision6DF = jsonToDF( diff --git a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java index 70f3b986d23b..7f2dc287ff5c 100644 --- a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java +++ b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java @@ -18,7 +18,7 @@ */ package org.apache.iceberg.spark.source; -import static org.apache.iceberg.TableProperties.SPARK_WRITE_PARTITIONED_FANOUT_ENABLED; +import static org.apache.iceberg.spark.SparkTableProperties.WRITE_PARTITIONED_FANOUT_ENABLED; import static org.apache.iceberg.types.Types.NestedField.optional; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -624,7 +624,7 @@ public void partitionedCreateWithTargetFileSizeViaOption(IcebergOptionsType opti .save(location.toString()); break; case TABLE: - table.updateProperties().set(SPARK_WRITE_PARTITIONED_FANOUT_ENABLED, "true").commit(); + table.updateProperties().set(WRITE_PARTITIONED_FANOUT_ENABLED, "true").commit(); df.select("id", "data") .write() .format("iceberg") diff --git a/spark/v4.1/build.gradle b/spark/v4.1/build.gradle index e6455fa34f88..9b6cac429efe 100644 --- a/spark/v4.1/build.gradle +++ b/spark/v4.1/build.gradle @@ -30,9 +30,9 @@ configure(sparkProjects) { configurations { all { resolutionStrategy { - force "com.fasterxml.jackson.module:jackson-module-scala_${scalaVersion}:${libs.versions.jackson215.get()}" - force "com.fasterxml.jackson.core:jackson-databind:${libs.versions.jackson215.get()}" - force "com.fasterxml.jackson.core:jackson-core:${libs.versions.jackson215.get()}" + force "com.fasterxml.jackson.module:jackson-module-scala_${scalaVersion}:${libs.versions.jackson221.get()}" + force "com.fasterxml.jackson.core:jackson-databind:${libs.versions.jackson221.get()}" + force "com.fasterxml.jackson.core:jackson-core:${libs.versions.jackson221.get()}" } } } @@ -306,6 +306,40 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio archiveClassifier.set(null) } + tasks.register('shadowSourceJar', Jar) { + archiveClassifier.set('sources') + + from(sourceSets.main.allSource) + + from(project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}").file('src/main/java')) + from(project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}").file('src/main/scala')) + from(project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}").file('src/main/java')) + from(project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}").file('src/main/scala')) + from(project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}").file('src/main/antlr')) + + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + } + + def sparkProject = project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}") + def sparkExtensionsProject = project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}") + + javadoc { + dependsOn sparkProject.tasks.named('classes') + dependsOn sparkExtensionsProject.tasks.named('classes') + + source = files( + sourceSets.main.allJava, + sparkProject.fileTree('src/main/java'), + sparkExtensionsProject.fileTree('src/main/java') + ) + + classpath = files( + sourceSets.main.compileClasspath, + sparkProject.sourceSets.main.compileClasspath, + sparkExtensionsProject.sourceSets.main.compileClasspath + ) + } + task integrationTest(type: Test) { useJUnitPlatform() description = "Test Spark3 Runtime Jar against Spark ${sparkMajorVersion}" @@ -322,6 +356,11 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio enabled = false } + tasks.matching { it.name == 'sourceJar' }.configureEach { + enabled = false + dependsOn shadowSourceJar + } + apply from: "${rootDir}/runtime-deps.gradle" } diff --git a/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java b/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java index 72988ae0ed9e..36f28acd9955 100644 --- a/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java +++ b/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java @@ -34,11 +34,11 @@ import static org.apache.iceberg.TableProperties.FORMAT_VERSION; import static org.apache.iceberg.TableProperties.ORC_VECTORIZATION_ENABLED; import static org.apache.iceberg.TableProperties.PARQUET_VECTORIZATION_ENABLED; -import static org.apache.iceberg.TableProperties.SPARK_WRITE_PARTITIONED_FANOUT_ENABLED; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_HASH; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_NONE; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_RANGE; +import static org.apache.iceberg.spark.SparkTableProperties.WRITE_PARTITIONED_FANOUT_ENABLED; import static org.assertj.core.api.Assertions.assertThat; import java.io.IOException; @@ -172,7 +172,7 @@ protected void initTable() { fileFormat, WRITE_DISTRIBUTION_MODE, distributionMode, - SPARK_WRITE_PARTITIONED_FANOUT_ENABLED, + WRITE_PARTITIONED_FANOUT_ENABLED, String.valueOf(fanoutEnabled), DATA_PLANNING_MODE, planningMode.modeName(), diff --git a/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMergeSchemaEvolution.java b/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMergeSchemaEvolution.java index 782321b588a7..fa8314ecaea4 100644 --- a/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMergeSchemaEvolution.java +++ b/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMergeSchemaEvolution.java @@ -24,9 +24,9 @@ import java.util.Map; import org.apache.iceberg.ParameterizedTestExtension; -import org.apache.iceberg.TableProperties; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.spark.SparkTableProperties; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.TestTemplate; @@ -272,7 +272,7 @@ public void testMergeWithSchemaEvolutionDisabledByTableProperty() { sql( "ALTER TABLE %s SET TBLPROPERTIES ('%s' = 'false')", - tableName, TableProperties.SPARK_WRITE_AUTO_SCHEMA_EVOLUTION); + tableName, SparkTableProperties.WRITE_AUTO_SCHEMA_EVOLUTION); createOrReplaceView( "source", diff --git a/spark/v4.1/spark-runtime/runtime-deps.txt b/spark/v4.1/spark-runtime/runtime-deps.txt index 1fcb7e284d8c..db0e53357807 100644 --- a/spark/v4.1/spark-runtime/runtime-deps.txt +++ b/spark/v4.1/spark-runtime/runtime-deps.txt @@ -1,6 +1,6 @@ com.fasterxml.jackson.core:jackson-annotations:2.22 -com.fasterxml.jackson.core:jackson-core:2.15 -com.fasterxml.jackson.core:jackson-databind:2.15 +com.fasterxml.jackson.core:jackson-core:2.21 +com.fasterxml.jackson.core:jackson-databind:2.21 com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.22 com.github.ben-manes.caffeine:caffeine:2.9 com.google.errorprone:error_prone_annotations:2.10 @@ -34,7 +34,7 @@ org.eclipse.collections:eclipse-collections-api:11.1 org.eclipse.collections:eclipse-collections:11.1 org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1 org.locationtech.jts:jts-core:1.20 -org.projectnessie.nessie:nessie-client:0.107 -org.projectnessie.nessie:nessie-model:0.107 +org.projectnessie.nessie:nessie-client:0.108 +org.projectnessie.nessie:nessie-model:0.108 org.roaringbitmap:RoaringBitmap:1.6 org.threeten:threeten-extra:1.7 diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/PruneColumnsWithoutReordering.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/PruneColumnsWithoutReordering.java index f4323f1c0350..90fa68594ade 100644 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/PruneColumnsWithoutReordering.java +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/PruneColumnsWithoutReordering.java @@ -37,7 +37,10 @@ import org.apache.spark.sql.types.DateType$; import org.apache.spark.sql.types.DecimalType; import org.apache.spark.sql.types.DoubleType$; +import org.apache.spark.sql.types.EdgeInterpolationAlgorithm; import org.apache.spark.sql.types.FloatType$; +import org.apache.spark.sql.types.GeographyType; +import org.apache.spark.sql.types.GeometryType; import org.apache.spark.sql.types.IntegerType$; import org.apache.spark.sql.types.LongType$; import org.apache.spark.sql.types.MapType; @@ -224,6 +227,33 @@ public Type primitive(Type.PrimitiveType primitive) { requestedDecimal.precision(), decimal.precision()); break; + case GEOMETRY: + Types.GeometryType geometry = (Types.GeometryType) primitive; + GeometryType requestedGeometry = (GeometryType) current; + Preconditions.checkArgument( + geometry.crs().equalsIgnoreCase(requestedGeometry.crs()), + "Cannot project geometry with incompatible CRS: %s != %s", + requestedGeometry.crs(), + geometry.crs()); + break; + case GEOGRAPHY: + Types.GeographyType geography = (Types.GeographyType) primitive; + GeographyType requestedGeography = (GeographyType) current; + Preconditions.checkArgument( + geography.crs().equalsIgnoreCase(requestedGeography.crs()), + "Cannot project geography with incompatible CRS: %s != %s", + requestedGeography.crs(), + geography.crs()); + // algorithm() is EdgeAlgorithm on Iceberg and EdgeInterpolationAlgorithm on Spark, so + // translate the table's algorithm into Spark's type and compare within one type system. + EdgeInterpolationAlgorithm tableAlgorithm = + TypeToSparkType.convertAlgorithm(geography.algorithm()); + Preconditions.checkArgument( + tableAlgorithm == requestedGeography.algorithm(), + "Cannot project geography with incompatible edge algorithm: %s != %s", + requestedGeography.algorithm(), + tableAlgorithm); + break; default: } @@ -244,6 +274,8 @@ public Type primitive(Type.PrimitiveType primitive) { .put(TypeID.STRING, ImmutableSet.of(StringType$.class)) .put(TypeID.FIXED, ImmutableSet.of(BinaryType$.class)) .put(TypeID.BINARY, ImmutableSet.of(BinaryType$.class)) + .put(TypeID.GEOMETRY, ImmutableSet.of(GeometryType.class)) + .put(TypeID.GEOGRAPHY, ImmutableSet.of(GeographyType.class)) .put(TypeID.UNKNOWN, ImmutableSet.of(NullType$.class)) .buildOrThrow(); } diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkTableProperties.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkTableProperties.java new file mode 100644 index 000000000000..6ca02220a8d8 --- /dev/null +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkTableProperties.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.spark; + +/** Spark-specific table properties that control Spark integration behavior. */ +public class SparkTableProperties { + + private SparkTableProperties() {} + + public static final String WRITE_PARTITIONED_FANOUT_ENABLED = "write.spark.fanout.enabled"; + public static final boolean WRITE_PARTITIONED_FANOUT_ENABLED_DEFAULT = false; + + public static final String WRITE_ACCEPT_ANY_SCHEMA = "write.spark.accept-any-schema"; + public static final boolean WRITE_ACCEPT_ANY_SCHEMA_DEFAULT = false; + + public static final String WRITE_AUTO_SCHEMA_EVOLUTION = + "write.spark.auto-schema-evolution.enabled"; + public static final boolean WRITE_AUTO_SCHEMA_EVOLUTION_DEFAULT = true; + + public static final String WRITE_ADVISORY_PARTITION_SIZE_BYTES = + "write.spark.advisory-partition-size-bytes"; +} diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkTypeToType.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkTypeToType.java index 54ad899ade77..6ce085a1b7a1 100644 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkTypeToType.java +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkTypeToType.java @@ -19,7 +19,9 @@ package org.apache.iceberg.spark; import java.util.List; +import java.util.Locale; import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.types.EdgeAlgorithm; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import org.apache.spark.sql.types.ArrayType; @@ -31,7 +33,10 @@ import org.apache.spark.sql.types.DateType; import org.apache.spark.sql.types.DecimalType; import org.apache.spark.sql.types.DoubleType; +import org.apache.spark.sql.types.EdgeInterpolationAlgorithm; import org.apache.spark.sql.types.FloatType; +import org.apache.spark.sql.types.GeographyType; +import org.apache.spark.sql.types.GeometryType; import org.apache.spark.sql.types.IntegerType; import org.apache.spark.sql.types.LongType; import org.apache.spark.sql.types.MapType; @@ -162,10 +167,38 @@ public Type atomic(DataType atomic) { ((DecimalType) atomic).precision(), ((DecimalType) atomic).scale()); } else if (atomic instanceof BinaryType) { return Types.BinaryType.get(); + } else if (atomic instanceof GeometryType) { + GeometryType geometry = (GeometryType) atomic; + if (geometry.isMixedSrid()) { + throw new UnsupportedOperationException( + "Cannot convert Spark geometry with mixed SRID to Iceberg"); + } + return Types.GeometryType.of(geometry.crs()); + } else if (atomic instanceof GeographyType) { + GeographyType geography = (GeographyType) atomic; + if (geography.isMixedSrid()) { + throw new UnsupportedOperationException( + "Cannot convert Spark geography with mixed SRID to Iceberg"); + } + return Types.GeographyType.of(geography.crs(), convertAlgorithm(geography.algorithm())); } else if (atomic instanceof NullType) { return Types.UnknownType.get(); } throw new UnsupportedOperationException("Not a supported type: " + atomic.catalogString()); } + + // Translates Spark's edge-interpolation algorithm to Iceberg's, mirroring + // TypeToSparkType#convertAlgorithm. Spark supports only the spherical algorithm today; anything + // else is rejected loudly rather than silently defaulting, so a new Spark algorithm surfaces here + // instead of being dropped. + private static EdgeAlgorithm convertAlgorithm(EdgeInterpolationAlgorithm algorithm) { + switch (algorithm.toString().toUpperCase(Locale.ROOT)) { + case "SPHERICAL": + return EdgeAlgorithm.SPHERICAL; + default: + throw new UnsupportedOperationException( + "Iceberg does not support Spark geography edge algorithm: " + algorithm); + } + } } diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkValueConverter.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkValueConverter.java index 28b717ac090e..af405bb04d9d 100644 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkValueConverter.java +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkValueConverter.java @@ -129,7 +129,7 @@ public static Object convertToSpark(Type type, Object object) { case STRUCT: case LIST: case MAP: - return new UnsupportedOperationException("Complex types currently not supported"); + throw new UnsupportedOperationException("Complex types currently not supported"); case DATE: return DateTimeUtils.daysToLocalDate((int) object); case TIMESTAMP: diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java index 80f93427805a..bf0474db6245 100644 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java @@ -239,7 +239,7 @@ private boolean fanoutWriterEnabled(boolean defaultValue) { return confParser .booleanConf() .option(SparkWriteOptions.FANOUT_ENABLED) - .tableProperty(TableProperties.SPARK_WRITE_PARTITIONED_FANOUT_ENABLED) + .tableProperty(SparkTableProperties.WRITE_PARTITIONED_FANOUT_ENABLED) .defaultValue(defaultValue) .parse(); } @@ -706,7 +706,7 @@ private long advisoryPartitionSize(long defaultValue) { .longConf() .option(SparkWriteOptions.ADVISORY_PARTITION_SIZE) .sessionConf(SparkSQLProperties.ADVISORY_PARTITION_SIZE) - .tableProperty(TableProperties.SPARK_WRITE_ADVISORY_PARTITION_SIZE_BYTES) + .tableProperty(SparkTableProperties.WRITE_ADVISORY_PARTITION_SIZE_BYTES) .defaultValue(defaultValue) .parse(); } diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/TypeToSparkType.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/TypeToSparkType.java index 09c89bbba813..dc077937577c 100644 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/TypeToSparkType.java +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/TypeToSparkType.java @@ -22,6 +22,7 @@ import org.apache.iceberg.MetadataColumns; import org.apache.iceberg.Schema; import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.types.EdgeAlgorithm; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types; @@ -33,7 +34,11 @@ import org.apache.spark.sql.types.DateType$; import org.apache.spark.sql.types.DecimalType$; import org.apache.spark.sql.types.DoubleType$; +import org.apache.spark.sql.types.EdgeInterpolationAlgorithm; +import org.apache.spark.sql.types.EdgeInterpolationAlgorithm.SPHERICAL$; import org.apache.spark.sql.types.FloatType$; +import org.apache.spark.sql.types.GeographyType$; +import org.apache.spark.sql.types.GeometryType$; import org.apache.spark.sql.types.IntegerType$; import org.apache.spark.sql.types.LongType$; import org.apache.spark.sql.types.MapType$; @@ -52,6 +57,9 @@ class TypeToSparkType extends TypeUtil.SchemaVisitor { public static final String METADATA_COL_ATTR_KEY = "__metadata_col"; + // Spark's only edge-interpolation algorithm. + private static final EdgeInterpolationAlgorithm SPARK_SPHERICAL = SPHERICAL$.MODULE$; + @Override public DataType schema(Schema schema, DataType structType) { return structType; @@ -145,6 +153,10 @@ public DataType primitive(Type.PrimitiveType primitive) { return BinaryType$.MODULE$; case BINARY: return BinaryType$.MODULE$; + case GEOMETRY: + return geometryType((Types.GeometryType) primitive); + case GEOGRAPHY: + return geographyType((Types.GeographyType) primitive); case DECIMAL: Types.DecimalType decimal = (Types.DecimalType) primitive; return DecimalType$.MODULE$.apply(decimal.precision(), decimal.scale()); @@ -156,6 +168,32 @@ public DataType primitive(Type.PrimitiveType primitive) { } } + private DataType geometryType(Types.GeometryType geometry) { + // The spec lets a geometry CRS be any string identifying a CRS, but Spark recognizes only a + // fixed set; a CRS Spark cannot resolve throws SparkIllegalArgumentException (an + // IllegalArgumentException). + return GeometryType$.MODULE$.apply(geometry.crs()); + } + + private DataType geographyType(Types.GeographyType geography) { + // The spec requires a geography CRS to be geographic; Spark recognizes only OGC:CRS84, so any + // other CRS throws SparkIllegalArgumentException (an IllegalArgumentException). + return GeographyType$.MODULE$.apply(geography.crs(), convertAlgorithm(geography.algorithm())); + } + + // Translates Iceberg's edge-interpolation algorithm to Spark's. Spark supports only the spherical + // algorithm (the Iceberg default); every other algorithm is rejected with a clear error. Shared + // with PruneColumnsWithoutReordering, which compares algorithms across the two type systems. + static EdgeInterpolationAlgorithm convertAlgorithm(EdgeAlgorithm algorithm) { + switch (algorithm) { + case SPHERICAL: + return SPARK_SPHERICAL; + default: + throw new UnsupportedOperationException( + "Spark does not support geography edge algorithm: " + algorithm); + } + } + private Metadata fieldMetadata(int fieldId) { if (MetadataColumns.metadataFieldIds().contains(fieldId)) { return new MetadataBuilder().putBoolean(METADATA_COL_ATTR_KEY, true).build(); diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java index aedb25e4a4a6..11935e815e76 100644 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.io.OutputStreamWriter; import java.nio.charset.StandardCharsets; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; @@ -31,9 +32,13 @@ import org.apache.iceberg.ContentFile; import org.apache.iceberg.DataFile; import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileContent; import org.apache.iceberg.FileFormat; import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.ManifestContent; import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.PartitionStatisticsFile; import org.apache.iceberg.RewriteTablePathUtil; @@ -69,13 +74,14 @@ import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.relocated.com.google.common.collect.Sets; import org.apache.iceberg.spark.JobGroupInfo; import org.apache.iceberg.spark.source.SerializableTableWithSize; import org.apache.iceberg.util.DeleteFileSet; import org.apache.iceberg.util.Pair; import org.apache.iceberg.util.Tasks; -import org.apache.spark.api.java.function.ForeachFunction; +import org.apache.spark.api.java.function.FlatMapFunction; import org.apache.spark.api.java.function.MapFunction; import org.apache.spark.api.java.function.ReduceFunction; import org.apache.spark.broadcast.Broadcast; @@ -87,6 +93,7 @@ import org.apache.spark.sql.functions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import scala.Tuple2; public class RewriteTablePathSparkAction extends BaseSparkAction implements RewriteTablePath { @@ -272,7 +279,8 @@ private String jobDesc() { *
    *
  • Rebuild version files to staging *
  • Rebuild manifest list files to staging - *
  • Rebuild manifest to staging + *
  • Rewrite referenced position delete files to staging + *
  • Rebuild manifests to staging *
  • Get all files needed to move *
*/ @@ -308,24 +316,29 @@ private Result rebuildMetadata() { RewriteResult rewriteManifestListResult = new RewriteResult<>(); manifestListResults.forEach(rewriteManifestListResult::append); - // rebuild manifest files - Set metaFiles = rewriteManifestListResult.toRewrite(); - RewriteContentFileResult rewriteManifestResult = - rewriteManifests(deltaSnapshots, endMetadata, metaFiles); + Set manifestFiles = rewriteManifestListResult.toRewrite(); // rebuild position delete files - Set deleteFiles = - rewriteManifestResult.toRewrite().stream() - .filter(e -> e instanceof DeleteFile) - .map(e -> (DeleteFile) e) - .collect(Collectors.toCollection(DeleteFileSet::create)); - rewritePositionDeletes(deleteFiles); + Set deleteManifests = + manifestFiles.stream() + .filter(manifest -> manifest.content() == ManifestContent.DELETES) + .collect(Collectors.toSet()); + Set deleteFilesToRewrite = positionDeletesToRewrite(deleteManifests); + Map rewrittenDeleteFileSizes = rewritePositionDeletes(deleteFilesToRewrite); + + // rebuild manifest files + RewriteContentFileResult rewriteManifestResult = + rewriteManifests( + deltaSnapshots, + endMetadata, + manifestFiles, + sparkContext().broadcast(rewrittenDeleteFileSizes)); ImmutableRewriteTablePath.Result.Builder builder = ImmutableRewriteTablePath.Result.builder() .stagingLocation(stagingDir) - .rewrittenDeleteFilePathsCount(deleteFiles.size()) - .rewrittenManifestFilePathsCount(metaFiles.size()) + .rewrittenDeleteFilePathsCount(deleteFilesToRewrite.size()) + .rewrittenManifestFilePathsCount(manifestFiles.size()) .latestVersion(RewriteTablePathUtil.fileName(endVersionName)); if (!createFileList) { @@ -561,7 +574,10 @@ public RewriteContentFileResult appendDeleteFile(RewriteResult r1) { /** Rewrite manifest files in a distributed manner and return rewritten data files path pairs. */ private RewriteContentFileResult rewriteManifests( - Set deltaSnapshots, TableMetadata tableMetadata, Set toRewrite) { + Set deltaSnapshots, + TableMetadata tableMetadata, + Set toRewrite, + Broadcast> rewrittenDeleteFileSizes) { if (toRewrite.isEmpty()) { return new RewriteContentFileResult(); } @@ -581,7 +597,8 @@ private RewriteContentFileResult rewriteManifests( stagingDir, tableMetadata.formatVersion(), sourcePrefix, - targetPrefix), + targetPrefix, + rewrittenDeleteFileSizes), Encoders.bean(RewriteContentFileResult.class)) // duplicates are expected here as the same data file can have different statuses // (e.g. added and deleted) @@ -594,7 +611,8 @@ private static MapFunction toManifests( String stagingLocation, int format, String sourcePrefix, - String targetPrefix) { + String targetPrefix, + Broadcast> rewrittenDeleteFileSizes) { return manifestFile -> { RewriteContentFileResult result = new RewriteContentFileResult(); @@ -619,7 +637,8 @@ private static MapFunction toManifests( stagingLocation, format, sourcePrefix, - targetPrefix)); + targetPrefix, + rewrittenDeleteFileSizes)); break; default: throw new UnsupportedOperationException( @@ -665,7 +684,8 @@ private static RewriteResult writeDeleteManifest( String stagingLocation, int format, String sourcePrefix, - String targetPrefix) { + String targetPrefix, + Broadcast> rewrittenDeleteFileSizes) { try { String stagingPath = RewriteTablePathUtil.stagingPath(manifestFile.path(), sourcePrefix, stagingLocation); @@ -682,27 +702,120 @@ private static RewriteResult writeDeleteManifest( specsById, sourcePrefix, targetPrefix, - stagingLocation); + stagingLocation, + rewrittenDeleteFileSizes.value()); } catch (IOException e) { throw new RuntimeIOException(e); } } - private void rewritePositionDeletes(Set toRewrite) { + /** + * Enumerate the distinct position delete files referenced by the given delete manifests. Deduped + * by identity (location, offset, size) so a file shared across manifests is counted once; the + * physical rewrite is further deduped by location in {@link #rewritePositionDeletes}. + */ + private Set positionDeletesToRewrite(Set deleteManifests) { + if (deleteManifests.isEmpty()) { + return Collections.emptySet(); + } + + Encoder manifestFileEncoder = Encoders.javaSerialization(ManifestFile.class); + Dataset manifestDS = + spark().createDataset(Lists.newArrayList(deleteManifests), manifestFileEncoder); + Encoder deleteFileEncoder = Encoders.javaSerialization(DeleteFile.class); + + List referencedDeleteFiles = + manifestDS + .repartition(deleteManifests.size()) + .flatMap(positionDeletesInManifest(tableBroadcast()), deleteFileEncoder) + .collectAsList(); + + return DeleteFileSet.of(referencedDeleteFiles); + } + + private static FlatMapFunction positionDeletesInManifest( + Broadcast
tableArg) { + return manifestFile -> { + Table table = tableArg.getValue(); + List deleteFiles = Lists.newArrayList(); + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifestFile, table.io(), table.specs())) { + for (DeleteFile deleteFile : reader) { + if (deleteFile.content() == FileContent.POSITION_DELETES) { + deleteFiles.add(deleteFile.copy()); + } + } + } + return deleteFiles.iterator(); + }; + } + + /** + * Rewrite the given position delete files in parallel, returning a map from each source delete + * file path to the size of its rewritten file. Physical files are deduped by location, so a + * Puffin file holding multiple DVs is rewritten once and its size keyed once. + */ + private Map rewritePositionDeletes(Set toRewrite) { if (toRewrite.isEmpty()) { - return; + return Collections.emptyMap(); + } + + // Multiple DVs can share one Puffin file at different blob offsets; rewrite each physical file + // once. The measured size is keyed by location and applied to every referencing manifest entry. + Map byLocation = Maps.newHashMapWithExpectedSize(toRewrite.size()); + for (DeleteFile deleteFile : toRewrite) { + byLocation.putIfAbsent(deleteFile.location(), deleteFile); } + List physicalFiles = Lists.newArrayList(byLocation.values()); Encoder deleteFileEncoder = Encoders.javaSerialization(DeleteFile.class); - Dataset deleteFileDs = - spark().createDataset(Lists.newArrayList(toRewrite), deleteFileEncoder); + Dataset deleteFileDS = spark().createDataset(physicalFiles, deleteFileEncoder); PositionDeleteReaderWriter posDeleteReaderWriter = new SparkPositionDeleteReaderWriter(); - deleteFileDs - .repartition(toRewrite.size()) - .foreach( - rewritePositionDelete( - tableBroadcast(), sourcePrefix, targetPrefix, stagingDir, posDeleteReaderWriter)); + List> rewrittenSizes = + deleteFileDS + .repartition(physicalFiles.size()) + .map( + rewritePositionDelete( + tableBroadcast(), + sourcePrefix, + targetPrefix, + stagingDir, + posDeleteReaderWriter), + Encoders.tuple(Encoders.STRING(), Encoders.LONG())) + .collectAsList(); + + Map sizesBySourcePath = Maps.newHashMapWithExpectedSize(rewrittenSizes.size()); + for (Tuple2 entry : rewrittenSizes) { + sizesBySourcePath.put(entry._1(), entry._2()); + } + return sizesBySourcePath; + } + + private static MapFunction> rewritePositionDelete( + Broadcast
tableArg, + String sourcePrefixArg, + String targetPrefixArg, + String stagingLocationArg, + PositionDeleteReaderWriter posDeleteReaderWriter) { + return deleteFile -> { + FileIO io = tableArg.getValue().io(); + String newPath = + RewriteTablePathUtil.stagingPath( + deleteFile.location(), sourcePrefixArg, stagingLocationArg); + OutputFile outputFile = io.newOutputFile(newPath); + PartitionSpec spec = tableArg.getValue().specs().get(deleteFile.specId()); + long rewrittenLength = + RewriteTablePathUtil.rewritePositionDelete( + deleteFile, + outputFile, + io, + spec, + sourcePrefixArg, + targetPrefixArg, + posDeleteReaderWriter); + return new Tuple2<>(deleteFile.location(), rewrittenLength); + }; } private static class SparkPositionDeleteReaderWriter implements PositionDeleteReaderWriter { @@ -724,30 +837,6 @@ public PositionDeleteWriter writer( } } - private ForeachFunction rewritePositionDelete( - Broadcast
tableArg, - String sourcePrefixArg, - String targetPrefixArg, - String stagingLocationArg, - PositionDeleteReaderWriter posDeleteReaderWriter) { - return deleteFile -> { - FileIO io = tableArg.getValue().io(); - String newPath = - RewriteTablePathUtil.stagingPath( - deleteFile.location(), sourcePrefixArg, stagingLocationArg); - OutputFile outputFile = io.newOutputFile(newPath); - PartitionSpec spec = tableArg.getValue().specs().get(deleteFile.specId()); - RewriteTablePathUtil.rewritePositionDeleteFile( - deleteFile, - outputFile, - io, - spec, - sourcePrefixArg, - targetPrefixArg, - posDeleteReaderWriter); - }; - } - private static CloseableIterable positionDeletesReader( InputFile inputFile, FileFormat format, PartitionSpec spec) { return FormatModelRegistry.readBuilder(format, Record.class, inputFile) diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java index 80a40d72c8d1..9cd170f352eb 100644 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java @@ -31,7 +31,6 @@ import org.apache.iceberg.Snapshot; import org.apache.iceberg.SnapshotRef; import org.apache.iceberg.Table; -import org.apache.iceberg.TableProperties; import org.apache.iceberg.TableScan; import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.expressions.Evaluator; @@ -49,6 +48,7 @@ import org.apache.iceberg.spark.CommitMetadata; import org.apache.iceberg.spark.Spark3Util; import org.apache.iceberg.spark.SparkReadConf; +import org.apache.iceberg.spark.SparkTableProperties; import org.apache.iceberg.spark.SparkTableUtil; import org.apache.iceberg.spark.SparkUtil; import org.apache.iceberg.spark.SparkV2Filters; @@ -375,15 +375,15 @@ private static Set computeCapabilities(Table table) { private static boolean acceptAnySchema(Table table) { return PropertyUtil.propertyAsBoolean( table.properties(), - TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA, - TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA_DEFAULT); + SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA, + SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA_DEFAULT); } private static boolean autoSchemaEvolution(Table table) { return PropertyUtil.propertyAsBoolean( table.properties(), - TableProperties.SPARK_WRITE_AUTO_SCHEMA_EVOLUTION, - TableProperties.SPARK_WRITE_AUTO_SCHEMA_EVOLUTION_DEFAULT); + SparkTableProperties.WRITE_AUTO_SCHEMA_EVOLUTION, + SparkTableProperties.WRITE_AUTO_SCHEMA_EVOLUTION_DEFAULT); } // returns latest snapshot for branch or current snapshot if branch is yet to be created diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestSparkDistributionAndOrderingUtil.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestSparkDistributionAndOrderingUtil.java index 21b67b89a99b..5fa9abde1225 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestSparkDistributionAndOrderingUtil.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestSparkDistributionAndOrderingUtil.java @@ -20,12 +20,12 @@ import static org.apache.iceberg.TableProperties.DELETE_DISTRIBUTION_MODE; import static org.apache.iceberg.TableProperties.MERGE_DISTRIBUTION_MODE; -import static org.apache.iceberg.TableProperties.SPARK_WRITE_PARTITIONED_FANOUT_ENABLED; import static org.apache.iceberg.TableProperties.UPDATE_DISTRIBUTION_MODE; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_HASH; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_NONE; import static org.apache.iceberg.TableProperties.WRITE_DISTRIBUTION_MODE_RANGE; +import static org.apache.iceberg.spark.SparkTableProperties.WRITE_PARTITIONED_FANOUT_ENABLED; import static org.apache.spark.sql.connector.write.RowLevelOperation.Command.DELETE; import static org.apache.spark.sql.connector.write.RowLevelOperation.Command.MERGE; import static org.apache.spark.sql.connector.write.RowLevelOperation.Command.UPDATE; @@ -3019,10 +3019,10 @@ private void checkPositionDeltaDistributionAndOrdering( } private void disableFanoutWriters(Table table) { - table.updateProperties().set(SPARK_WRITE_PARTITIONED_FANOUT_ENABLED, "false").commit(); + table.updateProperties().set(WRITE_PARTITIONED_FANOUT_ENABLED, "false").commit(); } private void enableFanoutWriters(Table table) { - table.updateProperties().set(SPARK_WRITE_PARTITIONED_FANOUT_ENABLED, "true").commit(); + table.updateProperties().set(WRITE_PARTITIONED_FANOUT_ENABLED, "true").commit(); } } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestSparkSchemaUtil.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestSparkSchemaUtil.java index d5f407a715ef..f107167a7a24 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestSparkSchemaUtil.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestSparkSchemaUtil.java @@ -20,6 +20,7 @@ import static org.apache.iceberg.types.Types.NestedField.optional; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.math.BigDecimal; import java.nio.ByteBuffer; @@ -29,12 +30,19 @@ import org.apache.iceberg.MetadataColumns; import org.apache.iceberg.Schema; import org.apache.iceberg.expressions.Literal; +import org.apache.iceberg.types.EdgeAlgorithm; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import org.apache.spark.sql.catalyst.expressions.AttributeReference; import org.apache.spark.sql.catalyst.expressions.MetadataAttribute; import org.apache.spark.sql.catalyst.types.DataTypeUtils; import org.apache.spark.sql.catalyst.util.ResolveDefaultColumnsUtils$; +import org.apache.spark.sql.types.DataType; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.GeographyType; +import org.apache.spark.sql.types.GeographyType$; +import org.apache.spark.sql.types.GeometryType; +import org.apache.spark.sql.types.GeometryType$; import org.apache.spark.sql.types.Metadata; import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; @@ -93,6 +101,121 @@ public void testSchemaConversionWithMetaDataColumnSchema() { } } + @Test + public void testGeospatialTypeConversion() { + // a default-CRS geometry round-trips through the null <-> OGC:CRS84 normalization + Types.GeometryType defaultGeometry = Types.GeometryType.crs84(); + DataType sparkDefaultGeometry = SparkSchemaUtil.convert(defaultGeometry); + assertThat(sparkDefaultGeometry).isInstanceOf(GeometryType.class); + assertThat(((GeometryType) sparkDefaultGeometry).crs()) + .isEqualTo(Types.GeometryType.DEFAULT_CRS); + assertThat(SparkSchemaUtil.convert(sparkDefaultGeometry)).isEqualTo(defaultGeometry); + + // a default-CRS geography round-trips through the null <-> OGC:CRS84 normalization + Types.GeographyType defaultGeography = Types.GeographyType.crs84(); + DataType sparkDefaultGeography = SparkSchemaUtil.convert(defaultGeography); + assertThat(sparkDefaultGeography).isInstanceOf(GeographyType.class); + assertThat(((GeographyType) sparkDefaultGeography).crs()) + .isEqualTo(Types.GeographyType.DEFAULT_CRS); + assertThat(SparkSchemaUtil.convert(sparkDefaultGeography)).isEqualTo(defaultGeography); + + Types.GeometryType geometry = Types.GeometryType.of("EPSG:3857"); + DataType sparkGeometry = SparkSchemaUtil.convert(geometry); + assertThat(sparkGeometry).isInstanceOf(GeometryType.class); + assertThat(((GeometryType) sparkGeometry).crs()).isEqualTo("EPSG:3857"); + assertThat(SparkSchemaUtil.convert(sparkGeometry)).isEqualTo(geometry); + + Types.GeographyType geography = Types.GeographyType.of("OGC:CRS84"); + DataType sparkGeography = SparkSchemaUtil.convert(geography); + assertThat(sparkGeography).isInstanceOf(GeographyType.class); + assertThat(((GeographyType) sparkGeography).crs()).isEqualTo("OGC:CRS84"); + assertThat(SparkSchemaUtil.convert(sparkGeography)).isEqualTo(geography); + + assertThat(SparkSchemaUtil.convert(GeometryType$.MODULE$.apply("EPSG:3857"))) + .isEqualTo(geometry); + assertThat(SparkSchemaUtil.convert(GeographyType$.MODULE$.apply("OGC:CRS84"))) + .isEqualTo(geography); + + Types.GeographyType vincentyGeography = + Types.GeographyType.of("OGC:CRS84", EdgeAlgorithm.VINCENTY); + assertThatThrownBy(() -> SparkSchemaUtil.convert(vincentyGeography)) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessage("Spark does not support geography edge algorithm: vincenty"); + } + + @Test + public void testGeospatialCrsUnsupportedBySparkIsRejected() { + // Iceberg permits any non-empty CRS, but Spark only recognizes a fixed set; a CRS Spark cannot + // resolve is rejected by Spark's SparkIllegalArgumentException (an IllegalArgumentException). + Types.GeometryType geometry = Types.GeometryType.of("EPSG:4269"); + assertThatThrownBy(() -> SparkSchemaUtil.convert(geometry)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("EPSG:4269"); + + Types.GeographyType geography = Types.GeographyType.of("EPSG:4269"); + assertThatThrownBy(() -> SparkSchemaUtil.convert(geography)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("EPSG:4269"); + } + + @Test + public void testGeospatialMixedSridIsRejected() { + // Spark models a mixed-SRID column with a sentinel CRS ("SRID:ANY"); Iceberg requires a + // concrete column-level CRS, so a mixed-SRID Spark type must be rejected, not persisted. + DataType mixedGeometry = GeometryType$.MODULE$.apply("ANY"); + assertThatThrownBy(() -> SparkSchemaUtil.convert(mixedGeometry)) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessage("Cannot convert Spark geometry with mixed SRID to Iceberg"); + + DataType mixedGeography = GeographyType$.MODULE$.apply("ANY"); + assertThatThrownBy(() -> SparkSchemaUtil.convert(mixedGeography)) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessage("Cannot convert Spark geography with mixed SRID to Iceberg"); + } + + @Test + public void testPruneGeospatialTypes() { + Schema schema = + new Schema( + optional(1, "geom", Types.GeometryType.of("EPSG:3857")), + optional(2, "geog", Types.GeographyType.of("OGC:CRS84")), + optional(3, "id", Types.LongType.get())); + + StructType requestedType = SparkSchemaUtil.convert(schema); + Schema pruned = SparkSchemaUtil.prune(schema, requestedType); + + assertThat(pruned.asStruct()).isEqualTo(schema.asStruct()); + } + + @Test + public void testPruneGeospatialTypeWithIncompatibleRequestedType() { + Schema schema = new Schema(optional(1, "geom", Types.GeometryType.of("EPSG:3857"))); + + // requesting a non-geo Spark type for a geometry column must be rejected + StructType incompatibleType = new StructType().add("geom", DataTypes.BinaryType, true); + + assertThatThrownBy(() -> SparkSchemaUtil.prune(schema, incompatibleType)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Cannot project") + .hasMessageContaining("incompatible type"); + } + + @Test + public void testPruneGeospatialTypeWithIncompatibleCrs() { + // Spark normally copies the type from the table schema, so this defends against a requested geo + // type whose CRS disagrees with the table's rather than a case reachable through normal reads. + // Both CRS values must be ones Spark recognizes (else construction fails first); OGC:CRS84 and + // EPSG:3857 are both valid and differ, exercising the prune-time CRS check. + Schema schema = new Schema(optional(1, "geom", Types.GeometryType.of("EPSG:3857"))); + + StructType mismatchedCrs = + new StructType().add("geom", GeometryType$.MODULE$.apply("OGC:CRS84"), true); + + assertThatThrownBy(() -> SparkSchemaUtil.prune(schema, mismatchedCrs)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Cannot project geometry with incompatible CRS"); + } + @Test public void testSchemaConversionWithOnlyWriteDefault() { Schema schema = diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestSparkValueConverter.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestSparkValueConverter.java index c7a2e6c18fca..3ffc303759db 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestSparkValueConverter.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestSparkValueConverter.java @@ -19,10 +19,13 @@ package org.apache.iceberg.spark; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import java.util.List; import org.apache.iceberg.Schema; import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.data.Record; +import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import org.apache.spark.sql.Row; import org.apache.spark.sql.RowFactory; @@ -83,6 +86,21 @@ public void testSparkNullPrimitiveConvert() { assertCorrectNullConversion(schema); } + @Test + public void testConvertToSparkComplexTypesThrow() { + Types.StructType struct = + Types.StructType.of(Types.NestedField.required(1, "lat", Types.FloatType.get())); + Types.ListType list = Types.ListType.ofOptional(1, Types.StringType.get()); + Types.MapType map = + Types.MapType.ofOptional(1, 2, Types.StringType.get(), Types.StringType.get()); + + for (Type type : List.of(struct, list, map)) { + assertThatThrownBy(() -> SparkValueConverter.convertToSpark(type, "unused")) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining("Complex types currently not supported"); + } + } + private void assertCorrectNullConversion(Schema schema) { Row sparkRow = RowFactory.create(1, null); Record record = GenericRecord.create(schema); diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java index df59a0ceefae..110e43ede1f9 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteDataFilesAction.java @@ -41,24 +41,19 @@ import java.io.File; import java.io.IOException; import java.io.UncheckedIOException; -import java.nio.file.Path; import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Random; import java.util.Set; -import java.util.TreeMap; import java.util.UUID; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.LongStream; import java.util.stream.Stream; import java.util.stream.StreamSupport; import org.apache.hadoop.conf.Configuration; -import org.apache.iceberg.AppendFiles; import org.apache.iceberg.BaseTable; import org.apache.iceberg.ContentFile; import org.apache.iceberg.DataFile; @@ -108,12 +103,10 @@ import org.apache.iceberg.io.OutputFile; import org.apache.iceberg.io.OutputFileFactory; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; import org.apache.iceberg.relocated.com.google.common.collect.Iterables; import org.apache.iceberg.relocated.com.google.common.collect.Lists; -import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.relocated.com.google.common.collect.Sets; import org.apache.iceberg.relocated.com.google.common.collect.Streams; import org.apache.iceberg.spark.FileRewriteCoordinator; @@ -137,7 +130,6 @@ import org.apache.spark.sql.catalyst.analysis.NoSuchTableException; import org.apache.spark.sql.internal.SQLConf; import org.apache.spark.sql.types.DataTypes; -import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.TestTemplate; @@ -151,18 +143,6 @@ public class TestRewriteDataFilesAction extends TestBase { @TempDir private File tableDir; private static final int SCALE = 400000; - // Row group size used by createTable(); part of the unpartitioned cache key so a future - // override of this property can't silently hand back cached files of a different shape. - private static final int INPUT_PARQUET_ROW_GROUP_SIZE_BYTES = 20 * 1024; - - // Cache of pre-written input data files keyed by table shape (schema/spec/props are - // fixed per key), so identical large inputs are materialized via Spark only once per JVM - // fork and reused by every test that asks for the same shape. The Spark write of SCALE - // rows dominates these tests; the rewrite under test still runs per test on a fresh table. - @TempDir private static Path inputCacheDir; - private static final Map> INPUT_FILE_CACHE = Maps.newConcurrentMap(); - private static final Map INPUT_CACHE_LOCKS = Maps.newConcurrentMap(); - private static final AtomicInteger INPUT_CACHE_SEQ = new AtomicInteger(); private static final HadoopTables TABLES = new HadoopTables(new Configuration()); private static final Schema SCHEMA = @@ -190,16 +170,6 @@ public static void setupSpark() { spark.conf().set(SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), "false"); } - @AfterAll - public static void clearInputFileCache() { - // inputCacheDir is a static @TempDir that JUnit recreates if the class runs twice in one JVM - // (IDE re-run, forkCount=0). Clear the cache so a second run can't return DataFiles pointing - // into the deleted first-run directory. - INPUT_FILE_CACHE.clear(); - INPUT_CACHE_LOCKS.clear(); - INPUT_CACHE_SEQ.set(0); - } - @BeforeEach public void setupTableLocation() { this.tableLocation = tableDir.toURI().toString(); @@ -2260,9 +2230,6 @@ protected void shouldHaveSnapshots(Table table, int expectedSnapshots) { } protected void shouldHaveNoOrphans(Table table) { - // Cached input files live under the static inputCacheDir, outside table.location(), so - // deleteOrphanFiles (which only scans the table prefix) never sees them by design. Orphan - // coverage therefore does not extend to the shared cached inputs. assertThat( actions() .deleteOrphanFiles(table) @@ -2375,9 +2342,7 @@ protected Table createTable() { Table table = TABLES.create(SCHEMA, spec, options, tableLocation); table .updateProperties() - .set( - TableProperties.PARQUET_ROW_GROUP_SIZE_BYTES, - Integer.toString(INPUT_PARQUET_ROW_GROUP_SIZE_BYTES)) + .set(TableProperties.PARQUET_ROW_GROUP_SIZE_BYTES, Integer.toString(20 * 1024)) .commit(); assertThat(table.currentSnapshot()).as("Table must be empty").isNull(); return table; @@ -2390,21 +2355,8 @@ protected Table createTable() { * @return the created table */ protected Table createTable(int files) { - String key = - String.format( - "unpartitioned|fv=%d|rowGroup=%d|files=%d|rows=%d", - formatVersion, INPUT_PARQUET_ROW_GROUP_SIZE_BYTES, files, SCALE); - List inputFiles = - cachedInputFiles( - key, - () -> { - Table golden = createTable(); - writeRecords(files, SCALE); - golden.refresh(); - return golden; - }); Table table = createTable(); - appendInputFiles(table, inputFiles); + writeRecords(files, SCALE); table.refresh(); return table; } @@ -2412,85 +2364,14 @@ protected Table createTable(int files) { protected Table createTablePartitioned( int partitions, int files, int numRecords, Map options) { PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).identity("c1").truncate("c2", 2).build(); - String key = - String.format( - "partitioned|fv=%d|spec=%s|opts=%s|files=%d|rows=%d|partitions=%d", - formatVersion, spec, new TreeMap<>(options), files, numRecords, partitions); - List inputFiles = - cachedInputFiles( - key, - () -> { - Table golden = TABLES.create(SCHEMA, spec, options, tableLocation); - assertThat(golden.currentSnapshot()).as("Table must be empty").isNull(); - writeRecords(files, numRecords, partitions); - golden.refresh(); - return golden; - }); Table table = TABLES.create(SCHEMA, spec, options, tableLocation); assertThat(table.currentSnapshot()).as("Table must be empty").isNull(); - appendInputFiles(table, inputFiles); + + writeRecords(files, numRecords, partitions); table.refresh(); return table; } - /** - * Returns the input data files for a given table shape, materializing them with Spark exactly - * once per JVM fork and reusing them afterwards. On a cache miss the {@code goldenBuilder} writes - * the data into a stable cache location (kept alive for the whole class via a static {@link - * TempDir}); on a hit the cached {@link DataFile}s are returned and re-appended to a fresh table - * by {@link #appendInputFiles}. The data is deterministic (fixed RNG seed) so reuse is - * byte-identical to regenerating it. - */ - private List cachedInputFiles(String key, Supplier
goldenBuilder) { - List cached = INPUT_FILE_CACHE.get(key); - if (cached != null) { - return cached; - } - // Serialize builds per key: concurrent callers requesting the same table shape block on the - // first build and then reuse its result, instead of materializing identical input twice. The - // heavy Spark write happens outside any map lock, so distinct shapes can still build in - // parallel. - Object lock = INPUT_CACHE_LOCKS.computeIfAbsent(key, ignored -> new Object()); - synchronized (lock) { - List existing = INPUT_FILE_CACHE.get(key); - if (existing != null) { - return existing; - } - String savedLocation = this.tableLocation; - try { - this.tableLocation = - inputCacheDir.resolve("input-" + INPUT_CACHE_SEQ.incrementAndGet()).toUri().toString(); - Table golden = goldenBuilder.get(); - // includeColumnStats() is required: a plain scan drops lower/upper bounds and - // value counts, and re-appending stat-less files breaks tests that read bounds. - // planFiles() returns a CloseableIterable holding manifest readers open, so close it via - // try-with-resources; otherwise every cache miss leaks file descriptors and can leave - // manifest files locked on Windows. - List built; - try (CloseableIterable tasks = - golden.newScan().includeColumnStats().planFiles()) { - built = - Streams.stream(tasks) - .map(FileScanTask::file) - .map(DataFile::copy) - .collect(ImmutableList.toImmutableList()); - } catch (IOException e) { - throw new UncheckedIOException("Failed to plan cached input files", e); - } - INPUT_FILE_CACHE.put(key, built); - return built; - } finally { - this.tableLocation = savedLocation; - } - } - } - - private static void appendInputFiles(Table table, List inputFiles) { - AppendFiles append = table.newAppend(); - inputFiles.forEach(append::appendFile); - append.commit(); - } - protected Table createTablePartitioned(int partitions, int files) { return createTablePartitioned( partitions, diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteTablePathsAction.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteTablePathsAction.java index c5db04762f21..9660beae187e 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteTablePathsAction.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteTablePathsAction.java @@ -30,6 +30,7 @@ import java.nio.file.Path; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.function.Predicate; @@ -41,15 +42,21 @@ import org.apache.iceberg.BaseTable; import org.apache.iceberg.DataFile; import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileFormat; import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; import org.apache.iceberg.Parameter; import org.apache.iceberg.ParameterizedTestExtension; import org.apache.iceberg.Parameters; import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.RowDelta; import org.apache.iceberg.Schema; import org.apache.iceberg.Snapshot; import org.apache.iceberg.SnapshotChanges; import org.apache.iceberg.StaticTableOperations; +import org.apache.iceberg.StructLike; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableProperties; @@ -62,14 +69,18 @@ import org.apache.iceberg.data.FileHelpers; import org.apache.iceberg.data.GenericRecord; import org.apache.iceberg.data.Record; +import org.apache.iceberg.deletes.BaseDVFileWriter; +import org.apache.iceberg.deletes.DVFileWriter; import org.apache.iceberg.deletes.PositionDelete; import org.apache.iceberg.hadoop.HadoopTables; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.OutputFile; +import org.apache.iceberg.io.OutputFileFactory; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.Iterables; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; import org.apache.iceberg.spark.SparkCatalog; import org.apache.iceberg.spark.TestBase; import org.apache.iceberg.spark.source.ThreeColumnRecord; @@ -629,12 +640,7 @@ public void testPositionDeletesDeduplication() throws Exception { // in a new manifest, which will cause duplicate DeleteFile objects when processing tableWithPosDeletes.newRowDelta().addDeletes(positionDeletes).commit(); - // This should NOT throw AlreadyExistsException - the fix uses DeleteFileSet to deduplicate - // Without the fix (using Collectors.toSet()), this would fail because: - // 1. Both manifests contain entries for the same delete file - // 2. Processing returns two different DeleteFile objects for the same file - // 3. HashSet doesn't deduplicate them (DeleteFile doesn't override equals()) - // 4. rewritePositionDeletes tries to write the same file twice -> AlreadyExistsException + // This should NOT throw AlreadyExistsException RewriteTablePath.Result result = actions() .rewriteTablePath(tableWithPosDeletes) @@ -642,13 +648,295 @@ public void testPositionDeletesDeduplication() throws Exception { .rewriteLocationPrefix(tableWithPosDeletes.location(), targetTableLocation()) .execute(); - // Verify the rewrite completed successfully - should have rewritten exactly 1 delete file - // (the duplicate should be deduplicated by DeleteFileSet) assertThat(result.rewrittenDeleteFilePathsCount()) .as("Should have rewritten exactly 1 delete file after deduplication") .isEqualTo(1); } + // Regression test: when the same position delete file is referenced from manifests in different + // snapshots, it must be rewritten once and the resulting size stamped consistently into every + // manifest that references it. The delete file is enumerated and deduped by path before the + // rewrite, so its measured size is shared across all referencing delete manifests. + @TestTemplate + public void testSharedDeleteFileSizeAcrossManifests() throws Exception { + assumeThat(formatVersion) + .as("Format versions 3+ use DVs with different validation rules") + .isEqualTo(2); + + Table tableWithPosDeletes = + createTableWithSnapshots( + tableDir.toFile().toURI().toString().concat("tableWithSharedDelete"), + 1, + Map.of(TableProperties.DELETE_DEFAULT_FILE_FORMAT, "parquet")); + + DataFile dataFile = + tableWithPosDeletes + .currentSnapshot() + .addedDataFiles(tableWithPosDeletes.io()) + .iterator() + .next(); + + List> deletes = Lists.newArrayList(Pair.of(dataFile.location(), 0L)); + File deleteFile = + new File( + removePrefix(tableWithPosDeletes.location() + "/data/deeply/nested/deletes.parquet")); + DeleteFile positionDeletes = + FileHelpers.writeDeleteFile( + tableWithPosDeletes, + tableWithPosDeletes.io().newOutputFile(deleteFile.toURI().toString()), + deletes, + formatVersion) + .first(); + + tableWithPosDeletes.newRowDelta().addDeletes(positionDeletes).commit(); + tableWithPosDeletes.newRowDelta().addDeletes(positionDeletes).commit(); + + RewriteTablePath.Result result = + actions() + .rewriteTablePath(tableWithPosDeletes) + .stagingLocation(stagingLocation()) + .rewriteLocationPrefix(tableWithPosDeletes.location(), targetTableLocation()) + .execute(); + copyTableFiles(result); + + Table targetTable = TABLES.load(targetTableLocation()); + List deleteManifests = + targetTable.currentSnapshot().deleteManifests(targetTable.io()); + assertThat(deleteManifests) + .as("Expected the shared delete file to be referenced by multiple manifests") + .hasSizeGreaterThanOrEqualTo(2); + for (ManifestFile manifest : deleteManifests) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, targetTable.io(), targetTable.specs())) { + for (DeleteFile df : reader) { + long manifestSize = df.fileSizeInBytes(); + long actualSize = targetTable.io().newInputFile(df.location()).getLength(); + assertThat(manifestSize) + .as( + "file_size_in_bytes in rewritten manifest should match actual file size for %s", + df.location()) + .isEqualTo(actualSize); + } + } + } + } + + // Regression test: a single delete manifest can reference multiple distinct position delete + // files, and each entry must be stamped with its own rewritten size. The two delete files carry + // a different number of records so they rewrite to different sizes, which catches a per-path size + // map that collapses entries to a single size or falls back to the stale original size. + @TestTemplate + public void testMultipleDistinctDeleteFileSizesAfterRewrite() throws Exception { + assumeThat(formatVersion) + .as("Format versions 3+ use DVs with different validation rules") + .isEqualTo(2); + + Table tableWithPosDeletes = + createTableWithSnapshots( + tableDir.toFile().toURI().toString().concat("tableWithDistinctDeletes"), + 2, + Map.of(TableProperties.DELETE_DEFAULT_FILE_FORMAT, "parquet")); + + List dataFiles = Lists.newArrayList(); + tableWithPosDeletes + .snapshots() + .forEach( + snapshot -> snapshot.addedDataFiles(tableWithPosDeletes.io()).forEach(dataFiles::add)); + assertThat(dataFiles).as("Expected two data files to reference from deletes").hasSize(2); + + // One delete file holds a single record, the other holds two, so they rewrite to distinct + // on-disk sizes. + File smallDeleteFile = + new File( + removePrefix( + tableWithPosDeletes.location() + "/data/deeply/nested/deletes-small.parquet")); + DeleteFile smallDelete = + FileHelpers.writeDeleteFile( + tableWithPosDeletes, + tableWithPosDeletes.io().newOutputFile(smallDeleteFile.toURI().toString()), + Lists.newArrayList(Pair.of(dataFiles.get(0).location(), 0L)), + formatVersion) + .first(); + + File largeDeleteFile = + new File( + removePrefix( + tableWithPosDeletes.location() + "/data/deeply/nested/deletes-large.parquet")); + DeleteFile largeDelete = + FileHelpers.writeDeleteFile( + tableWithPosDeletes, + tableWithPosDeletes.io().newOutputFile(largeDeleteFile.toURI().toString()), + Lists.newArrayList( + Pair.of(dataFiles.get(0).location(), 0L), + Pair.of(dataFiles.get(1).location(), 0L)), + formatVersion) + .first(); + + tableWithPosDeletes.newRowDelta().addDeletes(smallDelete).addDeletes(largeDelete).commit(); + + RewriteTablePath.Result result = + actions() + .rewriteTablePath(tableWithPosDeletes) + .stagingLocation(stagingLocation()) + .rewriteLocationPrefix(tableWithPosDeletes.location(), targetTableLocation()) + .execute(); + copyTableFiles(result); + + Table targetTable = TABLES.load(targetTableLocation()); + List rewrittenSizes = Lists.newArrayList(); + for (ManifestFile manifest : targetTable.currentSnapshot().deleteManifests(targetTable.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, targetTable.io(), targetTable.specs())) { + for (DeleteFile df : reader) { + long manifestSize = df.fileSizeInBytes(); + long actualSize = targetTable.io().newInputFile(df.location()).getLength(); + assertThat(manifestSize) + .as( + "file_size_in_bytes in rewritten manifest should match actual file size for %s", + df.location()) + .isEqualTo(actualSize); + rewrittenSizes.add(manifestSize); + } + } + } + + assertThat(rewrittenSizes) + .as( + "The two distinct delete files should rewrite to distinct, independently recorded sizes") + .hasSize(2) + .doesNotHaveDuplicates(); + } + + // Regression test: rewriting delete file paths changes the file size (since the + // embedded data file paths may differ in length), but file_size_in_bytes in the rewritten + // manifest was not updated. Readers that use file_size_in_bytes to elide a stat() call may + // fail. + @TestTemplate + public void testDeleteFileSizeInBytesAfterRewrite() throws Exception { + List> deletes = + Lists.newArrayList( + Pair.of( + SnapshotChanges.builderFor(table) + .build() + .addedDataFiles() + .iterator() + .next() + .location(), + 0L)); + + File file = new File(removePrefix(table.location() + "/data/deeply/nested/deletes.parquet")); + DeleteFile positionDeletes = + FileHelpers.writeDeleteFile( + table, table.io().newOutputFile(file.toURI().toString()), deletes, formatVersion) + .first(); + table.newRowDelta().addDeletes(positionDeletes).commit(); + + RewriteTablePath.Result result = + actions() + .rewriteTablePath(table) + .stagingLocation(stagingLocation()) + .rewriteLocationPrefix(table.location(), targetTableLocation()) + .execute(); + copyTableFiles(result); + + Table targetTable = TABLES.load(targetTableLocation()); + for (ManifestFile manifest : targetTable.currentSnapshot().deleteManifests(targetTable.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, targetTable.io(), targetTable.specs())) { + for (DeleteFile df : reader) { + long manifestSize = df.fileSizeInBytes(); + long actualSize = targetTable.io().newInputFile(df.location()).getLength(); + assertThat(manifestSize) + .as( + "file_size_in_bytes in rewritten manifest should match actual file size for %s", + df.location()) + .isEqualTo(actualSize); + } + } + } + } + + // Regression test: a single Puffin file can hold multiple DVs (one blob per data file) + // referenced by distinct DeleteFile entries that share the same location. The rewrite must + // rewrite + // the physical Puffin file once (rather than colliding on the staging path) and stamp the + // rewritten size into every referencing manifest entry. + @TestTemplate + public void testSharedPuffinDeleteFileSizeAfterRewrite() throws Exception { + assumeThat(formatVersion) + .as("DVs are introduced in v3; v4 writes parquet manifests the test setup cannot read") + .isEqualTo(3); + + Table tableWithDVs = + createTableWithSnapshots( + tableDir.toFile().toURI().toString().concat("tableWithSharedPuffin"), 2); + + List dataFilePaths = Lists.newArrayList(); + tableWithDVs + .snapshots() + .forEach( + snapshot -> + snapshot + .addedDataFiles(tableWithDVs.io()) + .forEach(dataFile -> dataFilePaths.add(dataFile.location()))); + assertThat(dataFilePaths).as("Expected two data files to back two DVs").hasSize(2); + + List dvs = writeDVsForDataFiles(tableWithDVs, dataFilePaths); + assertThat(dvs) + .as("Both DVs should live in a single Puffin file") + .hasSize(2) + .allSatisfy(dv -> assertThat(dv.location()).isEqualTo(dvs.get(0).location())); + + RowDelta rowDelta = tableWithDVs.newRowDelta(); + dvs.forEach(rowDelta::addDeletes); + rowDelta.commit(); + + RewriteTablePath.Result result = + actions() + .rewriteTablePath(tableWithDVs) + .stagingLocation(stagingLocation()) + .rewriteLocationPrefix(tableWithDVs.location(), targetTableLocation()) + .execute(); + assertThat(result.rewrittenDeleteFilePathsCount()) + .as("Two DVs are two delete files with rewritten paths") + .isEqualTo(2); + copyTableFiles(result); + + Table targetTable = TABLES.load(targetTableLocation()); + Set rewrittenLocations = Sets.newHashSet(); + for (ManifestFile manifest : targetTable.currentSnapshot().deleteManifests(targetTable.io())) { + try (ManifestReader reader = + ManifestFiles.readDeleteManifest(manifest, targetTable.io(), targetTable.specs())) { + for (DeleteFile df : reader) { + rewrittenLocations.add(df.location()); + long actualSize = targetTable.io().newInputFile(df.location()).getLength(); + assertThat(df.fileSizeInBytes()) + .as("file_size_in_bytes should match the rewritten Puffin size for %s", df.location()) + .isEqualTo(actualSize); + } + } + } + assertThat(rewrittenLocations) + .as("Both DVs should point at the single rewritten Puffin file") + .hasSize(1); + } + + // Writes one DV per data file path into a single Puffin file, returning the resulting DeleteFiles + // (which share a location but carry distinct blob offsets). + private List writeDVsForDataFiles(Table targetTable, List dataFilePaths) + throws IOException { + OutputFileFactory fileFactory = + OutputFileFactory.builderFor(targetTable, 1, 1).format(FileFormat.PUFFIN).build(); + DVFileWriter writer = new BaseDVFileWriter(fileFactory, p -> null); + try (DVFileWriter closeableWriter = writer) { + for (String path : dataFilePaths) { + closeableWriter.delete(path, 0L, targetTable.spec(), (StructLike) null); + } + } + + return writer.result().deleteFiles(); + } + @TestTemplate public void testEqualityDeletes() throws Exception { Table sourceTable = createTableWithSnapshots(newTableLocation(), 1); diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWriterV2.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWriterV2.java index 186d0b2b5204..c1adc405b89e 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWriterV2.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWriterV2.java @@ -25,10 +25,10 @@ import java.math.BigDecimal; import java.util.List; import org.apache.iceberg.ParameterizedTestExtension; -import org.apache.iceberg.TableProperties; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.spark.Spark3Util; +import org.apache.iceberg.spark.SparkTableProperties; import org.apache.iceberg.spark.TestBaseWithCatalog; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; @@ -61,7 +61,7 @@ public void removeTables() { public void testMergeSchemaFailsWithoutWriterOption() throws Exception { sql( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA); + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA); Dataset twoColDF = jsonToDF( @@ -120,7 +120,7 @@ public void testMergeSchemaWithoutAcceptAnySchema() throws Exception { public void testMergeSchemaSparkProperty() throws Exception { sql( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA); + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA); Dataset twoColDF = jsonToDF( @@ -154,7 +154,7 @@ public void testMergeSchemaSparkProperty() throws Exception { public void testMergeSchemaIcebergProperty() throws Exception { sql( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA); + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA); Dataset twoColDF = jsonToDF( @@ -196,7 +196,7 @@ public void testWriteWithCaseSensitiveOption() throws NoSuchTableException, Pars .sql( String.format( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA)) + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA)) .collect(); String schema = "ID bigint, DaTa string"; @@ -226,7 +226,7 @@ public void testWriteWithCaseSensitiveOption() throws NoSuchTableException, Pars public void testMergeSchemaSparkConfiguration() throws Exception { sql( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA); + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA); Dataset twoColDF = jsonToDF( "id bigint, data string", @@ -261,7 +261,7 @@ public void testMergeSchemaSparkConfiguration() throws Exception { public void testMergeSchemaIgnoreCastingLongToInt() throws Exception { sql( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA); + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA); Dataset bigintDF = jsonToDF( @@ -303,7 +303,7 @@ public void testMergeSchemaIgnoreCastingDoubleToFloat() throws Exception { sql("CREATE TABLE %s (id double, data string) USING iceberg", tableName); sql( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA); + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA); Dataset doubleDF = jsonToDF( @@ -345,7 +345,7 @@ public void testMergeSchemaIgnoreCastingDecimalToDecimalWithNarrowerPrecision() sql("CREATE TABLE %s (id decimal(6,2), data string) USING iceberg", tableName); sql( "ALTER TABLE %s SET TBLPROPERTIES ('%s'='true')", - tableName, TableProperties.SPARK_WRITE_ACCEPT_ANY_SCHEMA); + tableName, SparkTableProperties.WRITE_ACCEPT_ANY_SCHEMA); Dataset decimalPrecision6DF = jsonToDF( diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java index c2f5afef0e8c..aacdabfe6047 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java @@ -18,7 +18,7 @@ */ package org.apache.iceberg.spark.source; -import static org.apache.iceberg.TableProperties.SPARK_WRITE_PARTITIONED_FANOUT_ENABLED; +import static org.apache.iceberg.spark.SparkTableProperties.WRITE_PARTITIONED_FANOUT_ENABLED; import static org.apache.iceberg.types.Types.NestedField.optional; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -697,7 +697,7 @@ public void partitionedCreateWithTargetFileSizeViaOption(IcebergOptionsType opti .save(location.toString()); break; case TABLE: - table.updateProperties().set(SPARK_WRITE_PARTITIONED_FANOUT_ENABLED, "true").commit(); + table.updateProperties().set(WRITE_PARTITIONED_FANOUT_ENABLED, "true").commit(); df.select("id", "data") .write() .format("iceberg")