chore: migrate java-bigquery to monorepo (#12059)

chore: migrate java-bigquery to monorepo

Author: suztomo

.github/CODEOWNERS

@@ -8,3 +8,4 @@
 # java-vertexai has maintainers
 /java-vertexai/   @googleapis/vertexai-team @googleapis/cloud-sdk-java-team
 /java-bigquerystorage/ @googleapis/bigquery-team @googleapis/cloud-sdk-java-team
+/java-bigquery/ @googleapis/bigquery-team @googleapis/cloud-sdk-java-team

.github/workflows/ci.yaml

@@ -25,7 +25,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        java: [11, 17, 21, 25]
+        java: [8, 11, 17, 21, 25]
     steps:
     - name: Get current week within the year
       id: date
@@ -45,39 +45,6 @@ jobs:
       env:
         JOB_TYPE: test
         JOB_NAME: units-${{matrix.java}}
-  units-8-runtime:
-    runs-on: ubuntu-latest
-    name: "units (8)"
-    steps:
-    - name: Get current week within the year
-      id: date
-      run: echo "::set-output name=week_of_year::$(date +'%W' --utc)"
-    - uses: actions/checkout@v4
-    - uses: actions/setup-java@v4
-      with:
-        java-version: 11
-        distribution: temurin
-        cache: maven
-    - run: java -version
-    - uses: actions/cache@v4
-      id: mvn-cache
-      with:
-        path: ~/.m2/repository
-        key: ${{ runner.os }}-maven-unified-${{ steps.date.outputs.week_of_year }}
-    - name: Install all modules using Java 11
-      shell: bash
-      run: |
-        mvn -V -B -ntp clean install -DskipTests -Dfmt.skip -Dclirr.skip -Denforcer.skip -Dcheckstyle.skip -Dflatten.skip -Danimal-sniffer.skip -Djacoco.skip
-    - uses: actions/setup-java@v4
-      with:
-        java-version: 8
-        distribution: temurin
-    - run: java -version
-    - name: Run tests in Java 8 with the source compiled in Java 11
-      run: .kokoro/build.sh
-      env:
-        JOB_TYPE: test
-        JOB_NAME: units-11-runtime-8
   windows:
     runs-on: windows-latest
     steps:

.github/workflows/generated_files_sync.yaml

@@ -164,13 +164,15 @@ jobs:
         
         # java/com/google : This is the standard package
         # samples         : Samples are not shipped as a library
+        # benchmark       : Benchmarking tests are not shipped as a library
         # grafeas         : java-grafeas is known to have special package name
         # cloud-build v2  : java_package was not configured when we published
         #                   the Cloud Build V2 client library
         # the rest        : the same as above
         invalid_files=$(find . -name '*.java' \
             |grep --invert-match 'java/com/google' \
             |grep --invert-match samples \
+            |grep --invert-match benchmark \
             |grep --invert-match grafeas \
             |grep --invert-match 'cloud-build.*v2' \
             |grep --invert-match 'google/monitoring/v3/DroppedLabelsOuterClass.java' \

.github/workflows/java-bigquery-ci.yaml

@@ -0,0 +1,143 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# Github action job to test core java library features on
+# downstream client libraries before they are released.
+on:
+  push:
+    branches:
+    - main
+  pull_request:
+name: java-bigquery ci
+env:
+  BUILD_SUBDIR: java-bigquery
+jobs:
+  filter:
+    runs-on: ubuntu-latest
+    outputs:
+      library: ${{ steps.filter.outputs.library }}
+    steps:
+    - uses: actions/checkout@v4
+    - uses: dorny/paths-filter@v3
+      id: filter
+      with:
+        filters: |
+          library:
+            - 'java-bigquery/**'
+  units:
+    needs: filter
+    if: ${{ needs.filter.outputs.library == 'true' }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        java: [11, 17, 21, 25]
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-java@v4
+      with:
+        distribution: temurin
+        java-version: ${{matrix.java}}
+    - run: java -version
+    - run: .kokoro/build.sh
+      env:
+        JOB_TYPE: test
+  units-java8:
+    needs: filter
+    if: ${{ needs.filter.outputs.library == 'true' }}
+    # Building using Java 17 and run the tests with Java 8 runtime
+    name: "units (8)"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-java@v4
+        with:
+          java-version: 8
+          distribution: temurin
+      - name: "Set jvm system property environment variable for surefire plugin (unit tests)"
+        # Maven surefire plugin (unit tests) allows us to specify JVM to run the tests.
+        # https://maven.apache.org/surefire/maven-surefire-plugin/test-mojo.html#jvm
+        run: echo "SUREFIRE_JVM_OPT=-Djvm=${JAVA_HOME}/bin/java -P !java17" >> $GITHUB_ENV
+        shell: bash
+      - uses: actions/setup-java@v4
+        with:
+          java-version: 17
+          distribution: temurin
+      - run: .kokoro/build.sh
+        env:
+          JOB_TYPE: test
+  windows:
+    needs: filter
+    if: ${{ needs.filter.outputs.library == 'true' }}
+    runs-on: windows-latest
+    steps:
+    - name: Support longpaths
+      run: git config --system core.longpaths true
+    - name: Support longpaths
+      run: git config --system core.longpaths true
+    - uses: actions/checkout@v4
+    - uses: actions/setup-java@v4
+      with:
+        distribution: temurin
+        java-version: 8
+    - run: java -version
+    - run: .kokoro/build.sh
+      env:
+        JOB_TYPE: test
+  dependencies:
+    needs: filter
+    if: ${{ needs.filter.outputs.library == 'true' }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        java: [17]
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-java@v4
+      with:
+        distribution: temurin
+        java-version: ${{matrix.java}}
+    - run: java -version
+    - run: .kokoro/dependencies.sh
+  javadoc:
+    needs: filter
+    if: ${{ needs.filter.outputs.library == 'true' }}
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-java@v4
+      with:
+        distribution: temurin
+        java-version: 17
+    - run: java -version
+    - run: .kokoro/build.sh
+      env:
+        JOB_TYPE: javadoc
+  lint:
+    needs: filter
+    if: ${{ needs.filter.outputs.library == 'true' }}
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    - uses: actions/setup-java@v4
+      with:
+        distribution: temurin
+        java-version: 17
+    - run: java -version
+    - run: .kokoro/build.sh
+      env:
+        JOB_TYPE: lint
+        HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+        BASE_SHA: ${{ github.event.pull_request.base.sha }}

.github/workflows/java-bigquery-scorecard.yml

@@ -0,0 +1,88 @@
+# This workflow uses actions that are not certified by GitHub. They are provided
+# by a third-party and are governed by separate terms of service, privacy
+# policy, and support documentation.
+
+name: java-bigquery Scorecard supply-chain security
+on:
+  # For Branch-Protection check. Only the default branch is supported. See
+  # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection
+  branch_protection_rule:
+  # To guarantee Maintained check is occasionally updated. See
+  # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained
+  schedule:
+    - cron: '0 0 * * *'
+  push:
+    branches: [ "main" ]
+
+# Declare default permissions as read only.
+permissions: read-all
+
+env:
+  BUILD_SUBDIR: java-bigquery
+jobs:
+  filter:
+    runs-on: ubuntu-latest
+    outputs:
+      library: ${{ steps.filter.outputs.library }}
+    steps:
+    - uses: actions/checkout@v4
+    - uses: dorny/paths-filter@v3
+      id: filter
+      with:
+        filters: |
+          library:
+            - 'java-bigquery/**'
+  analysis:
+    needs: filter
+    if: ${{ needs.filter.outputs.library == 'true' }}
+    name: Scorecard analysis
+    runs-on: ubuntu-latest
+    permissions:
+      # Needed to upload the results to code-scanning dashboard.
+      security-events: write
+      # Needed to publish results and get a badge (see publish_results below).
+      id-token: write
+      # Uncomment the permissions below if installing in a private repository.
+      # contents: read
+      # actions: read
+
+    steps:
+      - name: "Checkout code"
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          persist-credentials: false
+
+      - name: "Run analysis"
+        uses: ossf/scorecard-action@05b42c624433fc40578a4040d5cf5e36ddca8cde # v2.4.2
+        with:
+          results_file: results.sarif
+          results_format: sarif
+          # (Optional) "write" PAT token. Uncomment the `repo_token` line below if:
+          # - you want to enable the Branch-Protection check on a *public* repository, or
+          # - you are installing Scorecard on a *private* repository
+          # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action#authentication-with-pat.
+          # repo_token: ${{ secrets.SCORECARD_TOKEN }}
+
+          # Public repositories:
+          #   - Publish results to OpenSSF REST API for easy access by consumers
+          #   - Allows the repository to include the Scorecard badge.
+          #   - See https://github.com/ossf/scorecard-action#publishing-results.
+          # For private repositories:
+          #   - `publish_results` will always be set to `false`, regardless
+          #     of the value entered here.
+          publish_results: true
+
+      # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
+      # format to the repository Actions tab.
+      - name: "Upload artifact"
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
+        with:
+          name: SARIF file
+          path: results.sarif
+          retention-days: 5
+
+      # Upload the results to GitHub's code scanning dashboard.
+      - name: "Upload to code-scanning"
+        uses: github/codeql-action/upload-sarif@b8d3b6e8af63cde30bdc382c0bc28114f4346c88 # v2.28.1
+        with:
+          sarif_file: results.sarif

.kokoro/build.sh

@@ -269,11 +269,19 @@ case ${JOB_TYPE} in
         exit 1
     fi
 
+    # The lint property is to activate the benchmark profile that declares the
+    # benchmark module in java-bigquery/benchmark. This module is not needed during
+    # other builds or releases.
     mvn -B -ntp \
       -T 1C \
       ${MODULE_FILTER} \
+      -Dlint \
       com.spotify.fmt:fmt-maven-plugin:check
-    mvn -B -ntp ${MODULE_FILTER} checkstyle:check@checkstyle
+    mvn -B -ntp \
+      -T 1C \
+      ${MODULE_FILTER} \
+      -Dlint \
+      checkstyle:check@checkstyle
 
     if [[ -n "${BUILD_SUBDIR}" ]]
     then

.kokoro/common.sh

@@ -19,6 +19,7 @@ excluded_modules=(
   'google-cloud-pom-parent'
   'java-vertexai'
   'java-logging'
+  'java-bigquery'
   'java-bigquerystorage'
   'java-datastore'
   'java-logging-logback'

.kokoro/presubmit/bigquery-graalvm-native-presubmit.cfg

@@ -0,0 +1,47 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Configure the docker image for kokoro-trampoline.
+env_vars: {
+  key: "TRAMPOLINE_IMAGE"
+  value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_a:3.57.0"
+}
+
+env_vars: {
+  key: "JOB_TYPE"
+  value: "graalvm-single"
+}
+
+# TODO: remove this after we've migrated all tests and scripts
+env_vars: {
+  key: "GCLOUD_PROJECT"
+  value: "gcloud-devel"
+}
+
+env_vars: {
+  key: "GOOGLE_CLOUD_PROJECT"
+  value: "gcloud-devel"
+}
+
+env_vars: {
+  key: "GOOGLE_APPLICATION_CREDENTIALS"
+  value: "secret_manager/java-it-service-account"
+}
+
+env_vars: {
+  key: "SECRET_MANAGER_KEYS"
+  value: "java-it-service-account"
+}
+
+env_vars: {
+  key: "ENABLE_FLAKYBOT"
+  value: "false"
+}
+env_vars: {
+  key: "BUILD_SUBDIR"
+  value: "java-bigquery"
+}
+
+env_vars: {
+  key: "INTEGRATION_TEST_ARGS"
+  value: "-pl !google-cloud-bigquery-jdbc"
+}