Explore performance improvements in golang generation pipeline

.github/workflows/reusable-examples.yml

@@ -10,8 +10,26 @@ on:
         default: ''
 
 jobs:
-  examples:
+  prepare:
     runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.split.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          repository: DataDog/datadog-api-client-go
+          ref: ${{ inputs.target-branch || github.ref }}
+      - name: Compute example groups
+        id: split
+        run: ./scripts/prepare-examples.sh
+        shell: bash
+
+  build:
+    needs: prepare
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        group: ${{ fromJson(needs.prepare.outputs.matrix) }}
     steps:
       - uses: actions/checkout@v3
         with:
@@ -23,6 +41,6 @@ jobs:
           go-version: 1.22.x
           cache: true
           cache-dependency-path: tests/go.sum
-      - name: Check examples
-        run: ./check-examples.sh
-        shell: bash
+      - name: Build examples (group ${{ matrix.group }})
+        run: ./scripts/build-examples-group.sh ${{ matrix.group }}
+        shell: bash

.github/workflows/reusable-go-test.yml

@@ -21,6 +21,37 @@ on:
         required: false
 
 jobs:
+  staticcheck:
+    strategy:
+      matrix:
+        go-version: ["1.22.x", "1.23.x"]
+        go-build-tags: ["--tags=goccy_gojson", ""]
+        platform: ["ubuntu-latest"]
+    runs-on: ${{ matrix.platform }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          repository: DataDog/datadog-api-client-go
+          ref: ${{ inputs.target-branch || github.ref }}
+      - name: Install Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: ${{ matrix.go-version }}
+          cache: true
+          cache-dependency-path: tests/go.sum
+      - name: Staticcheck (api module)
+        uses: dominikh/staticcheck-action@v1
+        with:
+          checks: "-SA1009"
+          working-directory: api
+          cache-key: ${{ matrix.go }}
+      - name: Staticcheck (tests module)
+        uses: dominikh/staticcheck-action@v1
+        with:
+          checks: "inherit,-SA1019"
+          cache-key: ${{ matrix.go }}
+          working-directory: tests
   test:
     strategy:
       matrix:
@@ -41,7 +72,7 @@ jobs:
           cache: true
           cache-dependency-path: tests/go.sum
       - name: Test
-        run: ./run-tests.sh
+        run: ./scripts/run-tests.sh
         env:
           TESTARGS: ${{ matrix.go-build-tags }}
 

.github/workflows/reusable-integration-test.yml

@@ -111,7 +111,7 @@ jobs:
           cache-dependency-path: tests/go.sum
       - name: Run integration tests
         shell: bash
-        run: ./run-tests.sh
+        run: ./scripts/run-tests.sh
         env:
           CI: "true"
           DD_AGENT_HOST: localhost

scripts/build-examples-group.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+set -e
+
+GROUP=${1:?usage: build-examples-group.sh <group-number>}
+EXAMPLES_PER_GROUP=${EXAMPLES_PER_GROUP:-400}
+
+ALL_EXAMPLES=()
+while IFS= read -r f; do ALL_EXAMPLES+=("$f"); done < <(find examples -mindepth 3 -maxdepth 3 -name "*.go" | sort)
+TOTAL=${#ALL_EXAMPLES[@]}
+
+START=$(( (GROUP - 1) * EXAMPLES_PER_GROUP ))
+SLICE=("${ALL_EXAMPLES[@]:$START:$EXAMPLES_PER_GROUP}")
+
+echo "Building group $GROUP: examples $((START + 1))-$((START + ${#SLICE[@]})) of $TOTAL"
+
+for f in "${SLICE[@]}"; do
+    df="build/$(dirname "$f")/$(basename "$f" .go)"
+    mkdir -p "$df"
+    cp "$f" "$df/main.go"
+done
+
+if go build -o /dev/null -ldflags "-s -w -linkmode internal" ./build/examples/*/*/*; then
+    echo "Examples are buildable"
+else
+    echo "Failed to build examples"
+    exit 1
+fi

scripts/prepare-examples.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+set -e
+
+EXAMPLES_PER_GROUP=${EXAMPLES_PER_GROUP:-400}
+
+ALL_EXAMPLES=()
+while IFS= read -r f; do ALL_EXAMPLES+=("$f"); done < <(find examples -mindepth 3 -maxdepth 3 -name "*.go" | sort)
+TOTAL=${#ALL_EXAMPLES[@]}
+NUM_GROUPS=$(( (TOTAL + EXAMPLES_PER_GROUP - 1) / EXAMPLES_PER_GROUP ))
+
+echo "Total examples: $TOTAL, groups: $NUM_GROUPS (max $EXAMPLES_PER_GROUP per group)"
+
+MATRIX_JSON="["
+for i in $(seq 1 "$NUM_GROUPS"); do
+    [ "$i" -gt 1 ] && MATRIX_JSON+=","
+    MATRIX_JSON+="$i"
+done
+MATRIX_JSON+="]"
+
+echo "matrix=$MATRIX_JSON"
+if [ -n "${GITHUB_OUTPUT:-}" ]; then
+    echo "matrix=$MATRIX_JSON" >> "$GITHUB_OUTPUT"
+fi

run-tests.sh → scripts/run-tests.sh

@@ -23,24 +23,26 @@ fi
 # make sure the below installed dependencies don't get added to go.mod/go.sum
 # unfortunately there's no better way to fix this than change directory
 # this might get solved in Go 1.14: https://github.com/golang/go/issues/30515
+echo "Installing gotestsum"
 cd "$(mktemp -d)"
-GO111MODULE=on go install honnef.co/go/tools/cmd/staticcheck@latest
 GO111MODULE=on go install gotest.tools/gotestsum@latest
 cd -
 
-staticcheck -checks -SA1009 ./api/...
+echo "Running mod tidy and cleanup test cache"
 go mod tidy
 go clean -testcache
 
 # Run the same in tests submodule
 cd tests
-staticcheck -checks inherit,-SA1019 ./...
+echo "Running mod tidy and cleanup test cache in test module"
 go mod tidy
 go clean -testcache
 
 if [ "$RECORD" == "none" ]; then
+    echo "Running with no RECORD"
     gotestsum --rerun-fails=1 --format short-verbose --packages ./... -- -timeout=20m $TESTARGS
 else
+    echo "Running with RECORD"
     gotestsum --format short-verbose --packages ./... -- $TESTARGS
 fi