code-scanning-pack-gen.yml

name: Code Scanning Query Pack Generation

on:
  pull_request:
    branches:
      - main
      - "rc/**"
      - next

  push:
    branches:
      - main
      - "rc/**"
      - next

env:
  XARGS_MAX_PROCS: 4

jobs:
  prepare-code-scanning-pack-matrix:
    name: Prepare CodeQL Code Scanning pack matrix
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.export-code-scanning-pack-matrix.outputs.matrix }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@v2

      - name: Export Code Scanning pack matrix
        id: export-code-scanning-pack-matrix
        run: |
          echo "::set-output name=matrix::$(
            jq --compact-output '.supported_environment | {include: .}' supported_codeql_configs.json
          )"

  create-code-scanning-pack-anon:
    name: Create anonymous Code Scanning pack
    needs: prepare-code-scanning-pack-matrix
    runs-on: ubuntu-20.04-xl
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.prepare-code-scanning-pack-matrix.outputs.matrix) }}
    steps:
      - uses: actions/checkout@v2

      - name: Cache CodeQL
        id: cache-codeql
        uses: actions/cache@v2.1.3
        with:
          path: ${{ github.workspace }}/codeql_home
          key: codeql-home-${{ matrix.os }}-${{ matrix.codeql_cli }}-${{ matrix.codeql_standard_library }}

      - name: Install CodeQL
        if: steps.cache-codeql.outputs.cache-hit != 'true'
        uses: ./.github/actions/install-codeql
        with:
          codeql-cli-version: ${{ matrix.codeql_cli }}
          codeql-stdlib-version: ${{ matrix.codeql_standard_library }}
          codeql-home: ${{ github.workspace }}/codeql_home
          add-to-path: false

      - name: Install Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.9"

      - name: Anonymising and pre-compiling queries
        env:
          CODEQL_HOME: ${{ github.workspace }}/codeql_home
        run: |
          PATH=$PATH:$CODEQL_HOME/codeql
          pip install -r scripts/requirements.txt
          find rule_packages/cpp -name '*.json' -exec basename {} .json \; | xargs --max-procs "$XARGS_MAX_PROCS" --max-args 1 python scripts/generate_rules/generate_package_files.py -a cpp
          find rule_packages/c -name '*.json' -exec basename {} .json \; | xargs --max-procs "$XARGS_MAX_PROCS" --max-args 1 python scripts/generate_rules/generate_package_files.py --skip-shared-test-generation -a c

          echo "Remove help files that cannot be freely distributed"
          find cpp/autosar/src/rules -name "*.md" -delete
          find c/misra/src/rules -name "*.md" -delete

          codeql query compile --search-path cpp --threads 0 cpp
          codeql query compile --search-path c --search-path cpp --threads 0 c
          cd ..
          zip -r codeql-coding-standards/code-scanning-cpp-query-pack-anon.zip codeql-coding-standards/c/ codeql-coding-standards/cpp/ codeql-coding-standards/.codeqlmanifest.json codeql-coding-standards/supported_codeql_configs.json codeql-coding-standards/scripts/deviations codeql-coding-standards/scripts/reports

      - name: Upload GHAS Query Pack
        uses: actions/upload-artifact@v2
        with:
          name: code-scanning-cpp-query-pack-anon.zip
          path: code-scanning-cpp-query-pack-anon.zip

      - name: Create LGTM query pack
        env:
          CODEQL_HOME: ${{ github.workspace }}/codeql_home
        run: |
          PATH=$PATH:$CODEQL_HOME/codeql
          mkdir -p lgtm-cpp-query-pack
          function copy_queries_for_pack {
            for q in $(codeql resolve queries $2/$1/src/codeql-suites/$1-default.qls)
            do
              copy_from_root_name="${q%.*}"
              copy_to_root_name=$(realpath --relative-to "./$2/$1/src/" "$copy_from_root_name")
              query_dir=$(dirname "lgtm-cpp-query-pack/$copy_to_root_name")
              mkdir -p "$query_dir"
              # Copy each selected ql file
              cp "$copy_from_root_name.ql" "lgtm-cpp-query-pack/$copy_to_root_name.ql"
            done
          }

          echo "Copying autosar-default queries (CPP)"
          copy_queries_for_pack "autosar" "cpp"
          echo "Copying cert-default queries (CPP)"
          copy_queries_for_pack "cert" "cpp"

          echo "Copying misra-default queries (C)"
          copy_queries_for_pack "misra" "c"
          echo "Copying cert-default queries (C)"
          copy_queries_for_pack "cert" "c"


          # Now copy all the .qll files
          lgtm_pack_dir=$(realpath lgtm-cpp-query-pack)
          for query_pack in autosar cert common
          do
            echo "Copying $query_pack qll files"
            pushd cpp/$query_pack/src
            for query_library in $(find . -name \*.qll)
            do
              qll_dir=$(dirname "$lgtm_pack_dir/$query_library")
              echo "Making $qll_dir"
              mkdir -p "$qll_dir"
              cp "$query_library" "$lgtm_pack_dir/$query_library"
            done
            popd
          done


          for query_pack in misra cert common
          do
            echo "Copying $query_pack qll files"
            pushd c/$query_pack/src
            for query_library in $(find . -name \*.qll)
            do
              qll_dir=$(dirname "$lgtm_pack_dir/$query_library")
              echo "Making $qll_dir"
              mkdir -p "$qll_dir"
              cp "$query_library" "$lgtm_pack_dir/$query_library"
            done
            popd
          done

          cd lgtm-cpp-query-pack
          zip -9 -r ../lgtm-cpp-query-pack-anon.zip *

      - name: Upload LGTM query pack
        uses: actions/upload-artifact@v2
        with:
          name: lgtm-cpp-query-pack-anon.zip
          path: lgtm-cpp-query-pack-anon.zip

  create-code-scanning-pack:
    name: Create Code Scanning pack
    needs: prepare-code-scanning-pack-matrix
    runs-on: ubuntu-20.04-xl
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.prepare-code-scanning-pack-matrix.outputs.matrix) }}
    steps:
      - uses: actions/checkout@v2

      - name: Cache CodeQL
        id: cache-codeql
        uses: actions/cache@v2.1.3
        with:
          path: ${{ github.workspace }}/codeql_home
          key: codeql-home-${{ matrix.os }}-${{ matrix.codeql_cli }}-${{ matrix.codeql_standard_library }}

      - name: Install CodeQL
        if: steps.cache-codeql.outputs.cache-hit != 'true'
        uses: ./.github/actions/install-codeql
        with:
          codeql-cli-version: ${{ matrix.codeql_cli }}
          codeql-stdlib-version: ${{ matrix.codeql_standard_library }}
          codeql-home: ${{ github.workspace }}/codeql_home
          add-to-path: false

      - name: Checkout external help files
        uses: actions/checkout@v2
        with:
          ssh-key: ${{ secrets.CODEQL_CODING_STANDARDS_HELP_KEY }}
          repository: "github/codeql-coding-standards-help"
          ref: ${{ github.head_ref }}
          path: external-help-files

      - name: Include external help files
        run: |
          pushd external-help-files
          find . -name '*.md' -exec rsync -av --relative {} "$GITHUB_WORKSPACE" \;
          popd

      - name: Pre-compiling queries
        env:
          CODEQL_HOME: ${{ github.workspace }}/codeql_home
        run: |
          PATH=$PATH:$CODEQL_HOME/codeql

          codeql query compile --search-path cpp --threads 0 cpp
          codeql query compile --search-path c --search-path cpp --threads 0 c

          cd ..
          zip -r codeql-coding-standards/code-scanning-cpp-query-pack.zip codeql-coding-standards/c/ codeql-coding-standards/cpp/ codeql-coding-standards/.codeqlmanifest.json codeql-coding-standards/supported_codeql_configs.json codeql-coding-standards/scripts/deviations codeql-coding-standards/scripts/reports

      - name: Upload GHAS Query Pack
        uses: actions/upload-artifact@v2
        with:
          name: code-scanning-cpp-query-pack.zip
          path: code-scanning-cpp-query-pack.zip

      - name: Create LGTM query pack
        env:
          CODEQL_HOME: ${{ github.workspace }}/codeql_home
        run: |
          PATH=$PATH:$CODEQL_HOME/codeql
          mkdir -p lgtm-cpp-query-pack
          function copy_queries_for_pack {
            for rule_dir in $(codeql resolve queries $2/$1/src/codeql-suites/$1-default.qls | xargs -L1 dirname | uniq)
            do
              copy_to_root="lgtm-cpp-query-pack/$(realpath --relative-to "./$2/$1/src/" "$rule_dir")"
              mkdir -p "$copy_to_root"
              # Copy each selected ql file, plus the related files
              find "$rule_dir" -name '*.ql' -o -name '*.c' -name '*.cpp' -o -name '*.png' -exec cp -n {} "$copy_to_root" \;
            done
          }
          echo "Copying autosar-default queries (CPP)"
          copy_queries_for_pack "autosar" "cpp"
          echo "Copying cert-default queries (CPP)"
          copy_queries_for_pack "cert" "cpp"

          echo "Copying misra-default queries (C)"
          copy_queries_for_pack "misra" "c"
          echo "Copying cert-default queries (C)"
          copy_queries_for_pack "cert" "c"


          # Now copy all the .qll files
          lgtm_pack_dir=$(realpath lgtm-cpp-query-pack)
          for query_pack in autosar cert common
          do
            echo "Copying $query_pack qll files"
            pushd cpp/$query_pack/src
            for query_library in $(find . -name \*.qll)
            do
              qll_dir=$(dirname "$lgtm_pack_dir/$query_library")
              echo "Making $qll_dir"
              mkdir -p "$qll_dir"
              cp "$query_library" "$lgtm_pack_dir/$query_library"
            done
            popd
          done

          for query_pack in misra cert common
          do
            echo "Copying $query_pack qll files"
            pushd c/$query_pack/src
            for query_library in $(find . -name \*.qll)
            do
              qll_dir=$(dirname "$lgtm_pack_dir/$query_library")
              echo "Making $qll_dir"
              mkdir -p "$qll_dir"
              cp "$query_library" "$lgtm_pack_dir/$query_library"
            done
            popd
          done

          cd lgtm-cpp-query-pack
          zip -9 -r ../lgtm-cpp-query-pack.zip *

      - name: Upload LGTM query pack
        uses: actions/upload-artifact@v2
        with:
          name: lgtm-cpp-query-pack.zip
          path: lgtm-cpp-query-pack.zip