Skip to content

Commit e17f940

Browse files
author
Dilan Bhalla
committed
iac windows extraction
1 parent e389dc1 commit e17f940

7 files changed

Lines changed: 179 additions & 21 deletions

File tree

‎.github/workflows/build.yml‎

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,10 @@ permissions:
1212

1313
jobs:
1414
tests:
15-
runs-on: ubuntu-latest
15+
runs-on: ${{ matrix.os }}
1616
strategy:
1717
matrix:
18+
os: [ubuntu-latest, windows-latest]
1819
test-folders: ["library-tests", "queries-tests"]
1920
steps:
2021
- name: "Checkout"
@@ -53,8 +54,8 @@ jobs:
5354
- uses: dtolnay/rust-toolchain@4305c38b25d97ef35a8ad1f985ccf2d2242004f2 # stable
5455
if: steps.extractor-changes.outputs.src == 'true'
5556

56-
- name: "Build Extractor"
57-
if: steps.extractor-changes.outputs.src == 'true'
57+
- name: "Build Extractor (Linux/macOS)"
58+
if: steps.extractor-changes.outputs.src == 'true' && matrix.os != 'windows-latest'
5859
env:
5960
GH_TOKEN: ${{ github.token }}
6061
run: |
@@ -66,12 +67,34 @@ jobs:
6667
6768
gh codeql resolve languages --format=json --search-path ./extractor-pack
6869
69-
- name: "Run Tests"
70+
- name: "Build Extractor (Windows)"
71+
if: steps.extractor-changes.outputs.src == 'true' && matrix.os == 'windows-latest'
72+
env:
73+
GH_TOKEN: ${{ github.token }}
74+
shell: pwsh
75+
run: |
76+
gh extension install github/gh-codeql
77+
gh codeql set-version latest
78+
79+
.\scripts\create-extractor-pack.ps1
80+
81+
gh codeql resolve languages --format=json --search-path .\extractor-pack
82+
83+
- name: "Run Tests (Linux/macOS)"
84+
if: matrix.os != 'windows-latest'
7085
env:
7186
GH_TOKEN: ${{ github.token }}
7287
run: |
7388
./scripts/run-tests.sh "ql/test/${{ matrix.test-folders }}"
7489
90+
- name: "Run Tests (Windows)"
91+
if: matrix.os == 'windows-latest'
92+
env:
93+
GH_TOKEN: ${{ github.token }}
94+
shell: pwsh
95+
run: |
96+
.\scripts\run-tests.ps1 "ql/test/${{ matrix.test-folders }}"
97+
7598
# scanning:
7699
# runs-on: ubuntu-latest
77100
# needs: [tests]

‎.github/workflows/publish.yml‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,7 @@ jobs:
4747
strategy:
4848
fail-fast: false
4949
matrix:
50-
# TODO: Add windows-latest
51-
os: [ubuntu-latest, macos-latest]
50+
os: [ubuntu-latest, macos-latest, windows-latest]
5251

5352
if: ${{ needs.release-check.outputs.release == 'true' }}
5453
steps:
@@ -59,14 +58,20 @@ jobs:
5958

6059
- name: "Set up Rust"
6160
uses: dtolnay/rust-toolchain@nightly
62-
if: ${{ matrix.os != 'windows-latest' }}
6361

64-
- name: "Build Extractor"
62+
- name: "Build Extractor (Linux/macOS)"
6563
if: ${{ matrix.os != 'windows-latest' }}
6664
env:
6765
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
6866
run: ./scripts/create-extractor-pack.sh
6967

68+
- name: "Build Extractor (Windows)"
69+
if: ${{ matrix.os == 'windows-latest' }}
70+
env:
71+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
72+
shell: pwsh
73+
run: .\scripts\create-extractor-pack.ps1
74+
7075
- name: "Upload bundle artifact"
7176
uses: actions/upload-artifact@v4
7277
with:
@@ -85,7 +90,7 @@ jobs:
8590
with:
8691
submodules: true
8792

88-
- name: "Downloadd all artifacts"
93+
- name: "Download all artifacts"
8994
uses: actions/download-artifact@v4
9095
with:
9196
path: "./extractor-pack"

‎scripts/create-extractor-pack.ps1‎

Lines changed: 55 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,58 @@
1-
cargo build --release
1+
$ErrorActionPreference = "Stop"
2+
3+
# Set platform
4+
$platform = "win64"
25

3-
cargo run --release -p ql-generator -- --dbscheme ql/src/ql.dbscheme --library ql/src/codeql_ql/ast/internal/TreeSitter.qll
4-
codeql query format -i ql\src\codeql_ql\ast\internal\TreeSitter.qll
6+
# Check for CodeQL binary
7+
if (Get-Command "codeql" -ErrorAction SilentlyContinue) {
8+
$CODEQL_BINARY = "codeql"
9+
}
10+
elseif (Get-Command "gh" -ErrorAction SilentlyContinue) {
11+
try {
12+
gh codeql version 2>&1 | Out-Null
13+
$CODEQL_BINARY = "gh codeql"
14+
}
15+
catch {
16+
Write-Host "Installing gh-codeql extension..."
17+
gh extension install github/gh-codeql
18+
$CODEQL_BINARY = "gh codeql"
19+
}
20+
}
21+
else {
22+
Write-Error "Neither 'codeql' nor 'gh' command found"
23+
exit 1
24+
}
525

26+
Write-Host "Creating extractor pack..."
627
if (Test-Path -Path extractor-pack) {
7-
rm -Recurse -Force extractor-pack
8-
}
9-
mkdir extractor-pack | Out-Null
10-
cp codeql-extractor.yml, ql\src\ql.dbscheme, ql\src\ql.dbscheme.stats extractor-pack
11-
cp -Recurse tools extractor-pack
12-
mkdir extractor-pack\tools\win64 | Out-Null
13-
cp target\release\ql-extractor.exe extractor-pack\tools\win64\extractor.exe
14-
cp target\release\ql-autobuilder.exe extractor-pack\tools\win64\autobuilder.exe
28+
Remove-Item -Recurse -Force extractor-pack
29+
}
30+
if (Test-Path -Path target) {
31+
Remove-Item -Recurse -Force target
32+
}
33+
34+
Write-Host "Update submodules..."
35+
git submodule update --init --recursive
36+
37+
Write-Host "Building extractor..."
38+
cargo build --release
39+
40+
Write-Host "Generating TreeSitter library..."
41+
cargo run --release --bin codeql-extractor-iac -- generate --dbscheme ql/lib/iac.dbscheme --library ql/lib/codeql/iac/ast/internal/TreeSitter.qll
42+
43+
Write-Host "Formatting generated library..."
44+
if ($CODEQL_BINARY -eq "gh codeql") {
45+
gh codeql query format -i ql/lib/codeql/iac/ast/internal/TreeSitter.qll
46+
}
47+
else {
48+
codeql query format -i ql/lib/codeql/iac/ast/internal/TreeSitter.qll
49+
}
50+
51+
New-Item -ItemType Directory -Path extractor-pack | Out-Null
52+
Copy-Item -Path codeql-extractor.yml, ql/lib/iac.dbscheme, ql/lib/iac.dbscheme.stats -Destination extractor-pack/
53+
Copy-Item -Recurse -Path downgrades, tools -Destination extractor-pack/
54+
55+
New-Item -ItemType Directory -Path "extractor-pack/tools/$platform" -Force | Out-Null
56+
Copy-Item -Path "target/release/codeql-extractor-iac.exe" -Destination "extractor-pack/tools/$platform/extractor.exe"
57+
58+
Write-Host "Extractor pack created successfully!"

‎scripts/install-extractor.ps1‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
param(
2+
[string]$ExtractorName = "iac",
3+
[string]$ExtractorLocations = "$env:USERPROFILE\.codeql\extractors"
4+
)
5+
6+
$ErrorActionPreference = "Stop"
7+
8+
Write-Host "Creating extractor directory..."
9+
if (!(Test-Path $ExtractorLocations)) {
10+
New-Item -ItemType Directory -Path $ExtractorLocations -Force | Out-Null
11+
}
12+
13+
Write-Host "Checking latest release..."
14+
gh release list -L 1 -R "advanced-security/codeql-extractor-$ExtractorName"
15+
16+
Write-Host "Downloading extractor pack..."
17+
gh release download `
18+
-R "advanced-security/codeql-extractor-$ExtractorName" `
19+
-D "$ExtractorLocations" `
20+
--clobber `
21+
--pattern 'extractor-*.tar.gz'
22+
23+
Write-Host "Extracting extractor pack..."
24+
tar -zxf "$ExtractorLocations/extractor-$ExtractorName.tar.gz" --directory "$ExtractorLocations"
25+
26+
Write-Host "Installation complete! Extractor installed to: $ExtractorLocations"

‎scripts/run-tests.ps1‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
param(
2+
[string]$TestsDir = "ql/test"
3+
)
4+
5+
$ErrorActionPreference = "Stop"
6+
7+
# Check for CodeQL binary
8+
if (Get-Command "codeql" -ErrorAction SilentlyContinue) {
9+
$CODEQL_BINARY = "codeql"
10+
}
11+
elseif (Get-Command "gh" -ErrorAction SilentlyContinue) {
12+
try {
13+
gh codeql version 2>&1 | Out-Null
14+
$CODEQL_BINARY = "gh codeql"
15+
}
16+
catch {
17+
Write-Host "Installing gh-codeql extension..."
18+
gh extension install github/gh-codeql
19+
$CODEQL_BINARY = "gh codeql"
20+
}
21+
}
22+
else {
23+
Write-Error "Neither 'codeql' nor 'gh' command found"
24+
exit 1
25+
}
26+
27+
Write-Host "Installing ql/test pack dependencies..."
28+
if ($CODEQL_BINARY -eq "gh codeql") {
29+
gh codeql pack install ql/test
30+
}
31+
else {
32+
codeql pack install ql/test
33+
}
34+
35+
Write-Host "Running tests in $TestsDir"
36+
37+
if ($CODEQL_BINARY -eq "gh codeql") {
38+
gh codeql test run `
39+
-j 0 `
40+
--check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition `
41+
--search-path ./extractor-pack `
42+
"$TestsDir"
43+
}
44+
else {
45+
codeql test run `
46+
-j 0 `
47+
--check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition `
48+
--search-path ./extractor-pack `
49+
"$TestsDir"
50+
}
51+
52+
if ($LASTEXITCODE -ne 0) {
53+
Write-Error "Tests failed with exit code $LASTEXITCODE"
54+
exit $LASTEXITCODE
55+
}
56+
57+
Write-Host "All tests passed!"

‎tools/index-files.cmd‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
@echo off
22

33
type NUL && "%CODEQL_EXTRACTOR_IAC_ROOT%\tools\win64\extractor.exe" ^
4+
extract ^
45
--file-list "%1" ^
56
--source-archive-dir "%CODEQL_EXTRACTOR_IAC_SOURCE_ARCHIVE_DIR%" ^
67
--output-dir "%CODEQL_EXTRACTOR_IAC_TRAP_DIR%"

‎tools/qltest.cmd‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,10 @@ type NUL && "%CODEQL_DIST%\codeql.exe" database index-files ^
44
--prune=**/*.testproj ^
55
--include-extension=.hcl ^
66
--include-extension=.tf ^
7+
--include-extension=.tfvars ^
8+
--include-extension=.bicep ^
79
--size-limit=5m ^
8-
--language=hcl ^
10+
--language=iac ^
911
--working-dir=. ^
1012
"%CODEQL_EXTRACTOR_IAC_WIP_DATABASE%"
1113

0 commit comments

Comments
 (0)