Skip to content

Commit f299fef

Browse files
chanel-ychanel-y
authored
Merge branch 'main' into api-graphs-improvements-for-chanel
2 parents 6c7715b + 74f46a9 commit f299fef

109 files changed

Lines changed: 30423 additions & 0 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎binary/.gitignore‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
**/bin/*
2+
**/obj/*
3+
**/*.dll
4+
**/myDB/*
5+
**/oatDB/*
6+
**/test-db-jvm/*
7+
**/test-db-jvm-create/*
8+
**/test-db-jvm*
9+
**/tools/*

‎binary/build-macos.sh‎

Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
#!/bin/bash
2+
3+
set -eu
4+
5+
# Build script for macOS (ARM64)
6+
# Usage:
7+
# ./build-macos.sh -cil -cliFolder /path/to/cli # Build CIL extractor
8+
# ./build-macos.sh -cil -clean # Clean CIL build artifacts
9+
# ./build-macos.sh -cil -init -cliFolder /path/to/cli # Initialize and build CIL
10+
#
11+
# Future: x86 extractor support will be added
12+
13+
# Defensive script directory detection
14+
if [[ -n "${BASH_SOURCE[0]:-}" ]]; then
15+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
16+
elif [[ -n "${0:-}" ]]; then
17+
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
18+
else
19+
echo "Error: Unable to determine script directory"
20+
exit 1
21+
fi
22+
23+
if [[ -z "${SCRIPT_DIR}" || "${SCRIPT_DIR}" != /* ]]; then
24+
echo "Error: Failed to determine absolute script directory"
25+
exit 1
26+
fi
27+
28+
# Verify we're in the expected directory by checking for a known file
29+
if [[ ! -f "${SCRIPT_DIR}/build-win64.ps1" ]]; then
30+
echo "Error: Script directory validation failed - expected files not found"
31+
echo "SCRIPT_DIR: ${SCRIPT_DIR}"
32+
exit 1
33+
fi
34+
35+
# Parse arguments
36+
BUILD_CIL=false
37+
BUILD_X86=false
38+
CLEAN=false
39+
INIT=false
40+
CLI_FOLDER=""
41+
42+
while [[ $# -gt 0 ]]; do
43+
case $1 in
44+
-cil)
45+
BUILD_CIL=true
46+
shift
47+
;;
48+
-x86)
49+
BUILD_X86=true
50+
shift
51+
;;
52+
-clean)
53+
CLEAN=true
54+
shift
55+
;;
56+
-init)
57+
INIT=true
58+
shift
59+
;;
60+
-cliFolder)
61+
CLI_FOLDER="$2"
62+
shift 2
63+
;;
64+
*)
65+
echo "Unknown option: $1"
66+
echo "Usage: $0 [-cil|-x86] [-clean|-init] [-cliFolder <path>]"
67+
exit 1
68+
;;
69+
esac
70+
done
71+
72+
# If no extractor specified, show usage
73+
if [[ "$BUILD_CIL" == false && "$BUILD_X86" == false ]]; then
74+
echo "Usage: $0 [-cil|-x86] [-clean|-init] [-cliFolder <path>]"
75+
echo ""
76+
echo "Options:"
77+
echo " -cil Build the CIL (C# IL) extractor"
78+
echo " -x86 Build the x86 extractor (not yet implemented)"
79+
echo " -clean Clean build artifacts"
80+
echo " -init Initialize dependencies (x86 only)"
81+
echo " -cliFolder Path to the CodeQL CLI folder (required for build)"
82+
exit 1
83+
fi
84+
85+
# Validate arguments
86+
if [[ "$CLEAN" == false && -z "$CLI_FOLDER" ]]; then
87+
echo "Error: -cliFolder is required unless -clean is specified"
88+
exit 1
89+
fi
90+
91+
build_cil() {
92+
local tools_folder="${CLI_FOLDER}/cil/tools/osx64"
93+
local cil_folder="${CLI_FOLDER}/cil"
94+
95+
pushd "${SCRIPT_DIR}/extractor/cil" > /dev/null
96+
97+
dotnet build Semmle.Extraction.CSharp.IL -o "${tools_folder}" -c Release --self-contained
98+
if [[ $? -ne 0 ]]; then
99+
echo "Build failed"
100+
popd > /dev/null
101+
exit 1
102+
fi
103+
104+
popd > /dev/null
105+
106+
# Create directories
107+
mkdir -p "${tools_folder}"
108+
mkdir -p "${cil_folder}"
109+
110+
# Copy extractor configuration
111+
cp "${SCRIPT_DIR}/extractor/cil/codeql-extractor.yml" "${cil_folder}/"
112+
113+
# Copy downgrades if they exist
114+
if [[ -d "${SCRIPT_DIR}/downgrades" ]]; then
115+
cp -r "${SCRIPT_DIR}/downgrades" "${cil_folder}/"
116+
fi
117+
118+
# Copy dbscheme files
119+
local ql_lib_folder="${SCRIPT_DIR}/ql/lib"
120+
cp "${ql_lib_folder}/semmlecode.binary.dbscheme" "${cil_folder}/"
121+
if [[ -f "${ql_lib_folder}/semmlecode.binary.dbscheme.stats" ]]; then
122+
cp "${ql_lib_folder}/semmlecode.binary.dbscheme.stats" "${cil_folder}/"
123+
fi
124+
125+
# Copy tool scripts
126+
mkdir -p "${cil_folder}/tools"
127+
cp "${SCRIPT_DIR}/tools/cil/"* "${cil_folder}/tools/"
128+
chmod +x "${cil_folder}/tools/"*.sh
129+
130+
echo "CIL extractor built successfully to ${cil_folder}"
131+
}
132+
133+
clean_cil() {
134+
echo "Cleaning CIL build artifacts..."
135+
136+
local bin_dir="${SCRIPT_DIR}/extractor/cil/Semmle.Extraction.CSharp.IL/bin"
137+
local obj_dir="${SCRIPT_DIR}/extractor/cil/Semmle.Extraction.CSharp.IL/obj"
138+
139+
[[ -d "${bin_dir}" ]] && rm -rf "${bin_dir}"
140+
[[ -d "${obj_dir}" ]] && rm -rf "${obj_dir}"
141+
142+
echo "CIL clean complete"
143+
}
144+
145+
build_x86() {
146+
echo "x86 extractor build for macOS is not yet implemented"
147+
echo "This will require:"
148+
echo " - LIEF library (build with cmake/make)"
149+
echo " - Zydis library (build with cmake/make)"
150+
echo " - fmt library"
151+
echo " - Boost headers"
152+
echo " - args library"
153+
echo " - clang++ compiler"
154+
exit 1
155+
}
156+
157+
clean_x86() {
158+
echo "Cleaning x86 build artifacts..."
159+
160+
local x86_dir="${SCRIPT_DIR}/extractor/x86"
161+
162+
[[ -d "${x86_dir}/args" ]] && rm -rf "${x86_dir}/args"
163+
[[ -d "${x86_dir}/boost-minimal" ]] && rm -rf "${x86_dir}/boost-minimal"
164+
[[ -d "${x86_dir}/fmt" ]] && rm -rf "${x86_dir}/fmt"
165+
[[ -d "${x86_dir}/LIEF" ]] && rm -rf "${x86_dir}/LIEF"
166+
[[ -d "${x86_dir}/zydis" ]] && rm -rf "${x86_dir}/zydis"
167+
[[ -f "${x86_dir}/extractor" ]] && rm -f "${x86_dir}/extractor"
168+
[[ -f "${x86_dir}/main.o" ]] && rm -f "${x86_dir}/main.o"
169+
170+
echo "x86 clean complete"
171+
}
172+
173+
init_x86() {
174+
echo "x86 extractor initialization for macOS is not yet implemented"
175+
exit 1
176+
}
177+
178+
# Execute requested builds
179+
if [[ "$BUILD_CIL" == true ]]; then
180+
if [[ "$CLEAN" == true ]]; then
181+
clean_cil
182+
else
183+
build_cil
184+
fi
185+
fi
186+
187+
if [[ "$BUILD_X86" == true ]]; then
188+
if [[ "$CLEAN" == true ]]; then
189+
clean_x86
190+
elif [[ "$INIT" == true ]]; then
191+
init_x86
192+
else
193+
build_x86
194+
fi
195+
fi

‎binary/build-win64.ps1‎

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
param (
2+
[Parameter(ParameterSetName = 'x86')]
3+
[Parameter(ParameterSetName = 'x86Clean')]
4+
[Parameter(ParameterSetName = 'x86Init')]
5+
[switch]$x86,
6+
7+
[Parameter(ParameterSetName = 'cil')]
8+
[Parameter(ParameterSetName = 'cilClean')]
9+
[Parameter(ParameterSetName = 'cilInit')]
10+
[switch]$cil,
11+
12+
[Parameter(Mandatory=$true, ParameterSetName = 'x86Init')]
13+
[Parameter(Mandatory=$true, ParameterSetName = 'cilInit')]
14+
[switch]$init,
15+
16+
[Parameter(Mandatory=$true, ParameterSetName = 'x86')]
17+
[Parameter(Mandatory=$true, ParameterSetName = 'cil')]
18+
[Parameter(Mandatory=$true, ParameterSetName = 'x86Init')]
19+
[Parameter(Mandatory=$true, ParameterSetName = 'cilInit')]
20+
[string]$cliFolder,
21+
22+
[Parameter(Mandatory = $true, ParameterSetName = 'x86Clean')]
23+
[Parameter(Mandatory = $true, ParameterSetName = 'cilClean')]
24+
[switch]$clean
25+
)
26+
27+
function x86 {
28+
function init-lief {
29+
git clone https://github.com/lief-project/LIEF.git
30+
cd LIEF
31+
cmake -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF -DCMAKE_MSVC_RUNTIME_LIBRARY="MultiThreaded" .
32+
nmake
33+
nmake install
34+
cd ..
35+
}
36+
37+
function init-zydis {
38+
git clone https://github.com/zyantific/zydis.git --recursive
39+
cd zydis
40+
cmake -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF -DCMAKE_MSVC_RUNTIME_LIBRARY="MultiThreaded" .
41+
nmake
42+
nmake install
43+
cd ..
44+
}
45+
46+
function init-fmt {
47+
git clone https://github.com/fmtlib/fmt.git
48+
}
49+
50+
function init-boost-library($name) {
51+
git clone https://github.com/boostorg/$name.git
52+
xcopy /E /I /Y $name\include\boost boost-minimal\boost
53+
Remove-Item -Path $name -Recurse -Force
54+
}
55+
56+
function init-boost {
57+
if (-not (Test-Path boost-minimal)) {
58+
mkdir boost-minimal
59+
}
60+
init-boost-library algorithm
61+
init-boost-library mpl
62+
init-boost-library range
63+
init-boost-library preprocessor
64+
init-boost-library type_traits
65+
init-boost-library iterator
66+
init-boost-library assert
67+
init-boost-library mp11
68+
init-boost-library static_assert
69+
init-boost-library core
70+
init-boost-library concept_check
71+
init-boost-library utility
72+
init-boost-library function
73+
init-boost-library bind
74+
init-boost-library throw_exception
75+
init-boost-library optional
76+
init-boost-library config
77+
}
78+
79+
function init-args {
80+
git clone https://github.com/Taywee/args.git
81+
}
82+
83+
if($init) {
84+
Push-Location extractor/x86
85+
init-lief
86+
init-zydis
87+
init-fmt
88+
init-boost
89+
init-args
90+
Pop-Location
91+
}
92+
93+
if($clean) {
94+
Push-Location extractor/x86
95+
if(Test-Path args) { Remove-Item -Path args -Recurse -Force }
96+
if(Test-Path boost-minimal) { Remove-Item -Path boost-minimal -Recurse -Force }
97+
if(Test-Path fmt) { Remove-Item -Path fmt -Recurse -Force }
98+
if(Test-Path LIEF) { Remove-Item -Path LIEF -Recurse -Force }
99+
if(Test-Path zydis) { Remove-Item -Path zydis -Recurse -Force }
100+
Pop-Location
101+
} else {
102+
Push-Location extractor/x86
103+
104+
cl.exe /DFMT_HEADER_ONLY /DZYDIS_STATIC_BUILD /I zydis\include /I zydis\dependencies\zycore\include /I LIEF/include /I fmt/include /I boost-minimal /I args /utf-8 src/main.cpp zydis\Zydis.lib zydis/zycore/zycore.lib LIEF/LIEF.lib /EHsc /std:c++17 /link /out:extractor.exe
105+
106+
if ($LASTEXITCODE -ne 0) {
107+
Write-Host "Build failed"
108+
Pop-Location
109+
exit 1
110+
}
111+
112+
Pop-Location
113+
114+
$toolsWin64Folder = Join-Path (Join-Path (Join-Path $cliFolder "binary") "tools") "win64"
115+
New-Item -ItemType Directory -Force -Path $toolsWin64Folder
116+
$binaryFolder = Join-Path -Path $cliFolder -ChildPath "binary"
117+
New-Item -ItemType Directory -Force -Path $binaryFolder
118+
Copy-Item -Path "$PSScriptRoot/extractor/x86/codeql-extractor.yml" -Destination $binaryFolder -Force
119+
Copy-Item -Path "$PSScriptRoot/downgrades" -Destination $binaryFolder -Recurse -Force
120+
$qlLibFolder = Join-Path -Path "$PSScriptRoot/ql" -ChildPath "lib"
121+
Copy-Item -Path (Join-Path $qlLibFolder "semmlecode.binary.dbscheme") -Destination $binaryFolder -Force
122+
Copy-Item -Path (Join-Path $qlLibFolder "semmlecode.binary.dbscheme.stats") -Destination $binaryFolder -Force
123+
Copy-Item -Path "$PSScriptRoot/tools/x86/*" -Destination (Join-Path $binaryFolder "tools") -Recurse -Force
124+
Copy-Item -Path "$PSScriptRoot/extractor/x86/extractor.exe" -Destination $toolsWin64Folder/extractor.exe
125+
}
126+
}
127+
128+
function cil {
129+
Push-Location extractor/cil
130+
$toolsWin64Folder = Join-Path (Join-Path (Join-Path $cliFolder "cil") "tools") "win64"
131+
dotnet build Semmle.Extraction.CSharp.IL -o $toolsWin64Folder -c Release --self-contained
132+
if ($LASTEXITCODE -ne 0) {
133+
Write-Host "Build failed"
134+
Pop-Location
135+
exit 1
136+
}
137+
138+
Pop-Location
139+
140+
New-Item -ItemType Directory -Force -Path $toolsWin64Folder
141+
$cilFolder = Join-Path -Path $cliFolder -ChildPath "cil"
142+
New-Item -ItemType Directory -Force -Path $cilFolder
143+
Copy-Item -Path "$PSScriptRoot/extractor/cil/codeql-extractor.yml" -Destination $cilFolder -Force
144+
Copy-Item -Path "$PSScriptRoot/downgrades" -Destination $cilFolder -Recurse -Force
145+
$qlLibFolder = Join-Path -Path "$PSScriptRoot/ql" -ChildPath "lib"
146+
Copy-Item -Path (Join-Path $qlLibFolder "semmlecode.binary.dbscheme") -Destination $cilFolder -Force
147+
Copy-Item -Path (Join-Path $qlLibFolder "semmlecode.binary.dbscheme.stats") -Destination $cilFolder -Force
148+
Copy-Item -Path "$PSScriptRoot/tools/cil/*" -Destination (Join-Path $cilFolder "tools") -Recurse -Force
149+
}
150+
151+
if($x86) {
152+
x86
153+
}
154+
155+
if($cil) {
156+
cil
157+
}

‎binary/clean.ps1‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
rm *.obj
2+
rm *.exe

‎binary/extractor/.gitignore‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
LIEF/
2+
args/
3+
boost-minimal/
4+
fmt/
5+
*.obj
6+
zydis/
7+
*.exe

0 commit comments

Comments
 (0)