Skip to content

Commit c28db23

Browse files
ManigurrManigurr
committed
Add upstream build and validation workflow
This change introduces a new upstream workflow to the branch. It implements automated build and testing processes. Signed-off-by: Manigurr <manigurr@qti.qualcomm.com>
1 parent 9cc527a commit c28db23

13 files changed

Lines changed: 1053 additions & 0 deletions

File tree

‎.github/actions/aws_s3_helper/action.yml‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
# name: AWS S3 Helper
2+
description: Upload and download files from AWS S3
3+
4+
inputs:
5+
s3_bucket:
6+
description: S3 Bucket Name
7+
required: true
8+
local_file:
9+
description: Local file paths
10+
required: false
11+
default: ../artifacts/file_list.txt
12+
download_file:
13+
description: Download file paths
14+
required: false
15+
default: ''
16+
download_location:
17+
description: File download location
18+
required: false
19+
default: .
20+
mode:
21+
description: Mode of operation (upload/download)
22+
required: true
23+
default: single-upload
24+
upload_location:
25+
description: Upload location
26+
required: true
27+
28+
outputs:
29+
presigned_url:
30+
description: Pre-signed URL for the uploaded file
31+
value: ${{ steps.sync-data.outputs.presigned_url }}
32+
s3_location:
33+
description: Upload location
34+
value: ${{ inputs.upload_location }}
35+
36+
runs:
37+
using: "composite"
38+
steps:
39+
- name: Sync Data
40+
id: sync-data
41+
shell: bash
42+
env:
43+
UPLOAD_LOCATION: ${{ inputs.upload_location }}
44+
run: |
45+
echo "::group::Uploading files to S3"
46+
case "${{ inputs.mode }}" in
47+
multi-upload)
48+
if [ ! -s "${{ inputs.local_file }}" ]; then
49+
echo "❌ File list is empty. No files to upload."
50+
exit 1
51+
fi
52+
53+
echo "📄 Contents of file list:"
54+
cat "${{ inputs.local_file }}"
55+
56+
first_line=true
57+
manifest="${{ github.workspace }}/presigned_urls.json"
58+
echo "{" > "${manifest}"
59+
60+
while IFS= read -r file; do
61+
resolved_file=$(readlink -f "$file")
62+
if [ -f "$resolved_file" ]; then
63+
filename=$(basename "$resolved_file")
64+
echo "📤 Uploading $filename..."
65+
aws s3 cp "$resolved_file" "s3://${{ inputs.s3_bucket }}/${{ env.UPLOAD_LOCATION }}/$filename"
66+
presigned_url=$(aws s3 presign "s3://${{ inputs.s3_bucket }}/${{ env.UPLOAD_LOCATION }}/$filename" --expires-in 259200)
67+
68+
if [ "$first_line" = true ]; then
69+
first_line=false
70+
else
71+
echo "," >> "${manifest}"
72+
fi
73+
74+
# Key = filename, Value = presigned_url
75+
echo " \"${filename}\": \"${presigned_url}\"" >> "${manifest}"
76+
echo "✅ Pre-signed URL for $filename: $presigned_url"
77+
else
78+
echo "⚠️ Skipping: $file is not a regular file or not accessible."
79+
fi
80+
done < "${{ inputs.local_file }}"
81+
82+
echo "}" >> "${manifest}"
83+
;;
84+
single-upload)
85+
resolved_file=$(readlink -f "${{ inputs.local_file }}")
86+
filename=$(basename "$resolved_file")
87+
aws s3 cp "$resolved_file" "s3://${{ inputs.s3_bucket }}/${{ env.UPLOAD_LOCATION }}/$filename"
88+
presigned_url=$(aws s3 presign "s3://${{ inputs.s3_bucket }}/${{ env.UPLOAD_LOCATION }}/$filename" --expires-in 259200)
89+
echo "presigned_url=${presigned_url}" >> "$GITHUB_OUTPUT"
90+
;;
91+
download)
92+
download_dir=$(realpath "${{ inputs.download_location }}")
93+
aws s3 cp "s3://${{ inputs.s3_bucket }}/${{ inputs.download_file }}" "$download_dir"
94+
;;
95+
*)
96+
echo "Invalid mode. Use 'upload', 'multi-upload', or 'download'."
97+
exit 1
98+
;;
99+
esac
100+
echo "::endgroup::"
101+
102+
- name: Upload presigned URL manifest
103+
if: ${{ inputs.mode == 'multi-upload' }}
104+
uses: actions/upload-artifact@v4
105+
with:
106+
name: presigned_urls.json
107+
path: ${{ github.workspace }}/presigned_urls.json
108+
retention-days: 3

‎.github/actions/build/action.yml‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
name: Build Workspace
2+
description: |
3+
Builds kernel and video-driver using a Docker image.
4+
5+
inputs:
6+
docker_image:
7+
description: Docker image to use
8+
required: true
9+
workspace_path:
10+
description: Path to workspace directory
11+
required: true
12+
13+
runs:
14+
using: "composite"
15+
steps:
16+
- name: Build kernel
17+
shell: bash
18+
run: |
19+
docker run --rm \
20+
-v "${{ inputs.workspace_path }}:${{ inputs.workspace_path }}" \
21+
-w "${{ inputs.workspace_path }}/kernel" \
22+
--user "$(id -u):$(id -g)" \
23+
${{ inputs.docker_image }} \
24+
bash -c "
25+
set -e
26+
27+
echo 'Cleaning existing iris and venus drivers...'
28+
rm -rf drivers/media/platform/qcom/iris
29+
rm -rf drivers/media/platform/qcom/venus
30+
31+
echo 'Listing qcom drivers directory...'
32+
ls -al drivers/media/platform/qcom/
33+
34+
echo 'Copying iris and venus from video-driver workspace...'
35+
cp -r ../video-driver/iris drivers/media/platform/qcom/
36+
cp -r ../video-driver/venus drivers/media/platform/qcom/
37+
38+
echo 'Listing qcom drivers directory after copy...'
39+
ls -al drivers/media/platform/qcom/
40+
41+
echo 'Building kernel...'
42+
make O=../kobj ARCH=arm64 defconfig &&
43+
make O=../kobj -j\$(nproc) &&
44+
make O=../kobj -j\$(nproc) dir-pkg INSTALL_MOD_STRIP=1
45+
"
46+
47+

‎.github/actions/lava_job_render/action.yml‎

Lines changed: 180 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,180 @@
1+
name: LAVA Job Render
2+
inputs:
3+
docker_image:
4+
description: Docker image
5+
required: true
6+
default: kmake-image:ver.1.0
7+
8+
runs:
9+
using: "composite"
10+
steps:
11+
- name: Process presigned_urls.json
12+
id: process_urls
13+
uses: actions/github-script@v7
14+
with:
15+
script: |
16+
const fs = require('fs');
17+
const p = require('path');
18+
19+
const filePath = p.join(process.env.GITHUB_WORKSPACE, 'presigned_urls.json');
20+
if (!fs.existsSync(filePath)) {
21+
core.setFailed(`File not found: ${filePath}`);
22+
}
23+
24+
// Read JSON mapping of uploaded file paths -> presigned URLs
25+
const data = JSON.parse(fs.readFileSync(filePath, 'utf-8'));
26+
27+
function findUrlByFilename(filename) {
28+
for (const [path, url] of Object.entries(data)) {
29+
if (path.endsWith(filename)) return url;
30+
}
31+
return null;
32+
}
33+
34+
const modulesTarUrl = findUrlByFilename('modules.tar.xz');
35+
const imageUrl = findUrlByFilename('Image');
36+
const mergedRamdiskUrl = findUrlByFilename('video-merged.cpio.gz');
37+
const vmlinuxUrl = findUrlByFilename('vmlinux');
38+
39+
// DTB is expected to be "<MACHINE>.dtb"
40+
const dtbFilename = `${process.env.MACHINE}.dtb`;
41+
const dtbUrl = findUrlByFilename(dtbFilename);
42+
43+
core.setOutput('modules_url', modulesTarUrl || '');
44+
core.setOutput('image_url', imageUrl || '');
45+
core.setOutput('vmlinux_url', vmlinuxUrl || '');
46+
core.setOutput('dtb_url', dtbUrl || '');
47+
core.setOutput('merged_ramdisk_url', mergedRamdiskUrl || '');
48+
49+
console.log(`Modules URL: ${modulesTarUrl}`);
50+
console.log(`Image URL: ${imageUrl}`);
51+
console.log(`Vmlinux URL: ${vmlinuxUrl}`);
52+
console.log(`Dtb URL: ${dtbUrl}`);
53+
console.log(`Merged Ramdisk URL: ${mergedRamdiskUrl}`);
54+
55+
- name: Create metadata.json
56+
id: create_metadata
57+
shell: bash
58+
run: |
59+
echo "Creating metadata.json from job_render templates"
60+
cd ../job_render
61+
docker run -i --rm \
62+
--user "$(id -u):$(id -g)" \
63+
--workdir="$PWD" \
64+
-v "$(dirname "$PWD")":"$(dirname "$PWD")" \
65+
-e dtb_url="${{ steps.process_urls.outputs.dtb_url }}" \
66+
${{ inputs.docker_image }} \
67+
jq '.artifacts["dtbs/qcom/${{ env.MACHINE }}.dtb"] = env.dtb_url' data/metadata.json > temp.json && mv temp.json data/metadata.json
68+
69+
- name: Upload metadata.json
70+
id: upload_metadata
71+
uses: qualcomm-linux/video-driver/.github/actions/aws_s3_helper@video.upstream(stage)
72+
with:
73+
local_file: ../job_render/data/metadata.json
74+
s3_bucket: qli-prd-video-gh-artifacts
75+
mode: single-upload
76+
77+
- name: Create template json (cloudData.json)
78+
shell: bash
79+
run: |
80+
echo "Populating cloudData.json with kernel, vmlinux, modules, metadata, ramdisk"
81+
metadata_url="${{ steps.upload_metadata.outputs.presigned_url }}"
82+
image_url="${{ steps.process_urls.outputs.image_url }}"
83+
vmlinux_url="${{ steps.process_urls.outputs.vmlinux_url }}"
84+
modules_url="${{ steps.process_urls.outputs.modules_url }}"
85+
merged_ramdisk_url="${{ steps.process_urls.outputs.merged_ramdisk_url }}"
86+
87+
cd ../job_render
88+
89+
# metadata
90+
docker run -i --rm \
91+
--user "$(id -u):$(id -g)" \
92+
--workdir="$PWD" \
93+
-v "$(dirname "$PWD")":"$(dirname "$PWD")" \
94+
-e metadata_url="$metadata_url" \
95+
${{ inputs.docker_image }} \
96+
jq '.artifacts.metadata = env.metadata_url' data/cloudData.json > temp.json && mv temp.json data/cloudData.json
97+
98+
# kernel Image
99+
docker run -i --rm \
100+
--user "$(id -u):$(id -g)" \
101+
--workdir="$PWD" \
102+
-v "$(dirname "$PWD")":"$(dirname "$PWD")" \
103+
-e image_url="$image_url" \
104+
${{ inputs.docker_image }} \
105+
jq '.artifacts.kernel = env.image_url' data/cloudData.json > temp.json && mv temp.json data/cloudData.json
106+
107+
# vmlinux (set only if present)
108+
docker run -i --rm \
109+
--user "$(id -u):$(id -g)" \
110+
--workdir="$PWD" \
111+
-v "$(dirname "$PWD")":"$(dirname "$PWD")" \
112+
-e vmlinux_url="$vmlinux_url" \
113+
${{ inputs.docker_image }} \
114+
sh -c 'if [ -n "$vmlinux_url" ]; then jq ".artifacts.vmlinux = env.vmlinux_url" data/cloudData.json > temp.json && mv temp.json data/cloudData.json; fi'
115+
116+
# modules
117+
docker run -i --rm \
118+
--user "$(id -u):$(id -g)" \
119+
--workdir="$PWD" \
120+
-v "$(dirname "$PWD")":"$(dirname "$PWD")" \
121+
-e modules_url="$modules_url" \
122+
${{ inputs.docker_image }} \
123+
jq '.artifacts.modules = env.modules_url' data/cloudData.json > temp.json && mv temp.json data/cloudData.json
124+
125+
# ramdisk: use merged only here (fallback added in next step if missing)
126+
docker run -i --rm \
127+
--user "$(id -u):$(id -g)" \
128+
--workdir="$PWD" \
129+
-v "$(dirname "$PWD")":"$(dirname "$PWD")" \
130+
-e merged_ramdisk_url="$merged_ramdisk_url" \
131+
${{ inputs.docker_image }} \
132+
sh -c 'if [ -n "$merged_ramdisk_url" ]; then jq ".artifacts.ramdisk = env.merged_ramdisk_url" data/cloudData.json > temp.json && mv temp.json data/cloudData.json; fi'
133+
134+
- name: Update firmware and ramdisk
135+
shell: bash
136+
run: |
137+
cd ../job_render
138+
# Fallback to stable kerneltest ramdisk only if merged ramdisk is not available
139+
if [ -z "${{ steps.process_urls.outputs.merged_ramdisk_url }}" ]; then
140+
echo "Merged ramdisk not found. Using stable kerneltest ramdisk fallback."
141+
ramdisk_url="$(aws s3 presign s3://qli-prd-video-gh-artifacts/qualcomm-linux/video-driver/artifacts/initramfs/initramfs-kerneltest-full-image-qcom-armv8a.cpio.gz --expires 7600)"
142+
docker run -i --rm \
143+
--user "$(id -u):$(id -g)" \
144+
--workdir="$PWD" \
145+
-v "$(dirname "$PWD")":"$(dirname "$PWD")" \
146+
-e ramdisk_url="$ramdisk_url" \
147+
${{ inputs.docker_image }} \
148+
jq '.artifacts.ramdisk = env.ramdisk_url' data/cloudData.json > temp.json && mv temp.json data/cloudData.json
149+
else
150+
echo "Ramdisk set from merged source; skipping kerneltest fallback."
151+
fi
152+
153+
# Optional board-specific firmware initramfs
154+
if [ -n "${{ env.FIRMWARE }}" ]; then
155+
firmware_url="$(aws s3 presign s3://qli-prd-video-gh-artifacts/qualcomm-linux/video-driver/artifacts/initramfs/initramfs-firmware-${{ env.FIRMWARE }}-image-qcom-armv8a.cpio.gz --expires 7600)"
156+
docker run -i --rm \
157+
--user "$(id -u):$(id -g)" \
158+
--workdir="$PWD" \
159+
-v "$(dirname "$PWD")":"$(dirname "$PWD")" \
160+
-e firmware_url="$firmware_url" \
161+
${{ inputs.docker_image }} \
162+
jq '.artifacts.firmware = env.firmware_url' data/cloudData.json > temp.json && mv temp.json data/cloudData.json
163+
fi
164+
165+
- name: Create lava_job_definition
166+
shell: bash
167+
run: |
168+
cd ../job_render
169+
mkdir -p renders
170+
docker run -i --rm \
171+
--user "$(id -u):$(id -g)" \
172+
--workdir="$PWD" \
173+
-v "$(dirname "$PWD")":"$(dirname "$PWD")" \
174+
-e TARGET="${{ env.LAVA_NAME }}" \
175+
-e TARGET_DTB="${{ env.MACHINE }}" \
176+
${{ inputs.docker_image }} \
177+
sh -c 'export BOOT_METHOD=fastboot && \
178+
export TARGET=${TARGET} && \
179+
export TARGET_DTB=${TARGET_DTB} && \
180+
python3 lava_Job_definition_generator.py --localjson ./data/cloudData.json --video-ci-sanity'

‎.github/actions/loading/action.yml‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
---
2+
name: Load Parameters
3+
description: Load parameters for the build job
4+
5+
outputs:
6+
build_matrix:
7+
description: Build matrix
8+
value: ${{ steps.set-matrix.outputs.build_matrix }}
9+
10+
full_matrix:
11+
description: full matrix containing lava devails
12+
value: ${{ steps.set-matrix.outputs.full_matrix }}
13+
14+
runs:
15+
using: "composite"
16+
steps:
17+
- name: Set Build Matrix
18+
id: set-matrix
19+
uses: actions/github-script@v7
20+
with:
21+
script: |
22+
const fs = require('fs');
23+
const path = require('path');
24+
const targetsPath = path.join(process.env.GITHUB_WORKSPACE, 'video-driver', 'ci', 'MACHINES.json');
25+
let targets;
26+
try {
27+
if (!fs.existsSync(targetsPath)) {
28+
core.setFailed(`MACHINES.json not found at ${targetsPath}`);
29+
return;
30+
}
31+
targets = JSON.parse(fs.readFileSync(targetsPath, 'utf-8'));
32+
} catch (err) {
33+
core.setFailed(`Failed to load or parse MACHINES.json: ${err.message}`);
34+
return;
35+
}
36+
// Build matrix: machine, firmware
37+
const build_matrix = Object.values(targets).map(({ machine, firmware }) => ({ machine, firmware }));
38+
core.setOutput('build_matrix', JSON.stringify(build_matrix));
39+
console.log("Build Matrix:", build_matrix);
40+
41+
// Full matrix: machine, firmware, lavaname
42+
const full_matrix = Object.values(targets).map(({ machine, firmware, lavaname }) => ({ machine, firmware, lavaname }));
43+
core.setOutput('full_matrix', JSON.stringify(full_matrix));
44+
console.log("Full Matrix:", full_matrix);
45+

0 commit comments

Comments
 (0)