Skip to content

Commit 12f256a

Browse files
committed
fix(ci): ccache bugs A/B/C and bump SYCL compute runtime to 26.x
Fix three root causes for ccache never hitting in CI: A) CCACHE_DIR/CCACHE_MAXSIZE not persisting across steps — write them to GITHUB_ENV in setup-ccache's Configure step. B) evict-old-files during setup destroyed restored cache entries before the build could use them — remove eviction from setup-ccache entirely (keep it only in save-rolling-ccache where it belongs). C) Relative CCACHE_DIR=.ccache resolved to build/.ccache during compilation (Ninja/Make run compilers from the build dir) while actions/cache saved from GITHUB_WORKSPACE/.ccache — add a Normalize ccache path step in both actions that resolves to an absolute path. Also bump SYCL Docker images from compute runtime 25.40 to 26.18 and IGC from v2.20.5 to v2.34.4 to fix a ~21% SYCL performance regression (ggml-org#23160). The 25.x versions are kept as comments for reference (needed for multi-GPU per ggml-org#21747 until kernel 7.1).
1 parent 13ad92a commit 12f256a

5 files changed

Lines changed: 57 additions & 34 deletions

File tree

.devops/intel.Dockerfile

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -65,11 +65,18 @@ RUN mkdir -p /app/full \
6565

6666
FROM intel/deep-learning-essentials:$ONEAPI_VERSION AS base
6767

68-
ARG IGC_VERSION=v2.20.5
69-
ARG IGC_VERSION_FULL=2_2.20.5+19972
70-
ARG COMPUTE_RUNTIME_VERSION=25.40.35563.10
71-
ARG COMPUTE_RUNTIME_VERSION_FULL=25.40.35563.10-0
72-
ARG IGDGMM_VERSION=22.8.2
68+
# 25.x causes ~21% SYCL regression: https://github.com/ggml-org/llama.cpp/issues/23160
69+
# 25.x is needed for multi-GPU: https://github.com/ggml-org/llama.cpp/issues/21747
70+
#ARG IGC_VERSION=v2.20.5
71+
#ARG IGC_VERSION_FULL=2_2.20.5+19972
72+
#ARG COMPUTE_RUNTIME_VERSION=25.40.35563.10
73+
#ARG COMPUTE_RUNTIME_VERSION_FULL=25.40.35563.10-0
74+
#ARG IGDGMM_VERSION=22.8.2
75+
ARG IGC_VERSION=v2.34.4
76+
ARG IGC_VERSION_FULL=2_2.34.4+21428
77+
ARG COMPUTE_RUNTIME_VERSION=26.18.38308.1
78+
ARG COMPUTE_RUNTIME_VERSION_FULL=26.18.38308.1-0
79+
ARG IGDGMM_VERSION=22.10.0
7380
RUN mkdir /tmp/neo/ && cd /tmp/neo/ \
7481
&& wget https://github.com/intel/intel-graphics-compiler/releases/download/$IGC_VERSION/intel-igc-core-${IGC_VERSION_FULL}_amd64.deb \
7582
&& wget https://github.com/intel/intel-graphics-compiler/releases/download/$IGC_VERSION/intel-igc-opencl-${IGC_VERSION_FULL}_amd64.deb \

.devops/runtime-intel-server.Dockerfile

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,18 @@ ARG ONEAPI_VERSION=2025.3.3-0-devel-ubuntu24.04
44

55
FROM intel/deep-learning-essentials:${ONEAPI_VERSION} AS server
66

7-
ARG IGC_VERSION=v2.20.5
8-
ARG IGC_VERSION_FULL=2_2.20.5+19972
9-
ARG COMPUTE_RUNTIME_VERSION=25.40.35563.10
10-
ARG COMPUTE_RUNTIME_VERSION_FULL=25.40.35563.10-0
11-
ARG IGDGMM_VERSION=22.8.2
7+
# 25.x causes ~21% SYCL regression: https://github.com/ggml-org/llama.cpp/issues/23160
8+
# 25.x is needed for multi-GPU: https://github.com/ggml-org/llama.cpp/issues/21747
9+
#ARG IGC_VERSION=v2.20.5
10+
#ARG IGC_VERSION_FULL=2_2.20.5+19972
11+
#ARG COMPUTE_RUNTIME_VERSION=25.40.35563.10
12+
#ARG COMPUTE_RUNTIME_VERSION_FULL=25.40.35563.10-0
13+
#ARG IGDGMM_VERSION=22.8.2
14+
ARG IGC_VERSION=v2.34.4
15+
ARG IGC_VERSION_FULL=2_2.34.4+21428
16+
ARG COMPUTE_RUNTIME_VERSION=26.18.38308.1
17+
ARG COMPUTE_RUNTIME_VERSION_FULL=26.18.38308.1-0
18+
ARG IGDGMM_VERSION=22.10.0
1219

1320
RUN mkdir /tmp/neo/ && cd /tmp/neo/ \
1421
&& wget https://github.com/intel/intel-graphics-compiler/releases/download/${IGC_VERSION}/intel-igc-core-${IGC_VERSION_FULL}_amd64.deb \

.github/actions/save-rolling-ccache/action.yml

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,24 @@ inputs:
1818
runs:
1919
using: composite
2020
steps:
21+
- name: Normalize ccache path
22+
id: ccache-path
23+
shell: bash
24+
env:
25+
INPUT_PATH: ${{ inputs.path }}
26+
run: |
27+
case "${INPUT_PATH}" in
28+
/*|[A-Za-z]:/*|[A-Za-z]:\\*) CCACHE_DIR_ABS="${INPUT_PATH}" ;;
29+
*) CCACHE_DIR_ABS="${GITHUB_WORKSPACE}/${INPUT_PATH}" ;;
30+
esac
31+
echo "path=${CCACHE_DIR_ABS}" >> "$GITHUB_OUTPUT"
32+
echo "Resolved ccache path: ${CCACHE_DIR_ABS}"
33+
2134
- name: Evict old cache files
2235
if: inputs.evict-old-files != ''
2336
shell: bash
2437
env:
25-
CCACHE_DIR: ${{ inputs.path }}
38+
CCACHE_DIR: ${{ steps.ccache-path.outputs.path }}
2639
EVICT_AGE: ${{ inputs.evict-old-files }}
2740
run: |
2841
if command -v ccache >/dev/null 2>&1; then
@@ -74,5 +87,5 @@ runs:
7487
- name: Save refreshed cache
7588
uses: actions/cache/save@v4
7689
with:
77-
path: ${{ inputs.path }}
78-
key: ${{ inputs.key }}
90+
path: ${{ steps.ccache-path.outputs.path }}
91+
key: ${{ inputs.key }}

.github/actions/setup-ccache/action.yml

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,6 @@ inputs:
1616
description: ccache sloppiness setting (e.g. time_macros, file_macro).
1717
required: false
1818
default: ""
19-
evict-old-files:
20-
description: Age threshold for evicting stale cache entries (e.g. 1d, 3600s).
21-
required: false
22-
default: ""
2319
path:
2420
description: ccache directory path.
2521
required: false
@@ -63,21 +59,34 @@ runs:
6359
choco install ccache -y
6460
}
6561
62+
- name: Normalize ccache path
63+
id: ccache-path
64+
shell: bash
65+
env:
66+
INPUT_PATH: ${{ inputs.path }}
67+
run: |
68+
case "${INPUT_PATH}" in
69+
/*|[A-Za-z]:/*|[A-Za-z]:\\*) CCACHE_DIR_ABS="${INPUT_PATH}" ;;
70+
*) CCACHE_DIR_ABS="${GITHUB_WORKSPACE}/${INPUT_PATH}" ;;
71+
esac
72+
echo "path=${CCACHE_DIR_ABS}" >> "$GITHUB_OUTPUT"
73+
echo "Resolved ccache path: ${CCACHE_DIR_ABS}"
74+
mkdir -p "${CCACHE_DIR_ABS}"
75+
6676
- name: Restore ccache
6777
id: restore-cache
6878
uses: actions/cache/restore@v4
6979
with:
70-
path: ${{ inputs.path }}
80+
path: ${{ steps.ccache-path.outputs.path }}
7181
key: ${{ inputs.key }}
7282
restore-keys: ${{ inputs.restore-keys }}
7383

7484
- name: Configure ccache
7585
shell: bash
7686
env:
77-
CCACHE_DIR: ${{ inputs.path }}
87+
CCACHE_DIR: ${{ steps.ccache-path.outputs.path }}
7888
CCACHE_MAXSIZE: ${{ inputs.max-size }}
7989
CCACHE_SLOPPINESS: ${{ inputs.sloppiness }}
80-
EVICT_AGE: ${{ inputs.evict-old-files }}
8190
run: |
8291
ccache --set-config=cache_dir="${CCACHE_DIR}"
8392
ccache --set-config=max_size="${CCACHE_MAXSIZE}"
@@ -86,10 +95,8 @@ runs:
8695
ccache --set-config=compiler_check=content
8796
fi
8897
ccache --set-config=sloppiness="${CCACHE_SLOPPINESS}"
89-
if [ -n "${EVICT_AGE}" ]; then
90-
ccache --evict-older-than "${EVICT_AGE}"
91-
fi
9298
ccache -z
99+
ccache --show-config | grep -E 'cache_dir|max_size'
93100
echo "CCACHE_DIR=${CCACHE_DIR}" >> "$GITHUB_ENV"
94101
echo "CCACHE_MAXSIZE=${CCACHE_MAXSIZE}" >> "$GITHUB_ENV"
95102

.github/workflows/release.yml

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -214,7 +214,6 @@ jobs:
214214
restore-keys: |
215215
release-${{ needs.release-meta.outputs.ccache_ref }}-macos-arm64-
216216
release-macos-arm64-
217-
evict-old-files: 1d
218217
219218
- name: Build
220219
run: |
@@ -293,7 +292,6 @@ jobs:
293292
restore-keys: |
294293
release-${{ needs.release-meta.outputs.ccache_ref }}-${{ matrix.os }}-cpu-
295294
release-${{ matrix.os }}-cpu-
296-
evict-old-files: 1d
297295
298296
- name: Dependencies
299297
run: |
@@ -377,7 +375,6 @@ jobs:
377375
restore-keys: |
378376
release-${{ needs.release-meta.outputs.ccache_ref }}-ubuntu-vulkan-x64-
379377
release-ubuntu-vulkan-x64-
380-
evict-old-files: 1d
381378
382379
- name: Dependencies
383380
run: |
@@ -464,7 +461,6 @@ jobs:
464461
restore-keys: |
465462
release-${{ needs.release-meta.outputs.ccache_ref }}-ubuntu-rocm-${{ env.ROCM_VERSION }}-x64-
466463
release-ubuntu-rocm-${{ env.ROCM_VERSION }}-x64-
467-
evict-old-files: 1d
468464
469465
- name: Dependencies
470466
run: |
@@ -580,7 +576,6 @@ jobs:
580576
restore-keys: |
581577
release-${{ needs.release-meta.outputs.ccache_ref }}-ubuntu-sycl-${{ env.ONEAPI_INSTALLER_VERSION }}-x64-
582578
release-ubuntu-sycl-${{ env.ONEAPI_INSTALLER_VERSION }}-x64-
583-
evict-old-files: 1d
584579
585580
- name: Dependencies
586581
run: |
@@ -695,7 +690,6 @@ jobs:
695690
release-ubuntu-cuda-${{ matrix.cuda }}-x64-
696691
max-size: 5G
697692
sloppiness: time_macros
698-
evict-old-files: 1d
699693

700694
- name: Build
701695
run: |
@@ -771,7 +765,6 @@ jobs:
771765
restore-keys: |
772766
release-${{ needs.release-meta.outputs.ccache_ref }}-windows-cpu-x64-
773767
release-windows-cpu-x64-
774-
evict-old-files: 1d
775768
776769
- name: Install Ninja
777770
run: choco install ninja -y
@@ -873,7 +866,6 @@ jobs:
873866
restore-keys: |
874867
release-${{ needs.release-meta.outputs.ccache_ref }}-windows-sycl-${{ env.ONEAPI_INSTALLER_VERSION }}-x64-
875868
release-windows-sycl-${{ env.ONEAPI_INSTALLER_VERSION }}-x64-
876-
evict-old-files: 1d
877869
878870
- name: Install Ninja
879871
run: choco install ninja -y
@@ -999,7 +991,6 @@ jobs:
999991
restore-keys: |
1000992
release-${{ needs.release-meta.outputs.ccache_ref }}-windows-vulkan-${{ env.VULKAN_VERSION }}-x64-
1001993
release-windows-vulkan-${{ env.VULKAN_VERSION }}-x64-
1002-
evict-old-files: 1d
1003994
1004995
- name: Install Ninja
1005996
run: choco install ninja -y
@@ -1071,7 +1062,6 @@ jobs:
10711062
release-windows-cuda-${{ matrix.cuda }}-
10721063
max-size: 5G
10731064
sloppiness: time_macros
1074-
evict-old-files: 1d
10751065

10761066
- name: Install CUDA Toolkit
10771067
uses: ./.github/actions/windows-setup-cuda
@@ -1196,7 +1186,6 @@ jobs:
11961186
restore-keys: |
11971187
release-${{ needs.release-meta.outputs.ccache_ref }}-windows-hip-${{ env.HIPSDK_INSTALLER_VERSION }}-x64-
11981188
release-windows-hip-${{ env.HIPSDK_INSTALLER_VERSION }}-x64-
1199-
evict-old-files: 1d
12001189
12011190
- name: Install ROCm
12021191
run: |

0 commit comments

Comments
 (0)