Skip to content

Commit 8e062d1

Browse files
authored
Update base docker image (#990)
* Update base docker image. Remove versioned env vars from Docker images. Added service Docker image to run unit tests on Windows. Update Clang to 20 in Linux Docker images * Cleanup * Revert zip utils file permissions handling
1 parent f2efdad commit 8e062d1

26 files changed

Lines changed: 214 additions & 84 deletions

.gitattributes

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
* text=auto
2+
*.sh text eol=lf
3+
*.yml text eol=lf
4+
*.yaml text eol=lf
5+
*.java text eol=lf
6+
*.gradle text eol=lf
7+
*.properties text eol=lf
8+
*.xml text eol=lf
9+
Dockerfile* text eol=lf
10+
gradlew text eol=lf
11+
12+
# Windows-specific files
13+
*.bat text eol=crlf
14+
*.cmd text eol=crlf
15+
*.ps1 text eol=crlf

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
**/build
2+
**/build-container
23
**/classes
34
/client/.buildcache
45
/tmp

build.gradle

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ subprojects {
44
}
55
}
66

7+
apply from: "$rootDir/gradle/docker-test.gradle"
8+
79
if (project.hasProperty('publishToGar')) {
810
apply plugin: 'maven-publish'
911
apply plugin: 'com.google.cloud.artifactregistry.gradle-plugin'

gradle/docker-test.gradle

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
// Windows hosts cannot run the docker-compose based integration/auth suite natively, and the
2+
// remaining (untagged) tests would run against a Windows JDK/filesystem, which can drift from CI.
3+
// So on Windows we route the whole non-integration suite through a small Linux container instead.
4+
//
5+
// Linux and macOS are untouched: useTestContainer is false there and every task below is a no-op.
6+
//
7+
// ./gradlew test on Windows -> one container run, all three modules
8+
// ./gradlew test -PnoDocker run natively (this is how you run integration/auth locally)
9+
10+
def testImage = 'extender-unit-test:1.0.0'
11+
12+
// EXTENDER_IN_TEST_CONTAINER guards against recursion. os.name is already "Linux" inside the
13+
// container so this is belt and braces, but it keeps the intent explicit.
14+
ext.useTestContainer = System.getProperty('os.name').toLowerCase().startsWith('windows') &&
15+
!project.hasProperty('noDocker') &&
16+
System.getenv('EXTENDER_IN_TEST_CONTAINER') == null
17+
18+
// Inside the container we build into build-container/ rather than build/. The two must not be
19+
// shared: the host Gradle daemon keeps its own build outputs open on Windows, and the container
20+
// then cannot overwrite them ("Unable to delete file .../extender-client-0.5.0.jar"). It also
21+
// keeps Linux and Windows compilation output from mixing.
22+
if (System.getenv('EXTENDER_IN_TEST_CONTAINER') != null) {
23+
allprojects {
24+
layout.buildDirectory = layout.projectDirectory.dir('build-container')
25+
}
26+
}
27+
28+
tasks.register('buildTestImage', Exec) {
29+
group = 'verification'
30+
description = 'Builds the container image used to run the tests on Windows.'
31+
// Docker layer caching makes this a no-op after the first run.
32+
commandLine 'docker', 'build',
33+
'-t', testImage,
34+
'-f', file('server/docker/Dockerfile.unit-test').absolutePath,
35+
file('server/docker').absolutePath
36+
}
37+
38+
tasks.register('dockerTest', Exec) {
39+
group = 'verification'
40+
description = 'Runs the non-integration tests of all modules inside a Linux container.'
41+
dependsOn 'buildTestImage'
42+
43+
// Docker wants forward slashes even on Windows.
44+
def mount = rootDir.absolutePath.replace('\\', '/')
45+
46+
commandLine 'docker', 'run', '--rm',
47+
'-v', "${mount}:/workspace",
48+
// Named volume, so the dependency cache survives between runs.
49+
'-v', 'extender-gradle-cache:/gradle-home',
50+
'-e', 'EXTENDER_IN_TEST_CONTAINER=1',
51+
'-w', '/workspace',
52+
testImage,
53+
// --project-cache-dir keeps the container out of the host's .gradle/, so a Windows-side
54+
// Gradle daemon and the container never fight over the same lock files. Everything else
55+
// (build/, reports, jacoco) stays on the bind mount and is visible from the host.
56+
'--no-daemon',
57+
'--project-cache-dir', '/tmp/gradle-project-cache',
58+
'-PexcludeTags=integration',
59+
':server:test', ':client:test', ':manifestmergetool:test'
60+
}
61+
62+
if (rootProject.ext.useTestContainer) {
63+
subprojects {
64+
tasks.matching { it.name == 'test' }.configureEach {
65+
// All three depend on the single root task, so `./gradlew test` still produces
66+
// exactly one container run.
67+
dependsOn ':dockerTest'
68+
enabled = false
69+
}
70+
// The container already wrote its own report under build-container/; there is no
71+
// execution data on the host side to report on.
72+
tasks.matching { it.name == 'jacocoTestReport' }.configureEach { enabled = false }
73+
}
74+
}
75+
// On Linux/macOS nothing is wired up - `test` runs natively as before. `dockerTest` stays
76+
// invokable by hand there if you want to reproduce the Windows path.

server/README.md

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,24 @@ During the testing local servers will be run. That's why it necessary to have pr
118118
* **test** - run for integration testing (see *IntegrationTest.java*)
119119
* **auth-test** - run for authentication testing (see *AuthenticationTest.java*)
120120

121+
### On Windows
122+
123+
On a Windows host `./gradlew test` (or `server:test`) does not run the tests locally. It builds
124+
`server/docker/Dockerfile.unit-test` and runs the whole non-integration suite - `:server`,
125+
`:client` and `:manifestmergetool`, with `-PexcludeTags=integration` - inside a Linux container, so
126+
the result matches CI. See `gradle/docker-test.gradle`. Everything is a no-op on Linux and macOS.
127+
128+
The container builds into `build-container/` rather than `build/` (the host Gradle daemon keeps its
129+
own outputs open on Windows), so reports end up in e.g.
130+
`server/build-container/reports/tests/test/index.html`.
131+
132+
Add `-PnoDocker` to run natively instead. This is what you need for the integration and auth tests,
133+
which use Git Bash to drive `scripts/start-test-server.sh`:
134+
135+
```sh
136+
./gradlew server:test -PnoDocker --tests '*IntegrationTest*'
137+
```
138+
121139
### Speeding up local runs (Apple Silicon)
122140

123141
A full `server:test` builds every target across every Defold version and, on an arm64 Mac, runs

server/build-docker.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ docker buildx build --network host --load --platform $MULTI_ARCH -t $DOCKER_REGI
2222

2323
# base images
2424
echo "Base image with archs: $MULTI_ARCH"
25-
docker buildx build --network host --load --platform $MULTI_ARCH -t $DOCKER_REGISTRY/extender-base-env:1.6.0 -t $DOCKER_REGISTRY/extender-base-env:latest -f $SCRIPT_DIR/docker/Dockerfile.base-env $SCRIPT_DIR/docker
25+
docker buildx build --network host --load --platform $MULTI_ARCH -t $DOCKER_REGISTRY/extender-base-env:1.7.0 -t $DOCKER_REGISTRY/extender-base-env:latest -f $SCRIPT_DIR/docker/Dockerfile.base-env $SCRIPT_DIR/docker
2626

2727
REQUESTED="$@"
2828
[ -z "$REQUESTED" ] && REQUESTED="android windows web ps4 ps5 nintendo xbox linux"
@@ -77,7 +77,7 @@ for request in $REQUESTED; do
7777
DM_PACKAGES_URL=$DM_PACKAGES_URL docker buildx build --network host --load --secret id=DM_PACKAGES_URL --platform linux/amd64 -t $DOCKER_XBOX_PRIVATE_REGISTRY/extender-${install}-env:latest -f $SCRIPT_DIR/docker/Dockerfile.$(echo $install | sed 's,-,.,')-env $SCRIPT_DIR/docker
7878
;;
7979
wine)
80-
DM_PACKAGES_URL=$DM_PACKAGES_URL docker buildx build --network host --load --secret id=DM_PACKAGES_URL --platform linux/amd64 -t $DOCKER_REGISTRY/extender-wine-env:1.8.0 -t $DOCKER_REGISTRY/extender-wine-env:latest -f $SCRIPT_DIR/docker/Dockerfile.wine-env $SCRIPT_DIR/docker
80+
DM_PACKAGES_URL=$DM_PACKAGES_URL docker buildx build --network host --load --secret id=DM_PACKAGES_URL --platform linux/amd64 -t $DOCKER_REGISTRY/extender-wine-env:1.9.0 -t $DOCKER_REGISTRY/extender-wine-env:latest -f $SCRIPT_DIR/docker/Dockerfile.wine-env $SCRIPT_DIR/docker
8181
;;
8282
android)
8383
DM_PACKAGES_URL=$DM_PACKAGES_URL docker buildx build --network host --load --secret id=DM_PACKAGES_URL --platform linux/amd64 -t $DOCKER_REGISTRY/extender-android-env:1.7.0 -t $DOCKER_REGISTRY/extender-android-env:latest -f $SCRIPT_DIR/docker/Dockerfile.android-env $SCRIPT_DIR/docker

server/docker/Dockerfile.android-env

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ RUN \
1515
unzip -q -d /opt/gradle gradle-${GRADLE_VERSION}-bin.zip && \
1616
rm gradle-${GRADLE_VERSION}-bin.zip
1717

18-
FROM europe-west1-docker.pkg.dev/extender-426409/extender-public-registry/extender-base-env:1.6.0
18+
FROM europe-west1-docker.pkg.dev/extender-426409/extender-public-registry/extender-base-env:1.7.0
1919

2020
ARG GRADLE_VERSION
2121

server/docker/Dockerfile.android.ndk25-env

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,14 @@ ENV ANDROID_ROOT=${ANDROID_ROOT} \
125125
ANDROID_NDK25_PATH=${ANDROID_NDK25_PATH} \
126126
ANDROID_NDK25_BIN_PATH=${ANDROID_NDK25_BIN_PATH} \
127127
ANDROID_NDK25_SYSROOT=${ANDROID_NDK25_PATH}/toolchains/llvm/prebuilt/linux-x86_64/sysroot \
128+
# Unversioned names are the canonical ones. The ANDROID_NDK25_* names above are
129+
# kept for backward compatibility with already released Defold SDKs.
130+
ANDROID_NDK_VERSION=${ANDROID_NDK25_VERSION} \
131+
ANDROID_NDK_API_VERSION=${ANDROID_NDK25_API_VERSION} \
132+
ANDROID_64_NDK_API_VERSION=${ANDROID_64_NDK25_API_VERSION} \
133+
ANDROID_NDK_PATH=${ANDROID_NDK25_PATH} \
134+
ANDROID_NDK_BIN_PATH=${ANDROID_NDK25_BIN_PATH} \
135+
ANDROID_NDK_SYSROOT=${ANDROID_NDK25_PATH}/toolchains/llvm/prebuilt/linux-x86_64/sysroot \
128136
ANDROID_PROGUARD=/usr/share/java/proguard.jar \
129137
# We specify it in build_input.yml by setting it the first in PATH for new SDK
130138
# But at least one SDK should be specified by default in Dockerfile

server/docker/Dockerfile.base-env

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,8 @@ RUN \
6060
export DOTNET_VERSION && \
6161
echo "${DOTNET_VERSION}" > "${DOTNET_VERSION_FILE}"
6262

63-
FROM ubuntu@sha256:ce4a593b4e323dcc3dd728e397e0a866a1bf516a1b7c31d6aa06991baec4f2e0
63+
# ubuntu:22.04
64+
FROM ubuntu@sha256:0e0a0fc6d18feda9db1590da249ac93e8d5abfea8f4c3c0c849ce512b5ef8982
6465

6566
ARG PLATFORMSDK_DIR
6667
ARG TARGETARCH

server/docker/Dockerfile.build-env

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# ubuntu:22.04
2-
FROM ubuntu@sha256:ce4a593b4e323dcc3dd728e397e0a866a1bf516a1b7c31d6aa06991baec4f2e0
2+
FROM ubuntu@sha256:0e0a0fc6d18feda9db1590da249ac93e8d5abfea8f4c3c0c849ce512b5ef8982
33

44
RUN \
55
apt-get update && \

0 commit comments

Comments
 (0)