Skip to content

Commit 84092cf

Browse files
michalChrobotmichalChrobot
committed
Comments
1 parent 75615db commit 84092cf

15 files changed

+378
-368
lines changed

‎.yamato/_run-all.yml‎

Lines changed: 8 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -2,41 +2,8 @@
22
---
33

44
# DESCRIPTION--------------------------------------------------------------------------
5-
# Defines aggregation jobs that group related test executions
6-
# Provides both full validation (all editors) and trunk-only variants
7-
# Groups tests by type (package, project) and platform (desktop, mobile, console)
8-
# Enables efficient test execution for different validation scenarios
9-
10-
# AGGREGATION PATTERNS---------------------------------------------------------------
11-
# Two main patterns for each test type:
12-
# 1. Complete Validation: All supported Unity Editor versions
13-
# 2. Trunk Validation: Only latest Unity Editor version
14-
#
15-
# Test Types:
16-
# - Package Tests: NGO package validation
17-
# - Project Tests: Project-specific validation
18-
# - Standards Tests: Code standards verification
19-
# - Platform Tests: Platform-specific builds and tests
20-
# * Desktop (Windows, macOS, Linux)
21-
# * Mobile (iOS, Android)
22-
# * Console (PS4, PS5, Switch, Xbox)
23-
# * WebGL (build-only validation)
24-
25-
# USAGE GUIDELINES------------------------------------------------------------------
26-
# Complete Validation:
27-
# - Use for release validation
28-
# - Weekly automated runs
29-
# - Major version updates
30-
#
31-
# Trunk Validation:
32-
# - Use for quick verification
33-
# - Daily automated runs
34-
# - PR validation
35-
#
36-
# Platform-Specific:
37-
# - Use when targeting specific platform changes
38-
# - Hardware-dependent validation
39-
# - Platform certification requirements
5+
# Those jobs group together related test executions
6+
# Enables efficient test execution for different validation scenarios
407

418
#-----------------------------------------------------------------------------------
429

@@ -112,7 +79,7 @@ run_all_webgl_builds:
11279
{% for project in projects.default -%}
11380
{% for platform in test_platforms.desktop -%}
11481
{% for editor in validation_editors.all -%}
115-
- .yamato/webgl-build.yml#webgl_build_{{ project.name }}_{{ platform }}_{{ editor }}
82+
- .yamato/webgl-build.yml#webgl_build_{{ project.name }}_{{ platform.name }}_{{ editor }}
11683
{% endfor -%}
11784
{% endfor -%}
11885
{% endfor -%}
@@ -125,7 +92,7 @@ run_all_webgl_builds_trunk:
12592
{% for project in projects.default -%}
12693
{% for platform in test_platforms.desktop -%}
12794
{% for editor in validation_editors.default -%}
128-
- .yamato/webgl-build.yml#webgl_build_{{ project.name }}_{{ platform }}_{{ editor }}
95+
- .yamato/webgl-build.yml#webgl_build_{{ project.name }}_{{ platform.name }}_{{ editor }}
12996
{% endfor -%}
13097
{% endfor -%}
13198
{% endfor -%}
@@ -167,7 +134,7 @@ run_all_project_tests_mobile_standalone:
167134
{% for project in projects.default -%}
168135
{% for platform in test_platforms.mobile_test -%}
169136
{% for editor in validation_editors.all -%}
170-
- .yamato/mobile-standalone-test.yml#run_{{ project.name }}_tests_{{ platform.name }}_{{ editor }}
137+
- .yamato/mobile-standalone-test.yml#mobile_standalone_test_{{ project.name }}_tests_{{ platform.name }}_{{ editor }}
171138
{% endfor -%}
172139
{% endfor -%}
173140
{% endfor -%}
@@ -180,7 +147,7 @@ run_all_project_tests_mobile_standalone_trunk:
180147
{% for project in projects.default -%}
181148
{% for platform in test_platforms.mobile_test -%}
182149
{% for editor in validation_editors.default -%}
183-
- .yamato/mobile-standalone-test.yml#run_{{ project.name }}_tests_{{ platform.name }}_{{ editor }}
150+
- .yamato/mobile-standalone-test.yml#mobile_standalone_test_{{ project.name }}_tests_{{ platform.name }}_{{ editor }}
184151
{% endfor -%}
185152
{% endfor -%}
186153
{% endfor -%}
@@ -193,7 +160,7 @@ run_all_project_tests_console_standalone:
193160
{% for project in projects.default -%}
194161
{% for platform in test_platforms.console_test -%}
195162
{% for editor in validation_editors.all -%}
196-
- .yamato/console-standalone-test.yml#run_{{ project.name }}_tests_{{ platform.name }}_{{ editor }}
163+
- .yamato/console-standalone-test.yml#console_standalone_test_{{ project.name }}_tests_{{ platform.name }}_{{ editor }}
197164
{% endfor -%}
198165
{% endfor -%}
199166
{% endfor -%}
@@ -206,7 +173,7 @@ run_all_project_tests_console_standalone_trunk:
206173
{% for project in projects.default -%}
207174
{% for platform in test_platforms.console_test -%}
208175
{% for editor in validation_editors.default -%}
209-
- .yamato/console-standalone-test.yml#run_{{ project.name }}_tests_{{ platform.name }}_{{ editor }}
176+
- .yamato/console-standalone-test.yml#console_standalone_test_{{ project.name }}_tests_{{ platform.name }}_{{ editor }}
210177
{% endfor -%}
211178
{% endfor -%}
212179
{% endfor -%}

‎.yamato/_triggers.yml‎

Lines changed: 20 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -3,28 +3,28 @@
33

44
# DESCRIPTION--------------------------------------------------------------------------
55
# Defines three main CI trigger patterns:
6-
# 1. Pull Request Validation: Basic validation for PR changes
7-
# 2. Nightly Development: Comprehensive testing on trunk
8-
# 3. Weekly Full Validation: Complete testing across all editor versions
9-
# Each pattern represents different balance between validation depth and execution time
6+
# 1. Pull Request Validation: Basic validation for PR changes
7+
# 2. Nightly Development: Comprehensive testing on trunk
8+
# 3. Weekly Full Validation: Complete testing across all editor versions
9+
# Each pattern represents different balance between validation depth, execution time and CI resource usage
1010

1111
# TRIGGER PATTERNS-------------------------------------------------------------------
12-
# Pull Request:
13-
# - Triggers on PRs to develop, develop-2.0.0, and release branches
14-
# - Focuses on critical validation paths
15-
# - Cancels previous runs on new commits
16-
# - Excludes draft PRs
17-
#
18-
# Nightly:
19-
# - Runs daily on develop-2.0.0
20-
# - Includes all test types but only on trunk
21-
# - Adds platform-specific and APV validation
22-
#
23-
# Weekly:
24-
# - Most comprehensive validation
25-
# - Runs across all supported editor versions
26-
# - Includes code coverage analysis
27-
# - Validates all projects and standards
12+
# Pull Request:
13+
# Triggers on PRs to develop, develop-2.0.0, and release branches
14+
# Focuses on critical validation paths that we should validate before merging PRs
15+
# Cancels previous runs on new commits
16+
# Excludes draft PRs
17+
18+
# Nightly:
19+
# Runs daily on develop-2.0.0 (local configuration)
20+
# Includes all test types but only on trunk. TODO: Add validation for minimum supported editor in nightly
21+
# Adds platform-specific and APV validation
22+
23+
# Weekly:
24+
# Most comprehensive validation
25+
# Runs across all supported editor versions
26+
# Includes code coverage analysis
27+
# Validates all projects and standards
2828

2929
#-----------------------------------------------------------------------------------
3030

‎.yamato/code-coverage.yml‎

Lines changed: 33 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,36 @@
11
{% metadata_file .yamato/project.metafile %} # All configuration that is used to create different configurations (used in for loops) is taken from this file.
22
---
3-
4-
# Runs package tests in order to determine code coverage of the NGO package.
5-
# In essence, it's performing the same task as .yamato/package-test jobs with the overhead being the measured code coverage
6-
# It's ok for code coverage to be performed only on one platform (default) since code coverage won't change much between those.
7-
# Default platform was chosen (ubuntu) since it's the fastest and most resource friendly with default editor.
3+
4+
# DESCRIPTION--------------------------------------------------------------------------
5+
# This job executes package tests with code coverage analysis enabled for the NGO (Netcode for GameObjects) package
6+
# Coverage analysis provides insights into:
7+
# Test coverage metrics for NGO assemblies
8+
# Line and branch coverage statistics
9+
# Generated HTML reports for coverage visualization
10+
# Additional metrics for coverage analysis
11+
12+
# CONFIGURATION STRUCTURE--------------------------------------------------------------
13+
# It may seem that there is only 1 job defined, but we are using for loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations
14+
# Jobs are generated using nested loops through:
15+
# 1. For default platform only (Ubuntu) since coverage would not vary between platforms (no need for checks on more platforms)
16+
# 2. For default editor version (trunk) since coverage would not vary between editors (no need for checks on more editors)
17+
18+
# In theory, we could manually write jobs for every configuration. However, this approach would be more error-prone, especially when modifications or fixes are needed, as it would require keeping track of all configurations.
19+
# The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements.
20+
21+
# The jobs name being generated would be for example "Code Coverage - NGO [testproject, 6000.0]" (name visible in Yamato) or "code_coverage_testproject_6000.0" (job name when querying for it)
22+
23+
#TECHNICAL CONSTRAINTS---------------------------------------------------------------
24+
# Requires Unity Editor installation
25+
# Uses UnifiedTestRunner (UTR) for test execution
26+
# Depends on successful package pack job completion
27+
# Burst compilation is disabled to ensure accurate coverage measurement
28+
29+
# QUALITY THOUGHTS--------------------------------------------------------------------
30+
# To see where this job is included (in trigger job definitions) look into _triggers.yml file
31+
32+
33+
834
{% for platform in test_platforms.default -%}
935
{% for editor in validation_editors.default -%}
1036
code_coverage_{{ platform.name }}_{{ editor }}:
@@ -14,9 +40,9 @@ code_coverage_{{ platform.name }}_{{ editor }}:
1440
image: {{ platform.image }}
1541
flavor: {{ platform.flavor }}
1642
commands:
17-
- unity-downloader-cli -u {{ editor }} -c Editor --fast --wait
43+
- unity-downloader-cli -u {{ editor }} -c Editor
1844
- upm-pvp create-test-project test-project --packages "upm-ci~/packages/*.tgz" --unity .Editor
19-
- UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --enable-code-coverage -coverage-results-path=$PWD/test-results/CodeCoverage --coverage-options="generateHtmlReport;generateAdditionalMetrics;assemblyFilters:+Unity.Netcode.Editor,+Unity.Netcode.Runtime" --extra-editor-arg=--burst-disable-compilation --extra-editor-arg=testCategory --extra-editor-arg=!Performance --timeout=1800 --reruncount=1 --clean-library-on-rerun --artifacts-path=test-results
45+
- UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --enable-code-coverage -coverage-results-path=$PWD/test-results/CodeCoverage --coverage-options="generateHtmlReport;generateAdditionalMetrics;assemblyFilters:+Unity.Netcode.Editor,+Unity.Netcode.Runtime" --extra-editor-arg=--burst-disable-compilation --timeout=1800 --reruncount=1 --clean-library-on-rerun --artifacts-path=test-results
2046
artifacts:
2147
logs:
2248
paths:

‎.yamato/console-standalone-test.yml‎

Lines changed: 37 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -2,55 +2,54 @@
22
---
33

44
# DESCRIPTION--------------------------------------------------------------------------
5-
# Console platform validation for NGO package using standalone builds
6-
# Executes runtime tests (PlayMode equivalent) in standalone console builds
7-
# Uses split build/run approach due to console-specific requirements
8-
# Covers PS4, PS5, Switch, XboxOne, and XboxSeriesX platforms
5+
# This file contains Console platform test validation for NGO package consisting of two phases:
6+
# 1. Build Phase: Creates standalone players for console platforms
7+
# 2. Run Phase: Executes runtime tests on actual console devices
8+
# Those phases need to be separated (in contrast to for example desktop tests)
99

10-
# PLATFORM SPECIFICS-----------------------------------------------------------------
10+
# CONFIGURATION STRUCTURE--------------------------------------------------------------
11+
# It may seem that there is only 1 job defined, but we are using for loops based on .yamato/project.metafile configuration. Yamato "compiles" this code in order to generate jobs with multiple given configurations
12+
# Jobs are generated using nested loops (separate build phase and run phase). Worth noting that run phase uses the build as dependency:
13+
# 1. For all console platform (Switch, ps4, ps5, xbox360, xboxOne) defined in test_platforms.console_ (in project.metafile)
14+
# 2. For all supported Unity Editor versions (for NGOv2.X this means 6000.0+ editors defined in project.metafile)
15+
# 3. For default project which serves as context
16+
17+
# In theory, we could manually write jobs for every configuration. However, this approach would be more error-prone, especially when modifications or fixes are needed, as it would require keeping track of all configurations.
18+
# The downside of our current approach is that it can sometimes impact readability due to the use of nested if and for statements.
19+
20+
# The jobs name being generated would be for example "Build testproject - [win, 6000.0, il2cpp]" (name visible in Yamato) or "console_standalone_build_testproject_win_6000.0" (job name when querying for it)
21+
22+
# PLATFORM SPECIFICS-----------------------------------------------------------------
1123
# Common Requirements:
12-
# - All consoles require IL2CPP scripting backend
13-
# - Windows-based build machines required
14-
# - Platform-specific SDK environment variables
15-
#
24+
# All consoles require IL2CPP scripting backend
25+
# Platform-specific SDK environment variables
26+
1627
# Platform-Specific:
17-
# - Switch: ARM64 architecture only
18-
# - Other Consoles: x64 architecture
19-
# - Each console requires specific SDK paths and tools
20-
# - Some consoles require specific hardware models for testing
21-
22-
# TECHNICAL CONSTRAINTS--------------------------------------------------------------
23-
# Build Phase:
24-
# - Windows build agents required
25-
# - Platform-specific SDK installations
26-
# - IL2CPP only (no Mono support)
27-
# - Architecture varies by console
28-
#
29-
# Run Phase:
30-
# - Requires specific console hardware
31-
# - Platform-specific environment setup
32-
# - Direct hardware access needed
33-
# - Specific SDK versions required
28+
# Switch: ARM64 architecture only
29+
# Other Consoles: x64 architecture
30+
# Each console requires specific SDK paths and tools
31+
32+
# Note: All builds can be made on x64 machines since those are compatible with ARM64 target devices
33+
# Note: More of a Unity specific but test assemblies need to be included in the build phase command
3434

35-
#-----------------------------------------------------------------------------------
35+
# QUALITY THOUGHTS--------------------------------------------------------------------
36+
# To see where this job is included (in trigger job definitions) look into _triggers.yml file
3637

38+
39+
40+
# BUILD PHASE CONFIGURATION------------------------------------------------------------------------------------
3741
{% for project in projects.default -%}
3842
{% for platform in test_platforms.console_build -%}
3943
{% for editor in validation_editors.all -%}
40-
build_{{ project.name }}_{{ platform.name }}_{{ editor }}:
44+
console_standalone_build_{{ project.name }}_{{ platform.name }}_{{ editor }}:
4145
name: Build {{ project.name }} - [{{ platform.name }}, {{ editor }}, il2cpp{% if platform.name == "switch" %}, arm64{% endif %}]
4246
agent:
4347
type: {{ platform.type }}
4448
image: {{ platform.image }}
4549
flavor: {{ platform.flavor }}
4650
commands:
47-
# Installing tools. unity-downloader-cli and utr should be already preinstalled on the image
48-
- sudo pip install unity-downloader-cli
49-
- unity-downloader-cli -u {{ editor }} -c Editor -c il2cpp -c {{ platform.name }} --fast --wait
50-
51-
# Platform specific Build
51+
- unity-downloader-cli -u {{ editor }} -c Editor -c il2cpp -c {{ platform.name }}
5252
- UnifiedTestRunner --testproject={{ project.path }} --architecture={% if platform.name == "switch" %}arm64{% else %}x64{% endif %} --scripting-backend=il2cpp --suite=playmode --platform={{ platform.standalone }} --editor-location=.Editor --artifacts-path=artifacts --player-save-path=build/players --testfilter="Unity.Netcode.RuntimeTests.*" --extra-editor-arg=-batchmode --extra-editor-arg=-nographics --reruncount=1 --clean-library-on-rerun --build-only --timeout=1800
53-
5453
variables:
5554
# PS4 related
5655
SCE_ORBIS_SDK_DIR: 'C:\Users\bokken\SCE\ps4_sdk_12_00'
@@ -74,11 +73,11 @@ build_{{ project.name }}_{{ platform.name }}_{{ editor }}:
7473

7574

7675

77-
# Executes RuntimeTests of the NGO package in the Standalone build for consoles
76+
# RUN PHASE CONFIGURATION------------------------------------------------------------------------------------
7877
{% for project in projects.default -%}
7978
{% for platform in test_platforms.console_test -%}
8079
{% for editor in validation_editors.all -%}
81-
run_{{ project.name }}_tests_{{ platform.name }}_{{ editor }}:
80+
console_standalone_test_{{ project.name }}_tests_{{ platform.name }}_{{ editor }}:
8281
name: Run {{ project.name }} Tests - [{{ platform.name }}, {{ editor }}, il2cpp]
8382
agent:
8483
type: {{ platform.type }}
@@ -88,13 +87,8 @@ run_{{ project.name }}_tests_{{ platform.name }}_{{ editor }}:
8887
model: {{ platform.model }}
8988
{% endif %}
9089
commands:
91-
# Installing tools.
92-
- sudo pip install unity-downloader-cli
93-
- unity-downloader-cli -u {{ editor }} -c Editor -c il2cpp -c {{ platform.name }} --fast --wait
94-
95-
# Platform specific Execution
90+
- unity-downloader-cli -u {{ editor }} -c Editor -c il2cpp -c {{ platform.name }}
9691
- UnifiedTestRunner --suite=playmode --testproject={{ project.path }} --editor-location=.Editor --artifacts-path=test-results --player-load-path=build/players --fail-on-assert --reruncount=1 --clean-library-on-rerun --timeout=1800
97-
9892
variables:
9993
# PS4 related
10094
SCE_ORBIS_SDK_DIR: 'C:\Users\bokken\SCE\ps4_sdk_12_00'
@@ -110,7 +104,7 @@ run_{{ project.name }}_tests_{{ platform.name }}_{{ editor }}:
110104
paths:
111105
- "test-results/**/*"
112106
dependencies:
113-
- .yamato/console-standalone-test.yml#build_{{ project.name }}_{{ platform.name }}_{{ editor }}
107+
- .yamato/console-standalone-test.yml#console_standalone_build_{{ project.name }}_{{ platform.name }}_{{ editor }}
114108
{% endfor -%}
115109
{% endfor -%}
116110
{% endfor -%}

0 commit comments

Comments
 (0)