Skip to content

Commit 2726346

Browse files
BreakthroughBreakthrough
committed
[release] Add new release test suite, workflow runner, and release plan template
1 parent 63fbf8b commit 2726346

16 files changed

Lines changed: 1389 additions & 0 deletions

‎.github/workflows/release-test.yml‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
name: Release Test Suite
2+
3+
on:
4+
workflow_dispatch:
5+
push:
6+
tags:
7+
- 'v*-release'
8+
9+
jobs:
10+
static:
11+
runs-on: ubuntu-latest
12+
steps:
13+
- uses: actions/checkout@v4
14+
- name: Set up Python
15+
uses: actions/setup-python@v5
16+
with:
17+
python-version: '3.10'
18+
- name: Install dependencies
19+
run: |
20+
python -m pip install --upgrade pip
21+
pip install build twine pip-audit
22+
- name: Version consistency check
23+
run: |
24+
VERSION=$(python -c "import scenedetect; print(scenedetect.__version__)")
25+
echo "scenedetect.__version__ = $VERSION"
26+
if [[ "${{ github.ref }}" == refs/tags/* ]]; then
27+
TAG_VERSION=${GITHUB_REF#refs/tags/v}
28+
TAG_VERSION=${TAG_VERSION%-release}
29+
if [[ "$VERSION" != "$TAG_VERSION" ]]; then
30+
echo "Version mismatch: scenedetect=$VERSION, tag=$TAG_VERSION"
31+
exit 1
32+
fi
33+
if ! grep -q "^## PySceneDetect $TAG_VERSION" website/pages/changelog.md; then
34+
echo "Changelog is missing a '## PySceneDetect $TAG_VERSION' heading"
35+
exit 1
36+
fi
37+
fi
38+
- name: Build and Check
39+
run: |
40+
python -m build
41+
twine check dist/*
42+
- name: pip-audit
43+
run: pip-audit
44+
45+
release-tests:
46+
needs: static
47+
strategy:
48+
matrix:
49+
os: [ubuntu-latest, macos-latest, windows-latest]
50+
python-version: ['3.10', '3.13']
51+
runs-on: ${{ matrix.os }}
52+
steps:
53+
- uses: actions/checkout@v4
54+
- name: Checkout resources branch
55+
run: |
56+
git fetch --depth=1 origin refs/heads/resources:refs/remotes/origin/resources
57+
git checkout refs/remotes/origin/resources -- tests/resources/
58+
git reset
59+
- name: Set up Python ${{ matrix.python-version }}
60+
uses: actions/setup-python@v5
61+
with:
62+
python-version: ${{ matrix.python-version }}
63+
- name: Install ffmpeg
64+
uses: ./.github/actions/setup-ffmpeg
65+
- name: Install dependencies
66+
run: |
67+
python -m pip install --upgrade pip
68+
pip install .[opencv,pyav,moviepy]
69+
pip install opentimelineio pillow psutil pytest
70+
- name: Run release tests
71+
run: pytest -m release -vv --ignore=tests/release/test_long_video_stress.py --ignore=tests/release/test_install_matrix.py
72+
73+
long-stress:
74+
needs: static
75+
runs-on: ubuntu-latest
76+
steps:
77+
- uses: actions/checkout@v4
78+
- name: Checkout resources branch
79+
run: |
80+
git fetch --depth=1 origin refs/heads/resources:refs/remotes/origin/resources
81+
git checkout refs/remotes/origin/resources -- tests/resources/
82+
git reset
83+
- name: Set up Python
84+
uses: actions/setup-python@v5
85+
with:
86+
python-version: '3.10'
87+
- name: Install ffmpeg
88+
uses: ./.github/actions/setup-ffmpeg
89+
- name: Install dependencies
90+
run: |
91+
python -m pip install --upgrade pip
92+
pip install .[opencv,pyav]
93+
pip install psutil pytest
94+
- name: Run long stress test
95+
run: pytest -m release -k long_video -vv
96+
97+
install-matrix:
98+
needs: static
99+
strategy:
100+
matrix:
101+
os: [ubuntu-latest, windows-latest]
102+
runs-on: ${{ matrix.os }}
103+
steps:
104+
- uses: actions/checkout@v4
105+
- name: Checkout resources branch
106+
run: |
107+
git fetch --depth=1 origin refs/heads/resources:refs/remotes/origin/resources
108+
git checkout refs/remotes/origin/resources -- tests/resources/
109+
git reset
110+
- name: Set up Python
111+
uses: actions/setup-python@v5
112+
with:
113+
python-version: '3.10'
114+
- name: Build wheel
115+
run: |
116+
pip install build
117+
python -m build --wheel
118+
- name: Test Bare Install
119+
shell: bash
120+
run: |
121+
WHEEL=$(ls dist/*.whl)
122+
PY=${{ matrix.os == 'windows-latest' && 'venv_bare/Scripts/python' || 'venv_bare/bin/python' }}
123+
PYTEST=${{ matrix.os == 'windows-latest' && 'venv_bare/Scripts/pytest' || 'venv_bare/bin/pytest' }}
124+
python -m venv venv_bare
125+
$PY -m pip install pytest "$WHEEL"
126+
$PYTEST -m release -k test_install_bare
127+
- name: Test OpenCV Install
128+
shell: bash
129+
run: |
130+
WHEEL=$(ls dist/*.whl)
131+
PY=${{ matrix.os == 'windows-latest' && 'venv_opencv/Scripts/python' || 'venv_opencv/bin/python' }}
132+
PYTEST=${{ matrix.os == 'windows-latest' && 'venv_opencv/Scripts/pytest' || 'venv_opencv/bin/pytest' }}
133+
python -m venv venv_opencv
134+
$PY -m pip install pytest "${WHEEL}[opencv]"
135+
$PYTEST -m release -k test_opencv_only
136+
- name: Test PyAV Install
137+
shell: bash
138+
run: |
139+
WHEEL=$(ls dist/*.whl)
140+
PY=${{ matrix.os == 'windows-latest' && 'venv_pyav/Scripts/python' || 'venv_pyav/bin/python' }}
141+
PYTEST=${{ matrix.os == 'windows-latest' && 'venv_pyav/Scripts/pytest' || 'venv_pyav/bin/pytest' }}
142+
python -m venv venv_pyav
143+
$PY -m pip install pytest "${WHEEL}[pyav]"
144+
$PYTEST -m release -k test_pyav_only

‎RELEASE-PLAN.md‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
# PySceneDetect Release Checklist
2+
3+
Use one copy per release (e.g. tick the boxes in a tracking issue or draft PR).
4+
Version referenced below as `X.Y[.Z]` — replace with the real version throughout.
5+
6+
## 0. Branch setup
7+
8+
- [ ] Create / fast-forward release branch: `releases/X.Y` off `main`.
9+
- [ ] All release-prep commits land on `releases/X.Y` (never directly on `main` during the freeze - commits are usually halted to `main` until the release branch is cut, after which the release branch is merged back into `main` and development resumes).
10+
11+
## 1. Code & version
12+
13+
- [ ] Bump `__version__` in `scenedetect/__init__.py`.
14+
- [ ] Bump `ProductVersion` in `dist/installer/PySceneDetect.aip` (must match `__version__` — `dist/pre_release.py --release` asserts this).
15+
- [ ] No `-dev` / pre-release suffix on the version string for a final release.
16+
17+
> **Note:** `setup.cfg` reads the package version dynamically via `version = attr: scenedetect.__version__`, and `pyproject.toml` does not declare a `version` field. The single source of truth is `scenedetect/__init__.py`; the `.aip` is the only other place to keep in sync.
18+
19+
## 2. Docs
20+
21+
- [ ] Docstrings / API docs reflect any signature changes (`cd docs/ && make html` builds clean).
22+
- [ ] `docs/api/migration_guide.rst` updated if any public API changed.
23+
- [ ] Docstring examples still run (nothing references removed symbols).
24+
25+
## 3. Website & changelog
26+
27+
- [ ] `website/pages/changelog.md`: rename the bottom **Development** section to `X.Y (YYYY-MM-DD)` and add a fresh empty **Development** section below it for post-release work.
28+
- [ ] Changelog entry covers: new features, breaking changes, bug fixes, known issues.
29+
- [ ] `website/pages/download.md` updated with the new version / installer link.
30+
- [ ] Any other version-stamped pages updated (`supporting.md`, `cli.md` if commands changed).
31+
32+
## 4. Tests
33+
34+
- [ ] Unit tests green locally and in CI: `pytest -vv` (should collect `-m 'not release'` by default).
35+
- [ ] `ruff check scenedetect/ tests/` and `ruff format --check scenedetect/ tests/` pass.
36+
- [ ] Release test suite green: tag a disposable `vX.Y.Z-release-rc` or use `workflow_dispatch` on `.github/workflows/release-test.yml` — all 4 jobs (`static`, `release-tests`, `install-matrix`, `long-stress`) green across the 3-OS × 2-Python matrix. See `RELEASE-TEST-PLAN.md` for what the suite covers.
37+
- [ ] `resources` branch has the artifacts the release tests need (goldens under `tests/resources/goldens/`, `tests/resources/stress_15min.mp4`). Re-push if any golden was regenerated.
38+
- [ ] Manual smoke: fresh venv, `pip install .` then `pip install .[opencv]` then `pip install .[pyav]`; run `scenedetect -i <video> detect-content list-scenes save-images` and eyeball the output.
39+
- [ ] `pip-audit` clean (or exceptions documented in the changelog).
40+
41+
## 5. Windows installer
42+
43+
- [ ] `python dist/pre_release.py --release` passes (enforces `.aip` ↔ `__version__` parity, writes `dist/.version_info`).
44+
- [ ] `pyinstaller dist/scenedetect.spec` produces a working `scenedetect.exe` — run it against a sample video.
45+
- [ ] Build the MSI via Advanced Installer (`dist/installer/PySceneDetect.aip`); install into a clean Windows VM and run the CLI.
46+
47+
## 6. Cut the release
48+
49+
- [ ] Final commit on `releases/X.Y`: "Release vX.Y[.Z]".
50+
- [ ] Tag `vX.Y[.Z]-release` on that commit and push — this fires `release-test.yml`. Wait for all jobs green.
51+
- [ ] Merge `releases/X.Y` into `main` (fast-forward or merge commit — keep history clean).
52+
- [ ] Tag the final release `vX.Y[.Z]` on the merged commit and push.
53+
54+
## 7. Publish
55+
56+
- [ ] `publish-pypi.yml` ran on the tag and uploaded successfully. Verify at https://pypi.org/project/scenedetect/.
57+
- [ ] Smoke-test PyPI: in a fresh venv, `pip install scenedetect==X.Y.Z` (bare), then with `[opencv]` and `[pyav]`. CLI launches.
58+
- [ ] Create GitHub Release from the `vX.Y[.Z]` tag, body = changelog section, attach Windows installer MSI + portable `.zip`.
59+
- [ ] Deploy website: `generate-website.yml` picks up the changelog / download page updates.
60+
- [ ] Deploy docs: `generate-docs.yml` publishes the new version.
61+
62+
## 8. Post-release
63+
64+
- [ ] On `main`: bump `__version__` to the next dev version (e.g. `X.(Y+1)-dev0` or `X.Y.(Z+1)-dev0`), matching `pyproject.toml` and `PySceneDetect.aip`.
65+
- [ ] Clear / archive release-scoped tracking files (`tracking.md`, any release-specific TODOs).
66+
- [ ] Announce: project site, relevant issues / discussions closed and linked to the release.
67+
- [ ] Delete `releases/X.Y` branch once nothing else targets it.
68+
69+
---
70+
71+
## Notes
72+
73+
- **Branching model**: work spans multiple commits on `releases/X.Y`; the final one gets the `vX.Y[.Z]-release` tag which gates the release-test workflow. A passing release-test is a hard prerequisite for publishing.
74+
- **Version consistency** is enforced in two places (`__init__.py`, `PySceneDetect.aip`). The `static` job of `release-test.yml` checks `__init__.py` against the tag and verifies the changelog has a matching `## PySceneDetect X.Y` heading; the installer parity is checked by `dist/pre_release.py --release`.
75+
- **Changelog convention**: the in-development section lives at the *bottom* of `website/pages/changelog.md` under the "Development" heading — don't move it to the top.

‎pyproject.toml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,12 @@
1212
requires = ["setuptools"]
1313
build-backend = "setuptools.build_meta"
1414

15+
[tool.pytest.ini_options]
16+
markers = [
17+
"release: opt-in release-validation tests; excluded by default (run with `pytest -m release`)",
18+
]
19+
addopts = "-m 'not release'"
20+
1521
[tool.ruff]
1622
line-length = 100
1723
indent-width = 4

‎scripts/generate_goldens.py‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
#
2+
# PySceneDetect: Python-Based Video Scene Detector
3+
# -------------------------------------------------------------------
4+
# [ Site: https://scenedetect.com ]
5+
# [ Docs: https://scenedetect.com/docs/ ]
6+
# [ Github: https://github.com/Breakthrough/PySceneDetect/ ]
7+
#
8+
# Copyright (C) 2026 Brandon Castellano <http://www.bcastell.com>.
9+
# PySceneDetect is licensed under the BSD 3-Clause License; see the
10+
# included LICENSE file, or visit one of the above pages for details.
11+
#
12+
"""Golden Cut-List Generator
13+
14+
This script generates golden cut-lists in JSON format for the release test suite.
15+
"""
16+
17+
import argparse
18+
import json
19+
import os
20+
21+
from scenedetect import (
22+
AdaptiveDetector,
23+
ContentDetector,
24+
HashDetector,
25+
HistogramDetector,
26+
SceneManager,
27+
ThresholdDetector,
28+
open_video,
29+
)
30+
31+
VIDEOS = [
32+
"tests/resources/testvideo.mp4",
33+
"tests/resources/goldeneye.mp4",
34+
"tests/resources/goldeneye-vfr.mp4",
35+
"tests/resources/goldeneye-vfr-drop3.mp4",
36+
"tests/resources/fades.mp4",
37+
"tests/resources/counter.mp4",
38+
]
39+
40+
# (DetectorClass, params, name_suffix)
41+
DETECTORS = [
42+
(ContentDetector, {}, "default"),
43+
(ContentDetector, {"threshold": 30.0}, "t30"),
44+
(AdaptiveDetector, {}, "default"),
45+
(AdaptiveDetector, {"adaptive_threshold": 5.0}, "t5"),
46+
(ThresholdDetector, {}, "default"),
47+
(HistogramDetector, {}, "default"),
48+
(HashDetector, {}, "default"),
49+
]
50+
51+
52+
def generate_golden(video_path: str, detector_class, params: dict) -> list[int]:
53+
video = open_video(video_path, backend="pyav")
54+
scene_manager = SceneManager()
55+
scene_manager.add_detector(detector_class(**params))
56+
scene_manager.detect_scenes(video)
57+
scene_list = scene_manager.get_scene_list()
58+
# Return start frame of each scene except the first one (which is 0)
59+
return [scene[0].get_frames() for scene in scene_list[1:]]
60+
61+
62+
def main():
63+
parser = argparse.ArgumentParser()
64+
parser.add_argument("--output-dir", default="tests/resources/goldens")
65+
args = parser.parse_args()
66+
67+
if not os.path.exists(args.output_dir):
68+
os.makedirs(args.output_dir)
69+
70+
for video_path in VIDEOS:
71+
if not os.path.exists(video_path):
72+
print(f"Skipping {video_path}, not found.")
73+
continue
74+
75+
video_name = os.path.basename(video_path)
76+
for detector_class, params, suffix in DETECTORS:
77+
detector_name = detector_class.__name__
78+
print(f"Generating golden for {video_name} with {detector_name} ({suffix})...")
79+
try:
80+
cuts = generate_golden(video_path, detector_class, params)
81+
output_filename = f"{video_name}.{detector_name}.{suffix}.json"
82+
output_path = os.path.join(args.output_dir, output_filename)
83+
with open(output_path, "w") as f:
84+
json.dump({"cuts": cuts}, f)
85+
except Exception as e:
86+
print(f"Failed to generate golden for {video_name} with {detector_name}: {e}")
87+
88+
89+
if __name__ == "__main__":
90+
main()

0 commit comments

Comments
 (0)