Add automatic output tests.

Author: SiegeLordEx

.github/workflows/continuous-integration.yml

@@ -16,13 +16,38 @@ jobs:
                    rustup component add rustfmt --toolchain nightly-x86_64-unknown-linux-gnu
             - name: Setup
               run: |
-                  sudo apt-get install python3-pip --force-yes
-                  sudo pip3 install toml
+                  curl -LsSf https://astral.sh/uv/install.sh | sh
+                  . setup_venv.sh
                   sudo apt-get install gnuplot
             - name: Tests
               run: |
+                  . venv/bin/activate
                   ./cargo_util.py --test
                   ./gnuplot/target/debug/examples/example1 --no-show
                   ./gnuplot/target/debug/examples/example2 --no-show
                   ./gnuplot/target/debug/examples/example3 --no-show
                   ./gnuplot/target/debug/examples/example4 --no-show
+
+                  if [ -n "${{ github.base_ref }}" ]; then
+                    CHANGED_OUTPUTS=$(echo "${{ github.event.pull_request.body }}" | sed -n 's/.*CHANGED_OUTPUTS=\([^ ]*\).*/\1/p')
+                    BASE_BRANCH="${{ github.base_ref }}"
+                    HEAD_BRANCH="${{ github.head_ref || github.ref_name }}"
+
+                    git fetch origin $BASE_BRANCH
+                    git checkout $BASE_BRANCH
+                    ./cargo_util.py --make_golden_outputs
+                    git checkout $HEAD_BRANCH
+                    ./cargo_util.py --test_outputs --ignore_new_outputs --changed_outputs $CHANGED_OUTPUTS
+                  fi
+            - name: Upload golden outputs
+              if: ${{ always() }}
+              uses: actions/upload-artifact@v4
+              with:
+                  name: golden_outputs
+                  path: golden_outputs
+            - name: Upload test outputs
+              if: ${{ always() }}
+              uses: actions/upload-artifact@v4
+              with:
+                  name: test_outputs
+                  path: test_outputs

cargo_util.py

@@ -25,7 +25,10 @@ def split(s):
 parser.add_argument('--publish', action='store_true', help='publish the crates')
 parser.add_argument('--build', action='store_true', help='build the crates')
 parser.add_argument('--test', action='store_true', help='test the crates')
-parser.add_argument('--test_outputs', action='store_true', help='run the unittests')
+parser.add_argument('--make_golden_outputs', action='store_true', help='make the golden outputs')
+parser.add_argument('--test_outputs', action='store_true', help='run the output tests')
+parser.add_argument('--ignore_new_outputs', action='store_true', help='whether to ignore new outputs')
+parser.add_argument('--changed_outputs', type=str, default='', help='comma separated list of outputs to ignore failures in, e.g. "foo.png,bar.png"')
 parser.add_argument('--clean', action='store_true', help='clean the crates')
 parser.add_argument('--doc', action='store_true', help='build the documentation')
 parser.add_argument('--format', action='store_true', help='format all the non-sys crates')
@@ -95,12 +98,16 @@ def cargo_cmd(*command):
 		check_call(cargo_cmd('test'), cwd=crate)
 		check_call(cargo_cmd('fmt', '--check'), cwd=crate)
 
-if args.test_outputs:
+if args.test_outputs or args.make_golden_outputs:
 	import numpy as np
 	from PIL import Image
 
-	os.makedirs('test_outputs', exist_ok=True)
-	output_dir = os.path.abspath('test_outputs')
+	if args.test_outputs:
+		output_dir = 'test_outputs'
+	else:
+		output_dir = 'golden_outputs'
+	os.makedirs(output_dir, exist_ok=True)
+	output_dir = os.path.abspath(output_dir)
 	metadata = json.loads(check_output(cargo_cmd('metadata', '--format-version=1', '--no-deps'), cwd='gnuplot').decode('utf8'))
 	for target in metadata['packages'][0]['targets']:
 		if target['kind'] != ['example']:
@@ -115,6 +122,9 @@ def cargo_cmd(*command):
 
 		check_call(cargo_cmd('run', '--example', target['name'], '--', '--no-show', '--output-dir', output_dir, '--save-png'), cwd='gnuplot')
 
+	if args.make_golden_outputs:
+		exit(0)
+
 	golden_images = [pathlib.Path(f) for f in glob.glob('golden_outputs/*.png')]
 	test_images = [pathlib.Path(f) for f in glob.glob(f'{output_dir}/*.png')]
 
@@ -123,19 +133,26 @@ def cargo_cmd(*command):
 	if golden_filenames != test_filenames:
 		missing = set(golden_filenames) - set(test_filenames)
 		extra = set(test_filenames) - set(golden_filenames)
-		assert False, f"Test images don't match golden images.\nExtra: {extra}\nMissing: {missing}"
+		if not args.ignore_new_outputs or missing:
+			assert False, f"Test images don't match golden images.\nExtra: {extra}\nMissing: {missing}"
 
+	changed_outputs = args.changed_outputs.split(',')
+	failed = False
 	for image_name in golden_images:
 		golden_image_path = pathlib.Path(image_name)
 		test_image_path = pathlib.Path(output_dir) / golden_image_path.name
 		assert test_image_path.exists(), f"{test_image_path} not found"
-
+		if golden_image_path.name in changed_outputs:
+			continue
 		golden_image = np.array(Image.open(golden_image_path)).astype(np.float32)
 		test_image = np.array(Image.open(test_image_path)).astype(np.float32)
 		try:
 			np.testing.assert_allclose(golden_image, test_image, atol=5, err_msg=f"{golden_image_path.resolve()}\n{test_image_path.resolve()}")
 		except AssertionError as e:
+			failed = True
 			print(e)
+	if failed:
+		exit(1)
 
 
 if args.clean:

gnuplot/src/axes2d.rs

@@ -662,11 +662,7 @@ impl Axes2D
 	{
 		let (data, num_rows, num_cols) = generate_data!(options, x, y);
 		self.common.elems.push(PlotElement::new_plot(
-			Polygons,
-			data,
-			num_rows,
-			num_cols,
-			options,
+			Polygons, data, num_rows, num_cols, options,
 		));
 		self
 	}