Test snapshots (rustnn#166)

## Summary

- [x] Describe the user-visible and code-level changes.

- Remove bug trtx cuda_host_register
- snapshot onnx and trtx test results

## Validation

- [x] `make test`
- [x] Relevant WPT or integration checks

## Documentation

- [ ] Updated docs if behavior changed
- [ ] If backend converter/executor operator support changed, ran `make
docs-backend-ops` and committed
`docs/development/backend-operator-support.md`

Author: theHamsta

.gitignore

@@ -50,3 +50,4 @@ env/
 *.swo
 *~
 .DS_Store
+*.snap.new

README.md

@@ -192,7 +192,9 @@ Contributions welcome! Please see:
 2. Install hooks (optional): `./scripts/install-git-hooks.sh`
 3. Make changes and test: `make test && make python-test`
 4. If Snapshots need to be updated (will be indicated by test failures),
-   run `cargo insta review` (see https://insta.rs/docs/cli/ to install)
+   run `cargo insta review` (see https://insta.rs/docs/cli/ to install) or
+   `INSTA_UPDATE=always cargo test` to update all snapshots automatically
+   (review before committing!)
 5. Format code: `make fmt`
 6. Commit and push
 

src/backends/trtx.rs

@@ -1,5 +1,5 @@
 use std::borrow::Cow;
-use std::collections::{HashMap, HashSet};
+use std::collections::HashMap;
 use std::ffi::c_void;
 use std::mem::MaybeUninit;
 use std::ops::Deref;
@@ -244,8 +244,6 @@ pub(crate) struct TrtxContext<'context> {
     cuda_ctx: Arc<CudaContext>,
     tensors: Vec<TrtxTensor>,
     events: Vec<CudaEvent>,
-    attempted_host_registrations: HashSet<usize>,
-    registered_host_pointers: HashMap<usize, usize>,
     runtime: Arc<Mutex<trtx::Runtime<'context>>>,
     config: Arc<Mutex<trtx::BuilderConfig<'context>>>, // needs to be destroyed before builder
     builder: Arc<Mutex<trtx::Builder<'context>>>,
@@ -283,34 +281,11 @@ impl<'context> TrtxContext<'context> {
             cuda_ctx,
             tensors: vec![],
             events: vec![],
-            attempted_host_registrations: HashSet::new(),
-            registered_host_pointers: HashMap::new(),
             runtime,
             builder: Arc::new(builder.into()),
             config,
         })
     }
-
-    fn try_register_host_memory(&mut self, pointer: *const u8, size: usize) {
-        let pointer_address = pointer as usize;
-        if !self.attempted_host_registrations.insert(pointer_address) {
-            return;
-        }
-
-        let registration_result = self.cuda_ctx.bind_to_thread().and_then(|()| unsafe {
-            sys::cuMemHostRegister_v2(pointer.cast_mut().cast(), size, 0).result()
-        });
-        match registration_result {
-            Ok(()) => {
-                self.registered_host_pointers.insert(pointer_address, size);
-            }
-            Err(error) => {
-                warn!(
-                    "Failed to register CPU tensor pointer {pointer:p} (size={size}) with CUDA: {error}"
-                );
-            }
-        }
-    }
 }
 
 #[allow(dead_code)]
@@ -560,7 +535,6 @@ impl<'context> MLBackendContext<'context> for TrtxContext<'context> {
         tensor: &crate::mlcontext::MLTensor,
         array: &mut [u8],
     ) -> crate::error::Result<()> {
-        self.try_register_host_memory(array.as_ptr(), array.len());
         let cuda_tensor = &self.tensors[tensor.id];
         let stream = &cuda_tensor.stream;
         debug!(
@@ -582,7 +556,6 @@ impl<'context> MLBackendContext<'context> for TrtxContext<'context> {
         tensor: &crate::mlcontext::MLTensor,
         array: &[u8],
     ) -> crate::error::Result<()> {
-        self.try_register_host_memory(array.as_ptr(), array.len());
         let cuda_tensor = &mut self.tensors[tensor.id];
         let stream = &cuda_tensor.stream;
         debug!(

tests/run_wpt_conformance.rs

@@ -4,7 +4,9 @@ use std::time::Instant;
 
 use libtest_mimic::{Arguments, Completion, Failed, Trial};
 use wpt_conformance::wpt_backend::WptBackend;
-use wpt_conformance::wpt_js_loader::{default_wpt_dir, load_wpt_corpus, trial_name};
+use wpt_conformance::wpt_js_loader::{
+    default_wpt_dir, load_wpt_corpus, sanitize_test_id, trial_name,
+};
 use wpt_conformance::wpt_report::{WptReportCollector, report_output_path};
 use wpt_conformance::wpt_types::WptLoadedCase;
 use wpt_conformance::{run_one_test_case, should_skip_test, wpt_types::WptTestCase};
@@ -37,14 +39,20 @@ fn push_backend_trials(
         let file_name = case.file_name.clone();
         let test_name = case.name.clone();
         let backend_prefix = prefix.to_string();
+        let snapshot_name = format!("{backend_prefix}_{}", sanitize_test_id(&test_name));
         let report = report.clone();
         let backend = backend.clone();
         trials.push(Trial::ignorable_test(name, move || {
             let started = Instant::now();
             let result = run_trial(&backend, &operation, &test_case);
             let duration = started.elapsed();
+
             match &result {
                 Ok(Completion::Completed) => {
+                    insta::assert_debug_snapshot!(
+                        snapshot_name,
+                        (&file_name, &test_name, &backend_prefix, "PASS")
+                    );
                     report.record_pass(&file_name, &test_name, &backend_prefix, duration);
                 }
                 Ok(Completion::Ignored { reason }) => {
@@ -53,6 +61,10 @@ fn push_backend_trials(
                 }
                 Err(err) => {
                     let msg = err.message().unwrap_or("test failed").to_string();
+                    insta::assert_snapshot!(
+                        snapshot_name,
+                        format!("{file_name} {test_name}, {backend_prefix}\n {msg}")
+                    );
                     report.record_fail(&file_name, &test_name, &backend_prefix, msg, duration);
                 }
             }

tests/snapshots/run_wpt_conformance__onnx_abs_float16_1D_constant_tensor.snap

@@ -0,0 +1,10 @@
+---
+source: tests/run_wpt_conformance.rs
+expression: "(&file_name, &test_name, &backend_prefix, \"PASS\")"
+---
+(
+    "abs.https.any.js",
+    "abs float16 1D constant tensor",
+    "onnx",
+    "PASS",
+)

tests/snapshots/run_wpt_conformance__onnx_abs_float16_1D_tensor.snap

@@ -0,0 +1,10 @@
+---
+source: tests/run_wpt_conformance.rs
+expression: "(&file_name, &test_name, &backend_prefix, \"PASS\")"
+---
+(
+    "abs.https.any.js",
+    "abs float16 1D tensor",
+    "onnx",
+    "PASS",
+)

tests/snapshots/run_wpt_conformance__onnx_abs_float16_2D_tensor.snap

@@ -0,0 +1,10 @@
+---
+source: tests/run_wpt_conformance.rs
+expression: "(&file_name, &test_name, &backend_prefix, \"PASS\")"
+---
+(
+    "abs.https.any.js",
+    "abs float16 2D tensor",
+    "onnx",
+    "PASS",
+)

tests/snapshots/run_wpt_conformance__onnx_abs_float16_3D_tensor.snap

@@ -0,0 +1,10 @@
+---
+source: tests/run_wpt_conformance.rs
+expression: "(&file_name, &test_name, &backend_prefix, \"PASS\")"
+---
+(
+    "abs.https.any.js",
+    "abs float16 3D tensor",
+    "onnx",
+    "PASS",
+)

tests/snapshots/run_wpt_conformance__onnx_abs_float16_4D_tensor.snap

@@ -0,0 +1,10 @@
+---
+source: tests/run_wpt_conformance.rs
+expression: "(&file_name, &test_name, &backend_prefix, \"PASS\")"
+---
+(
+    "abs.https.any.js",
+    "abs float16 4D tensor",
+    "onnx",
+    "PASS",
+)

tests/snapshots/run_wpt_conformance__onnx_abs_float16_5D_tensor.snap

@@ -0,0 +1,10 @@
+---
+source: tests/run_wpt_conformance.rs
+expression: "(&file_name, &test_name, &backend_prefix, \"PASS\")"
+---
+(
+    "abs.https.any.js",
+    "abs float16 5D tensor",
+    "onnx",
+    "PASS",
+)