Refactor

Author: kripken

test/unit/test_fuzz_preserve.py

@@ -8,84 +8,108 @@
 from . import utils
 
 
-class PreserveFuzzTest(utils.BinaryenTestCase):
-    def test_against_js(self):
-        # When --fuzz-against-js is used, the wasm is only going to be fuzzed
-        # against JS, so the fuzzer mutates the boundary in valid ways, even if
-        # --fuzz-preserve-imports-exports is set.
-        #
-        # Testing this deterministically is too hard (as the fuzzer evolves, it
-        # will handle random data differently, and the test would constantly get
-        # out of date). Instead, test randomly, but in a way that the chance of
-        # a flake is unrealistic.
-        max_size = 1024
+# Runs the fuzzer many times and allows checking for specific variety in the
+# output. Calls hooks:
+#
+#  self.found_variety()  - checks if we found what we are looking for
+#  self.process_wat(wat) - receives the current fuzz wat
+#
+class FuzzerVarietyTester:
+    # Run until we find what we want. Stop only if we reached a max number
+    # of iterations and a timeout.
+    max_time = 60
+    min_iters = 200
+
+    # The maximum size of the wasm-generating input
+    max_size = 1024
+
+    def __init__(self, initial):
+        self.initial = initial
+
+    def test(self):
         temp_dat = tempfile.NamedTemporaryFile(suffix='.dat')
-        initial = self.input_path('fuzz.wat')
-
-        # The set of all params we see, for the import that is refinable. Ditto
-        # for export results.
-        import_params = set()
-        export_results = set()
 
-        # Run until we find what we want. Stop only if we reached a max number
-        # of iterations and a timeout.
-        min_iters = 200
         start_time = time.time()
-        # Locally this succeeds in less than 1 second. Give it a very wide
-        # margin of error to avoid flakes.
-        max_time = start_time + 60
+        stop_time = start_time + self.max_time
 
         i = 0
         while True:
             i += 1
 
-            if self.found_expected(import_params) and self.found_expected(export_results):
+            # Stop early if we found what we are looking for.
+            if self.found_variety():
                 print(f"{i} iterations {round(time.time() - start_time, 2)} seconds)")
-                print(f'proper import_params : {import_params}')
-                print(f'proper export_results: {export_results}')
+                print(f'proper import_params : {self.import_params}')
+                print(f'proper export_results: {self.export_results}')
                 return
 
-            if i > min_iters and time.time() > max_time:
+            if i > self.min_iters and time.time() > stop_time:
                 raise Exception('looked too long and still failed')
 
             # Generate raw random data
-            size = random.randint(1, max_size)
+            size = random.randint(1, self.max_size)
             with open(temp_dat.name, 'wb') as f:
                 f.write(bytes([random.randint(0, 255) for x in range(size)]))
 
             # Generate the fuzz testcase from the random data + the initial
             # contents.
-            args = ['-ttf', temp_dat.name, '--initial-fuzz=' + initial, '-all']
-            args += ['--fuzz-preserve-imports-exports', '--fuzz-against-js']
+            args = ['-ttf', temp_dat.name, '--initial-fuzz=' + self.initial, '-all']
+            args += self.ttf_args
             args += ['--print']
             wat = shared.run_process(shared.WASM_OPT + args,
                                    stdout=subprocess.PIPE).stdout
 
-            # The things that begin reffed might end up not reffed, if mutation
-            # removes the refs. Check for that.
-            import_reffed_is_reffed = '(ref.func $import-reffed)' in wat
-            export_reffed_is_reffed = '(ref.func $export-reffed)' in wat
-
-            # Find the params/results that might be refined.
-            for line in wat.splitlines():
-                if line.startswith(' (import "module" "base" (func $import '):
-                    params, results = self.parse_params_results(line)
-                    import_params.add(params)
-                    assert results == '(result eqref)', 'cannot refine import result'
-                elif line.startswith(' (import "module" "base" (func $import-reffed '):
-                    params, results = self.parse_params_results(line)
-                    if import_reffed_is_reffed:
-                        assert params == '(param i32 anyref)', 'cannot refine reffed stuff'
-                    assert results == '(result eqref)', 'cannot refine import result'
-                if line.startswith(' (func $export '):
-                    params, results = self.parse_params_results(line)
-                    assert params == '(param $0 i32) (param $1 anyref)', 'cannot refine export params'
-                    export_results.add(results)
-                if line.startswith(' (func $export-reffed '):
-                    params, results = self.parse_params_results(line)
-                    assert params == '(param $0 i32) (param $1 anyref)', 'cannot refine export params'
-                    if export_reffed_is_reffed:
-                        assert results == '(result eqref)', 'cannot refine reffed stuff'
+            self.process_wat(wat)
+
+
+class FuzzAgainstJSVarietyTester(FuzzerVarietyTester):
+    # When --fuzz-against-js is used, the wasm is only going to be fuzzed
+    # against JS, so the fuzzer mutates the boundary in valid ways, even if
+    # --fuzz-preserve-imports-exports is set.
+    #
+    # Testing this deterministically is too hard (as the fuzzer evolves, it
+    # will handle random data differently, and the test would constantly get
+    # out of date). Instead, test randomly, but in a way that the chance of
+    # a flake is unrealistic.
+    ttf_args = ['--fuzz-preserve-imports-exports', '--fuzz-against-js']
+
+    def __init__(self, initial):
+        super().__init__(initial)
+
+        # The set of all params we see, for the import that is refinable. Ditto
+        # for export results.
+        self.import_params = set()
+        self.export_results = set()
+
+    def found_variety(self):
+        return self.found_expected(self.import_params) and self.found_expected(self.export_results)
+
+    def process_wat(self, wat):
+        # The things that begin reffed might end up not reffed, if mutation
+        # removes the refs. Check for that.
+        import_reffed_is_reffed = '(ref.func $import-reffed)' in wat
+        export_reffed_is_reffed = '(ref.func $export-reffed)' in wat
+
+        # Find the params/results that might be refined.
+        for line in wat.splitlines():
+            if line.startswith(' (import "module" "base" (func $import '):
+                params, results = self.parse_params_results(line)
+                self.import_params.add(params)
+                assert results == '(result eqref)', 'cannot refine import result'
+            elif line.startswith(' (import "module" "base" (func $import-reffed '):
+                params, results = self.parse_params_results(line)
+                if import_reffed_is_reffed:
+                    assert params == '(param i32 anyref)', 'cannot refine reffed stuff'
+                assert results == '(result eqref)', 'cannot refine import result'
+            if line.startswith(' (func $export '):
+                params, results = self.parse_params_results(line)
+                assert params == '(param $0 i32) (param $1 anyref)', 'cannot refine export params'
+                self.export_results.add(results)
+            if line.startswith(' (func $export-reffed '):
+                params, results = self.parse_params_results(line)
+                assert params == '(param $0 i32) (param $1 anyref)', 'cannot refine export params'
+                if export_reffed_is_reffed:
+                    assert results == '(result eqref)', 'cannot refine reffed stuff'
 
     # Given the types we saw for params or results, look in detail for the
     # things we expect to see.
@@ -161,3 +185,9 @@ def get(what, line):
             return ret
 
         return get('(param', line), get('(result', line)
+
+
+class PreserveFuzzTest(utils.BinaryenTestCase):
+    def test_against_js(self):
+        FuzzAgainstJSVarietyTester(self.input_path('fuzz.wat')).test()
+