diff --git a/qubes/tests/integ/dispvm.py b/qubes/tests/integ/dispvm.py
index a3613b51c..e718626a7 100644
--- a/qubes/tests/integ/dispvm.py
+++ b/qubes/tests/integ/dispvm.py
@@ -410,6 +410,8 @@ async def wait_preload(
         preload_unfinished = preload_dispvm
         for _ in range(timeout):
             for qube in preload_unfinished.copy():
+                if qube not in self.app.domains:
+                    break
                 if self.app.domains[qube].preload_complete.is_set():
                     logger.info("preload completed for '%s'", qube)
                     preload_unfinished.remove(qube)
@@ -1104,6 +1106,53 @@ def mock_open_mem_threshold(file, *args, **kwargs):
             )
             self.assertEqual(1, len(self.disp_base.get_feat_preload()))
 
+    def test_012_preload_low_mem_early_startup(self):
+        """Test preloading with low memory on early startup"""
+        self.loop.run_until_complete(
+            self._test_012_preload_low_mem_early_startup()
+        )
+
+    async def _test_012_preload_low_mem_early_startup(self):
+        # pylint: disable=unspecified-encoding
+        logger.info("start")
+        unpatched_open = open
+
+        def mock_open_mem_raise(file, *args, **kwargs):
+            if file == qubes.config.qmemman_avail_mem_file:
+                raise FileNotFoundError(2, "No such file or directory", file)
+            return unpatched_open(file, *args, **kwargs)
+
+        preload_max = 2
+        with patch("builtins.open", side_effect=mock_open_mem_raise):
+            logger.info("insufficient memory reserve (early startup failure)")
+            old_memory = self.disp_base.memory
+            old_dispvms = [
+                qube.name
+                for qube in self.app.domains
+                if getattr(qube, "is_preload", False)
+            ]
+            self.disp_base.memory = 999999999999999999999
+            self.disp_base.features["preload-dispvm-max"] = str(preload_max)
+            await self.wait_preload(
+                preload_max, fail_on_timeout=False, timeout=15
+            )
+            self.assertEqual(0, len(self.disp_base.get_feat_preload()))
+            new_dispvms = [
+                qube.name
+                for qube in self.app.domains
+                if getattr(qube, "is_preload", False)
+            ]
+            self.assertEqual(new_dispvms, old_dispvms)
+
+        # Nothing will be done here, just to prepare to the next test.
+        self.disp_base.features["preload-dispvm-max"] = "0"
+        with patch("builtins.open", side_effect=mock_open_mem_raise):
+            logger.info("enough memory reserve but no avail-mem file")
+            self.disp_base.memory = old_memory
+            self.disp_base.features["preload-dispvm-max"] = str(preload_max)
+            await self.wait_preload(preload_max)
+            self.assertEqual(2, len(self.disp_base.get_feat_preload()))
+
         logger.info("end")
 
     def test_015_preload_race_more(self):
diff --git a/qubes/vm/dispvm.py b/qubes/vm/dispvm.py
index b51206054..e61fb5bce 100644
--- a/qubes/vm/dispvm.py
+++ b/qubes/vm/dispvm.py
@@ -1011,7 +1011,7 @@ async def start(self, **kwargs):
             await super().start(**kwargs)
         except:
             # Cleanup also on failed startup
-            await self.cleanup()
+            await self.cleanup(force=True)
             raise
 
     def create_qdb_entries(self) -> None:
diff --git a/qubes/vm/mix/dvmtemplate.py b/qubes/vm/mix/dvmtemplate.py
index e5abfdc55..1b8795764 100644
--- a/qubes/vm/mix/dvmtemplate.py
+++ b/qubes/vm/mix/dvmtemplate.py
@@ -516,7 +516,6 @@ async def on_domain_preload_dispvm_used(
             return
 
         avail_mem_file = qubes.config.qmemman_avail_mem_file
-        available_memory = None
         try:
             with open(avail_mem_file, "r", encoding="ascii") as file:
                 available_memory = max(
@@ -525,7 +524,7 @@ async def on_domain_preload_dispvm_used(
         except FileNotFoundError:
             can_preload = want_preload
             self.log.warning("File containing available memory was not found")
-        if available_memory is not None:
+        else:
             memory = getattr(self, "memory", 0) * 1024**2
             unrestricted_preload = int(available_memory / memory)
             can_preload = min(unrestricted_preload, want_preload)