Convert our use of SQLite to use RETURNING clauses

spinn_front_end_common/interface/buffer_management/storage_objects/buffer_database.py

@@ -17,6 +17,7 @@
 from spinnman.spalloc.spalloc_job import SpallocJob
 from spinn_front_end_common.data import FecDataView
 from spinn_front_end_common.utilities.base_database import BaseDatabase
+from spinn_front_end_common.utilities.exceptions import DatabaseException
 
 _SECONDS_TO_MICRO_SECONDS_CONVERSION = 1000
 #: Name of the database in the data folder
@@ -140,13 +141,15 @@ def _get_region_id(self, x, y, p, region):
                 """, (x, y, p, region)):
             return row["region_id"]
         core_id = self._get_core_id(x, y, p)
-        self.execute(
-            """
-            INSERT INTO region(
-                core_id, local_region_index, content, content_len, fetches)
-            VALUES(?, ?, CAST('' AS BLOB), 0, 0)
-            """, (core_id, region))
-        return self.lastrowid
+        for row in self.execute(
+                """
+                INSERT INTO region(
+                    core_id, local_region_index, content, content_len, fetches)
+                VALUES(?, ?, CAST('' AS BLOB), 0, 0)
+                RETURNING region_id
+                """, (core_id, region)):
+            return row["region_id"]
+        raise DatabaseException("database insert failed (region)")
 
     def store_data_in_region_buffer(self, x, y, p, region, missing, data):
         """

spinn_front_end_common/interface/ds/data_specification_generator.py

@@ -39,7 +39,7 @@ def __init__(self, x, y, p, vertex, ds_db, report_writer=None):
             Determines if a text version of the specification is to be
             written and, if so, where. No report is written if this is `None`.
         :type report_writer: ~io.TextIOBase or None
-        :raises DsDatabaseException: If this core is not known
+        :raises DatabaseException: If this core is not known
             and no vertex supplied (during reload)
         :raises AttributeError:
             If the vertex is not an AbstractHasAssociatedBinary

spinn_front_end_common/interface/ds/ds_sqllite_database.py

@@ -22,7 +22,7 @@
 from spinn_front_end_common.data import FecDataView
 from spinn_front_end_common.utilities.constants import (
     APP_PTR_TABLE_BYTE_SIZE)
-from spinn_front_end_common.utilities.exceptions import DsDatabaseException
+from spinn_front_end_common.utilities.exceptions import DatabaseException
 from spinn_front_end_common.utilities.sqlite_db import SQLiteDB
 
 _DDL_FILE = os.path.join(os.path.dirname(__file__), "dse.sql")
@@ -162,15 +162,18 @@ def set_memory_region(
         :param reference: A globally unique reference for this region
         :type reference: int or None
         :param label:
-        :return:
+        :return: The row ID (usually ignored)
+        :rtype: int
         """
-        self.execute(
-            """
-            INSERT INTO region(
-                x, y, p, region_num, size, reference_num, region_label)
-            VALUES(?, ?, ?, ?, ?, ?, ?)
-            """, (x, y, p, region_num, size, reference, label))
-        return self.lastrowid
+        for row in self.execute(
+                """
+                INSERT INTO region(
+                    x, y, p, region_num, size, reference_num, region_label)
+                VALUES(?, ?, ?, ?, ?, ?, ?)
+                RETURNING rowid AS row_num
+                """, (x, y, p, region_num, size, reference, label)):
+            return row["row_num"]
+        raise DatabaseException("database insert failed (region)")
 
     def get_region_size(self, x, y, p, region_num):
         """
@@ -191,11 +194,11 @@ def get_region_size(self, x, y, p, region_num):
                 LIMIT 1
                 """, (x, y, p, region_num)):
             return row["size"]
-        raise DsDatabaseException(f"Region {region_num} not set")
+        raise DatabaseException(f"Region {region_num} not set")
 
     def set_reference(self, x, y, p, region_num, reference, ref_label):
         """
-        Writes a outgoing region_reference into the database
+        Writes a outgoing region_reference into the database.
 
         :param int x: X coordinate of the core
         :param int y: Y coordinate of the core
@@ -215,7 +218,7 @@ def set_reference(self, x, y, p, region_num, reference, ref_label):
 
     def get_reference_pointers(self, x, y, p):
         """
-        Yields the reference regions and where they point for this core
+        Yields the reference regions and where they point for this core.
 
         This may yield nothing if there are no reference pointers or
         if the core is not known
@@ -239,7 +242,7 @@ def get_reference_pointers(self, x, y, p):
 
     def get_unlinked_references(self):
         """
-        Finds and yields info on unreferenced links
+        Finds and yields info on unreferenced links.
 
         If all is well this method yields nothing!
 
@@ -251,20 +254,20 @@ def get_unlinked_references(self):
         """
         for row in self.execute(
                 """
-                SELECT  x, y, ref_p, ref_region, reference_num,
-                    COALESCE(ref_label, "") as ref_label
+                SELECT x, y, ref_p, ref_region, reference_num,
+                    COALESCE(ref_label, "") AS ref_label
                 FROM linked_reference_view
                 WHERE act_region IS NULL
-                 """):
+                """):
             yield (row["x"], row["y"], row["ref_p"], row["ref_region"],
                    row["reference_num"], str(row["ref_label"], "utf8"))
 
     def get_double_region(self):
         """
         Finds and yields any region that was used in both region definition
-            and a reference
+        and a reference.
 
-         If all is well this method yields nothing!
+        If all is well this method yields nothing!
 
         .. note::
             Do not use the database for anything else while iterating.
@@ -292,29 +295,29 @@ def set_region_content(self, x, y, p, region_num, content, content_debug):
         :param bytearray content: content to write
         :param content_debug: debug text
         :type content_debug: str or None
-        :raises DsDatabaseException: If the region already has content
+        :raises DatabaseException: If the region already has content
         """
         # check for previous content
         for row in self.execute(
                 """
                 SELECT content
                 FROM region
-                WHERE x = ? AND y = ? and p = ? and region_num = ?
+                WHERE x = ? AND y = ? AND p = ? AND region_num = ?
                 LIMIT 1
-                 """, (x, y, p, region_num)):
+                """, (x, y, p, region_num)):
             if row["content"]:
-                raise DsDatabaseException(
+                raise DatabaseException(
                     f"Illegal attempt to overwrite content for "
                     f"{x=} {y=} {p=} {region_num=}")
 
         self.execute(
             """
             UPDATE region
             SET content = ?, content_debug = ?
-            WHERE x = ? AND y = ? and p = ? and region_num = ?
+            WHERE x = ? AND y = ? AND p = ? AND region_num = ?
             """, (content, content_debug, x, y, p, region_num))
         if self.rowcount == 0:
-            raise DsDatabaseException(
+            raise DatabaseException(
                 f"No region {x=} {y=} {p=} {region_num=}")
 
     def get_region_pointer(self, x, y, p, region_num):
@@ -330,7 +333,7 @@ def get_region_pointer(self, x, y, p, region_num):
         :param int region_num: The DS region number
         :return: The pointer set during the original load
         :rtype: int or None
-        :raises DsDatabaseException: if the region is not known
+        :raises DatabaseException: if the region is not known
         """
         for row in self.execute(
                 """
@@ -340,7 +343,7 @@ def get_region_pointer(self, x, y, p, region_num):
                 LIMIT 1
                 """, (x, y, p, region_num)):
             return row["pointer"]
-        raise DsDatabaseException(f"No region {x=} {y=} {p=} {region_num=}")
+        raise DatabaseException(f"No region {x=} {y=} {p=} {region_num=}")
 
     def get_region_sizes(self, x, y, p):
         """
@@ -382,14 +385,14 @@ def get_total_regions_size(self, x, y, p):
         :rtype: int
         """
         for row in self.execute(
-            """
-                SELECT COALESCE(sum(size), 0) as total
+                """
+                SELECT COALESCE(sum(size), 0) AS total
                 FROM region
                 WHERE x = ? AND y = ? AND p = ?
                 LIMIT 1
                 """, (x, y, p)):
             return row["total"]
-        raise DsDatabaseException("Query failed unexpectedly")
+        raise DatabaseException("Query failed unexpectedly")
 
     def set_start_address(self, x, y, p, start_address):
         """
@@ -401,7 +404,7 @@ def set_start_address(self, x, y, p, start_address):
         :param int start_address: The base address for the whole core
         :return: The expected size of the malloced_area
         :rtype: int
-        :raises DsDatabaseException: if the region is not known
+        :raises DatabaseException: if the region is not known
         """
         self.execute(
             """
@@ -410,8 +413,7 @@ def set_start_address(self, x, y, p, start_address):
             WHERE x = ? AND y = ? AND p = ?
             """, (start_address, x, y, p))
         if self.rowcount == 0:
-            raise DsDatabaseException(
-                f"No core {x=} {y=} {p=}")
+            raise DatabaseException(f"No core {x=} {y=} {p=}")
 
     def get_start_address(self, x, y, p):
         """
@@ -427,21 +429,21 @@ def get_start_address(self, x, y, p):
                 """
                 SELECT start_address
                 FROM core
-                WHERE x = ? AND y = ? and p = ?
+                WHERE x = ? AND y = ? AND p = ?
                 LIMIT 1
                 """, (x, y, p)):
             return row["start_address"]
-        raise DsDatabaseException(f"No core {x=} {y=} {p=}")
+        raise DatabaseException(f"No core {x=} {y=} {p=}")
 
     def set_region_pointer(self, x, y, p, region_num, pointer):
         self.execute(
             """
             UPDATE region
             SET pointer = ?
-            WHERE x = ? AND y = ? and p = ? and region_num = ?
+            WHERE x = ? AND y = ? AND p = ? AND region_num = ?
             """, (pointer, x, y, p, region_num))
         if self.rowcount == 0:
-            raise DsDatabaseException(
+            raise DatabaseException(
                 f"No region {x=} {y=} {p=} {region_num=}")
 
     def get_region_pointers_and_content(self, x, y, p):
@@ -453,6 +455,9 @@ def get_region_pointers_and_content(self, x, y, p):
         Will yield nothing if there are no regions reserved or if the core if
         not known
 
+        .. note::
+            Do not use the database for anything else while iterating.
+
         :param int x: X coordinate of the core
         :param int y: Y coordinate of the core
         :param int p: Processor ID of the core
@@ -488,7 +493,8 @@ def get_ds_cores(self):
         """
         for row in self.execute(
                 """
-                SELECT x, y, p FROM core
+                SELECT x, y, p
+                FROM core
                 """):
             yield (row["x"], row["y"], row["p"])
 
@@ -499,15 +505,16 @@ def get_n_ds_cores(self):
         Includes cores where DataSpecs started even if no regions reserved
 
         :rtype: int
-        :raises DsDatabaseException:
+        :raises DatabaseException:
         """
         for row in self.execute(
                 """
-                SELECT COUNT(*) as count FROM core
+                SELECT COUNT(*) AS count
+                FROM core
                 LIMIT 1
                 """):
             return row["count"]
-        raise DsDatabaseException("Count query failed")
+        raise DatabaseException("Count query failed")
 
     def get_memory_to_malloc(self, x, y, p):
         """
@@ -571,7 +578,7 @@ def get_info_for_cores(self):
         """
         for row in self.execute(
                 """
-                SELECT x, y, p, start_address, to_write,malloc_size
+                SELECT x, y, p, start_address, to_write, malloc_size
                 FROM core_summary_view
                 """):
             yield ((row["x"], row["y"], row["p"]), row["start_address"],
@@ -599,13 +606,14 @@ def write_session_credentials_to_db(self):
     def set_app_id(self):
         """
         Sets the app id
-
         """
         # check for previous content
-        self.execute(
-            """
-            INSERT INTO app_id(app_id)
-            VALUES(?)
-            """, (FecDataView.get_app_id(), ))
-        if self.rowcount == 0:
-            raise DsDatabaseException("Unable to set app id")
+        for _row in self.execute(
+                """
+                INSERT INTO app_id(app_id)
+                VALUES(?)
+                RETURNING *
+                """, (FecDataView.get_app_id(), )):
+            break
+        else:
+            raise DatabaseException("Unable to set app id")

spinn_front_end_common/interface/provenance/global_provenance.py

@@ -21,6 +21,7 @@
 from spinn_front_end_common.utilities.constants import (
     MICRO_TO_MILLISECOND_CONVERSION)
 from spinn_front_end_common.utilities.sqlite_db import SQLiteDB
+from spinn_front_end_common.utilities.exceptions import DatabaseException
 
 logger = FormatAdapter(logging.getLogger(__name__))
 
@@ -102,16 +103,19 @@ def insert_category(self, category, machine_on):
         :param bool machine_on: If the machine was done during all
             or some of the time
         """
-        self.execute(
-            """
-            INSERT INTO category_timer_provenance(
-                category, machine_on, n_run, n_loop)
-            VALUES(?, ?, ?, ?)
-            """,
-            [category.category_name, machine_on,
-             FecDataView.get_run_number(),
-             FecDataView.get_run_step()])
-        return self.lastrowid
+        for row in self.execute(
+                """
+                INSERT INTO category_timer_provenance(
+                    category, machine_on, n_run, n_loop)
+                VALUES(?, ?, ?, ?)
+                RETURNING category_id
+                """,
+                [category.category_name, machine_on,
+                 FecDataView.get_run_number(),
+                 FecDataView.get_run_step()]):
+            return row[0]
+        raise DatabaseException(
+            "database insert failed (category_timer_provenance)")
 
     def insert_category_timing(self, category_id, timedelta):
         """