diff --git a/spinn_front_end_common/interface/buffer_management/storage_objects/buffer_database.py b/spinn_front_end_common/interface/buffer_management/storage_objects/buffer_database.py index e3456d9593..be85f5a28a 100644 --- a/spinn_front_end_common/interface/buffer_management/storage_objects/buffer_database.py +++ b/spinn_front_end_common/interface/buffer_management/storage_objects/buffer_database.py @@ -17,6 +17,7 @@ from spinnman.spalloc.spalloc_job import SpallocJob from spinn_front_end_common.data import FecDataView from spinn_front_end_common.utilities.base_database import BaseDatabase +from spinn_front_end_common.utilities.exceptions import DatabaseException _SECONDS_TO_MICRO_SECONDS_CONVERSION = 1000 #: Name of the database in the data folder @@ -140,13 +141,15 @@ def _get_region_id(self, x, y, p, region): """, (x, y, p, region)): return row["region_id"] core_id = self._get_core_id(x, y, p) - self.execute( - """ - INSERT INTO region( - core_id, local_region_index, content, content_len, fetches) - VALUES(?, ?, CAST('' AS BLOB), 0, 0) - """, (core_id, region)) - return self.lastrowid + for row in self.execute( + """ + INSERT INTO region( + core_id, local_region_index, content, content_len, fetches) + VALUES(?, ?, CAST('' AS BLOB), 0, 0) + RETURNING region_id + """, (core_id, region)): + return row["region_id"] + raise DatabaseException("database insert failed (region)") def store_data_in_region_buffer(self, x, y, p, region, missing, data): """ diff --git a/spinn_front_end_common/interface/ds/data_specification_generator.py b/spinn_front_end_common/interface/ds/data_specification_generator.py index 205360d5b1..d783aa4ba6 100644 --- a/spinn_front_end_common/interface/ds/data_specification_generator.py +++ b/spinn_front_end_common/interface/ds/data_specification_generator.py @@ -39,7 +39,7 @@ def __init__(self, x, y, p, vertex, ds_db, report_writer=None): Determines if a text version of the specification is to be written and, if so, where. No report is written if this is `None`. :type report_writer: ~io.TextIOBase or None - :raises DsDatabaseException: If this core is not known + :raises DatabaseException: If this core is not known and no vertex supplied (during reload) :raises AttributeError: If the vertex is not an AbstractHasAssociatedBinary diff --git a/spinn_front_end_common/interface/ds/ds_sqllite_database.py b/spinn_front_end_common/interface/ds/ds_sqllite_database.py index 0b6c99b093..0484ba327f 100644 --- a/spinn_front_end_common/interface/ds/ds_sqllite_database.py +++ b/spinn_front_end_common/interface/ds/ds_sqllite_database.py @@ -22,7 +22,7 @@ from spinn_front_end_common.data import FecDataView from spinn_front_end_common.utilities.constants import ( APP_PTR_TABLE_BYTE_SIZE) -from spinn_front_end_common.utilities.exceptions import DsDatabaseException +from spinn_front_end_common.utilities.exceptions import DatabaseException from spinn_front_end_common.utilities.sqlite_db import SQLiteDB _DDL_FILE = os.path.join(os.path.dirname(__file__), "dse.sql") @@ -162,15 +162,18 @@ def set_memory_region( :param reference: A globally unique reference for this region :type reference: int or None :param label: - :return: + :return: The row ID (usually ignored) + :rtype: int """ - self.execute( - """ - INSERT INTO region( - x, y, p, region_num, size, reference_num, region_label) - VALUES(?, ?, ?, ?, ?, ?, ?) - """, (x, y, p, region_num, size, reference, label)) - return self.lastrowid + for row in self.execute( + """ + INSERT INTO region( + x, y, p, region_num, size, reference_num, region_label) + VALUES(?, ?, ?, ?, ?, ?, ?) + RETURNING rowid AS row_num + """, (x, y, p, region_num, size, reference, label)): + return row["row_num"] + raise DatabaseException("database insert failed (region)") def get_region_size(self, x, y, p, region_num): """ @@ -191,11 +194,11 @@ def get_region_size(self, x, y, p, region_num): LIMIT 1 """, (x, y, p, region_num)): return row["size"] - raise DsDatabaseException(f"Region {region_num} not set") + raise DatabaseException(f"Region {region_num} not set") def set_reference(self, x, y, p, region_num, reference, ref_label): """ - Writes a outgoing region_reference into the database + Writes a outgoing region_reference into the database. :param int x: X coordinate of the core :param int y: Y coordinate of the core @@ -215,7 +218,7 @@ def set_reference(self, x, y, p, region_num, reference, ref_label): def get_reference_pointers(self, x, y, p): """ - Yields the reference regions and where they point for this core + Yields the reference regions and where they point for this core. This may yield nothing if there are no reference pointers or if the core is not known @@ -239,7 +242,7 @@ def get_reference_pointers(self, x, y, p): def get_unlinked_references(self): """ - Finds and yields info on unreferenced links + Finds and yields info on unreferenced links. If all is well this method yields nothing! @@ -251,20 +254,20 @@ def get_unlinked_references(self): """ for row in self.execute( """ - SELECT x, y, ref_p, ref_region, reference_num, - COALESCE(ref_label, "") as ref_label + SELECT x, y, ref_p, ref_region, reference_num, + COALESCE(ref_label, "") AS ref_label FROM linked_reference_view WHERE act_region IS NULL - """): + """): yield (row["x"], row["y"], row["ref_p"], row["ref_region"], row["reference_num"], str(row["ref_label"], "utf8")) def get_double_region(self): """ Finds and yields any region that was used in both region definition - and a reference + and a reference. - If all is well this method yields nothing! + If all is well this method yields nothing! .. note:: Do not use the database for anything else while iterating. @@ -292,18 +295,18 @@ def set_region_content(self, x, y, p, region_num, content, content_debug): :param bytearray content: content to write :param content_debug: debug text :type content_debug: str or None - :raises DsDatabaseException: If the region already has content + :raises DatabaseException: If the region already has content """ # check for previous content for row in self.execute( """ SELECT content FROM region - WHERE x = ? AND y = ? and p = ? and region_num = ? + WHERE x = ? AND y = ? AND p = ? AND region_num = ? LIMIT 1 - """, (x, y, p, region_num)): + """, (x, y, p, region_num)): if row["content"]: - raise DsDatabaseException( + raise DatabaseException( f"Illegal attempt to overwrite content for " f"{x=} {y=} {p=} {region_num=}") @@ -311,10 +314,10 @@ def set_region_content(self, x, y, p, region_num, content, content_debug): """ UPDATE region SET content = ?, content_debug = ? - WHERE x = ? AND y = ? and p = ? and region_num = ? + WHERE x = ? AND y = ? AND p = ? AND region_num = ? """, (content, content_debug, x, y, p, region_num)) if self.rowcount == 0: - raise DsDatabaseException( + raise DatabaseException( f"No region {x=} {y=} {p=} {region_num=}") def get_region_pointer(self, x, y, p, region_num): @@ -330,7 +333,7 @@ def get_region_pointer(self, x, y, p, region_num): :param int region_num: The DS region number :return: The pointer set during the original load :rtype: int or None - :raises DsDatabaseException: if the region is not known + :raises DatabaseException: if the region is not known """ for row in self.execute( """ @@ -340,7 +343,7 @@ def get_region_pointer(self, x, y, p, region_num): LIMIT 1 """, (x, y, p, region_num)): return row["pointer"] - raise DsDatabaseException(f"No region {x=} {y=} {p=} {region_num=}") + raise DatabaseException(f"No region {x=} {y=} {p=} {region_num=}") def get_region_sizes(self, x, y, p): """ @@ -382,14 +385,14 @@ def get_total_regions_size(self, x, y, p): :rtype: int """ for row in self.execute( - """ - SELECT COALESCE(sum(size), 0) as total + """ + SELECT COALESCE(sum(size), 0) AS total FROM region WHERE x = ? AND y = ? AND p = ? LIMIT 1 """, (x, y, p)): return row["total"] - raise DsDatabaseException("Query failed unexpectedly") + raise DatabaseException("Query failed unexpectedly") def set_start_address(self, x, y, p, start_address): """ @@ -401,7 +404,7 @@ def set_start_address(self, x, y, p, start_address): :param int start_address: The base address for the whole core :return: The expected size of the malloced_area :rtype: int - :raises DsDatabaseException: if the region is not known + :raises DatabaseException: if the region is not known """ self.execute( """ @@ -410,8 +413,7 @@ def set_start_address(self, x, y, p, start_address): WHERE x = ? AND y = ? AND p = ? """, (start_address, x, y, p)) if self.rowcount == 0: - raise DsDatabaseException( - f"No core {x=} {y=} {p=}") + raise DatabaseException(f"No core {x=} {y=} {p=}") def get_start_address(self, x, y, p): """ @@ -427,21 +429,21 @@ def get_start_address(self, x, y, p): """ SELECT start_address FROM core - WHERE x = ? AND y = ? and p = ? + WHERE x = ? AND y = ? AND p = ? LIMIT 1 """, (x, y, p)): return row["start_address"] - raise DsDatabaseException(f"No core {x=} {y=} {p=}") + raise DatabaseException(f"No core {x=} {y=} {p=}") def set_region_pointer(self, x, y, p, region_num, pointer): self.execute( """ UPDATE region SET pointer = ? - WHERE x = ? AND y = ? and p = ? and region_num = ? + WHERE x = ? AND y = ? AND p = ? AND region_num = ? """, (pointer, x, y, p, region_num)) if self.rowcount == 0: - raise DsDatabaseException( + raise DatabaseException( f"No region {x=} {y=} {p=} {region_num=}") def get_region_pointers_and_content(self, x, y, p): @@ -453,6 +455,9 @@ def get_region_pointers_and_content(self, x, y, p): Will yield nothing if there are no regions reserved or if the core if not known + .. note:: + Do not use the database for anything else while iterating. + :param int x: X coordinate of the core :param int y: Y coordinate of the core :param int p: Processor ID of the core @@ -488,7 +493,8 @@ def get_ds_cores(self): """ for row in self.execute( """ - SELECT x, y, p FROM core + SELECT x, y, p + FROM core """): yield (row["x"], row["y"], row["p"]) @@ -499,15 +505,16 @@ def get_n_ds_cores(self): Includes cores where DataSpecs started even if no regions reserved :rtype: int - :raises DsDatabaseException: + :raises DatabaseException: """ for row in self.execute( """ - SELECT COUNT(*) as count FROM core + SELECT COUNT(*) AS count + FROM core LIMIT 1 """): return row["count"] - raise DsDatabaseException("Count query failed") + raise DatabaseException("Count query failed") def get_memory_to_malloc(self, x, y, p): """ @@ -571,7 +578,7 @@ def get_info_for_cores(self): """ for row in self.execute( """ - SELECT x, y, p, start_address, to_write,malloc_size + SELECT x, y, p, start_address, to_write, malloc_size FROM core_summary_view """): yield ((row["x"], row["y"], row["p"]), row["start_address"], @@ -599,13 +606,14 @@ def write_session_credentials_to_db(self): def set_app_id(self): """ Sets the app id - """ # check for previous content - self.execute( - """ - INSERT INTO app_id(app_id) - VALUES(?) - """, (FecDataView.get_app_id(), )) - if self.rowcount == 0: - raise DsDatabaseException("Unable to set app id") + for _row in self.execute( + """ + INSERT INTO app_id(app_id) + VALUES(?) + RETURNING * + """, (FecDataView.get_app_id(), )): + break + else: + raise DatabaseException("Unable to set app id") diff --git a/spinn_front_end_common/interface/provenance/global_provenance.py b/spinn_front_end_common/interface/provenance/global_provenance.py index 4213a5c35a..7373522d75 100644 --- a/spinn_front_end_common/interface/provenance/global_provenance.py +++ b/spinn_front_end_common/interface/provenance/global_provenance.py @@ -21,6 +21,7 @@ from spinn_front_end_common.utilities.constants import ( MICRO_TO_MILLISECOND_CONVERSION) from spinn_front_end_common.utilities.sqlite_db import SQLiteDB +from spinn_front_end_common.utilities.exceptions import DatabaseException logger = FormatAdapter(logging.getLogger(__name__)) @@ -102,16 +103,19 @@ def insert_category(self, category, machine_on): :param bool machine_on: If the machine was done during all or some of the time """ - self.execute( - """ - INSERT INTO category_timer_provenance( - category, machine_on, n_run, n_loop) - VALUES(?, ?, ?, ?) - """, - [category.category_name, machine_on, - FecDataView.get_run_number(), - FecDataView.get_run_step()]) - return self.lastrowid + for row in self.execute( + """ + INSERT INTO category_timer_provenance( + category, machine_on, n_run, n_loop) + VALUES(?, ?, ?, ?) + RETURNING category_id + """, + [category.category_name, machine_on, + FecDataView.get_run_number(), + FecDataView.get_run_step()]): + return row[0] + raise DatabaseException( + "database insert failed (category_timer_provenance)") def insert_category_timing(self, category_id, timedelta): """ diff --git a/spinn_front_end_common/interface/provenance/provenance_writer.py b/spinn_front_end_common/interface/provenance/provenance_writer.py index 17d0ae1f5a..ecdc970dde 100644 --- a/spinn_front_end_common/interface/provenance/provenance_writer.py +++ b/spinn_front_end_common/interface/provenance/provenance_writer.py @@ -16,6 +16,7 @@ from spinn_utilities.config_holder import get_config_int_or_none from spinn_utilities.log import FormatAdapter from spinn_front_end_common.utilities.base_database import BaseDatabase +from spinn_front_end_common.utilities.exceptions import DatabaseException logger = FormatAdapter(logging.getLogger(__name__)) @@ -144,12 +145,16 @@ def insert_report(self, message): :param str message: """ - self.execute( - """ - INSERT INTO reports(message) - VALUES(?) - """, [message]) - recorded = self.lastrowid + for row in self.execute( + """ + INSERT INTO reports(message) + VALUES(?) + RETURNING rowid AS row_num + """, [message]): + recorded = row["row_num"] + break + else: + raise DatabaseException("database insert failed (reports)") cutoff = get_config_int_or_none("Reports", "provenance_report_cutoff") if cutoff is None or recorded < cutoff: logger.warning(message) @@ -192,7 +197,7 @@ def insert_board_provenance(self, connections): self.executemany( """ INSERT OR IGNORE INTO boards_provenance( - ethernet_x, ethernet_y, ip_addres) + ethernet_x, ethernet_y, ip_addres) VALUES (?, ?, ?) """, ((x, y, ipaddress) for ((x, y), ipaddress) in connections.items())) diff --git a/spinn_front_end_common/utilities/base_database.py b/spinn_front_end_common/utilities/base_database.py index 280245109d..540baa79a9 100644 --- a/spinn_front_end_common/utilities/base_database.py +++ b/spinn_front_end_common/utilities/base_database.py @@ -18,6 +18,7 @@ from spinn_utilities.abstract_context_manager import AbstractContextManager from spinn_front_end_common.data import FecDataView from spinn_front_end_common.utilities.sqlite_db import SQLiteDB +from spinn_front_end_common.utilities.exceptions import DatabaseException _DDL_FILE = os.path.join(os.path.dirname(__file__), "db.sql") @@ -82,8 +83,11 @@ def _get_core_id(self, x, y, p): LIMIT 1 """, (x, y, p)): return row["core_id"] - self.execute( - """ - INSERT INTO core(x, y, processor) VALUES(?, ?, ?) - """, (x, y, p)) - return self.lastrowid + for row in self.execute( + """ + INSERT INTO core(x, y, processor) + VALUES(?, ?, ?) + RETURNING core_id + """, (x, y, p)): + return row["core_id"] + raise DatabaseException("database insert failed (core)") diff --git a/spinn_front_end_common/utilities/database/database_writer.py b/spinn_front_end_common/utilities/database/database_writer.py index f9b0a1b7d4..aac3e91ed2 100644 --- a/spinn_front_end_common/utilities/database/database_writer.py +++ b/spinn_front_end_common/utilities/database/database_writer.py @@ -25,6 +25,7 @@ AbstractSupportsDatabaseInjection, HasCustomAtomKeyMap) from spinn_front_end_common.utility_models import LivePacketGather from spinn_front_end_common.utility_models import LivePacketGatherMachineVertex +from spinn_front_end_common.utilities.exceptions import DatabaseException logger = FormatAdapter(logging.getLogger(__name__)) DB_NAME = "input_output_database.sqlite3" @@ -99,8 +100,9 @@ def __insert(self, sql, *args): :rtype: int """ try: - self.execute(sql, args) - return self.lastrowid + for row in self.execute(sql, args): + return row["inserted"] + raise DatabaseException("database insert failed") except Exception: logger.exception("problem with insertion; argument types are {}", str(map(type, args))) @@ -116,6 +118,7 @@ def add_machine_objects(self): INSERT INTO Machine_layout( x_dimension, y_dimension) VALUES(?, ?) + RETURNING machine_id AS inserted """, machine.width, machine.height) self.executemany( """ @@ -136,8 +139,11 @@ def add_application_vertices(self): # add vertices for vertex in FecDataView.iterate_vertices(): vertex_id = self.__insert( - "INSERT INTO Application_vertices(vertex_label) VALUES(?)", - vertex.label) + """ + INSERT INTO Application_vertices(vertex_label) + VALUES(?) + RETURNING vertex_id AS inserted + """, vertex.label) self.__vertex_to_id[vertex] = vertex_id for m_vertex in vertex.machine_vertices: m_vertex_id = self.__add_machine_vertex(m_vertex) @@ -146,13 +152,16 @@ def add_application_vertices(self): INSERT INTO graph_mapper_vertex ( application_vertex_id, machine_vertex_id) VALUES(?, ?) - """, - vertex_id, m_vertex_id) + RETURNING rowid AS inserted + """, vertex_id, m_vertex_id) def __add_machine_vertex(self, m_vertex): m_vertex_id = self.__insert( - "INSERT INTO Machine_vertices (label) VALUES(?)", - str(m_vertex.label)) + """ + INSERT INTO Machine_vertices (label) + VALUES(?) + RETURNING vertex_id AS inserted + """, str(m_vertex.label)) self.__vertex_to_id[m_vertex] = m_vertex_id return m_vertex_id @@ -261,8 +270,7 @@ def create_atom_to_event_id_mapping(self, machine_vertices): INSERT INTO event_to_atom_mapping( vertex_id, event_id, atom_id) VALUES (?, ?, ?) - """, ((m_vertex_id, int(key), i) for i, key in atom_keys) - ) + """, ((m_vertex_id, int(key), i) for i, key in atom_keys)) def _get_machine_lpg_mappings(self, part): """ Get places where an LPG Machine vertex has been added to a graph diff --git a/spinn_front_end_common/utilities/exceptions.py b/spinn_front_end_common/utilities/exceptions.py index cc96ddd1f3..cff4894acc 100644 --- a/spinn_front_end_common/utilities/exceptions.py +++ b/spinn_front_end_common/utilities/exceptions.py @@ -70,12 +70,6 @@ class CantFindSDRAMToUseException(SpinnFrontEndException): """ -class DsDatabaseException(SpinnFrontEndException): - """ - Raise when a query in the Data Specification database failed. - """ - - class DatabaseException(SpinnFrontEndException): """ Raise when something in a database failed. diff --git a/unittests/interface/ds/test_ds.py b/unittests/interface/ds/test_ds.py index 37d00228c6..2cc58766cc 100644 --- a/unittests/interface/ds/test_ds.py +++ b/unittests/interface/ds/test_ds.py @@ -29,7 +29,7 @@ from spinn_front_end_common.utilities.constants import ( APP_PTR_TABLE_BYTE_SIZE) from spinn_front_end_common.utilities.exceptions import ( - DataSpecException, DsDatabaseException) + DataSpecException, DatabaseException) class _TestVertexWithBinary(SimpleMachineVertex, AbstractHasAssociatedBinary): @@ -167,7 +167,7 @@ def test_switch_write_focus(self): # check internal fields used later are correct self.assertEqual(123456, dsg._size) # Error is switching into a region not reserved - with self.assertRaises(DsDatabaseException): + with self.assertRaises(DatabaseException): dsg.switch_write_focus(8) def test_pointers(self): @@ -196,10 +196,10 @@ def test_pointers(self): dsg4 = DataSpecificationGenerator(1, 1, 4, vertex, db) dsg4.reference_memory_region(8, 3, "oops") - with self.assertRaises(DsDatabaseException): + with self.assertRaises(DatabaseException): db.set_start_address(1, 3, 4, 123) - with self.assertRaises(DsDatabaseException): + with self.assertRaises(DatabaseException): db.get_start_address(1, 3, 4) p2 = 1000 + APP_PTR_TABLE_BYTE_SIZE @@ -230,9 +230,9 @@ def test_pointers(self): # region_num, pointer, content - with self.assertRaises(DsDatabaseException): + with self.assertRaises(DatabaseException): db.set_region_pointer(1, 2, 3, 9, 1400) - with self.assertRaises(DsDatabaseException): + with self.assertRaises(DatabaseException): db.get_region_pointer(1, 2, 3, 9) def test_write(self): @@ -283,7 +283,7 @@ def test_write(self): self.assertIn( ((0, 1, 3), None, size3, APP_PTR_TABLE_BYTE_SIZE), info) - with self.assertRaises(DsDatabaseException): + with self.assertRaises(DatabaseException): db.set_region_content( 0, 1, 4, 5, bytearray(b'\x0c\x00\x00\x00'), "test")