From af1fd6312981566e3e41bec6f820235a7bfa6138 Mon Sep 17 00:00:00 2001 From: "Donal K. Fellows" Date: Mon, 25 Sep 2023 13:17:39 +0100 Subject: [PATCH 1/4] Convert our use of SQLite to use RETURNING clauses Note that this requires that we are running against a new enough Python to support RETURNING clauses. --- .../storage_objects/buffer_database.py | 17 +++++---- .../interface/ds/ds_sqllite_database.py | 19 +++++----- .../interface/provenance/global_provenance.py | 23 ++++++------ .../interface/provenance/provenance_writer.py | 30 +++++++++------- .../utilities/base_database.py | 14 +++++--- .../utilities/database/database_writer.py | 35 +++++++++++-------- 6 files changed, 81 insertions(+), 57 deletions(-) diff --git a/spinn_front_end_common/interface/buffer_management/storage_objects/buffer_database.py b/spinn_front_end_common/interface/buffer_management/storage_objects/buffer_database.py index d9d6b6c677..45b14a6c03 100644 --- a/spinn_front_end_common/interface/buffer_management/storage_objects/buffer_database.py +++ b/spinn_front_end_common/interface/buffer_management/storage_objects/buffer_database.py @@ -17,6 +17,7 @@ from spinnman.spalloc.spalloc_job import SpallocJob from spinn_front_end_common.data import FecDataView from spinn_front_end_common.utilities.base_database import BaseDatabase +from spinn_front_end_common.utilities.exceptions import SpinnFrontEndException _SECONDS_TO_MICRO_SECONDS_CONVERSION = 1000 #: Name of the database in the data folder @@ -143,13 +144,15 @@ def _get_region_id(self, cursor, x, y, p, region): """, (x, y, p, region)): return row["region_id"] core_id = self._get_core_id(cursor, x, y, p) - cursor.execute( - """ - INSERT INTO region( - core_id, local_region_index, content, content_len, fetches) - VALUES(?, ?, CAST('' AS BLOB), 0, 0) - """, (core_id, region)) - return cursor.lastrowid + for row in cursor.execute( + """ + INSERT INTO region( + core_id, local_region_index, content, content_len, fetches) + VALUES(?, ?, CAST('' AS BLOB), 0, 0) + RETURNING region_id + """, (core_id, region)): + return row["region_id"] + raise SpinnFrontEndException("database insert failed") def store_data_in_region_buffer(self, x, y, p, region, missing, data): """ diff --git a/spinn_front_end_common/interface/ds/ds_sqllite_database.py b/spinn_front_end_common/interface/ds/ds_sqllite_database.py index 4f5422c60e..bffe09fb5c 100644 --- a/spinn_front_end_common/interface/ds/ds_sqllite_database.py +++ b/spinn_front_end_common/interface/ds/ds_sqllite_database.py @@ -22,7 +22,8 @@ from spinn_front_end_common.data import FecDataView from spinn_front_end_common.utilities.constants import ( APP_PTR_TABLE_BYTE_SIZE) -from spinn_front_end_common.utilities.exceptions import DsDatabaseException +from spinn_front_end_common.utilities.exceptions import DsDatabaseException,\ + SpinnFrontEndException from spinn_front_end_common.utilities.sqlite_db import SQLiteDB _DDL_FILE = os.path.join(os.path.dirname(__file__), "dse.sql") @@ -181,13 +182,15 @@ def set_memory_region( :return: """ with self.transaction() as cursor: - cursor.execute( - """ - INSERT INTO region( - x, y, p, region_num, size, reference_num, region_label) - VALUES(?, ?, ?, ?, ?, ?, ?) - """, (x, y, p, region_num, size, reference, label)) - return cursor.lastrowid + for row in cursor.execute( + """ + INSERT INTO region( + x, y, p, region_num, size, reference_num, region_label) + VALUES(?, ?, ?, ?, ?, ?, ?) + RETURNING region_id + """, (x, y, p, region_num, size, reference, label)): + return row["region_id"] + raise SpinnFrontEndException("database insert failed") def get_region_size(self, x, y, p, region_num): """ diff --git a/spinn_front_end_common/interface/provenance/global_provenance.py b/spinn_front_end_common/interface/provenance/global_provenance.py index 2f3f8ef7a0..606d60063d 100644 --- a/spinn_front_end_common/interface/provenance/global_provenance.py +++ b/spinn_front_end_common/interface/provenance/global_provenance.py @@ -21,6 +21,7 @@ from spinn_front_end_common.utilities.constants import ( MICRO_TO_MILLISECOND_CONVERSION) from spinn_front_end_common.utilities.sqlite_db import SQLiteDB +from spinn_front_end_common.utilities.exceptions import SpinnFrontEndException logger = FormatAdapter(logging.getLogger(__name__)) @@ -104,16 +105,18 @@ def insert_category(self, category, machine_on): or some of the time """ with self.transaction() as cur: - cur.execute( - """ - INSERT INTO category_timer_provenance( - category, machine_on, n_run, n_loop) - VALUES(?, ?, ?, ?) - """, - [category.category_name, machine_on, - FecDataView.get_run_number(), - FecDataView.get_run_step()]) - return cur.lastrowid + for row in cur.execute( + """ + INSERT INTO category_timer_provenance( + category, machine_on, n_run, n_loop) + VALUES(?, ?, ?, ?) + RETURNING category_id + """, + [category.category_name, machine_on, + FecDataView.get_run_number(), + FecDataView.get_run_step()]): + return row["category_id"] + raise SpinnFrontEndException("database insert failed") def insert_category_timing(self, category_id, timedelta): """ diff --git a/spinn_front_end_common/interface/provenance/provenance_writer.py b/spinn_front_end_common/interface/provenance/provenance_writer.py index 720f378a35..d705d1d1f6 100644 --- a/spinn_front_end_common/interface/provenance/provenance_writer.py +++ b/spinn_front_end_common/interface/provenance/provenance_writer.py @@ -16,6 +16,7 @@ from spinn_utilities.config_holder import get_config_int_or_none from spinn_utilities.log import FormatAdapter from spinn_front_end_common.utilities.base_database import BaseDatabase +from spinn_front_end_common.utilities.exceptions import SpinnFrontEndException logger = FormatAdapter(logging.getLogger(__name__)) @@ -150,12 +151,16 @@ def insert_report(self, message): :param str message: """ with self.transaction() as cur: - cur.execute( - """ - INSERT INTO reports(message) - VALUES(?) - """, [message]) - recorded = cur.lastrowid + for row in cur.execute( + """ + INSERT INTO reports(message) + VALUES(?) + RETURNING rowid AS row_num + """, [message]): + recorded = row["row_num"] + break + else: + raise SpinnFrontEndException("database insert failed") cutoff = get_config_int_or_none("Reports", "provenance_report_cutoff") if cutoff is None or recorded < cutoff: logger.warning(message) @@ -213,11 +218,12 @@ def _test_log_locked(self, text): """ with self.transaction() as cur: # lock the database - cur.execute( - """ - INSERT INTO reports(message) - VALUES(?) - """, [text]) - cur.lastrowid # pylint: disable=pointless-statement + for row in cur.execute( + """ + INSERT INTO reports(message) + VALUES(?) + RETURNING rowid AS row_num + """, [text]): + row["row_num"] # pylint: disable=pointless-statement # try logging and storing while locked. logger.warning(text) diff --git a/spinn_front_end_common/utilities/base_database.py b/spinn_front_end_common/utilities/base_database.py index d9a984b016..351dd7d189 100644 --- a/spinn_front_end_common/utilities/base_database.py +++ b/spinn_front_end_common/utilities/base_database.py @@ -18,6 +18,7 @@ from spinn_utilities.abstract_context_manager import AbstractContextManager from spinn_front_end_common.data import FecDataView from spinn_front_end_common.utilities.sqlite_db import SQLiteDB +from spinn_front_end_common.utilities.exceptions import SpinnFrontEndException _DDL_FILE = os.path.join(os.path.dirname(__file__), "db.sql") @@ -83,8 +84,11 @@ def _get_core_id(self, cursor, x, y, p): LIMIT 1 """, (x, y, p)): return row["core_id"] - cursor.execute( - """ - INSERT INTO core(x, y, processor) VALUES(?, ?, ?) - """, (x, y, p)) - return cursor.lastrowid + for row in cursor.execute( + """ + INSERT INTO core(x, y, processor) + VALUES(?, ?, ?) + RETURNING core_id + """, (x, y, p)): + return row["core_id"] + raise SpinnFrontEndException("database insert failed") diff --git a/spinn_front_end_common/utilities/database/database_writer.py b/spinn_front_end_common/utilities/database/database_writer.py index 4a92dea8a9..85be9a033d 100644 --- a/spinn_front_end_common/utilities/database/database_writer.py +++ b/spinn_front_end_common/utilities/database/database_writer.py @@ -25,6 +25,7 @@ AbstractSupportsDatabaseInjection, HasCustomAtomKeyMap) from spinn_front_end_common.utility_models import LivePacketGather from spinn_front_end_common.utility_models import LivePacketGatherMachineVertex +from spinn_front_end_common.utilities.exceptions import SpinnFrontEndException logger = FormatAdapter(logging.getLogger(__name__)) DB_NAME = "input_output_database.sqlite3" @@ -99,8 +100,9 @@ def __insert(self, cur, sql, *args): :rtype: int """ try: - cur.execute(sql, args) - return cur.lastrowid + for row in cur.execute(sql, args): + return row["inserted"] + raise SpinnFrontEndException("database insert failed") except Exception: logger.exception("problem with insertion; argument types are {}", str(map(type, args))) @@ -113,11 +115,11 @@ def add_machine_objects(self): machine = FecDataView.get_machine() with self.transaction() as cur: self.__machine_to_id[machine] = self._machine_id = self.__insert( - cur, - """ + cur, """ INSERT INTO Machine_layout( x_dimension, y_dimension) VALUES(?, ?) + RETURNING machine_id AS inserted """, machine.width, machine.height) cur.executemany( """ @@ -139,25 +141,29 @@ def add_application_vertices(self): # add vertices for vertex in FecDataView.iterate_vertices(): vertex_id = self.__insert( - cur, - "INSERT INTO Application_vertices(vertex_label) VALUES(?)", - vertex.label) + cur, """ + INSERT INTO Application_vertices(vertex_label) + VALUES(?) + RETURNING vertex_id AS inserted + """, vertex.label) self.__vertex_to_id[vertex] = vertex_id for m_vertex in vertex.machine_vertices: m_vertex_id = self.__add_machine_vertex(cur, m_vertex) self.__insert( - cur, - """ + cur, """ INSERT INTO graph_mapper_vertex ( application_vertex_id, machine_vertex_id) VALUES(?, ?) - """, - vertex_id, m_vertex_id) + RETURNING rowid AS inserted + """, vertex_id, m_vertex_id) def __add_machine_vertex(self, cur, m_vertex): m_vertex_id = self.__insert( - cur, "INSERT INTO Machine_vertices (label) VALUES(?)", - str(m_vertex.label)) + cur, """ + INSERT INTO Machine_vertices (label) + VALUES(?) + RETURNING vertex_id AS inserted + """, str(m_vertex.label)) self.__vertex_to_id[m_vertex] = m_vertex_id return m_vertex_id @@ -267,8 +273,7 @@ def create_atom_to_event_id_mapping(self, machine_vertices): INSERT INTO event_to_atom_mapping( vertex_id, event_id, atom_id) VALUES (?, ?, ?) - """, ((m_vertex_id, int(key), i) for i, key in atom_keys) - ) + """, ((m_vertex_id, int(key), i) for i, key in atom_keys)) def _get_machine_lpg_mappings(self, part): """ Get places where an LPG Machine vertex has been added to a graph From 29047da2e1375ea16467f6e582801f63a7ad19c1 Mon Sep 17 00:00:00 2001 From: "Donal K. Fellows" Date: Mon, 25 Sep 2023 14:07:11 +0100 Subject: [PATCH 2/4] Bleah --- .../interface/provenance/global_provenance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spinn_front_end_common/interface/provenance/global_provenance.py b/spinn_front_end_common/interface/provenance/global_provenance.py index 606d60063d..f003572843 100644 --- a/spinn_front_end_common/interface/provenance/global_provenance.py +++ b/spinn_front_end_common/interface/provenance/global_provenance.py @@ -115,7 +115,7 @@ def insert_category(self, category, machine_on): [category.category_name, machine_on, FecDataView.get_run_number(), FecDataView.get_run_step()]): - return row["category_id"] + return row[0] raise SpinnFrontEndException("database insert failed") def insert_category_timing(self, category_id, timedelta): From c89af9460328b765cea97744970a09a7ba618542 Mon Sep 17 00:00:00 2001 From: "Donal K. Fellows" Date: Mon, 25 Sep 2023 14:13:04 +0100 Subject: [PATCH 3/4] Ick --- spinn_front_end_common/interface/ds/ds_sqllite_database.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/spinn_front_end_common/interface/ds/ds_sqllite_database.py b/spinn_front_end_common/interface/ds/ds_sqllite_database.py index bffe09fb5c..c2c629762f 100644 --- a/spinn_front_end_common/interface/ds/ds_sqllite_database.py +++ b/spinn_front_end_common/interface/ds/ds_sqllite_database.py @@ -179,7 +179,8 @@ def set_memory_region( :param reference: A globally unique reference for this region :type reference: int or None :param label: - :return: + :return: The row ID (usually ignored) + :rtype: int """ with self.transaction() as cursor: for row in cursor.execute( @@ -187,9 +188,9 @@ def set_memory_region( INSERT INTO region( x, y, p, region_num, size, reference_num, region_label) VALUES(?, ?, ?, ?, ?, ?, ?) - RETURNING region_id + RETURNING rowid AS row_num """, (x, y, p, region_num, size, reference, label)): - return row["region_id"] + return row["row_num"] raise SpinnFrontEndException("database insert failed") def get_region_size(self, x, y, p, region_num): From 010434c961829cb627dbb8f9a76c6a9872d3a33d Mon Sep 17 00:00:00 2001 From: "Donal K. Fellows" Date: Mon, 25 Sep 2023 15:03:09 +0100 Subject: [PATCH 4/4] Rename DsDatabaseException to DatabaseException we have more DBs than we used to --- .../storage_objects/buffer_database.py | 4 +- .../ds/data_specification_generator.py | 2 +- .../interface/ds/ds_sqllite_database.py | 80 ++++++++++--------- .../interface/provenance/global_provenance.py | 5 +- .../interface/provenance/provenance_writer.py | 4 +- .../utilities/base_database.py | 4 +- .../utilities/database/database_writer.py | 4 +- .../utilities/exceptions.py | 4 +- unittests/interface/ds/test_ds.py | 14 ++-- 9 files changed, 63 insertions(+), 58 deletions(-) diff --git a/spinn_front_end_common/interface/buffer_management/storage_objects/buffer_database.py b/spinn_front_end_common/interface/buffer_management/storage_objects/buffer_database.py index 45b14a6c03..1f3d5b995c 100644 --- a/spinn_front_end_common/interface/buffer_management/storage_objects/buffer_database.py +++ b/spinn_front_end_common/interface/buffer_management/storage_objects/buffer_database.py @@ -17,7 +17,7 @@ from spinnman.spalloc.spalloc_job import SpallocJob from spinn_front_end_common.data import FecDataView from spinn_front_end_common.utilities.base_database import BaseDatabase -from spinn_front_end_common.utilities.exceptions import SpinnFrontEndException +from spinn_front_end_common.utilities.exceptions import DatabaseException _SECONDS_TO_MICRO_SECONDS_CONVERSION = 1000 #: Name of the database in the data folder @@ -152,7 +152,7 @@ def _get_region_id(self, cursor, x, y, p, region): RETURNING region_id """, (core_id, region)): return row["region_id"] - raise SpinnFrontEndException("database insert failed") + raise DatabaseException("database insert failed (region)") def store_data_in_region_buffer(self, x, y, p, region, missing, data): """ diff --git a/spinn_front_end_common/interface/ds/data_specification_generator.py b/spinn_front_end_common/interface/ds/data_specification_generator.py index 205360d5b1..d783aa4ba6 100644 --- a/spinn_front_end_common/interface/ds/data_specification_generator.py +++ b/spinn_front_end_common/interface/ds/data_specification_generator.py @@ -39,7 +39,7 @@ def __init__(self, x, y, p, vertex, ds_db, report_writer=None): Determines if a text version of the specification is to be written and, if so, where. No report is written if this is `None`. :type report_writer: ~io.TextIOBase or None - :raises DsDatabaseException: If this core is not known + :raises DatabaseException: If this core is not known and no vertex supplied (during reload) :raises AttributeError: If the vertex is not an AbstractHasAssociatedBinary diff --git a/spinn_front_end_common/interface/ds/ds_sqllite_database.py b/spinn_front_end_common/interface/ds/ds_sqllite_database.py index c2c629762f..bf47347cf5 100644 --- a/spinn_front_end_common/interface/ds/ds_sqllite_database.py +++ b/spinn_front_end_common/interface/ds/ds_sqllite_database.py @@ -22,8 +22,7 @@ from spinn_front_end_common.data import FecDataView from spinn_front_end_common.utilities.constants import ( APP_PTR_TABLE_BYTE_SIZE) -from spinn_front_end_common.utilities.exceptions import DsDatabaseException,\ - SpinnFrontEndException +from spinn_front_end_common.utilities.exceptions import DatabaseException from spinn_front_end_common.utilities.sqlite_db import SQLiteDB _DDL_FILE = os.path.join(os.path.dirname(__file__), "dse.sql") @@ -191,7 +190,7 @@ def set_memory_region( RETURNING rowid AS row_num """, (x, y, p, region_num, size, reference, label)): return row["row_num"] - raise SpinnFrontEndException("database insert failed") + raise DatabaseException("database insert failed (region)") def get_region_size(self, x, y, p, region_num): """ @@ -213,11 +212,11 @@ def get_region_size(self, x, y, p, region_num): LIMIT 1 """, (x, y, p, region_num)): return row["size"] - raise DsDatabaseException(f"Region {region_num} not set") + raise DatabaseException(f"Region {region_num} not set") def set_reference(self, x, y, p, region_num, reference, ref_label): """ - Writes a outgoing region_reference into the database + Writes a outgoing region_reference into the database. :param int x: X coordinate of the core :param int y: Y coordinate of the core @@ -237,7 +236,7 @@ def set_reference(self, x, y, p, region_num, reference, ref_label): def get_reference_pointers(self, x, y, p): """ - Yields the reference regions and where they point for this core + Yields the reference regions and where they point for this core. This may yield nothing if there are no reference pointers or if the core is not known @@ -262,7 +261,7 @@ def get_reference_pointers(self, x, y, p): def get_unlinked_references(self): """ - Finds and yields info on unreferenced links + Finds and yields info on unreferenced links. If all is well this method yields nothing! @@ -276,7 +275,7 @@ def get_unlinked_references(self): for row in cursor.execute( """ SELECT x, y, ref_p, ref_region, reference_num, - COALESCE(ref_label, "") as ref_label + COALESCE(ref_label, "") AS ref_label FROM linked_reference_view WHERE act_region IS NULL """): @@ -286,7 +285,7 @@ def get_unlinked_references(self): def get_double_region(self): """ Finds and yields any region that was used in both region definition - and a reference + and a reference. If all is well this method yields nothing! @@ -317,7 +316,7 @@ def set_region_content(self, x, y, p, region_num, content, content_debug): :param bytearray content: content to write :param content_debug: debug text :type content_debug: str or None - :raises DsDatabaseException: If the region already has content + :raises DatabaseException: If the region already has content """ with self.transaction() as cursor: # check for previous content @@ -325,11 +324,11 @@ def set_region_content(self, x, y, p, region_num, content, content_debug): """ SELECT content FROM region - WHERE x = ? AND y = ? and p = ? and region_num = ? + WHERE x = ? AND y = ? AND p = ? AND region_num = ? LIMIT 1 """, (x, y, p, region_num)): if row["content"]: - raise DsDatabaseException( + raise DatabaseException( f"Illegal attempt to overwrite content for " f"{x=} {y=} {p=} {region_num=}") @@ -337,10 +336,10 @@ def set_region_content(self, x, y, p, region_num, content, content_debug): """ UPDATE region SET content = ?, content_debug = ? - WHERE x = ? AND y = ? and p = ? and region_num = ? + WHERE x = ? AND y = ? AND p = ? AND region_num = ? """, (content, content_debug, x, y, p, region_num)) if cursor.rowcount == 0: - raise DsDatabaseException( + raise DatabaseException( f"No region {x=} {y=} {p=} {region_num=}") def get_region_pointer(self, x, y, p, region_num): @@ -356,7 +355,7 @@ def get_region_pointer(self, x, y, p, region_num): :param int region_num: The DS region number :return: The pointer set during the original load :rtype: int or None - :raises DsDatabaseException: if the region is not known + :raises DatabaseException: if the region is not known """ with self.transaction() as cursor: for row in cursor.execute( @@ -367,7 +366,7 @@ def get_region_pointer(self, x, y, p, region_num): LIMIT 1 """, (x, y, p, region_num)): return row["pointer"] - raise DsDatabaseException(f"No region {x=} {y=} {p=} {region_num=}") + raise DatabaseException(f"No region {x=} {y=} {p=} {region_num=}") def get_region_sizes(self, x, y, p): """ @@ -412,13 +411,13 @@ def get_total_regions_size(self, x, y, p): with self.transaction() as cursor: for row in cursor.execute( """ - SELECT COALESCE(sum(size), 0) as total + SELECT COALESCE(sum(size), 0) AS total FROM region WHERE x = ? AND y = ? AND p = ? LIMIT 1 """, (x, y, p)): return row["total"] - raise DsDatabaseException("Query failed unexpectedly") + raise DatabaseException("Query failed unexpectedly") def set_start_address(self, x, y, p, start_address): """ @@ -430,7 +429,7 @@ def set_start_address(self, x, y, p, start_address): :param int start_address: The base address for the whole core :return: The expected size of the malloced_area :rtype: int - :raises DsDatabaseException: if the region is not known + :raises DatabaseException: if the region is not known """ with self.transaction() as cursor: cursor.execute( @@ -440,8 +439,7 @@ def set_start_address(self, x, y, p, start_address): WHERE x = ? AND y = ? AND p = ? """, (start_address, x, y, p)) if cursor.rowcount == 0: - raise DsDatabaseException( - f"No core {x=} {y=} {p=}") + raise DatabaseException(f"No core {x=} {y=} {p=}") def get_start_address(self, x, y, p): """ @@ -458,11 +456,11 @@ def get_start_address(self, x, y, p): """ SELECT start_address FROM core - WHERE x = ? AND y = ? and p = ? + WHERE x = ? AND y = ? AND p = ? LIMIT 1 """, (x, y, p)): return row["start_address"] - raise DsDatabaseException(f"No core {x=} {y=} {p=}") + raise DatabaseException(f"No core {x=} {y=} {p=}") def set_region_pointer(self, x, y, p, region_num, pointer): with self.transaction() as cursor: @@ -470,10 +468,10 @@ def set_region_pointer(self, x, y, p, region_num, pointer): """ UPDATE region SET pointer = ? - WHERE x = ? AND y = ? and p = ? and region_num = ? + WHERE x = ? AND y = ? AND p = ? AND region_num = ? """, (pointer, x, y, p, region_num)) if cursor.rowcount == 0: - raise DsDatabaseException( + raise DatabaseException( f"No region {x=} {y=} {p=} {region_num=}") def get_region_pointers_and_content(self, x, y, p): @@ -485,6 +483,9 @@ def get_region_pointers_and_content(self, x, y, p): Will yield nothing if there are no regions reserved or if the core if not known + .. note:: + Do not use the database for anything else while iterating. + :param int x: X coordinate of the core :param int y: Y coordinate of the core :param int p: Processor ID of the core @@ -522,7 +523,8 @@ def get_ds_cores(self): with self.transaction() as cursor: for row in cursor.execute( """ - SELECT x, y, p FROM core + SELECT x, y, p + FROM core """): yield (row["x"], row["y"], row["p"]) @@ -533,16 +535,17 @@ def get_n_ds_cores(self): Includes cores where DataSpecs started even if no regions reserved :rtype: int - :raises DsDatabaseException: + :raises DatabaseException: """ with self.transaction() as cursor: for row in cursor.execute( """ - SELECT COUNT(*) as count FROM core + SELECT COUNT(*) AS count + FROM core LIMIT 1 """): return row["count"] - raise DsDatabaseException("Count query failed") + raise DatabaseException("Count query failed") def get_memory_to_malloc(self, x, y, p): """ @@ -609,7 +612,7 @@ def get_info_for_cores(self): with self.transaction() as cursor: for row in cursor.execute( """ - SELECT x, y, p, start_address, to_write,malloc_size + SELECT x, y, p, start_address, to_write, malloc_size FROM core_summary_view """): yield ((row["x"], row["y"], row["p"]), row["start_address"], @@ -634,14 +637,15 @@ def write_session_credentials_to_db(self): def set_app_id(self): """ Sets the app id - """ with self.transaction() as cursor: # check for previous content - cursor.execute( - """ - INSERT INTO app_id(app_id) - VALUES(?) - """, (FecDataView.get_app_id(), )) - if cursor.rowcount == 0: - raise DsDatabaseException("Unable to set app id") + for _row in cursor.execute( + """ + INSERT INTO app_id(app_id) + VALUES(?) + RETURNING * + """, (FecDataView.get_app_id(), )): + break + else: + raise DatabaseException("Unable to set app id") diff --git a/spinn_front_end_common/interface/provenance/global_provenance.py b/spinn_front_end_common/interface/provenance/global_provenance.py index f003572843..26aafbc3ae 100644 --- a/spinn_front_end_common/interface/provenance/global_provenance.py +++ b/spinn_front_end_common/interface/provenance/global_provenance.py @@ -21,7 +21,7 @@ from spinn_front_end_common.utilities.constants import ( MICRO_TO_MILLISECOND_CONVERSION) from spinn_front_end_common.utilities.sqlite_db import SQLiteDB -from spinn_front_end_common.utilities.exceptions import SpinnFrontEndException +from spinn_front_end_common.utilities.exceptions import DatabaseException logger = FormatAdapter(logging.getLogger(__name__)) @@ -116,7 +116,8 @@ def insert_category(self, category, machine_on): FecDataView.get_run_number(), FecDataView.get_run_step()]): return row[0] - raise SpinnFrontEndException("database insert failed") + raise DatabaseException( + "database insert failed (category_timer_provenance)") def insert_category_timing(self, category_id, timedelta): """ diff --git a/spinn_front_end_common/interface/provenance/provenance_writer.py b/spinn_front_end_common/interface/provenance/provenance_writer.py index d705d1d1f6..0b6d712c3f 100644 --- a/spinn_front_end_common/interface/provenance/provenance_writer.py +++ b/spinn_front_end_common/interface/provenance/provenance_writer.py @@ -16,7 +16,7 @@ from spinn_utilities.config_holder import get_config_int_or_none from spinn_utilities.log import FormatAdapter from spinn_front_end_common.utilities.base_database import BaseDatabase -from spinn_front_end_common.utilities.exceptions import SpinnFrontEndException +from spinn_front_end_common.utilities.exceptions import DatabaseException logger = FormatAdapter(logging.getLogger(__name__)) @@ -160,7 +160,7 @@ def insert_report(self, message): recorded = row["row_num"] break else: - raise SpinnFrontEndException("database insert failed") + raise DatabaseException("database insert failed (reports)") cutoff = get_config_int_or_none("Reports", "provenance_report_cutoff") if cutoff is None or recorded < cutoff: logger.warning(message) diff --git a/spinn_front_end_common/utilities/base_database.py b/spinn_front_end_common/utilities/base_database.py index 351dd7d189..59230e1b6e 100644 --- a/spinn_front_end_common/utilities/base_database.py +++ b/spinn_front_end_common/utilities/base_database.py @@ -18,7 +18,7 @@ from spinn_utilities.abstract_context_manager import AbstractContextManager from spinn_front_end_common.data import FecDataView from spinn_front_end_common.utilities.sqlite_db import SQLiteDB -from spinn_front_end_common.utilities.exceptions import SpinnFrontEndException +from spinn_front_end_common.utilities.exceptions import DatabaseException _DDL_FILE = os.path.join(os.path.dirname(__file__), "db.sql") @@ -91,4 +91,4 @@ def _get_core_id(self, cursor, x, y, p): RETURNING core_id """, (x, y, p)): return row["core_id"] - raise SpinnFrontEndException("database insert failed") + raise DatabaseException("database insert failed (core)") diff --git a/spinn_front_end_common/utilities/database/database_writer.py b/spinn_front_end_common/utilities/database/database_writer.py index 85be9a033d..ff7e9a5e83 100644 --- a/spinn_front_end_common/utilities/database/database_writer.py +++ b/spinn_front_end_common/utilities/database/database_writer.py @@ -25,7 +25,7 @@ AbstractSupportsDatabaseInjection, HasCustomAtomKeyMap) from spinn_front_end_common.utility_models import LivePacketGather from spinn_front_end_common.utility_models import LivePacketGatherMachineVertex -from spinn_front_end_common.utilities.exceptions import SpinnFrontEndException +from spinn_front_end_common.utilities.exceptions import DatabaseException logger = FormatAdapter(logging.getLogger(__name__)) DB_NAME = "input_output_database.sqlite3" @@ -102,7 +102,7 @@ def __insert(self, cur, sql, *args): try: for row in cur.execute(sql, args): return row["inserted"] - raise SpinnFrontEndException("database insert failed") + raise DatabaseException("database insert failed") except Exception: logger.exception("problem with insertion; argument types are {}", str(map(type, args))) diff --git a/spinn_front_end_common/utilities/exceptions.py b/spinn_front_end_common/utilities/exceptions.py index b69cfa9bab..3437454b3a 100644 --- a/spinn_front_end_common/utilities/exceptions.py +++ b/spinn_front_end_common/utilities/exceptions.py @@ -70,9 +70,9 @@ class CantFindSDRAMToUseException(SpinnFrontEndException): """ -class DsDatabaseException(SpinnFrontEndException): +class DatabaseException(SpinnFrontEndException): """ - Raise when a query in the Data Specification database failed. + Raised when a query to a database failed. """ diff --git a/unittests/interface/ds/test_ds.py b/unittests/interface/ds/test_ds.py index 0b03e421c6..7879ae637f 100644 --- a/unittests/interface/ds/test_ds.py +++ b/unittests/interface/ds/test_ds.py @@ -29,7 +29,7 @@ from spinn_front_end_common.utilities.constants import ( APP_PTR_TABLE_BYTE_SIZE) from spinn_front_end_common.utilities.exceptions import ( - DataSpecException, DsDatabaseException) + DataSpecException, DatabaseException) class _TestVertexWithBinary(SimpleMachineVertex, AbstractHasAssociatedBinary): @@ -167,7 +167,7 @@ def test_switch_write_focus(self): # check internal fields used later are correct self.assertEqual(123456, dsg._size) # Error is switching into a region not reserved - with self.assertRaises(DsDatabaseException): + with self.assertRaises(DatabaseException): dsg.switch_write_focus(8) def test_pointers(self): @@ -197,10 +197,10 @@ def test_pointers(self): dsg4 = DataSpecificationGenerator(1, 1, 4, vertex, db) dsg4.reference_memory_region(8, 3, "oops") - with self.assertRaises(DsDatabaseException): + with self.assertRaises(DatabaseException): db.set_start_address(1, 3, 4, 123) - with self.assertRaises(DsDatabaseException): + with self.assertRaises(DatabaseException): db.get_start_address(1, 3, 4) p2 = 1000 + APP_PTR_TABLE_BYTE_SIZE @@ -231,9 +231,9 @@ def test_pointers(self): # region_num, pointer, content - with self.assertRaises(DsDatabaseException): + with self.assertRaises(DatabaseException): db.set_region_pointer(1, 2, 3, 9, 1400) - with self.assertRaises(DsDatabaseException): + with self.assertRaises(DatabaseException): db.get_region_pointer(1, 2, 3, 9) def test_write(self): @@ -282,7 +282,7 @@ def test_write(self): self.assertIn(((0, 1, 2), None, size2, 404), info) self.assertIn(((0, 1, 3), None, size3, APP_PTR_TABLE_BYTE_SIZE), info) - with self.assertRaises(DsDatabaseException): + with self.assertRaises(DatabaseException): db.set_region_content( 0, 1, 4, 5, bytearray(b'\x0c\x00\x00\x00'), "test")