From f21721eb52367af20ac5e9c46fe72bf235068f1b Mon Sep 17 00:00:00 2001 From: George Kourtidis Date: Sun, 19 Feb 2023 18:05:46 +0200 Subject: [PATCH 01/16] Added NOT & Between --- miniDB/database.py | 10 ++++++++-- miniDB/misc.py | 46 ++++++++++++++++++++++++++++++++++++++-------- miniDB/table.py | 29 ++++++++++++++++++++++++++--- 3 files changed, 72 insertions(+), 13 deletions(-) diff --git a/miniDB/database.py b/miniDB/database.py index a3ac6be7..4a4bd3c5 100644 --- a/miniDB/database.py +++ b/miniDB/database.py @@ -4,7 +4,8 @@ import os,sys import logging import warnings -import readline +import pyreadline +import re from tabulate import tabulate sys.path.append(f'{os.path.dirname(os.path.dirname(os.path.abspath(__file__)))}/miniDB') @@ -358,7 +359,12 @@ def select(self, columns, table_name, condition, distinct=None, order_by=None, \ return table_name._select_where(columns, condition, distinct, order_by, desc, limit) if condition is not None: - condition_column = split_condition(condition)[0] + if "between" in condition.split() or "and" in condition.split() or "or" in condition.split(): + + condition_column = condition.split()[0] + else: + + condition_column = split_condition(condition)[0] else: condition_column = '' diff --git a/miniDB/misc.py b/miniDB/misc.py index aefada74..bc49091c 100644 --- a/miniDB/misc.py +++ b/miniDB/misc.py @@ -1,4 +1,5 @@ import operator +import re def get_op(op, a, b): ''' @@ -8,24 +9,40 @@ def get_op(op, a, b): '<': operator.lt, '>=': operator.ge, '<=': operator.le, - '=': operator.eq} + '=': operator.eq, + '!=': operator.ne} try: return ops[op](a,b) except TypeError: # if a or b is None (deleted record), python3 raises typerror return False -def split_condition(condition): - ops = {'>=': operator.ge, - '<=': operator.le, - '=': operator.eq, - '>': operator.gt, - '<': operator.lt} +def find_condition_operator(condition): + if condition.startswith("not "): + return -1, condition[4:].strip() + + + +def split_condition(condition,negate=0): + if condition.startswith("not "): + negate=-1 + condition = condition[4:].strip() + + + ops = {'>=': operator.ge, + '<=': operator.le, + '=': operator.eq, + '>': operator.gt, + '<': operator.lt, + '!=': operator.ne} + for op_key in ops.keys(): + splt=condition.split(op_key) if len(splt)>1: left, right = splt[0].strip(), splt[1].strip() + if right[0] == '"' == right[-1]: # If the value has leading and trailing quotes, remove them. right = right.strip('"') @@ -34,7 +51,20 @@ def split_condition(condition): if right.find('"') != -1: # If there are any double quotes in the value, throw. (Notice we've already removed the leading and trailing ones) raise ValueError(f'Invalid condition: {condition}\nDouble quotation marks are not allowed inside values.') - + if negate==-1: + if op_key == '<=': + op_key = '>' + elif op_key == '>=': + op_key= '<' + elif op_key == '=': + op_key ='!=' + elif op_key=='>': + op_key = '<=' + elif op_key=='<': + op_key='>=' + elif op_key=='!=': + op_key='=' + return left, op_key, right def reverse_op(op): diff --git a/miniDB/table.py b/miniDB/table.py index f5c7d937..d16eb59a 100644 --- a/miniDB/table.py +++ b/miniDB/table.py @@ -3,6 +3,7 @@ import pickle import os import sys +import re sys.path.append(f'{os.path.dirname(os.path.dirname(os.path.abspath(__file__)))}/miniDB') @@ -233,9 +234,31 @@ def _select_where(self, return_columns, condition=None, distinct=False, order_by # if condition is None, return all rows # if not, return the rows with values where condition is met for value if condition is not None: - column_name, operator, value = self._parse_condition(condition) - column = self.column_by_name(column_name) - rows = [ind for ind, x in enumerate(column) if get_op(operator, x, value)] + + if re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition) or condition.startswith("not "):#simple condition + column_name, operator, value = self._parse_condition(condition) + column = self.column_by_name(column_name) + rows = [ind for ind, x in enumerate(column) if get_op(operator, x, value)] + elif re.match(r"^\w+\s+between\s+\w+\s+and\s+\w+$", condition): + + query = condition.split() + index = query.index("between") + megalutero = query[index+1] + mikrotero = query[index+3] + column_name = query[index-1] + column = self.column_by_name(column_name) + rows = [] + for i,j in enumerate(column): + if int(j) >= int(megalutero) and int(j) <= int(mikrotero): + rows.append(i) + elif re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+\s+AND\s+\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition): + + elif re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+\s+OR\s+\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition): + return 1, condition + #else: + # column_name, operator, value = self._parse_condition(condition) + #column = self.column_by_name(column_name) + #rows = [ind for ind, x in enumerate(column) if get_op(operator, x, value)] else: rows = [i for i in range(len(self.data))] From e927ff42bfec5e6694de810ffae4c9b9bdb9e699 Mon Sep 17 00:00:00 2001 From: Kapiniaris-Xrisovalantis <92021533+Kapiniaris-Xrisovalantis@users.noreply.github.com> Date: Sun, 19 Feb 2023 18:20:55 +0200 Subject: [PATCH 02/16] added or/and --- miniDB/table.py | 37 +++++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/miniDB/table.py b/miniDB/table.py index d16eb59a..cd855670 100644 --- a/miniDB/table.py +++ b/miniDB/table.py @@ -235,7 +235,7 @@ def _select_where(self, return_columns, condition=None, distinct=False, order_by # if not, return the rows with values where condition is met for value if condition is not None: - if re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition) or condition.startswith("not "):#simple condition + if re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition) or condition.startswith("not "):#simple condition or not column_name, operator, value = self._parse_condition(condition) column = self.column_by_name(column_name) rows = [ind for ind, x in enumerate(column) if get_op(operator, x, value)] @@ -251,14 +251,35 @@ def _select_where(self, return_columns, condition=None, distinct=False, order_by for i,j in enumerate(column): if int(j) >= int(megalutero) and int(j) <= int(mikrotero): rows.append(i) - elif re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+\s+AND\s+\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition): + elif re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+\s+or\s+\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition): - elif re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+\s+OR\s+\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition): - return 1, condition - #else: - # column_name, operator, value = self._parse_condition(condition) - #column = self.column_by_name(column_name) - #rows = [ind for ind, x in enumerate(column) if get_op(operator, x, value)] + column_name = condition.split()[0] + conditions = condition.split("or") + rows_L=[] + for condition_ in conditions: + column_name, operator, value = self._parse_condition(condition_) + column = self.column_by_name(column_name) + rows_L.append([ind for ind, x in enumerate(column) if get_op(operator, x, value)]) + + rows=[] + for rlist in rows_L: + for row in rlist: + rows.append(row) + elif re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+\s+and\s+\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition): + column_name = condition.split()[0] + conditions = condition.split("and") + rows_L=[] + for condition_ in conditions: + column_name, operator, value = self._parse_condition(condition_) + column = self.column_by_name(column_name) + rows_L.append([ind for ind, x in enumerate(column) if get_op(operator, x, value)]) + + + rows = set(rows_L[0]).intersection(*rows_L) + rows = list(rows) + + else: + raise("invalid where condition") else: rows = [i for i in range(len(self.data))] From cad5c10a269c70907ceeef06c75f60f4f94dca9a Mon Sep 17 00:00:00 2001 From: Kapiniaris-Xrisovalantis <92021533+Kapiniaris-Xrisovalantis@users.noreply.github.com> Date: Sun, 19 Feb 2023 18:23:36 +0200 Subject: [PATCH 03/16] added or/and --- miniDB/table.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/miniDB/table.py b/miniDB/table.py index cd855670..296f54e4 100644 --- a/miniDB/table.py +++ b/miniDB/table.py @@ -234,15 +234,19 @@ def _select_where(self, return_columns, condition=None, distinct=False, order_by # if condition is None, return all rows # if not, return the rows with values where condition is met for value if condition is not None: - + #Regex για τα σύμβολα ή για το αν υπάρχει not μέσα if re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition) or condition.startswith("not "):#simple condition or not column_name, operator, value = self._parse_condition(condition) column = self.column_by_name(column_name) rows = [ind for ind, x in enumerate(column) if get_op(operator, x, value)] elif re.match(r"^\w+\s+between\s+\w+\s+and\s+\w+$", condition): - + #Regex για να παίρνουμε το between + #Σπάμε το condition query = condition.split() + #Αναθέτουμε στο index το index του between (δλδ που βρίσκεται στο query) index = query.index("between") + #megalutero είναι η 2η συνθήκη ενώ μικρότερο είναι η 1η π.χ select * from table where column between x and y + #Το megalutero = x , το μικρότερο = y και το column_name = column megalutero = query[index+1] mikrotero = query[index+3] column_name = query[index-1] @@ -252,7 +256,7 @@ def _select_where(self, return_columns, condition=None, distinct=False, order_by if int(j) >= int(megalutero) and int(j) <= int(mikrotero): rows.append(i) elif re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+\s+or\s+\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition): - + #σπάμε τα δύο conditions και για καθένα βάζουμε τα rows που θα επιστραφούν σε ένα list. Στο τέλος συνδέουμε τα δυο lists που θα γυρίσουν column_name = condition.split()[0] conditions = condition.split("or") rows_L=[] @@ -266,6 +270,7 @@ def _select_where(self, return_columns, condition=None, distinct=False, order_by for row in rlist: rows.append(row) elif re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+\s+and\s+\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition): + #σπάμε τα δύο conditions και για καθένα βάζουμε τα rows που θα επιστραφούν σε ένα list. Στο τέλος πέρνουμε τα κοινα των δυο λιστών column_name = condition.split()[0] conditions = condition.split("and") rows_L=[] From c5a3d3e2cff9b7f3f20ea374e0f994eaa06a2fef Mon Sep 17 00:00:00 2001 From: Maria Peristeraki Date: Sun, 19 Feb 2023 19:39:35 +0200 Subject: [PATCH 04/16] Added Update and Delete support for OR, AND & Between --- ' | 0 miniDB/database.py | 22 +++++-- miniDB/misc.py | 7 +- miniDB/table.py | 156 +++++++++++++++++++++++++++++++++++++-------- 4 files changed, 151 insertions(+), 34 deletions(-) create mode 100644 ' diff --git a/' b/' new file mode 100644 index 00000000..e69de29b diff --git a/miniDB/database.py b/miniDB/database.py index 4a4bd3c5..44c699e4 100644 --- a/miniDB/database.py +++ b/miniDB/database.py @@ -268,6 +268,7 @@ def insert_into(self, table_name, row_str): self.load_database() # fetch the insert_stack. For more info on the insert_stack # check the insert_stack meta table + lock_ownership = self.lock_table(table_name, mode='x') insert_stack = self._get_insert_stack_for_table(table_name) try: @@ -357,14 +358,27 @@ def select(self, columns, table_name, condition, distinct=None, order_by=None, \ self.load_database() if isinstance(table_name,Table): return table_name._select_where(columns, condition, distinct, order_by, desc, limit) - + if condition is not None: - if "between" in condition.split() or "and" in condition.split() or "or" in condition.split(): - + if "between" in condition.split(): condition_column = condition.split()[0] + elif "and" in condition.split() : + columns2=[] + conditions=[] + conditions=condition.split("and") + for con in conditions: + col=self.tables[table_name]._parse_condition(con) + columns2.append(col) + for con in columns2: + if con[0] == self.tables[table_name].column_names[self.tables[table_name].pk_idx] :#since only pk supports index , and only one pk per table we keep the column if it is pk + condition_column = con[0] + + + elif " or " in condition.split(): + ca=[] else: - condition_column = split_condition(condition)[0] + condition_column = self.tables[table_name]._parse_condition(condition) else: condition_column = '' diff --git a/miniDB/misc.py b/miniDB/misc.py index bc49091c..e3b6ccfa 100644 --- a/miniDB/misc.py +++ b/miniDB/misc.py @@ -17,9 +17,7 @@ def get_op(op, a, b): except TypeError: # if a or b is None (deleted record), python3 raises typerror return False -def find_condition_operator(condition): - if condition.startswith("not "): - return -1, condition[4:].strip() + @@ -76,5 +74,6 @@ def reverse_op(op): '>=' : '<=', '<' : '>', '<=' : '>=', - '=' : '=' + '=' : '=', + '!=': '!=' }.get(op) diff --git a/miniDB/table.py b/miniDB/table.py index 296f54e4..c291ce3c 100644 --- a/miniDB/table.py +++ b/miniDB/table.py @@ -7,7 +7,7 @@ sys.path.append(f'{os.path.dirname(os.path.dirname(os.path.abspath(__file__)))}/miniDB') -from misc import get_op, split_condition +from misc import get_op, split_condition, reverse_op class Table: @@ -151,19 +151,77 @@ def _update_rows(self, set_value, set_column, condition): Operatores supported: (<,<=,=,>=,>) ''' + if condition is not None: + + if re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition) or condition.startswith("not "):#simple condition or not + column_name, operator, value = self._parse_condition(condition) + column = self.column_by_name(column_name) + set_column_idx = self.column_names.index(set_column) + for row_ind, column_value in enumerate(column): + if get_op(operator, column_value, value): + self.data[row_ind][set_column_idx] = set_value + elif re.match(r"^\w+\s+between\s+\w+\s+and\s+\w+$", condition): + + query = condition.split() + index = query.index("between") + megalutero = query[index+1] + mikrotero = query[index+3] + column_name = query[index-1] + column = self.column_by_name(column_name) + set_column_idx = self.column_names.index(set_column) + + for i,j in enumerate(column): + if j >= megalutero and j <= mikrotero: + self.data[i][set_column_idx] = set_value + elif re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+\s+or\s+\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition): + + + conditions = condition.split("or") + set_column_idx = self.column_names.index(set_column) + for condition_ in conditions: + column_name, operator, value = self._parse_condition(condition_) + column = self.column_by_name(column_name) + + for row_ind, column_value in enumerate(column): + if get_op(operator, column_value, value): + self.data[row_ind][set_column_idx] = set_value + + + elif re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+\s+and\s+\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition): + column_name = condition.split()[0] + conditions = condition.split("and") + set_column_idx = self.column_names.index(set_column) + rows_L=[] + for condition_ in conditions: + column_name, operator, value = self._parse_condition(condition_) + column = self.column_by_name(column_name) + + rows_L.append([ind for ind, x in enumerate(column) if get_op(operator, x, value)]) + + print(rows_L) + + rows = set(rows_L[0]).intersection(*rows_L) + + for row in rows: + self.data[row][set_column_idx] = set_value + + else: + raise("invalid where condition") + #else: + #rows = [i for i in range(len(self.data))] # parse the condition - column_name, operator, value = self._parse_condition(condition) + ###column_name, operator, value = self._parse_condition(condition) # get the condition and the set column - column = self.column_by_name(column_name) - set_column_idx = self.column_names.index(set_column) + ### column = self.column_by_name(column_name) + ### set_column_idx = self.column_names.index(set_column) # set_columns_indx = [self.column_names.index(set_column_name) for set_column_name in set_column_names] # for each value in column, if condition, replace it with set_value - for row_ind, column_value in enumerate(column): - if get_op(operator, column_value, value): - self.data[row_ind][set_column_idx] = set_value + ### for row_ind, column_value in enumerate(column): + ### if get_op(operator, column_value, value): + #### self.data[row_ind][set_column_idx] = set_value # self._update() # print(f"Updated {len(indexes_to_del)} rows") @@ -182,15 +240,59 @@ def _delete_where(self, condition): 'value[<,<=,==,>=,>]column'. Operatores supported: (<,<=,==,>=,>) + ''' - column_name, operator, value = self._parse_condition(condition) - indexes_to_del = [] + if re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition) or condition.startswith("not "):#simple condition or not + column_name, operator, value = self._parse_condition(condition) + column = self.column_by_name(column_name) + for index, row_value in enumerate(column): + if get_op(operator, row_value, value): + indexes_to_del.append(index) + elif re.match(r"^\w+\s+between\s+\w+\s+and\s+\w+$", condition): + + query = condition.split() + index = query.index("between") + megalutero = query[index+1] + mikrotero = query[index+3] + column_name = query[index-1] + column = self.column_by_name(column_name) + + for i,j in enumerate(column): + if str(j) >= megalutero and str(j) <= mikrotero: + indexes_to_del.append(i) + elif re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+\s+or\s+\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition): + + column_name = condition.split()[0] + conditions = condition.split(" or ") + rows_L=[] + for condition_ in conditions: + + column_name, operator, value = self._parse_condition(condition_) + column = self.column_by_name(column_name) + rows_L.append([ind for ind, x in enumerate(column) if get_op(operator, x, value)]) - column = self.column_by_name(column_name) - for index, row_value in enumerate(column): - if get_op(operator, row_value, value): - indexes_to_del.append(index) + + for rlist in rows_L: + for index in rlist: + indexes_to_del.append(index) + elif re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+\s+and\s+\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition): + column_name = condition.split()[0] + conditions = condition.split(" and ") + rows_L=[] + for condition_ in conditions: + column_name, operator, value = self._parse_condition(condition_) + column = self.column_by_name(column_name) + rows_L.append([ind for ind, x in enumerate(column) if get_op(operator, x, value)]) + + + indexes_to_del = set(rows_L[0]).intersection(*rows_L) + rows = list(rows) + + else: + raise("invalid where condition") + + # we pop from highest to lowest index in order to avoid removing the wrong item # since we dont delete, we dont have to to pop in that order, but since delete is used @@ -234,29 +336,25 @@ def _select_where(self, return_columns, condition=None, distinct=False, order_by # if condition is None, return all rows # if not, return the rows with values where condition is met for value if condition is not None: - #Regex για τα σύμβολα ή για το αν υπάρχει not μέσα + if re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition) or condition.startswith("not "):#simple condition or not column_name, operator, value = self._parse_condition(condition) column = self.column_by_name(column_name) rows = [ind for ind, x in enumerate(column) if get_op(operator, x, value)] elif re.match(r"^\w+\s+between\s+\w+\s+and\s+\w+$", condition): - #Regex για να παίρνουμε το between - #Σπάμε το condition + query = condition.split() - #Αναθέτουμε στο index το index του between (δλδ που βρίσκεται στο query) index = query.index("between") - #megalutero είναι η 2η συνθήκη ενώ μικρότερο είναι η 1η π.χ select * from table where column between x and y - #Το megalutero = x , το μικρότερο = y και το column_name = column megalutero = query[index+1] mikrotero = query[index+3] column_name = query[index-1] column = self.column_by_name(column_name) rows = [] for i,j in enumerate(column): - if int(j) >= int(megalutero) and int(j) <= int(mikrotero): + if j >= megalutero and j <= mikrotero: rows.append(i) elif re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+\s+or\s+\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition): - #σπάμε τα δύο conditions και για καθένα βάζουμε τα rows που θα επιστραφούν σε ένα list. Στο τέλος συνδέουμε τα δυο lists που θα γυρίσουν + column_name = condition.split()[0] conditions = condition.split("or") rows_L=[] @@ -270,7 +368,6 @@ def _select_where(self, return_columns, condition=None, distinct=False, order_by for row in rlist: rows.append(row) elif re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+\s+and\s+\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition): - #σπάμε τα δύο conditions και για καθένα βάζουμε τα rows που θα επιστραφούν σε ένα list. Στο τέλος πέρνουμε τα κοινα των δυο λιστών column_name = condition.split()[0] conditions = condition.split("and") rows_L=[] @@ -607,11 +704,18 @@ def _parse_condition(self, condition, join=False): # cast the value with the specified column's type and return the column name, the operator and the casted value left, op, right = split_condition(condition) - if left not in self.column_names: + if right in self.column_names:#'value[<,<=,==,>=,>]column' fromat + coltype = self.column_types[self.column_names.index(right)] + op =reverse_op(op) + return right, op, coltype(left) + + elif left in self.column_names:#'column[<,<=,==,>=,>]value' format + coltype = self.column_types[self.column_names.index(left)] + + return left, op, coltype(right) + else: + #raise ValueError(f'Condition is not valid (cant find column name)') raise ValueError(f'Condition is not valid (cant find column name)') - coltype = self.column_types[self.column_names.index(left)] - - return left, op, coltype(right) def _load_from_file(self, filename): From 15b6cdbc5b089927111dab0f051b97790ff4c7bb Mon Sep 17 00:00:00 2001 From: Kapiniaris-Xrisovalantis <92021533+Kapiniaris-Xrisovalantis@users.noreply.github.com> Date: Sun, 19 Feb 2023 19:54:45 +0200 Subject: [PATCH 05/16] Delete ' --- ' | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 ' diff --git a/' b/' deleted file mode 100644 index e69de29b..00000000 From 140ea86f42ad34803e26596fb8445373d9bb6f43 Mon Sep 17 00:00:00 2001 From: Kapiniaris-Xrisovalantis <92021533+Kapiniaris-Xrisovalantis@users.noreply.github.com> Date: Mon, 4 Sep 2023 10:46:41 +0300 Subject: [PATCH 06/16] Add files via upload updated the where statements for select,delete,update and search throught btree, added unique support, added btree unique support and some hash index support --- btree.py | 350 ++++++++++++++++++ database.py | 873 ++++++++++++++++++++++++++++++++++++++++++++ hashidx.py | 82 +++++ mdb.py | 50 ++- table.py | 1005 +++++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 2356 insertions(+), 4 deletions(-) create mode 100644 btree.py create mode 100644 database.py create mode 100644 hashidx.py create mode 100644 table.py diff --git a/btree.py b/btree.py new file mode 100644 index 00000000..6b3a6fdc --- /dev/null +++ b/btree.py @@ -0,0 +1,350 @@ +''' +https://en.wikipedia.org/wiki/B%2B_tree +''' + +class Node: + ''' + Node abstraction. Represents a single bucket + ''' + def __init__(self, b, values=None, ptrs=None,left_sibling=None, right_sibling=None, parent=None, is_leaf=False): + self.b = b # branching factor + self.values = [] if values is None else values # Values (the data from the pk column) + self.ptrs = [] if ptrs is None else ptrs # ptrs (the indexes of each datapoint or the index of another bucket) + self.left_sibling = left_sibling # the index of a buckets left sibling + self.right_sibling = right_sibling # the index of a buckets right sibling + self.parent = parent # the index of a buckets parent + self.is_leaf = is_leaf # a boolean value signaling whether the node is a leaf or not + + + def find(self, value, return_ops=False): + ''' + Returns the index of the next node to search for a value if the node is not a leaf (a ptrs of the available ones). + If it is a leaf (we have found the appropriate node), nothing is returned. + + Args: + value: float. The value being searched for. + return_ops: boolean. Set to True if you want to use the number of operations (for benchmarking). + ''' + ops = 0 # number of operations (<>= etc). Used for benchmarking + if self.is_leaf: # + return + + # for each value in the node, if the user supplied value is smaller, return the btrees value index + # else (no value in the node is larger) return the last ptr + for index, existing_val in enumerate(self.values): + ops+=1 + if value is None or existing_val is None: + continue + if value= etc). Used for benchmarking + + #start with the root node + node = self.nodes[self.root] + # while the node that we are searching in is not a leaf + # keep searching + while not node.is_leaf: + idx, ops1 = node.find(value, return_ops=True) + node = self.nodes[idx] + ops += ops1 + + # finally return the index of the appropriate node (and the ops if you want to) + if return_ops: + return self.nodes.index(node), ops + else: + return self.nodes.index(node) + + + def split(self, node_id): + ''' + Split the node with index=node_id. + + Args: + node_id: float. The corresponding ID of the node. + ''' + # fetch the node to be split + node = self.nodes[node_id] + # the value that will be propagated to the parent is the middle one. + new_parent_value = node.values[len(node.values)//2] + if node.is_leaf: + # if the node is a leaf, the parent value should be a part of the new node (right) + # Important: in a b+tree, every value should appear in a leaf + right_values = node.values[len(node.values)//2:] + right_ptrs = node.ptrs[len(node.ptrs)//2:] + + # create the new node with the right half of the old nodes values and ptrs (including the middle ones) + right = Node(self.b, right_values, right_ptrs,\ + left_sibling=node_id, right_sibling=node.right_sibling, parent=node.parent, is_leaf=node.is_leaf) + # since the new node (right) will be the next one to be appended to the nodes list + # its index will be equal to the length of the nodes list. + # Thus we set the old nodes (now left) right sibling to the right nodes future index (len of nodes) + if node.right_sibling is not None: + self.nodes[node.right_sibling].left_sibling = len(self.nodes) + node.right_sibling = len(self.nodes) + + + else: + # if the node is not a leaf, the parent value shoudl NOT be part of the new node + right_values = node.values[len(node.values)//2+1:] + if self.b%2==1: + right_ptrs = node.ptrs[len(node.ptrs)//2:] + else: + right_ptrs = node.ptrs[len(node.ptrs)//2+1:] + + # if nonleafs should be connected change the following two lines and add siblings + right = Node(self.b, right_values, right_ptrs,\ + parent=node.parent, is_leaf=node.is_leaf) + # make sure that a non leaf node doesnt have a parent + node.right_sibling = None + # the right node's kids should have him as a parent (if not all nodes will have left as parent) + for ptr in right_ptrs: + self.nodes[ptr].parent = len(self.nodes) + + # old node (left) keeps only the first half of the values/ptrs + node.values = node.values[:len(node.values)//2] + if self.b%2==1: + node.ptrs = node.ptrs[:len(node.ptrs)//2] + else: + node.ptrs = node.ptrs[:len(node.ptrs)//2+1] + + # append the new node (right) to the nodes list + self.nodes.append(right) + + # If the new nodes have no parents (a new level needs to be added + if node.parent is None: + # its the root that is split + # new root contains the parent value and ptrs to the two recently split nodes + parent = Node(self.b, [new_parent_value], [node_id, len(self.nodes)-1]\ + ,parent=node.parent, is_leaf=False) + + # set root, and parent of split celss to the index of the new root node (len of nodes-1) + self.nodes.append(parent) + self.root = len(self.nodes)-1 + node.parent = len(self.nodes)-1 + right.parent = len(self.nodes)-1 + else: + # insert the parent value to the parent + + self.nodes[node.parent].insert(new_parent_value, len(self.nodes)-1) + # check whether the parent needs to be split + if len(self.nodes[node.parent].values)==self.b: + self.split(node.parent) + + + + + def show(self): + ''' + Show important info for each node (sort by level - root first, then left to right). + ''' + nds = [] + nds.append(self.root) + for ptr in nds: + if self.nodes[ptr].is_leaf: + continue + nds.extend(self.nodes[ptr].ptrs) + + for ptr in nds: + print(f'## {ptr} ##') + self.nodes[ptr].show() + print('----') + + + def plot(self): + ## arrange the nodes top to bottom left to right + nds = [] + nds.append(self.root) + for ptr in nds: + if self.nodes[ptr].is_leaf: + continue + nds.extend(self.nodes[ptr].ptrs) + + # add each node and each link + g = 'digraph G{\nforcelabels=true;\n' + + for i in nds: + node = self.nodes[i] + g+=f'{i} [label="{node.values}"]\n' + if node.is_leaf: + continue + # if node.left_sibling is not None: + # g+=f'"{node.values}"->"{self.nodes[node.left_sibling].values}" [color="blue" constraint=false];\n' + # if node.right_sibling is not None: + # g+=f'"{node.values}"->"{self.nodes[node.right_sibling].values}" [color="green" constraint=false];\n' + # + # g+=f'"{node.values}"->"{self.nodes[node.parent].values}" [color="red" constraint=false];\n' + else: + for child in node.ptrs: + g+=f'{child} [label="{self.nodes[child].values}"]\n' + g+=f'{i}->{child};\n' + g +="}" + + try: + from graphviz import Source + src = Source(g) + src.render('bplustree', view=True) + except ImportError: + print('"graphviz" package not found. Writing to graph.gv.') + with open('graph.gv','w') as f: + f.write(g) + + def find(self, operator, value): + ''' + Return ptrs of elements where btree_value"operator"value. + Important, the user supplied "value" is the right value of the operation. That is why the operation are reversed below. + The left value of the op is the btree value. + + Args: + operator: string. The provided evaluation operator. + value: float. The value being searched for. + ''' + results = [] + # find the index of the node that the element should exist in + leaf_idx, ops = self._search(value, True) + target_node = self.nodes[leaf_idx] + + if operator == '=': + # if the element exist, append to list, else pass and return + try: + results.append(target_node.ptrs[target_node.values.index(value)]) + # print('Found') + except: + # print('Not found') + pass + + # for all other ops, the code is the same, only the operations themselves and the sibling indexes change + # for > and >= (btree value is >/>= of user supplied value), we return all the right siblings (all values are larger than current cell) + # for < and <= (btree value is ': + for idx, node_value in enumerate(target_node.values): + ops+=1 + + if node_value > value: + results.append(target_node.ptrs[idx]) + while target_node.right_sibling is not None: + target_node = self.nodes[target_node.right_sibling] + results.extend(target_node.ptrs) + + + if operator == '>=': + for idx, node_value in enumerate(target_node.values): + ops+=1 + if node_value >= value: + results.append(target_node.ptrs[idx]) + while target_node.right_sibling is not None: + target_node = self.nodes[target_node.right_sibling] + results.extend(target_node.ptrs) + + if operator == '<': + for idx, node_value in enumerate(target_node.values): + ops+=1 + if node_value < value: + results.append(target_node.ptrs[idx]) + while target_node.left_sibling is not None: + target_node = self.nodes[target_node.left_sibling] + results.extend(target_node.ptrs) + + if operator == '<=': + for idx, node_value in enumerate(target_node.values): + ops+=1 + if node_value <= value: + results.append(target_node.ptrs[idx]) + while target_node.left_sibling is not None: + target_node = self.nodes[target_node.left_sibling] + results.extend(target_node.ptrs) + + # print the number of operations (usefull for benchamrking) + # print(f'With BTree -> {ops} comparison operations') + return results diff --git a/database.py b/database.py new file mode 100644 index 00000000..054f6cc4 --- /dev/null +++ b/database.py @@ -0,0 +1,873 @@ +from __future__ import annotations +import pickle +from time import sleep, localtime, strftime +import os,sys +import logging +import warnings +#import pyreadline +import re +from tabulate import tabulate + +sys.path.append(f'{os.path.dirname(os.path.dirname(os.path.abspath(__file__)))}/miniDB') +from miniDB import table +sys.modules['table'] = table + +from hashidx import ExtendibleHashing as hash +from joins import Inlj, Smj +from btree import Btree +from misc import split_condition +from table import Table + + +# readline.clear_history() + +class Database: + ''' + Main Database class, containing tables. + ''' + + def __init__(self, name, load=True, verbose = True): + self.tables = {} + self._name = name + self.verbose = verbose + + self.savedir = f'dbdata/{name}_db' + + if load: + try: + self.load_database() + logging.info(f'Loaded "{name}".') + return + except: + if verbose: + warnings.warn(f'Database "{name}" does not exist. Creating new.') + + # create dbdata directory if it doesnt exist + if not os.path.exists('dbdata'): + os.mkdir('dbdata') + + # create new dbs save directory + try: + os.mkdir(self.savedir) + except: + pass + + # create all the meta tables + self.create_table('meta_length', 'table_name,no_of_rows', 'str,int') + self.create_table('meta_locks', 'table_name,pid,mode', 'str,int,str') + self.create_table('meta_insert_stack', 'table_name,indexes', 'str,list') + self.create_table('meta_indexes', 'table_name,index_name,index_type,column', 'str,str,str,str') + self.save_database() + + def save_database(self): + ''' + Save database as a pkl file. This method saves the database object, including all tables and attributes. + ''' + for name, table in self.tables.items(): + with open(f'{self.savedir}/{name}.pkl', 'wb') as f: + pickle.dump(table, f) + + def _save_locks(self): + ''' + Stores the meta_locks table to file as meta_locks.pkl. + ''' + with open(f'{self.savedir}/meta_locks.pkl', 'wb') as f: + pickle.dump(self.tables['meta_locks'], f) + + def load_database(self): + ''' + Load all tables that are part of the database (indices noted here are loaded). + + Args: + path: string. Directory (path) of the database on the system. + ''' + path = f'dbdata/{self._name}_db' + for file in os.listdir(path): + + if file[-3:]!='pkl': # if used to load only pkl files + continue + f = open(path+'/'+file, 'rb') + tmp_dict = pickle.load(f) + f.close() + name = f'{file.split(".")[0]}' + self.tables.update({name: tmp_dict}) + # setattr(self, name, self.tables[name]) + + #### IO #### + + def _update(self): + ''' + Update all meta tables. + ''' + self._update_meta_length() + self._update_meta_insert_stack() + + + def create_table(self, name, column_names, column_types, primary_key=None, unique=None,load=None): + ''' + This method create a new table. This table is saved and can be accessed via db_object.tables['table_name'] or db_object.table_name + + Args: + name: string. Name of table. + column_names: list. Names of columns. + column_types: list. Types of columns. + primary_key: string. The primary key (if it exists). + load: boolean. Defines table object parameters as the name of the table and the column names. + ''' + # print('here -> ', column_names.split(',')) + self.tables.update({name: Table(name=name, column_names=column_names.split(','), column_types=column_types.split(','), primary_key=primary_key,unique=unique, load=load)}) + # self._name = Table(name=name, column_names=column_names, column_types=column_types, load=load) + # check that new dynamic var doesnt exist already + # self.no_of_tables += 1 + self._update() + self.save_database() + # (self.tables[name]) + if self.verbose: + print(f'Created table "{name}".') + + + def drop_table(self, table_name): + ''' + Drop table from current database. + + Args: + table_name: string. Name of table. + ''' + self.load_database() + self.lock_table(table_name) + + self.tables.pop(table_name) + if os.path.isfile(f'{self.savedir}/{table_name}.pkl'): + os.remove(f'{self.savedir}/{table_name}.pkl') + else: + warnings.warn(f'"{self.savedir}/{table_name}.pkl" not found.') + self.delete_from('meta_locks', f'table_name={table_name}') + self.delete_from('meta_length', f'table_name={table_name}') + self.delete_from('meta_insert_stack', f'table_name={table_name}') + + if self._has_index(table_name): + to_be_deleted = [] + for key, table in enumerate(self.tables['meta_indexes'].column_by_name('table_name')): + if table == table_name: + to_be_deleted.append(key) + + for i in reversed(to_be_deleted): + self.drop_index(self.tables['meta_indexes'].data[i][1]) + + try: + delattr(self, table_name) + except AttributeError: + pass + # self._update() + self.save_database() + + + def import_table(self, table_name, filename, column_types=None, primary_key=None, unique=None): + ''' + Creates table from CSV file. + + Args: + filename: string. CSV filename. If not specified, filename's name will be used. + column_types: list. Types of columns. If not specified, all will be set to type str. + primary_key: string. The primary key (if it exists). + ''' + file = open(filename, 'r') + + first_line=True + for line in file.readlines(): + if first_line: + colnames = line.strip('\n') + if column_types is None: + column_types = ",".join(['str' for _ in colnames.split(',')]) + self.create_table(name=table_name, column_names=colnames, column_types=column_types, primary_key=primary_key,unique=unique) + lock_ownership = self.lock_table(table_name, mode='x') + first_line = False + continue + self.tables[table_name]._insert(line.strip('\n').split(',')) + + if lock_ownership: + self.unlock_table(table_name) + self._update() + self.save_database() + + + def export(self, table_name, filename=None): + ''' + Transform table to CSV. + + Args: + table_name: string. Name of table. + filename: string. Output CSV filename. + ''' + res = '' + for row in [self.tables[table_name].column_names]+self.tables[table_name].data: + res+=str(row)[1:-1].replace('\'', '').replace('"','').replace(' ','')+'\n' + + if filename is None: + filename = f'{table_name}.csv' + + with open(filename, 'w') as file: + file.write(res) + + def table_from_object(self, new_table): + ''' + Add table object to database. + + Args: + new_table: string. Name of new table. + ''' + + self.tables.update({new_table._name: new_table}) + if new_table._name not in self.__dir__(): + setattr(self, new_table._name, new_table) + else: + raise Exception(f'"{new_table._name}" attribute already exists in class "{self.__class__.__name__}".') + self._update() + self.save_database() + + + + ##### table functions ##### + + # In every table function a load command is executed to fetch the most recent table. + # In every table function, we first check whether the table is locked. Since we have implemented + # only the X lock, if the tables is locked we always abort. + # After every table function, we update and save. Update updates all the meta tables and save saves all + # tables. + + # these function calls are named close to the ones in postgres + + def cast(self, column_name, table_name, cast_type): + ''' + Modify the type of the specified column and cast all prexisting values. + (Executes type() for every value in column and saves) + + Args: + table_name: string. Name of table (must be part of database). + column_name: string. The column that will be casted (must be part of database). + cast_type: type. Cast type (do not encapsulate in quotes). + ''' + self.load_database() + + lock_ownership = self.lock_table(table_name, mode='x') + self.tables[table_name]._cast_column(column_name, eval(cast_type)) + if lock_ownership: + self.unlock_table(table_name) + self._update() + self.save_database() + + def insert_into(self, table_name, row_str): + ''' + Inserts data to given table. + + Args: + table_name: string. Name of table (must be part of database). + row: list. A list of values to be inserted (will be casted to a predifined type automatically). + lock_load_save: boolean. If False, user needs to load, lock and save the states of the database (CAUTION). Useful for bulk-loading. + ''' + row = row_str.strip().split(',') + self.load_database() + # fetch the insert_stack. For more info on the insert_stack + # check the insert_stack meta table + + lock_ownership = self.lock_table(table_name, mode='x') + insert_stack = self._get_insert_stack_for_table(table_name) + try: + self.tables[table_name]._insert(row, insert_stack) + except Exception as e: + logging.info(e) + logging.info('ABORTED') + + self._update_meta_insert_stack_for_tb(table_name, insert_stack[:-1]) + + if lock_ownership: + self.unlock_table(table_name) + self._update() + self.save_database() + + + def update_table(self, table_name, set_args, condition): + ''' + Update the value of a column where a condition is met. + + Args: + table_name: string. Name of table (must be part of database). + set_value: string. New value of the predifined column name. + set_column: string. The column to be altered. + condition: string. A condition using the following format: + 'column[<,<=,==,>=,>]value' or + 'value[<,<=,==,>=,>]column'. + + Operatores supported: (<,<=,==,>=,>) + ''' + set_column, set_value = set_args.replace(' ','').split('=') + self.load_database() + + lock_ownership = self.lock_table(table_name, mode='x') + self.tables[table_name]._update_rows(set_value, set_column, condition) + if lock_ownership: + self.unlock_table(table_name) + self._update() + self.save_database() + + def delete_from(self, table_name, condition): + ''' + Delete rows of table where condition is met. + + Args: + table_name: string. Name of table (must be part of database). + condition: string. A condition using the following format: + 'column[<,<=,==,>=,>]value' or + 'value[<,<=,==,>=,>]column'. + + Operatores supported: (<,<=,==,>=,>) + ''' + self.load_database() + + lock_ownership = self.lock_table(table_name, mode='x') + deleted = self.tables[table_name]._delete_where(condition) + if lock_ownership: + self.unlock_table(table_name) + self._update() + self.save_database() + # we need the save above to avoid loading the old database that still contains the deleted elements + if table_name[:4]!='meta': + self._add_to_insert_stack(table_name, deleted) + self.save_database() + + def select(self, columns, table_name, condition, distinct=None, order_by=None, \ + limit=True, desc=None, save_as=None, return_object=True): + ''' + Selects and outputs a table's data where condtion is met. + + Args: + table_name: string. Name of table (must be part of database). + columns: list. The columns that will be part of the output table (use '*' to select all available columns) + condition: string. A condition using the following format: + 'column[<,<=,==,>=,>]value' or + 'value[<,<=,==,>=,>]column'. + + Operatores supported: (<,<=,==,>=,>) + order_by: string. A column name that signals that the resulting table should be ordered based on it (no order if None). + desc: boolean. If True, order_by will return results in descending order (True by default). + limit: int. An integer that defines the number of rows that will be returned (all rows if None). + save_as: string. The name that will be used to save the resulting table into the database (no save if None). + return_object: boolean. If True, the result will be a table object (useful for internal use - the result will be printed by default). + distinct: boolean. If True, the resulting table will contain only unique rows. + ''' + + # print(table_name) + self.load_database() + if isinstance(table_name,Table): + return table_name._select_where(columns, condition, distinct, order_by, desc, limit) + + condition_column="" + if condition is not None: + if "between" in condition.split(): + condition_column = condition.split()[0] + elif "and" in condition.split() : + columns2=[] + conditions=[] + conditions=condition.split(" and ") + for con in conditions: + col=self.tables[table_name]._parse_condition(con) + columns2.append(col) + for con in columns2: + if (self.tables[table_name].pk_idx is not None and con[0] == self.tables[table_name].column_names[self.tables[table_name].pk_idx]) or (self.tables[table_name].unique is not None and con[0] in self.tables[table_name].unique ):#since only pk supports index , and only one pk per table we keep the column if it is pk + condition_column = con[0] + else: + condition_column="" + + elif "or" in condition.split(): + + columns2=[] + conditions=[] + conditions=condition.split(" or ") + for con in conditions: + + col=self.tables[table_name]._parse_condition(con) + + columns2.append(col) + + for con in columns2: + + + if self.tables[table_name].pk_idx is not None and con[0] == self.tables[table_name].column_names[self.tables[table_name].pk_idx]: + condition_column = con[0] + else: + condition_column = "" + else: + + col,op=self.tables[table_name]._parse_condition(condition) + if op=="=": + condition_column=col + + + + + + + + # self.lock_table(table_name, mode='x') + if self.is_locked(table_name): + return + + if self.tables[table_name].pk_idx is not None and self._has_index(table_name,condition_column): + print("here") + index_name = self.select('*', 'meta_indexes', f'table_name={table_name}',return_object=True).column_by_name('index_name')[0] + index_type = self.select('*', 'meta_indexes', f'table_name={table_name}',return_object=True).column_by_name('index_type')[0] + if index_type=="btree": + bt = self._load_idx(index_name) + table = self.tables[table_name]._select_where_with_btree(columns, bt, condition, distinct, order_by, desc, limit) + elif index_type=="hash": + bt = self._load_idx(index_name) + table = self.tables[table_name]._select_where_with_hash(columns, bt, condition, distinct, order_by, desc, limit) + + elif self.tables[table_name].unique_idx is not None and self._has_index(table_name): + + found = False + for j in range (len(self.tables[table_name].unique)): + + if self.tables[table_name].unique[j] in condition_column: + found = True + index_name = self.select('*', 'meta_indexes', f'table_name={table_name}', return_object=True).column_by_name('index_name')[0] + print(index_name) + bt = self._load_idx(index_name) + bt.show() + table = self.tables[table_name]._select_where_with_btree(columns, bt, condition, distinct, order_by, desc, limit) + break + if not found: + + table = self.tables[table_name]._select_where(columns, condition, distinct, order_by, desc, limit) + else: + + table = self.tables[table_name]._select_where(columns, condition, distinct, order_by, desc, limit) + # self.unlock_table(table_name) + if save_as is not None: + table._name = save_as + self.table_from_object(table) + else: + if return_object: + return table + else: + return table.show() + + + def show_table(self, table_name, no_of_rows=None): + ''' + Print table in a readable tabular design (using tabulate). + + Args: + table_name: string. Name of table (must be part of database). + ''' + self.load_database() + + self.tables[table_name].show(no_of_rows, self.is_locked(table_name)) + + + def sort(self, table_name, column_name, asc=False): + ''' + Sorts a table based on a column. + + Args: + table_name: string. Name of table (must be part of database). + column_name: string. the column name that will be used to sort. + asc: If True sort will return results in ascending order (False by default). + ''' + + self.load_database() + + lock_ownership = self.lock_table(table_name, mode='x') + self.tables[table_name]._sort(column_name, asc=asc) + if lock_ownership: + self.unlock_table(table_name) + self._update() + self.save_database() + + def create_view(self, table_name, table): + ''' + Create a virtual table based on the result-set of the SQL statement provided. + + Args: + table_name: string. Name of the table that will be saved. + table: table. The table that will be saved. + ''' + table._name = table_name + self.table_from_object(table) + + def join(self, mode, left_table, right_table, condition, save_as=None, return_object=True): + ''' + Join two tables that are part of the database where condition is met. + + Args: + left_table: string. Name of the left table (must be in DB) or Table obj. + right_table: string. Name of the right table (must be in DB) or Table obj. + condition: string. A condition using the following format: + 'column[<,<=,==,>=,>]value' or + 'value[<,<=,==,>=,>]column'. + + Operators supported: (<,<=,==,>=,>) + save_as: string. The output filename that will be used to save the resulting table in the database (won't save if None). + return_object: boolean. If True, the result will be a table object (useful for internal usage - the result will be printed by default). + ''' + self.load_database() + if self.is_locked(left_table) or self.is_locked(right_table): + return + + left_table = left_table if isinstance(left_table, Table) else self.tables[left_table] + right_table = right_table if isinstance(right_table, Table) else self.tables[right_table] + + + if mode=='inner': + res = left_table._inner_join(right_table, condition) + + elif mode=='left': + res = left_table._left_join(right_table, condition) + + elif mode=='right': + res = left_table._right_join(right_table, condition) + + elif mode=='full': + res = left_table._full_join(right_table, condition) + + elif mode=='inl': + # Check if there is an index of either of the two tables available, as if there isn't we can't use inlj + leftIndexExists = self._has_index(left_table._name) + rightIndexExists = self._has_index(right_table._name) + + if not leftIndexExists and not rightIndexExists: + res = None + raise Exception('Index-nested-loop join cannot be executed. Use inner join instead.\n') + elif rightIndexExists: + index_name = self.select('*', 'meta_indexes', f'table_name={right_table._name}', return_object=True).column_by_name('index_name')[0] + res = Inlj(condition, left_table, right_table, self._load_idx(index_name), 'right').join() + elif leftIndexExists: + index_name = self.select('*', 'meta_indexes', f'table_name={left_table._name}', return_object=True).column_by_name('index_name')[0] + res = Inlj(condition, left_table, right_table, self._load_idx(index_name), 'left').join() + + elif mode=='sm': + res = Smj(condition, left_table, right_table).join() + + else: + raise NotImplementedError + + if save_as is not None: + res._name = save_as + self.table_from_object(res) + else: + if return_object: + return res + else: + res.show() + + if return_object: + return res + else: + res.show() + + def lock_table(self, table_name, mode='x'): + ''' + Locks the specified table using the exclusive lock (X). + + Args: + table_name: string. Table name (must be part of database). + ''' + if table_name[:4]=='meta' or table_name not in self.tables.keys() or isinstance(table_name,Table): + return + + with open(f'{self.savedir}/meta_locks.pkl', 'rb') as f: + self.tables.update({'meta_locks': pickle.load(f)}) + + try: + pid = self.tables['meta_locks']._select_where('pid',f'table_name={table_name}').data[0][0] + if pid!=os.getpid(): + raise Exception(f'Table "{table_name}" is locked by process with pid={pid}') + else: + return False + + except IndexError: + pass + + if mode=='x': + self.tables['meta_locks']._insert([table_name, os.getpid(), mode]) + else: + raise NotImplementedError + self._save_locks() + return True + # print(f'Locking table "{table_name}"') + + def unlock_table(self, table_name, force=False): + ''' + Unlocks the specified table that is exclusively locked (X). + + Args: + table_name: string. Table name (must be part of database). + ''' + if table_name not in self.tables.keys(): + raise Exception(f'Table "{table_name}" is not in database') + + if not force: + try: + # pid = self.select('*','meta_locks', f'table_name={table_name}', return_object=True).data[0][1] + pid = self.tables['meta_locks']._select_where('pid',f'table_name={table_name}').data[0][0] + if pid!=os.getpid(): + raise Exception(f'Table "{table_name}" is locked by the process with pid={pid}') + except IndexError: + pass + self.tables['meta_locks']._delete_where(f'table_name={table_name}') + self._save_locks() + # print(f'Unlocking table "{table_name}"') + + def is_locked(self, table_name): + ''' + Check whether the specified table is exclusively locked (X). + + Args: + table_name: string. Table name (must be part of database). + ''' + if isinstance(table_name,Table) or table_name[:4]=='meta': # meta tables will never be locked (they are internal) + return False + + with open(f'{self.savedir}/meta_locks.pkl', 'rb') as f: + self.tables.update({'meta_locks': pickle.load(f)}) + + try: + pid = self.tables['meta_locks']._select_where('pid',f'table_name={table_name}').data[0][0] + if pid!=os.getpid(): + raise Exception(f'Table "{table_name}" is locked by the process with pid={pid}') + + except IndexError: + pass + return False + + + #### META #### + + # The following functions are used to update, alter, load and save the meta tables. + # Important: Meta tables contain info regarding the NON meta tables ONLY. + # i.e. meta_length will not show the number of rows in meta_locks etc. + + def _update_meta_length(self): + ''' + Updates the meta_length table. + ''' + + for table in self.tables.values(): + + if table._name[:4]=='meta': #skip meta tables + continue + if table._name not in self.tables['meta_length'].column_by_name('table_name'): # if new table, add record with 0 no. of rows + self.tables['meta_length']._insert([table._name, 0]) + + # the result needs to represent the rows that contain data. Since we use an insert_stack + # some rows are filled with Nones. We skip these rows. + non_none_rows = len([row for row in table.data if any(row)]) + self.tables['meta_length']._update_rows(non_none_rows, 'no_of_rows', f'table_name={table._name}') + # self.update_row('meta_length', len(table.data), 'no_of_rows', 'table_name', '==', table._name) + + def _update_meta_locks(self): + ''' + Updates the meta_locks table. + ''' + for table in self.tables.values(): + if table._name[:4]=='meta': #skip meta tables + continue + if table._name not in self.tables['meta_locks'].column_by_name('table_name'): + + self.tables['meta_locks']._insert([table._name, False]) + # self.insert('meta_locks', [table._name, False]) + + def _update_meta_insert_stack(self): + ''' + Updates the meta_insert_stack table. + ''' + for table in self.tables.values(): + if table._name[:4]=='meta': #skip meta tables + continue + if table._name not in self.tables['meta_insert_stack'].column_by_name('table_name'): + self.tables['meta_insert_stack']._insert([table._name, []]) + + + def _add_to_insert_stack(self, table_name, indexes): + ''' + Adds provided indices to the insert stack of the specified table. + + Args: + table_name: string. Table name (must be part of database). + indexes: list. The list of indices that will be added to the insert stack (the indices of the newly deleted elements). + ''' + old_lst = self._get_insert_stack_for_table(table_name) + self._update_meta_insert_stack_for_tb(table_name, old_lst+indexes) + + def _get_insert_stack_for_table(self, table_name): + ''' + Returns the insert stack of the specified table. + + Args: + table_name: string. Table name (must be part of database). + ''' + return self.tables['meta_insert_stack']._select_where('*', f'table_name={table_name}').column_by_name('indexes')[0] + # res = self.select('meta_insert_stack', '*', f'table_name={table_name}', return_object=True).indexes[0] + # return res + + def _update_meta_insert_stack_for_tb(self, table_name, new_stack): + ''' + Replaces the insert stack of a table with the one supplied by the user. + + Args: + table_name: string. Table name (must be part of database). + new_stack: string. The stack that will be used to replace the existing one. + ''' + self.tables['meta_insert_stack']._update_rows(new_stack, 'indexes', f'table_name={table_name}') + + + # indexes + def create_index(self, index_name, table_name,index_type='btree' ,column='pkey'): + ''' + Creates an index on a specified table with a given name. + Important: An index can only be created on a primary key (the user does not specify the column). + + Args: + table_name: string. Table name (must be part of database). + index_name: string. Name of the created index. + ''' + #if the user didn't specify a column, make the index on primary key + if self.tables[table_name].pk_idx is not None and column=='pkey': + column = self.tables[table_name].pk + + + if self.tables[table_name].pk_idx is None and self.tables[table_name].unique is None: # if no primary key, no index + raise Exception('Cannot create index. Table has no primary key or unique columns.') + + if index_name not in self.tables['meta_indexes'].column_by_name('index_name'): + # currently only btree is supported. This can be changed by adding another if. + if index_type=='btree': + logging.info('Creating Btree index.') + print('Creating BTREE index') + # insert a record with the name of the index and the table on which it's created to the meta_indexes table + self.tables['meta_indexes']._insert([table_name, index_name,index_type,column]) + # crate the actual index + self._construct_index(table_name, index_name,column,index_type) + #self._construct_index(table_name, index_name) + self.save_database() + if index_type=='hash': + logging.info('Creating Hash index.') + print('Creating HASH index') + # insert a record with the name of the index and the table on which it's created to the meta_indexes table + self.tables['meta_indexes']._insert([table_name, index_name,index_type,column]) + # crate the actual index + self._construct_index(table_name, index_name,column,index_type) + #self._construct_index(table_name, index_name) + self.save_database() + else: + raise Exception('Cannot create index. Another index with the same name already exists.') + + def _construct_index(self, table_name, index_name,column,index_type='btree'): + ''' + Construct a btree on a table and save. + + Args: + table_name: string. Table name (must be part of database). + index_name: string. Name of the created index. + ''' + if index_type=='btree': + bt = Btree(1) # 3 is arbitrary + + # for each record in the primary key of the table, insert its value and index to the btree + if self.tables[table_name].pk is not None and column==self.tables[table_name].pk: + for idx, key in enumerate(self.tables[table_name].column_by_name(column)): + if key is None: + continue + bt.insert(key, idx) + + elif self.tables[table_name].unique is not None and column in self.tables[table_name].unique: + + for idx, key in enumerate(self.tables[table_name].column_by_name(column)): + + if key is None: + continue + bt.insert(key, idx) + + else: + raise ValueError(f'##ERROR->{column} is not primary key or unique') + # save the btree + + self._save_index(index_name, bt) + elif index_type=='hash': + hi = hash(1) + if self.tables[table_name].pk is not None and column==self.tables[table_name].pk: + for idx, key in enumerate(self.tables[table_name].column_by_name(column)): + if key is None: + + continue + + hi.insert(idx, key)#idx will be used as the hashing key, which is the order that the values are in the table + #key is the value that will be inserted + elif self.tables[table_name].unique is not None and column in self.tables[table_name].unique: + + for idx, key in enumerate(self.tables[table_name].column_by_name(column)): + + if key is None: + continue + hi.insert(idx,key) + self._save_index(index_name, hi) + hi.show() + def _has_index(self, table_name, column): + ''' + Check whether the specified table's primary key column is indexed. + + Args: + table_name: string. Table name (must be part of the database). + column: string. Column name to check for indexing. + ''' + if table_name in self.tables['meta_indexes'].column_by_name('table_name') and column in self.tables['meta_indexes'].column_by_name('column'): + return True + else: + return False + + + def _save_index(self, index_name, index): + ''' + Save the index object. + + Args: + index_name: string. Name of the created index. + index: obj. The actual index object (btree object). + ''' + try: + os.mkdir(f'{self.savedir}/indexes') + except: + pass + + with open(f'{self.savedir}/indexes/meta_{index_name}_index.pkl', 'wb') as f: + pickle.dump(index, f) + + def _load_idx(self, index_name): + ''' + Load and return the specified index. + + Args: + index_name: string. Name of created index. + ''' + f = open(f'{self.savedir}/indexes/meta_{index_name}_index.pkl', 'rb') + index = pickle.load(f) + f.close() + return index + + def drop_index(self, index_name): + ''' + Drop index from current database. + + Args: + index_name: string. Name of index. + ''' + if index_name in self.tables['meta_indexes'].column_by_name('index_name'): + self.delete_from('meta_indexes', f'index_name = {index_name}') + + if os.path.isfile(f'{self.savedir}/indexes/meta_{index_name}_index.pkl'): + os.remove(f'{self.savedir}/indexes/meta_{index_name}_index.pkl') + else: + warnings.warn(f'"{self.savedir}/indexes/meta_{index_name}_index.pkl" not found.') + + self.save_database() + \ No newline at end of file diff --git a/hashidx.py b/hashidx.py new file mode 100644 index 00000000..1fa908a1 --- /dev/null +++ b/hashidx.py @@ -0,0 +1,82 @@ +class ExtendibleHashing: + def __init__(self, global_depth): + self.global_depth = global_depth + self.directory = {} + self.bucket_size = 4 # Number of elements in each bucket + + def hash_function(self, key): + return key % (2 ** self.global_depth) + + def insert(self, key, value): + hashed_key = self.hash_function(key) + if hashed_key in self.directory: + bucket = self.directory[hashed_key] + for i, (k, v) in enumerate(bucket): + if k == key: + bucket[i] = (k, value) # Update value for existing key + return + if len(bucket) < self.bucket_size: + bucket.append((key, value)) + else: + if self.global_depth == len(bin(hashed_key)) - 2: + self.double_directory() + self.split_bucket(hashed_key) + self.insert(key, value) + else: + if self.global_depth == len(bin(hashed_key)) - 2: + self.double_directory() + self.directory[hashed_key] = [(key, value)] + + + def double_directory(self): + self.global_depth += 1 + directory_size = 2 ** (self.global_depth - 1) + for i in range(directory_size): + if i in self.directory: + self.directory[i + directory_size] = self.directory[i] + else: + self.directory[i + directory_size] = [] + + def split_bucket(self, hashed_key): + bucket = self.directory[hashed_key] + new_bucket = [] + split_index = len(bucket) // 2 + + # Split the bucket into two by creating a new bucket and updating the directory + new_bucket = bucket[split_index:] + bucket = bucket[:split_index] + self.directory[hashed_key] = bucket + + # Update the hashed keys of the new bucket and its duplicates + new_hashed_key = hashed_key + (2 ** (self.global_depth - 1)) + for key in range(new_hashed_key, new_hashed_key + (2 ** (self.global_depth - 1)), 2 ** (self.global_depth - 1)): + self.directory[key] = new_bucket.copy() + + + + + def find(self, key): + hashed_key = self.hash_function(key) + if hashed_key in self.directory: + bucket = self.directory[hashed_key] + for item in bucket: + if item[0] == key: + return item[1] + return None + + def delete(self, key): + hashed_key = self.hash_function(key) + if hashed_key in self.directory: + bucket = self.directory[hashed_key] + for i, item in enumerate(bucket): + if item[0] == key: + del bucket[i] + return True + return False + + def show(self): + for hashed_key, bucket in self.directory.items(): + print(f"Hashed Key: {hashed_key}") + for item in bucket: + print(f" Key: {item[0]}, Value: {item[1]}") + diff --git a/mdb.py b/mdb.py index a981e5be..bb9323cc 100644 --- a/mdb.py +++ b/mdb.py @@ -2,7 +2,7 @@ import re from pprint import pprint import sys -import readline +#import pyreadline import traceback import shutil sys.path.append('miniDB') @@ -96,16 +96,39 @@ def create_query_plan(query, keywords, action): if action=='create table': args = dic['create table'][dic['create table'].index('('):dic['create table'].index(')')+1] dic['create table'] = dic['create table'].removesuffix(args).strip() + arg_nopk = args.replace('primary key', '')[1:-1] + arglist = [val.strip().split(' ') for val in arg_nopk.split(',')] + dic['column_names'] = ','.join([val[0] for val in arglist]) + dic['column_types'] = ','.join([val[1] for val in arglist]) if 'primary key' in args: - arglist = args[1:-1].split(' ') - dic['primary key'] = arglist[arglist.index('primary')-2] + arglist_has_pkey = args[1:-1].split(' ') + + dic['primary key'] = arglist_has_pkey[arglist_has_pkey.index('primary')-2] else: dic['primary key'] = None + + + unique_columns=[] + + + + for col in arglist: + if 'unique' in col: + + unique_columns.append(col[0]) + + if len(unique_columns)!=0: + dic['unique'] = ','.join(unique_columns) + else: + dic['unique']=None + + + if action=='import': dic = {'import table' if key=='import' else key: val for key, val in dic.items()} @@ -120,6 +143,23 @@ def create_query_plan(query, keywords, action): dic['force'] = True else: dic['force'] = False + + if action=='create index': + + args = dic['on'].split(' ') + dic['on'] = dic['on'].split(' ')[0] + if len(args) > 1: + + dic['column'] = args[2] + else: + dic['column'] = 'pkey' + args = dic['using'].split(' ') + dic['using'] = dic['using'].split(' ')[0] + if len(args) > 1: + dic['index_type'] = args[1] + + + return dic @@ -200,6 +240,7 @@ def execute_dic(dic): dic[key] = execute_dic(dic[key]) action = list(dic.keys())[0].replace(' ','_') + return getattr(db, action)(*dic.values()) def interpret_meta(command): @@ -251,7 +292,7 @@ def remove_db(db_name): dbname = os.getenv('DB') db = Database(dbname, load=True) - + if fname is not None: @@ -262,6 +303,7 @@ def remove_db(db_name): pprint(dic, sort_dicts=False) else : dic = interpret(line.lower()) + result = execute_dic(dic) if isinstance(result,Table): result.show() diff --git a/table.py b/table.py new file mode 100644 index 00000000..755d4585 --- /dev/null +++ b/table.py @@ -0,0 +1,1005 @@ +from __future__ import annotations +from tabulate import tabulate +import pickle +import os +import sys +import re + +sys.path.append(f'{os.path.dirname(os.path.dirname(os.path.abspath(__file__)))}/miniDB') + +from misc import get_op, split_condition, reverse_op + + +class Table: + ''' + Table object represents a table inside a database + + A Table object can be created either by assigning: + - a table name (string) + - column names (list of strings) + - column types (list of functions like str/int etc) + - primary (name of the primary key column) + + OR + + - by assigning a value to the variable called load. This value can be: + - a path to a Table file saved using the save function + - a dictionary that includes the appropriate info (all the attributes in __init__) + + ''' + def __init__(self, name=None, column_names=None, column_types=None, primary_key=None,unique=None, load=None): + + if load is not None: + + # if load is a dict, replace the object dict with it (replaces the object with the specified one) + if isinstance(load, dict): + self.__dict__.update(load) + # self._update() + # if load is str, load from a file + elif isinstance(load, str): + self._load_from_file(load) + + + # if name, columns_names and column types are not none + elif (name is not None) and (column_names is not None) and (column_types is not None): + + self._name = name + + if len(column_names)!=len(column_types): + raise ValueError('Need same number of column names and types.') + + self.column_names = column_names + + self.columns = [] + + for col in self.column_names: + if col not in self.__dir__(): + # this is used in order to be able to call a column using its name as an attribute. + # example: instead of table.columns['column_name'], we do table.column_name + setattr(self, col, []) + self.columns.append([]) + else: + raise Exception(f'"{col}" attribute already exists in "{self.__class__.__name__} "class.') + + self.column_types = [eval(ct) if not isinstance(ct, type) else ct for ct in column_types] + self.data = [] # data is a list of lists, a list of rows that is. + + # if primary key is set, keep its index as an attribute + if primary_key is not None: + self.pk_idx = self.column_names.index(primary_key) + + else: + self.pk_idx = None + + self.pk = primary_key + + self.unique_idx = [] + + if unique is not None: + self.unique = unique.split(',') + unique=unique.split(',') + for unq in unique: + if unq in self.column_names: + self.unique_idx.append(self.column_names.index(unq)) + else: + self.unique_idx=None + self.unique=None + + + # self._update() + + # if any of the name, columns_names and column types are none. return an empty table object + + def column_by_name(self, column_name): + return [row[self.column_names.index(column_name)] for row in self.data] + + + def _update(self): + ''' + Update all the available columns with the appended rows. + ''' + self.columns = [[row[i] for row in self.data] for i in range(len(self.column_names))] + for ind, col in enumerate(self.column_names): + setattr(self, col, self.columns[ind]) + + def _cast_column(self, column_name, cast_type): + ''' + Cast all values of a column using a specified type. + + Args: + column_name: string. The column that will be casted. + cast_type: type. Cast type (do not encapsulate in quotes). + ''' + # get the column from its name + column_idx = self.column_names.index(column_name) + # for every column's value in each row, replace it with itself but casted as the specified type + for i in range(len(self.data)): + self.data[i][column_idx] = cast_type(self.data[i][column_idx]) + # change the type of the column + self.column_types[column_idx] = cast_type + # self._update() + + + def _insert(self, row, insert_stack=[]): + ''' + Insert row to table. + + Args: + row: list. A list of values to be inserted (will be casted to a predifined type automatically). + insert_stack: list. The insert stack (empty by default). + ''' + + if len(row)!=len(self.column_names): + raise ValueError(f'ERROR -> Cannot insert {len(row)} values. Only {len(self.column_names)} columns exist') + + for i in range(len(row)): + # for each value, cast and replace it in row. + try: + row[i] = self.column_types[i](row[i]) + except ValueError: + if row[i] != 'NULL': + raise ValueError(f'ERROR -> Value {row[i]} of type {type(row[i])} is not of type {self.column_types[i]}.') + except TypeError as exc: + if row[i] != None: + print(exc) + + # if value is to be appended to the primary_key column, check that it doesnt alrady exist (no duplicate primary keys) + if i==self.pk_idx and row[i] in self.column_by_name(self.pk): + raise ValueError(f'## ERROR -> Value {row[i]} already exists in primary key column.') + + elif i==self.pk_idx and row[i] is None: + raise ValueError(f'ERROR -> The value of the primary key cannot be None.') + + + if self.unique_idx is not None: + if i in self.unique_idx: + for j in range (len(self.unique)): + if row[i] in self.column_by_name(self.unique[j]): + raise ValueError(f'## ERROR -> Value {row[i]} already exists in unique column.') + + + # if insert_stack is not empty, append to its last index + if insert_stack != []: + self.data[insert_stack[-1]] = row + else: # else append to the end + self.data.append(row) + # self._update() + + def _update_rows(self, set_value, set_column, condition): + ''' + Update where Condition is met. + + Args: + set_value: string. The provided set value. + set_column: string. The column to be altered. + condition: string. A condition using the following format: + 'column[<,<=,=,>=,>]value' or + 'value[<,<=,=,>=,>]column'. + + Operatores supported: (<,<=,=,>=,>) + ''' + if condition is not None: + condition = self.replace_between(condition) + sub_conditions = condition.split(" or ") + print(sub_conditions) + rows = set() + for sub_cond in sub_conditions: + is_not = False + if sub_cond.startswith("not "): + is_not = True + sub_cond = sub_cond[4:] + sub_cond=sub_cond.replace("( ", "").replace(" )", "") + + conditions = sub_cond.split(" and ") + rows_L=[] + for condition_ in conditions: + column_name, operator, value = self._parse_condition(condition_) + column = self.column_by_name(column_name) + rows_L.append([ind for ind, x in enumerate(column) if get_op(operator, x, value)]) + + and_rows = set(rows_L[0]).intersection(*rows_L) if rows_L else set() + + if is_not: + not_rows = set(range(len(column))) - and_rows + rows.update(not_rows) + else: + rows.update(and_rows) + #rows.update(self.find_rows(sub_cond)) + + + rows_to_upd = list(rows) + set_column_idx = self.column_names.index(set_column) + for idx in rows_to_upd: + self.data[idx][set_column_idx] = set_value + + ''' + if condition is not None: + + if re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition) or condition.startswith("not "):#simple condition or not + column_name, operator, value = self._parse_condition(condition) + column = self.column_by_name(column_name) + set_column_idx = self.column_names.index(set_column) + for row_ind, column_value in enumerate(column): + if get_op(operator, column_value, value): + self.data[row_ind][set_column_idx] = set_value + elif re.match(r"^\w+\s+between\s+\w+\s+and\s+\w+$", condition): + + query = condition.split() + index = query.index("between") + megalutero = query[index+1] + mikrotero = query[index+3] + column_name = query[index-1] + column = self.column_by_name(column_name) + set_column_idx = self.column_names.index(set_column) + + for i,j in enumerate(column): + if j >= megalutero and j <= mikrotero: + self.data[i][set_column_idx] = set_value + elif re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+\s+or\s+\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition): + + + conditions = condition.split(" or ") + set_column_idx = self.column_names.index(set_column) + for condition_ in conditions: + column_name, operator, value = self._parse_condition(condition_) + column = self.column_by_name(column_name) + + for row_ind, column_value in enumerate(column): + if get_op(operator, column_value, value): + self.data[row_ind][set_column_idx] = set_value + + + elif re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+\s+and\s+\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition): + column_name = condition.split()[0] + conditions = condition.split(" and ") + set_column_idx = self.column_names.index(set_column) + rows_L=[] + for condition_ in conditions: + column_name, operator, value = self._parse_condition(condition_) + column = self.column_by_name(column_name) + + rows_L.append([ind for ind, x in enumerate(column) if get_op(operator, x, value)]) + + + + rows = set(rows_L[0]).intersection(*rows_L) + + for row in rows: + self.data[row][set_column_idx] = set_value + + else: + raise("invalid where condition") + #else: + #rows = [i for i in range(len(self.data))] + ''' + # parse the condition + ###column_name, operator, value = self._parse_condition(condition) + + # get the condition and the set column + ### column = self.column_by_name(column_name) + ### set_column_idx = self.column_names.index(set_column) + + # set_columns_indx = [self.column_names.index(set_column_name) for set_column_name in set_column_names] + + # for each value in column, if condition, replace it with set_value + ### for row_ind, column_value in enumerate(column): + ### if get_op(operator, column_value, value): + #### self.data[row_ind][set_column_idx] = set_value + + # self._update() + # print(f"Updated {len(indexes_to_del)} rows") + + + def _delete_where(self, condition): + ''' + Deletes rows where condition is met. + + Important: delete replaces the rows to be deleted with rows filled with Nones. + These rows are then appended to the insert_stack. + + Args: + condition: string. A condition using the following format: + 'column[<,<=,==,>=,>]value' or + 'value[<,<=,==,>=,>]column'. + + Operatores supported: (<,<=,==,>=,>) + + ''' + indexes_to_del = [] + if condition is not None: + condition = self.replace_between(condition) + sub_conditions = condition.split(" or ") + print(sub_conditions) + rows = set() + for sub_cond in sub_conditions: + is_not = False + if sub_cond.startswith("not "): + is_not = True + sub_cond = sub_cond[4:] + sub_cond=sub_cond.replace("( ", "").replace(" )", "") + + conditions = sub_cond.split(" and ") + rows_L=[] + for condition_ in conditions: + column_name, operator, value = self._parse_condition(condition_) + column = self.column_by_name(column_name) + rows_L.append([ind for ind, x in enumerate(column) if get_op(operator, x, value)]) + + and_rows = set(rows_L[0]).intersection(*rows_L) if rows_L else set() + + if is_not: + not_rows = set(range(len(column))) - and_rows + rows.update(not_rows) + else: + rows.update(and_rows) + #rows.update(self.find_rows(sub_cond)) + + + indexes_to_del = list(rows) + ''' + if re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition) or condition.startswith("not "):#simple condition or not + column_name, operator, value = self._parse_condition(condition) + column = self.column_by_name(column_name) + for index, row_value in enumerate(column): + if get_op(operator, row_value, value): + indexes_to_del.append(index) + elif re.match(r"^\w+\s+between\s+\w+\s+and\s+\w+$", condition): + + query = condition.split() + index = query.index("between") + + column_name = query[index-1] + column = self.column_by_name(column_name) + for i in range (len(self.columns)): + if column_name==self.column_names[i]: + + megalutero = self.column_types[i](query[index+1]) + mikrotero = self.column_types[i](query[index+3]) + + for i,j in enumerate(column): + if j >= megalutero and j <= mikrotero: + indexes_to_del.append(i) + elif re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+\s+or\s+\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition): + + column_name = condition.split()[0] + conditions = condition.split(" or ") + rows_L=[] + for condition_ in conditions: + + column_name, operator, value = self._parse_condition(condition_) + column = self.column_by_name(column_name) + rows_L.append([ind for ind, x in enumerate(column) if get_op(operator, x, value)]) + + + for rlist in rows_L: + for index in rlist: + indexes_to_del.append(index) + elif re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+\s+and\s+\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition): + column_name = condition.split()[0] + conditions = condition.split(" and ") + rows_L=[] + for condition_ in conditions: + column_name, operator, value = self._parse_condition(condition_) + column = self.column_by_name(column_name) + rows_L.append([ind for ind, x in enumerate(column) if get_op(operator, x, value)]) + + + indexes_to_del = set(rows_L[0]).intersection(*rows_L) + indexes_to_del = list(indexes_to_del) + + else: + raise("invalid where condition") + ''' + + + # we pop from highest to lowest index in order to avoid removing the wrong item + # since we dont delete, we dont have to to pop in that order, but since delete is used + # to delete from meta tables too, we still implement it. + + for index in sorted(indexes_to_del, reverse=True): + if self._name[:4] != 'meta': + # if the table is not a metatable, replace the row with a row of nones + self.data[index] = [None for _ in range(len(self.column_names))] + else: + self.data.pop(index) + + # self._update() + # we have to return the deleted indexes, since they will be appended to the insert_stack + return indexes_to_del + + + def _select_where(self, return_columns, condition=None, distinct=False, order_by=None, desc=True, limit=None): + ''' + Select and return a table containing specified columns and rows where condition is met. + + Args: + return_columns: list. The columns to be returned. + condition: string. A condition using the following format: + 'column[<,<=,==,>=,>]value' or + 'value[<,<=,==,>=,>]column'. + + Operatores supported: (<,<=,==,>=,>) + distinct: boolean. If True, the resulting table will contain only unique rows (False by default). + order_by: string. A column name that signals that the resulting table should be ordered based on it (no order if None). + desc: boolean. If True, order_by will return results in descending order (False by default). + limit: int. An integer that defines the number of rows that will be returned (all rows if None). + ''' + + # if * return all columns, else find the column indexes for the columns specified + if return_columns == '*': + return_cols = [i for i in range(len(self.column_names))] + else: + return_cols = [self.column_names.index(col.strip()) for col in return_columns.split(',')] + + # if condition is None, return all rows + # if not, return the rows with values where condition is met for value + if condition is not None: + condition = self.replace_between(condition) + sub_conditions = condition.split(" or ") + print(sub_conditions) + rows = set() + for sub_cond in sub_conditions: + is_not = False + if sub_cond.startswith("not "): + is_not = True + sub_cond = sub_cond[4:] + sub_cond=sub_cond.replace("( ", "").replace(" )", "") + + conditions = sub_cond.split(" and ") + rows_L=[] + for condition_ in conditions: + column_name, operator, value = self._parse_condition(condition_) + column = self.column_by_name(column_name) + rows_L.append([ind for ind, x in enumerate(column) if get_op(operator, x, value)]) + + and_rows = set(rows_L[0]).intersection(*rows_L) if rows_L else set() + + if is_not: + not_rows = set(range(len(column))) - and_rows + rows.update(not_rows) + else: + rows.update(and_rows) + #rows.update(self.find_rows(sub_cond)) + + + rows = list(rows) + ''' + if re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition) or condition.startswith("not "):#simple condition or not + column_name, operator, value = self._parse_condition(condition) + column = self.column_by_name(column_name) + rows = [ind for ind, x in enumerate(column) if get_op(operator, x, value)] + elif re.match(r"^\w+\s+between\s+\w+\s+and\s+\w+$", condition): + + query = condition.split() + index = query.index("between") + + #megalutero = query[index+1] + #mikrotero = query[index+3] + column_name = query[index-1] + + + + column = self.column_by_name(column_name) + for i in range (len(self.columns)): + if column_name==self.column_names[i]: + + megalutero = self.column_types[i](query[index+1]) + mikrotero = self.column_types[i](query[index+3]) + + rows = [] + for i,j in enumerate(column): + if j >= megalutero and j <= mikrotero: + rows.append(i) + elif re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+\s+or\s+\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition): + + column_name = condition.split()[0] + conditions = condition.split(" or ") + rows_L=[] + for condition_ in conditions: + + column_name, operator, value = self._parse_condition(condition_) + column = self.column_by_name(column_name) + rows_L.append([ind for ind, x in enumerate(column) if get_op(operator, x, value)]) + + rows=[] + for rlist in rows_L: + for row in rlist: + rows.append(row) + elif re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+\s+and\s+\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition): + column_name = condition.split()[0] + conditions = condition.split(" and ") + rows_L=[] + for condition_ in conditions: + column_name, operator, value = self._parse_condition(condition_) + column = self.column_by_name(column_name) + rows_L.append([ind for ind, x in enumerate(column) if get_op(operator, x, value)]) + + + rows = set(rows_L[0]).intersection(*rows_L) + rows = list(rows) + + else: + raise("invalid where condition") + ''' + else: + rows = [i for i in range(len(self.data))] + + # copy the old dict, but only the rows and columns of data with index in rows/columns (the indexes that we want returned) + dict = {(key):([[self.data[i][j] for j in return_cols] for i in rows] if key=="data" else value) for key,value in self.__dict__.items()} + + # we need to set the new column names/types and no of columns, since we might + # only return some columns + dict['column_names'] = [self.column_names[i] for i in return_cols] + dict['column_types'] = [self.column_types[i] for i in return_cols] + + s_table = Table(load=dict) + + s_table.data = list(set(map(lambda x: tuple(x), s_table.data))) if distinct else s_table.data + + if order_by: + s_table.order_by(order_by, desc) + + # if isinstance(limit, str): + # try: + # k = int(limit) + # except ValueError: + # raise Exception("The value following 'top' in the query should be a number.") + + # # Remove from the table's data all the None-filled rows, as they are not shown by default + # # Then, show the first k rows + # s_table.data.remove(len(s_table.column_names) * [None]) + # s_table.data = s_table.data[:k] + if isinstance(limit,str): + s_table.data = [row for row in s_table.data if any(row)][:int(limit)] + + return s_table + + + + def replace_between(self,condition): + + query = condition.split() + try: + index = query.index("between") + except: + return condition + + + + + megalutero = query[index+1] + mikrotero = query[index+3] + column_name = query[index-1] + between_condition = str(column_name) + ">=" + str(megalutero) + " and " + str(column_name) + "<=" + str(mikrotero) + + del query[index-1:index+4] + query.insert(index-1, between_condition) + blank =" " + new_condition = blank.join(query) + + new_condition = self.replace_between(new_condition) + return new_condition + + + + + def _select_where_with_btree(self, return_columns, bt, condition, distinct=False, order_by=None, desc=True, limit=None): + print("select from btree") + # if * return all columns, else find the column indexes for the columns specified + if return_columns == '*': + return_cols = [i for i in range(len(self.column_names))] + else: + return_cols = [self.column_names.index(colname) for colname in return_columns] + + + #column_name, operator, value = self._parse_condition(condition) + if condition is not None: + condition = self.replace_between(condition) + sub_conditions = condition.split(" or ") + print(sub_conditions) + rows = set() + for sub_cond in sub_conditions: + is_not = False + if sub_cond.startswith("not "): + is_not = True + sub_cond = sub_cond[4:] + sub_cond=sub_cond.replace("( ", "").replace(" )", "") + + conditions = sub_cond.split(" and ") + rows_L=[] + for condition_ in conditions: + column_name, operator, value = self._parse_condition(condition_) + column = self.column_by_name(column_name) + rows_L.append(bt.find(operator, value)) + + + + and_rows = set(rows_L[0]).intersection(*rows_L) if rows_L else set() + + if is_not: + not_rows = set(range(len(column))) - and_rows + rows.update(not_rows) + else: + rows.update(and_rows) + + + + rows = list(rows) + + + + '''if re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition) or condition.startswith("not "):#simple condition or not + column_name, operator, value = self._parse_condition(condition) + column = self.column_by_name(column_name) + if (self.pk_idx is not None and column_name != self.column_names[self.pk_idx]) and (self.unique is not None and column_name not in self.unique): + #print('Column is not PK. Aborting') + raise ValueError('Column is not PK or UNIQUE') + rows1 = [] + opsseq = 0 + for ind, x in enumerate(column): + opsseq+=1 + if get_op(operator, x, value): + rows1.append(ind) + rows = bt.find(operator, value) + + + elif re.match(r"^\w+\s+between\s+\w+\s+and\s+\w+$", condition): + + query = condition.split() + index = query.index("between") + + column_name = query[index-1] + if (self.pk_idx is not None and column_name != self.column_names[self.pk_idx]) and (self.unique is not None and column_name not in self.unique): + #print('Column is not PK. Aborting') + raise ValueError('Column is not PK or UNIQUE') + column = self.column_by_name(column_name) + for i in range (len(self.columns)): + if column_name==self.column_names[i]: + + megalutero = self.column_types[i](query[index+1]) + mikrotero = self.column_types[i](query[index+3]) + rows_greater = bt.find('>=',str(megalutero)) + rows_less = bt.find('<=',str(mikrotero)) + rows = set(rows_greater).intersection(rows_less) + rows = list(rows) + + elif re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+\s+or\s+\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition): + + print(condition) + conditions = condition.split(" or ") + rows_L=[] + for condition_ in conditions: + + column_name, operator, value = self._parse_condition(condition_) + if (self.pk_idx is not None and column_name != self.column_names[self.pk_idx]) and (self.unique is not None and column_name not in self.unique): + #print('Column is not PK. Aborting') + raise ValueError('Column is not PK or UNIQUE') + + rows_L.append(bt.find(operator, value)) + rows=[] + print(rows_L) + for rlist in rows_L: + for row in rlist: + rows.append(row) + elif re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+\s+and\s+\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition): + + conditions = condition.split(" and ") + rows_L=[] + for condition_ in conditions: + column_name, operator, value = self._parse_condition(condition_) + if (self.pk_idx is not None and column_name != self.column_names[self.pk_idx]) and (self.unique is not None and column_name not in self.unique): + #print('Column is not PK. Aborting') + continue + column = self.column_by_name(column_name) + rows_L.append(bt.find(operator, value)) + + rows = set(rows_L[0]).intersection(*rows_L) + rows = list(rows)''' + # if the column in condition is not a primary key, abort the select + + + # here we run the same select twice, sequentially and using the btree. + # we then check the results match and compare performance (number of operation) + ''' + column = self.column_by_name(column_name) + + # sequential + rows1 = [] + opsseq = 0 + for ind, x in enumerate(column): + opsseq+=1 + if get_op(operator, x, value): + rows1.append(ind) + + # btree find + rows = bt.find(operator, value) + ''' + try: + k = int(limit) + except TypeError: + k = None + # same as simple select from now on + rows = rows[:k] + + # TODO: this needs to be dumbed down + dict = {(key):([[self.data[i][j] for j in return_cols] for i in rows] if key=="data" else value) for key,value in self.__dict__.items()} + + dict['column_names'] = [self.column_names[i] for i in return_cols] + dict['column_types'] = [self.column_types[i] for i in return_cols] + + s_table = Table(load=dict) + + s_table.data = list(set(map(lambda x: tuple(x), s_table.data))) if distinct else s_table.data + + if order_by: + s_table.order_by(order_by, desc) + + if isinstance(limit,str): + s_table.data = [row for row in s_table.data if row is not None][:int(limit)] + + return s_table + + def order_by(self, column_name, desc=True): + ''' + Order table based on column. + + Args: + column_name: string. Name of column. + desc: boolean. If True, order_by will return results in descending order (False by default). + ''' + column = [val if val is not None else 0 for val in self.column_by_name(column_name)] + idx = sorted(range(len(column)), key=lambda k: column[k], reverse=desc) + # print(idx) + self.data = [self.data[i] for i in idx] + # self._update() + + + def _general_join_processing(self, table_right:Table, condition, join_type): + ''' + Performs the processes all the join operations need (regardless of type) so that there is no code repetition. + + Args: + condition: string. A condition using the following format: + 'column[<,<=,==,>=,>]value' or + 'value[<,<=,==,>=,>]column'. + + Operators supported: (<,<=,==,>=,>) + ''' + # get columns and operator + column_name_left, operator, column_name_right = self._parse_condition(condition, join=True) + # try to find both columns, if you fail raise error + + if(operator != '=' and join_type in ['left','right','full']): + class CustomFailException(Exception): + pass + raise CustomFailException('Outer Joins can only be used if the condition operator is "=".\n') + + try: + column_index_left = self.column_names.index(column_name_left) + except: + raise Exception(f'Column "{column_name_left}" dont exist in left table. Valid columns: {self.column_names}.') + + try: + column_index_right = table_right.column_names.index(column_name_right) + except: + raise Exception(f'Column "{column_name_right}" dont exist in right table. Valid columns: {table_right.column_names}.') + + # get the column names of both tables with the table name in front + # ex. for left -> name becomes left_table_name_name etc + left_names = [f'{self._name}.{colname}' if self._name!='' else colname for colname in self.column_names] + right_names = [f'{table_right._name}.{colname}' if table_right._name!='' else colname for colname in table_right.column_names] + + # define the new tables name, its column names and types + join_table_name = '' + join_table_colnames = left_names+right_names + join_table_coltypes = self.column_types+table_right.column_types + join_table = Table(name=join_table_name, column_names=join_table_colnames, column_types= join_table_coltypes) + + return join_table, column_index_left, column_index_right, operator + + + def _inner_join(self, table_right: Table, condition): + ''' + Join table (left) with a supplied table (right) where condition is met. + + Args: + condition: string. A condition using the following format: + 'column[<,<=,==,>=,>]value' or + 'value[<,<=,==,>=,>]column'. + + Operators supported: (<,<=,==,>=,>) + ''' + join_table, column_index_left, column_index_right, operator = self._general_join_processing(table_right, condition, 'inner') + + # count the number of operations (<,> etc) + no_of_ops = 0 + # this code is dumb on purpose... it needs to illustrate the underline technique + # for each value in left column and right column, if condition, append the corresponding row to the new table + for row_left in self.data: + left_value = row_left[column_index_left] + for row_right in table_right.data: + right_value = row_right[column_index_right] + if(left_value is None and right_value is None): + continue + no_of_ops+=1 + if get_op(operator, left_value, right_value): #EQ_OP + join_table._insert(row_left+row_right) + + return join_table + + def _left_join(self, table_right: Table, condition): + ''' + Perform a left join on the table with the supplied table (right). + + Args: + condition: string. A condition using the following format: + 'column[<,<=,==,>=,>]value' or + 'value[<,<=,==,>=,>]column'. + + Operators supported: (<,<=,==,>=,>) + ''' + join_table, column_index_left, column_index_right, operator = self._general_join_processing(table_right, condition, 'left') + + right_column = table_right.column_by_name(table_right.column_names[column_index_right]) + right_table_row_length = len(table_right.column_names) + + for row_left in self.data: + left_value = row_left[column_index_left] + if left_value is None: + continue + elif left_value not in right_column: + join_table._insert(row_left + right_table_row_length*["NULL"]) + else: + for row_right in table_right.data: + right_value = row_right[column_index_right] + if left_value == right_value: + join_table._insert(row_left + row_right) + + return join_table + + def _right_join(self, table_right: Table, condition): + ''' + Perform a right join on the table with the supplied table (right). + + Args: + condition: string. A condition using the following format: + 'column[<,<=,==,>=,>]value' or + 'value[<,<=,==,>=,>]column'. + + Operators supported: (<,<=,==,>=,>) + ''' + join_table, column_index_left, column_index_right, operator = self._general_join_processing(table_right, condition, 'right') + + left_column = self.column_by_name(self.column_names[column_index_left]) + left_table_row_length = len(self.column_names) + + for row_right in table_right.data: + right_value = row_right[column_index_right] + if right_value is None: + continue + elif right_value not in left_column: + join_table._insert(left_table_row_length*["NULL"] + row_right) + else: + for row_left in self.data: + left_value = row_left[column_index_left] + if left_value == right_value: + join_table._insert(row_left + row_right) + + return join_table + + def _full_join(self, table_right: Table, condition): + ''' + Perform a full join on the table with the supplied table (right). + + Args: + condition: string. A condition using the following format: + 'column[<,<=,==,>=,>]value' or + 'value[<,<=,==,>=,>]column'. + + Operators supported: (<,<=,==,>=,>) + ''' + join_table, column_index_left, column_index_right, operator = self._general_join_processing(table_right, condition, 'full') + + right_column = table_right.column_by_name(table_right.column_names[column_index_right]) + left_column = self.column_by_name(self.column_names[column_index_left]) + + right_table_row_length = len(table_right.column_names) + left_table_row_length = len(self.column_names) + + for row_left in self.data: + left_value = row_left[column_index_left] + if left_value is None: + continue + if left_value not in right_column: + join_table._insert(row_left + right_table_row_length*["NULL"]) + else: + for row_right in table_right.data: + right_value = row_right[column_index_right] + if left_value == right_value: + join_table._insert(row_left + row_right) + + for row_right in table_right.data: + right_value = row_right[column_index_right] + + if right_value is None: + continue + elif right_value not in left_column: + join_table._insert(left_table_row_length*["NULL"] + row_right) + + return join_table + + def show(self, no_of_rows=None, is_locked=False): + ''' + Print the table in a nice readable format. + + Args: + no_of_rows: int. Number of rows. + is_locked: boolean. Whether it is locked (False by default). + ''' + output = "" + # if the table is locked, add locked keyword to title + if is_locked: + output += f"\n## {self._name} (locked) ##\n" + else: + output += f"\n## {self._name} ##\n" + + # headers -> "column name (column type)" + headers = [f'{col} ({tp.__name__})' for col, tp in zip(self.column_names, self.column_types)] + if self.pk_idx is not None: + # table has a primary key, add PK next to the appropriate column + headers[self.pk_idx] = headers[self.pk_idx]+' #PK#' + + if self.unique_idx is not None: + for unq in self.unique_idx: + headers[unq]=headers[unq]+' #UNQ#' + # detect the rows that are no tfull of nones (these rows have been deleted) + # if we dont skip these rows, the returning table has empty rows at the deleted positions + non_none_rows = [row for row in self.data if any(row)] + # print using tabulate + print(tabulate(non_none_rows[:no_of_rows], headers=headers)+'\n') + + + def _parse_condition(self, condition, join=False): + ''' + Parse the single string condition and return the value of the column and the operator. + + Args: + condition: string. A condition using the following format: + 'column[<,<=,==,>=,>]value' or + 'value[<,<=,==,>=,>]column'. + + Operatores supported: (<,<=,==,>=,>) + join: boolean. Whether to join or not (False by default). + ''' + # if both_columns (used by the join function) return the names of the names of the columns (left first) + if join: + return split_condition(condition) + + # cast the value with the specified column's type and return the column name, the operator and the casted value + left, op, right = split_condition(condition) + if right in self.column_names:#'value[<,<=,==,>=,>]column' fromat + coltype = self.column_types[self.column_names.index(right)] + op =reverse_op(op) + return right, op, coltype(left) + + elif left in self.column_names:#'column[<,<=,==,>=,>]value' format + coltype = self.column_types[self.column_names.index(left)] + + return left, op, coltype(right) + else: + #raise ValueError(f'Condition is not valid (cant find column name)') + raise ValueError(f'Condition is not valid (cant find column name)') + + + def _load_from_file(self, filename): + ''' + Load table from a pkl file (not used currently). + + Args: + filename: string. Name of pkl file. + ''' + f = open(filename, 'rb') + tmp_dict = pickle.load(f) + f.close() + + self.__dict__.update(tmp_dict.__dict__) From 88839b363b7f4db12ddaa9eaf1d099068950e7ce Mon Sep 17 00:00:00 2001 From: Kapiniaris-Xrisovalantis <92021533+Kapiniaris-Xrisovalantis@users.noreply.github.com> Date: Mon, 4 Sep 2023 10:48:54 +0300 Subject: [PATCH 07/16] Delete table.py added in wrong folder --- table.py | 1005 ------------------------------------------------------ 1 file changed, 1005 deletions(-) delete mode 100644 table.py diff --git a/table.py b/table.py deleted file mode 100644 index 755d4585..00000000 --- a/table.py +++ /dev/null @@ -1,1005 +0,0 @@ -from __future__ import annotations -from tabulate import tabulate -import pickle -import os -import sys -import re - -sys.path.append(f'{os.path.dirname(os.path.dirname(os.path.abspath(__file__)))}/miniDB') - -from misc import get_op, split_condition, reverse_op - - -class Table: - ''' - Table object represents a table inside a database - - A Table object can be created either by assigning: - - a table name (string) - - column names (list of strings) - - column types (list of functions like str/int etc) - - primary (name of the primary key column) - - OR - - - by assigning a value to the variable called load. This value can be: - - a path to a Table file saved using the save function - - a dictionary that includes the appropriate info (all the attributes in __init__) - - ''' - def __init__(self, name=None, column_names=None, column_types=None, primary_key=None,unique=None, load=None): - - if load is not None: - - # if load is a dict, replace the object dict with it (replaces the object with the specified one) - if isinstance(load, dict): - self.__dict__.update(load) - # self._update() - # if load is str, load from a file - elif isinstance(load, str): - self._load_from_file(load) - - - # if name, columns_names and column types are not none - elif (name is not None) and (column_names is not None) and (column_types is not None): - - self._name = name - - if len(column_names)!=len(column_types): - raise ValueError('Need same number of column names and types.') - - self.column_names = column_names - - self.columns = [] - - for col in self.column_names: - if col not in self.__dir__(): - # this is used in order to be able to call a column using its name as an attribute. - # example: instead of table.columns['column_name'], we do table.column_name - setattr(self, col, []) - self.columns.append([]) - else: - raise Exception(f'"{col}" attribute already exists in "{self.__class__.__name__} "class.') - - self.column_types = [eval(ct) if not isinstance(ct, type) else ct for ct in column_types] - self.data = [] # data is a list of lists, a list of rows that is. - - # if primary key is set, keep its index as an attribute - if primary_key is not None: - self.pk_idx = self.column_names.index(primary_key) - - else: - self.pk_idx = None - - self.pk = primary_key - - self.unique_idx = [] - - if unique is not None: - self.unique = unique.split(',') - unique=unique.split(',') - for unq in unique: - if unq in self.column_names: - self.unique_idx.append(self.column_names.index(unq)) - else: - self.unique_idx=None - self.unique=None - - - # self._update() - - # if any of the name, columns_names and column types are none. return an empty table object - - def column_by_name(self, column_name): - return [row[self.column_names.index(column_name)] for row in self.data] - - - def _update(self): - ''' - Update all the available columns with the appended rows. - ''' - self.columns = [[row[i] for row in self.data] for i in range(len(self.column_names))] - for ind, col in enumerate(self.column_names): - setattr(self, col, self.columns[ind]) - - def _cast_column(self, column_name, cast_type): - ''' - Cast all values of a column using a specified type. - - Args: - column_name: string. The column that will be casted. - cast_type: type. Cast type (do not encapsulate in quotes). - ''' - # get the column from its name - column_idx = self.column_names.index(column_name) - # for every column's value in each row, replace it with itself but casted as the specified type - for i in range(len(self.data)): - self.data[i][column_idx] = cast_type(self.data[i][column_idx]) - # change the type of the column - self.column_types[column_idx] = cast_type - # self._update() - - - def _insert(self, row, insert_stack=[]): - ''' - Insert row to table. - - Args: - row: list. A list of values to be inserted (will be casted to a predifined type automatically). - insert_stack: list. The insert stack (empty by default). - ''' - - if len(row)!=len(self.column_names): - raise ValueError(f'ERROR -> Cannot insert {len(row)} values. Only {len(self.column_names)} columns exist') - - for i in range(len(row)): - # for each value, cast and replace it in row. - try: - row[i] = self.column_types[i](row[i]) - except ValueError: - if row[i] != 'NULL': - raise ValueError(f'ERROR -> Value {row[i]} of type {type(row[i])} is not of type {self.column_types[i]}.') - except TypeError as exc: - if row[i] != None: - print(exc) - - # if value is to be appended to the primary_key column, check that it doesnt alrady exist (no duplicate primary keys) - if i==self.pk_idx and row[i] in self.column_by_name(self.pk): - raise ValueError(f'## ERROR -> Value {row[i]} already exists in primary key column.') - - elif i==self.pk_idx and row[i] is None: - raise ValueError(f'ERROR -> The value of the primary key cannot be None.') - - - if self.unique_idx is not None: - if i in self.unique_idx: - for j in range (len(self.unique)): - if row[i] in self.column_by_name(self.unique[j]): - raise ValueError(f'## ERROR -> Value {row[i]} already exists in unique column.') - - - # if insert_stack is not empty, append to its last index - if insert_stack != []: - self.data[insert_stack[-1]] = row - else: # else append to the end - self.data.append(row) - # self._update() - - def _update_rows(self, set_value, set_column, condition): - ''' - Update where Condition is met. - - Args: - set_value: string. The provided set value. - set_column: string. The column to be altered. - condition: string. A condition using the following format: - 'column[<,<=,=,>=,>]value' or - 'value[<,<=,=,>=,>]column'. - - Operatores supported: (<,<=,=,>=,>) - ''' - if condition is not None: - condition = self.replace_between(condition) - sub_conditions = condition.split(" or ") - print(sub_conditions) - rows = set() - for sub_cond in sub_conditions: - is_not = False - if sub_cond.startswith("not "): - is_not = True - sub_cond = sub_cond[4:] - sub_cond=sub_cond.replace("( ", "").replace(" )", "") - - conditions = sub_cond.split(" and ") - rows_L=[] - for condition_ in conditions: - column_name, operator, value = self._parse_condition(condition_) - column = self.column_by_name(column_name) - rows_L.append([ind for ind, x in enumerate(column) if get_op(operator, x, value)]) - - and_rows = set(rows_L[0]).intersection(*rows_L) if rows_L else set() - - if is_not: - not_rows = set(range(len(column))) - and_rows - rows.update(not_rows) - else: - rows.update(and_rows) - #rows.update(self.find_rows(sub_cond)) - - - rows_to_upd = list(rows) - set_column_idx = self.column_names.index(set_column) - for idx in rows_to_upd: - self.data[idx][set_column_idx] = set_value - - ''' - if condition is not None: - - if re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition) or condition.startswith("not "):#simple condition or not - column_name, operator, value = self._parse_condition(condition) - column = self.column_by_name(column_name) - set_column_idx = self.column_names.index(set_column) - for row_ind, column_value in enumerate(column): - if get_op(operator, column_value, value): - self.data[row_ind][set_column_idx] = set_value - elif re.match(r"^\w+\s+between\s+\w+\s+and\s+\w+$", condition): - - query = condition.split() - index = query.index("between") - megalutero = query[index+1] - mikrotero = query[index+3] - column_name = query[index-1] - column = self.column_by_name(column_name) - set_column_idx = self.column_names.index(set_column) - - for i,j in enumerate(column): - if j >= megalutero and j <= mikrotero: - self.data[i][set_column_idx] = set_value - elif re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+\s+or\s+\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition): - - - conditions = condition.split(" or ") - set_column_idx = self.column_names.index(set_column) - for condition_ in conditions: - column_name, operator, value = self._parse_condition(condition_) - column = self.column_by_name(column_name) - - for row_ind, column_value in enumerate(column): - if get_op(operator, column_value, value): - self.data[row_ind][set_column_idx] = set_value - - - elif re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+\s+and\s+\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition): - column_name = condition.split()[0] - conditions = condition.split(" and ") - set_column_idx = self.column_names.index(set_column) - rows_L=[] - for condition_ in conditions: - column_name, operator, value = self._parse_condition(condition_) - column = self.column_by_name(column_name) - - rows_L.append([ind for ind, x in enumerate(column) if get_op(operator, x, value)]) - - - - rows = set(rows_L[0]).intersection(*rows_L) - - for row in rows: - self.data[row][set_column_idx] = set_value - - else: - raise("invalid where condition") - #else: - #rows = [i for i in range(len(self.data))] - ''' - # parse the condition - ###column_name, operator, value = self._parse_condition(condition) - - # get the condition and the set column - ### column = self.column_by_name(column_name) - ### set_column_idx = self.column_names.index(set_column) - - # set_columns_indx = [self.column_names.index(set_column_name) for set_column_name in set_column_names] - - # for each value in column, if condition, replace it with set_value - ### for row_ind, column_value in enumerate(column): - ### if get_op(operator, column_value, value): - #### self.data[row_ind][set_column_idx] = set_value - - # self._update() - # print(f"Updated {len(indexes_to_del)} rows") - - - def _delete_where(self, condition): - ''' - Deletes rows where condition is met. - - Important: delete replaces the rows to be deleted with rows filled with Nones. - These rows are then appended to the insert_stack. - - Args: - condition: string. A condition using the following format: - 'column[<,<=,==,>=,>]value' or - 'value[<,<=,==,>=,>]column'. - - Operatores supported: (<,<=,==,>=,>) - - ''' - indexes_to_del = [] - if condition is not None: - condition = self.replace_between(condition) - sub_conditions = condition.split(" or ") - print(sub_conditions) - rows = set() - for sub_cond in sub_conditions: - is_not = False - if sub_cond.startswith("not "): - is_not = True - sub_cond = sub_cond[4:] - sub_cond=sub_cond.replace("( ", "").replace(" )", "") - - conditions = sub_cond.split(" and ") - rows_L=[] - for condition_ in conditions: - column_name, operator, value = self._parse_condition(condition_) - column = self.column_by_name(column_name) - rows_L.append([ind for ind, x in enumerate(column) if get_op(operator, x, value)]) - - and_rows = set(rows_L[0]).intersection(*rows_L) if rows_L else set() - - if is_not: - not_rows = set(range(len(column))) - and_rows - rows.update(not_rows) - else: - rows.update(and_rows) - #rows.update(self.find_rows(sub_cond)) - - - indexes_to_del = list(rows) - ''' - if re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition) or condition.startswith("not "):#simple condition or not - column_name, operator, value = self._parse_condition(condition) - column = self.column_by_name(column_name) - for index, row_value in enumerate(column): - if get_op(operator, row_value, value): - indexes_to_del.append(index) - elif re.match(r"^\w+\s+between\s+\w+\s+and\s+\w+$", condition): - - query = condition.split() - index = query.index("between") - - column_name = query[index-1] - column = self.column_by_name(column_name) - for i in range (len(self.columns)): - if column_name==self.column_names[i]: - - megalutero = self.column_types[i](query[index+1]) - mikrotero = self.column_types[i](query[index+3]) - - for i,j in enumerate(column): - if j >= megalutero and j <= mikrotero: - indexes_to_del.append(i) - elif re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+\s+or\s+\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition): - - column_name = condition.split()[0] - conditions = condition.split(" or ") - rows_L=[] - for condition_ in conditions: - - column_name, operator, value = self._parse_condition(condition_) - column = self.column_by_name(column_name) - rows_L.append([ind for ind, x in enumerate(column) if get_op(operator, x, value)]) - - - for rlist in rows_L: - for index in rlist: - indexes_to_del.append(index) - elif re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+\s+and\s+\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition): - column_name = condition.split()[0] - conditions = condition.split(" and ") - rows_L=[] - for condition_ in conditions: - column_name, operator, value = self._parse_condition(condition_) - column = self.column_by_name(column_name) - rows_L.append([ind for ind, x in enumerate(column) if get_op(operator, x, value)]) - - - indexes_to_del = set(rows_L[0]).intersection(*rows_L) - indexes_to_del = list(indexes_to_del) - - else: - raise("invalid where condition") - ''' - - - # we pop from highest to lowest index in order to avoid removing the wrong item - # since we dont delete, we dont have to to pop in that order, but since delete is used - # to delete from meta tables too, we still implement it. - - for index in sorted(indexes_to_del, reverse=True): - if self._name[:4] != 'meta': - # if the table is not a metatable, replace the row with a row of nones - self.data[index] = [None for _ in range(len(self.column_names))] - else: - self.data.pop(index) - - # self._update() - # we have to return the deleted indexes, since they will be appended to the insert_stack - return indexes_to_del - - - def _select_where(self, return_columns, condition=None, distinct=False, order_by=None, desc=True, limit=None): - ''' - Select and return a table containing specified columns and rows where condition is met. - - Args: - return_columns: list. The columns to be returned. - condition: string. A condition using the following format: - 'column[<,<=,==,>=,>]value' or - 'value[<,<=,==,>=,>]column'. - - Operatores supported: (<,<=,==,>=,>) - distinct: boolean. If True, the resulting table will contain only unique rows (False by default). - order_by: string. A column name that signals that the resulting table should be ordered based on it (no order if None). - desc: boolean. If True, order_by will return results in descending order (False by default). - limit: int. An integer that defines the number of rows that will be returned (all rows if None). - ''' - - # if * return all columns, else find the column indexes for the columns specified - if return_columns == '*': - return_cols = [i for i in range(len(self.column_names))] - else: - return_cols = [self.column_names.index(col.strip()) for col in return_columns.split(',')] - - # if condition is None, return all rows - # if not, return the rows with values where condition is met for value - if condition is not None: - condition = self.replace_between(condition) - sub_conditions = condition.split(" or ") - print(sub_conditions) - rows = set() - for sub_cond in sub_conditions: - is_not = False - if sub_cond.startswith("not "): - is_not = True - sub_cond = sub_cond[4:] - sub_cond=sub_cond.replace("( ", "").replace(" )", "") - - conditions = sub_cond.split(" and ") - rows_L=[] - for condition_ in conditions: - column_name, operator, value = self._parse_condition(condition_) - column = self.column_by_name(column_name) - rows_L.append([ind for ind, x in enumerate(column) if get_op(operator, x, value)]) - - and_rows = set(rows_L[0]).intersection(*rows_L) if rows_L else set() - - if is_not: - not_rows = set(range(len(column))) - and_rows - rows.update(not_rows) - else: - rows.update(and_rows) - #rows.update(self.find_rows(sub_cond)) - - - rows = list(rows) - ''' - if re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition) or condition.startswith("not "):#simple condition or not - column_name, operator, value = self._parse_condition(condition) - column = self.column_by_name(column_name) - rows = [ind for ind, x in enumerate(column) if get_op(operator, x, value)] - elif re.match(r"^\w+\s+between\s+\w+\s+and\s+\w+$", condition): - - query = condition.split() - index = query.index("between") - - #megalutero = query[index+1] - #mikrotero = query[index+3] - column_name = query[index-1] - - - - column = self.column_by_name(column_name) - for i in range (len(self.columns)): - if column_name==self.column_names[i]: - - megalutero = self.column_types[i](query[index+1]) - mikrotero = self.column_types[i](query[index+3]) - - rows = [] - for i,j in enumerate(column): - if j >= megalutero and j <= mikrotero: - rows.append(i) - elif re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+\s+or\s+\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition): - - column_name = condition.split()[0] - conditions = condition.split(" or ") - rows_L=[] - for condition_ in conditions: - - column_name, operator, value = self._parse_condition(condition_) - column = self.column_by_name(column_name) - rows_L.append([ind for ind, x in enumerate(column) if get_op(operator, x, value)]) - - rows=[] - for rlist in rows_L: - for row in rlist: - rows.append(row) - elif re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+\s+and\s+\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition): - column_name = condition.split()[0] - conditions = condition.split(" and ") - rows_L=[] - for condition_ in conditions: - column_name, operator, value = self._parse_condition(condition_) - column = self.column_by_name(column_name) - rows_L.append([ind for ind, x in enumerate(column) if get_op(operator, x, value)]) - - - rows = set(rows_L[0]).intersection(*rows_L) - rows = list(rows) - - else: - raise("invalid where condition") - ''' - else: - rows = [i for i in range(len(self.data))] - - # copy the old dict, but only the rows and columns of data with index in rows/columns (the indexes that we want returned) - dict = {(key):([[self.data[i][j] for j in return_cols] for i in rows] if key=="data" else value) for key,value in self.__dict__.items()} - - # we need to set the new column names/types and no of columns, since we might - # only return some columns - dict['column_names'] = [self.column_names[i] for i in return_cols] - dict['column_types'] = [self.column_types[i] for i in return_cols] - - s_table = Table(load=dict) - - s_table.data = list(set(map(lambda x: tuple(x), s_table.data))) if distinct else s_table.data - - if order_by: - s_table.order_by(order_by, desc) - - # if isinstance(limit, str): - # try: - # k = int(limit) - # except ValueError: - # raise Exception("The value following 'top' in the query should be a number.") - - # # Remove from the table's data all the None-filled rows, as they are not shown by default - # # Then, show the first k rows - # s_table.data.remove(len(s_table.column_names) * [None]) - # s_table.data = s_table.data[:k] - if isinstance(limit,str): - s_table.data = [row for row in s_table.data if any(row)][:int(limit)] - - return s_table - - - - def replace_between(self,condition): - - query = condition.split() - try: - index = query.index("between") - except: - return condition - - - - - megalutero = query[index+1] - mikrotero = query[index+3] - column_name = query[index-1] - between_condition = str(column_name) + ">=" + str(megalutero) + " and " + str(column_name) + "<=" + str(mikrotero) - - del query[index-1:index+4] - query.insert(index-1, between_condition) - blank =" " - new_condition = blank.join(query) - - new_condition = self.replace_between(new_condition) - return new_condition - - - - - def _select_where_with_btree(self, return_columns, bt, condition, distinct=False, order_by=None, desc=True, limit=None): - print("select from btree") - # if * return all columns, else find the column indexes for the columns specified - if return_columns == '*': - return_cols = [i for i in range(len(self.column_names))] - else: - return_cols = [self.column_names.index(colname) for colname in return_columns] - - - #column_name, operator, value = self._parse_condition(condition) - if condition is not None: - condition = self.replace_between(condition) - sub_conditions = condition.split(" or ") - print(sub_conditions) - rows = set() - for sub_cond in sub_conditions: - is_not = False - if sub_cond.startswith("not "): - is_not = True - sub_cond = sub_cond[4:] - sub_cond=sub_cond.replace("( ", "").replace(" )", "") - - conditions = sub_cond.split(" and ") - rows_L=[] - for condition_ in conditions: - column_name, operator, value = self._parse_condition(condition_) - column = self.column_by_name(column_name) - rows_L.append(bt.find(operator, value)) - - - - and_rows = set(rows_L[0]).intersection(*rows_L) if rows_L else set() - - if is_not: - not_rows = set(range(len(column))) - and_rows - rows.update(not_rows) - else: - rows.update(and_rows) - - - - rows = list(rows) - - - - '''if re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition) or condition.startswith("not "):#simple condition or not - column_name, operator, value = self._parse_condition(condition) - column = self.column_by_name(column_name) - if (self.pk_idx is not None and column_name != self.column_names[self.pk_idx]) and (self.unique is not None and column_name not in self.unique): - #print('Column is not PK. Aborting') - raise ValueError('Column is not PK or UNIQUE') - rows1 = [] - opsseq = 0 - for ind, x in enumerate(column): - opsseq+=1 - if get_op(operator, x, value): - rows1.append(ind) - rows = bt.find(operator, value) - - - elif re.match(r"^\w+\s+between\s+\w+\s+and\s+\w+$", condition): - - query = condition.split() - index = query.index("between") - - column_name = query[index-1] - if (self.pk_idx is not None and column_name != self.column_names[self.pk_idx]) and (self.unique is not None and column_name not in self.unique): - #print('Column is not PK. Aborting') - raise ValueError('Column is not PK or UNIQUE') - column = self.column_by_name(column_name) - for i in range (len(self.columns)): - if column_name==self.column_names[i]: - - megalutero = self.column_types[i](query[index+1]) - mikrotero = self.column_types[i](query[index+3]) - rows_greater = bt.find('>=',str(megalutero)) - rows_less = bt.find('<=',str(mikrotero)) - rows = set(rows_greater).intersection(rows_less) - rows = list(rows) - - elif re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+\s+or\s+\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition): - - print(condition) - conditions = condition.split(" or ") - rows_L=[] - for condition_ in conditions: - - column_name, operator, value = self._parse_condition(condition_) - if (self.pk_idx is not None and column_name != self.column_names[self.pk_idx]) and (self.unique is not None and column_name not in self.unique): - #print('Column is not PK. Aborting') - raise ValueError('Column is not PK or UNIQUE') - - rows_L.append(bt.find(operator, value)) - rows=[] - print(rows_L) - for rlist in rows_L: - for row in rlist: - rows.append(row) - elif re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+\s+and\s+\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition): - - conditions = condition.split(" and ") - rows_L=[] - for condition_ in conditions: - column_name, operator, value = self._parse_condition(condition_) - if (self.pk_idx is not None and column_name != self.column_names[self.pk_idx]) and (self.unique is not None and column_name not in self.unique): - #print('Column is not PK. Aborting') - continue - column = self.column_by_name(column_name) - rows_L.append(bt.find(operator, value)) - - rows = set(rows_L[0]).intersection(*rows_L) - rows = list(rows)''' - # if the column in condition is not a primary key, abort the select - - - # here we run the same select twice, sequentially and using the btree. - # we then check the results match and compare performance (number of operation) - ''' - column = self.column_by_name(column_name) - - # sequential - rows1 = [] - opsseq = 0 - for ind, x in enumerate(column): - opsseq+=1 - if get_op(operator, x, value): - rows1.append(ind) - - # btree find - rows = bt.find(operator, value) - ''' - try: - k = int(limit) - except TypeError: - k = None - # same as simple select from now on - rows = rows[:k] - - # TODO: this needs to be dumbed down - dict = {(key):([[self.data[i][j] for j in return_cols] for i in rows] if key=="data" else value) for key,value in self.__dict__.items()} - - dict['column_names'] = [self.column_names[i] for i in return_cols] - dict['column_types'] = [self.column_types[i] for i in return_cols] - - s_table = Table(load=dict) - - s_table.data = list(set(map(lambda x: tuple(x), s_table.data))) if distinct else s_table.data - - if order_by: - s_table.order_by(order_by, desc) - - if isinstance(limit,str): - s_table.data = [row for row in s_table.data if row is not None][:int(limit)] - - return s_table - - def order_by(self, column_name, desc=True): - ''' - Order table based on column. - - Args: - column_name: string. Name of column. - desc: boolean. If True, order_by will return results in descending order (False by default). - ''' - column = [val if val is not None else 0 for val in self.column_by_name(column_name)] - idx = sorted(range(len(column)), key=lambda k: column[k], reverse=desc) - # print(idx) - self.data = [self.data[i] for i in idx] - # self._update() - - - def _general_join_processing(self, table_right:Table, condition, join_type): - ''' - Performs the processes all the join operations need (regardless of type) so that there is no code repetition. - - Args: - condition: string. A condition using the following format: - 'column[<,<=,==,>=,>]value' or - 'value[<,<=,==,>=,>]column'. - - Operators supported: (<,<=,==,>=,>) - ''' - # get columns and operator - column_name_left, operator, column_name_right = self._parse_condition(condition, join=True) - # try to find both columns, if you fail raise error - - if(operator != '=' and join_type in ['left','right','full']): - class CustomFailException(Exception): - pass - raise CustomFailException('Outer Joins can only be used if the condition operator is "=".\n') - - try: - column_index_left = self.column_names.index(column_name_left) - except: - raise Exception(f'Column "{column_name_left}" dont exist in left table. Valid columns: {self.column_names}.') - - try: - column_index_right = table_right.column_names.index(column_name_right) - except: - raise Exception(f'Column "{column_name_right}" dont exist in right table. Valid columns: {table_right.column_names}.') - - # get the column names of both tables with the table name in front - # ex. for left -> name becomes left_table_name_name etc - left_names = [f'{self._name}.{colname}' if self._name!='' else colname for colname in self.column_names] - right_names = [f'{table_right._name}.{colname}' if table_right._name!='' else colname for colname in table_right.column_names] - - # define the new tables name, its column names and types - join_table_name = '' - join_table_colnames = left_names+right_names - join_table_coltypes = self.column_types+table_right.column_types - join_table = Table(name=join_table_name, column_names=join_table_colnames, column_types= join_table_coltypes) - - return join_table, column_index_left, column_index_right, operator - - - def _inner_join(self, table_right: Table, condition): - ''' - Join table (left) with a supplied table (right) where condition is met. - - Args: - condition: string. A condition using the following format: - 'column[<,<=,==,>=,>]value' or - 'value[<,<=,==,>=,>]column'. - - Operators supported: (<,<=,==,>=,>) - ''' - join_table, column_index_left, column_index_right, operator = self._general_join_processing(table_right, condition, 'inner') - - # count the number of operations (<,> etc) - no_of_ops = 0 - # this code is dumb on purpose... it needs to illustrate the underline technique - # for each value in left column and right column, if condition, append the corresponding row to the new table - for row_left in self.data: - left_value = row_left[column_index_left] - for row_right in table_right.data: - right_value = row_right[column_index_right] - if(left_value is None and right_value is None): - continue - no_of_ops+=1 - if get_op(operator, left_value, right_value): #EQ_OP - join_table._insert(row_left+row_right) - - return join_table - - def _left_join(self, table_right: Table, condition): - ''' - Perform a left join on the table with the supplied table (right). - - Args: - condition: string. A condition using the following format: - 'column[<,<=,==,>=,>]value' or - 'value[<,<=,==,>=,>]column'. - - Operators supported: (<,<=,==,>=,>) - ''' - join_table, column_index_left, column_index_right, operator = self._general_join_processing(table_right, condition, 'left') - - right_column = table_right.column_by_name(table_right.column_names[column_index_right]) - right_table_row_length = len(table_right.column_names) - - for row_left in self.data: - left_value = row_left[column_index_left] - if left_value is None: - continue - elif left_value not in right_column: - join_table._insert(row_left + right_table_row_length*["NULL"]) - else: - for row_right in table_right.data: - right_value = row_right[column_index_right] - if left_value == right_value: - join_table._insert(row_left + row_right) - - return join_table - - def _right_join(self, table_right: Table, condition): - ''' - Perform a right join on the table with the supplied table (right). - - Args: - condition: string. A condition using the following format: - 'column[<,<=,==,>=,>]value' or - 'value[<,<=,==,>=,>]column'. - - Operators supported: (<,<=,==,>=,>) - ''' - join_table, column_index_left, column_index_right, operator = self._general_join_processing(table_right, condition, 'right') - - left_column = self.column_by_name(self.column_names[column_index_left]) - left_table_row_length = len(self.column_names) - - for row_right in table_right.data: - right_value = row_right[column_index_right] - if right_value is None: - continue - elif right_value not in left_column: - join_table._insert(left_table_row_length*["NULL"] + row_right) - else: - for row_left in self.data: - left_value = row_left[column_index_left] - if left_value == right_value: - join_table._insert(row_left + row_right) - - return join_table - - def _full_join(self, table_right: Table, condition): - ''' - Perform a full join on the table with the supplied table (right). - - Args: - condition: string. A condition using the following format: - 'column[<,<=,==,>=,>]value' or - 'value[<,<=,==,>=,>]column'. - - Operators supported: (<,<=,==,>=,>) - ''' - join_table, column_index_left, column_index_right, operator = self._general_join_processing(table_right, condition, 'full') - - right_column = table_right.column_by_name(table_right.column_names[column_index_right]) - left_column = self.column_by_name(self.column_names[column_index_left]) - - right_table_row_length = len(table_right.column_names) - left_table_row_length = len(self.column_names) - - for row_left in self.data: - left_value = row_left[column_index_left] - if left_value is None: - continue - if left_value not in right_column: - join_table._insert(row_left + right_table_row_length*["NULL"]) - else: - for row_right in table_right.data: - right_value = row_right[column_index_right] - if left_value == right_value: - join_table._insert(row_left + row_right) - - for row_right in table_right.data: - right_value = row_right[column_index_right] - - if right_value is None: - continue - elif right_value not in left_column: - join_table._insert(left_table_row_length*["NULL"] + row_right) - - return join_table - - def show(self, no_of_rows=None, is_locked=False): - ''' - Print the table in a nice readable format. - - Args: - no_of_rows: int. Number of rows. - is_locked: boolean. Whether it is locked (False by default). - ''' - output = "" - # if the table is locked, add locked keyword to title - if is_locked: - output += f"\n## {self._name} (locked) ##\n" - else: - output += f"\n## {self._name} ##\n" - - # headers -> "column name (column type)" - headers = [f'{col} ({tp.__name__})' for col, tp in zip(self.column_names, self.column_types)] - if self.pk_idx is not None: - # table has a primary key, add PK next to the appropriate column - headers[self.pk_idx] = headers[self.pk_idx]+' #PK#' - - if self.unique_idx is not None: - for unq in self.unique_idx: - headers[unq]=headers[unq]+' #UNQ#' - # detect the rows that are no tfull of nones (these rows have been deleted) - # if we dont skip these rows, the returning table has empty rows at the deleted positions - non_none_rows = [row for row in self.data if any(row)] - # print using tabulate - print(tabulate(non_none_rows[:no_of_rows], headers=headers)+'\n') - - - def _parse_condition(self, condition, join=False): - ''' - Parse the single string condition and return the value of the column and the operator. - - Args: - condition: string. A condition using the following format: - 'column[<,<=,==,>=,>]value' or - 'value[<,<=,==,>=,>]column'. - - Operatores supported: (<,<=,==,>=,>) - join: boolean. Whether to join or not (False by default). - ''' - # if both_columns (used by the join function) return the names of the names of the columns (left first) - if join: - return split_condition(condition) - - # cast the value with the specified column's type and return the column name, the operator and the casted value - left, op, right = split_condition(condition) - if right in self.column_names:#'value[<,<=,==,>=,>]column' fromat - coltype = self.column_types[self.column_names.index(right)] - op =reverse_op(op) - return right, op, coltype(left) - - elif left in self.column_names:#'column[<,<=,==,>=,>]value' format - coltype = self.column_types[self.column_names.index(left)] - - return left, op, coltype(right) - else: - #raise ValueError(f'Condition is not valid (cant find column name)') - raise ValueError(f'Condition is not valid (cant find column name)') - - - def _load_from_file(self, filename): - ''' - Load table from a pkl file (not used currently). - - Args: - filename: string. Name of pkl file. - ''' - f = open(filename, 'rb') - tmp_dict = pickle.load(f) - f.close() - - self.__dict__.update(tmp_dict.__dict__) From 0070e5a0a6c04c3678becc76b7e02dc5c4393dd1 Mon Sep 17 00:00:00 2001 From: Kapiniaris-Xrisovalantis <92021533+Kapiniaris-Xrisovalantis@users.noreply.github.com> Date: Mon, 4 Sep 2023 10:49:16 +0300 Subject: [PATCH 08/16] Delete hashidx.py added in wrong folder --- hashidx.py | 82 ------------------------------------------------------ 1 file changed, 82 deletions(-) delete mode 100644 hashidx.py diff --git a/hashidx.py b/hashidx.py deleted file mode 100644 index 1fa908a1..00000000 --- a/hashidx.py +++ /dev/null @@ -1,82 +0,0 @@ -class ExtendibleHashing: - def __init__(self, global_depth): - self.global_depth = global_depth - self.directory = {} - self.bucket_size = 4 # Number of elements in each bucket - - def hash_function(self, key): - return key % (2 ** self.global_depth) - - def insert(self, key, value): - hashed_key = self.hash_function(key) - if hashed_key in self.directory: - bucket = self.directory[hashed_key] - for i, (k, v) in enumerate(bucket): - if k == key: - bucket[i] = (k, value) # Update value for existing key - return - if len(bucket) < self.bucket_size: - bucket.append((key, value)) - else: - if self.global_depth == len(bin(hashed_key)) - 2: - self.double_directory() - self.split_bucket(hashed_key) - self.insert(key, value) - else: - if self.global_depth == len(bin(hashed_key)) - 2: - self.double_directory() - self.directory[hashed_key] = [(key, value)] - - - def double_directory(self): - self.global_depth += 1 - directory_size = 2 ** (self.global_depth - 1) - for i in range(directory_size): - if i in self.directory: - self.directory[i + directory_size] = self.directory[i] - else: - self.directory[i + directory_size] = [] - - def split_bucket(self, hashed_key): - bucket = self.directory[hashed_key] - new_bucket = [] - split_index = len(bucket) // 2 - - # Split the bucket into two by creating a new bucket and updating the directory - new_bucket = bucket[split_index:] - bucket = bucket[:split_index] - self.directory[hashed_key] = bucket - - # Update the hashed keys of the new bucket and its duplicates - new_hashed_key = hashed_key + (2 ** (self.global_depth - 1)) - for key in range(new_hashed_key, new_hashed_key + (2 ** (self.global_depth - 1)), 2 ** (self.global_depth - 1)): - self.directory[key] = new_bucket.copy() - - - - - def find(self, key): - hashed_key = self.hash_function(key) - if hashed_key in self.directory: - bucket = self.directory[hashed_key] - for item in bucket: - if item[0] == key: - return item[1] - return None - - def delete(self, key): - hashed_key = self.hash_function(key) - if hashed_key in self.directory: - bucket = self.directory[hashed_key] - for i, item in enumerate(bucket): - if item[0] == key: - del bucket[i] - return True - return False - - def show(self): - for hashed_key, bucket in self.directory.items(): - print(f"Hashed Key: {hashed_key}") - for item in bucket: - print(f" Key: {item[0]}, Value: {item[1]}") - From 946011e7b869fb546bcf2a60ab715b30ba723665 Mon Sep 17 00:00:00 2001 From: Kapiniaris-Xrisovalantis <92021533+Kapiniaris-Xrisovalantis@users.noreply.github.com> Date: Mon, 4 Sep 2023 10:49:35 +0300 Subject: [PATCH 09/16] Delete btree.py added in wrong folder --- btree.py | 350 ------------------------------------------------------- 1 file changed, 350 deletions(-) delete mode 100644 btree.py diff --git a/btree.py b/btree.py deleted file mode 100644 index 6b3a6fdc..00000000 --- a/btree.py +++ /dev/null @@ -1,350 +0,0 @@ -''' -https://en.wikipedia.org/wiki/B%2B_tree -''' - -class Node: - ''' - Node abstraction. Represents a single bucket - ''' - def __init__(self, b, values=None, ptrs=None,left_sibling=None, right_sibling=None, parent=None, is_leaf=False): - self.b = b # branching factor - self.values = [] if values is None else values # Values (the data from the pk column) - self.ptrs = [] if ptrs is None else ptrs # ptrs (the indexes of each datapoint or the index of another bucket) - self.left_sibling = left_sibling # the index of a buckets left sibling - self.right_sibling = right_sibling # the index of a buckets right sibling - self.parent = parent # the index of a buckets parent - self.is_leaf = is_leaf # a boolean value signaling whether the node is a leaf or not - - - def find(self, value, return_ops=False): - ''' - Returns the index of the next node to search for a value if the node is not a leaf (a ptrs of the available ones). - If it is a leaf (we have found the appropriate node), nothing is returned. - - Args: - value: float. The value being searched for. - return_ops: boolean. Set to True if you want to use the number of operations (for benchmarking). - ''' - ops = 0 # number of operations (<>= etc). Used for benchmarking - if self.is_leaf: # - return - - # for each value in the node, if the user supplied value is smaller, return the btrees value index - # else (no value in the node is larger) return the last ptr - for index, existing_val in enumerate(self.values): - ops+=1 - if value is None or existing_val is None: - continue - if value= etc). Used for benchmarking - - #start with the root node - node = self.nodes[self.root] - # while the node that we are searching in is not a leaf - # keep searching - while not node.is_leaf: - idx, ops1 = node.find(value, return_ops=True) - node = self.nodes[idx] - ops += ops1 - - # finally return the index of the appropriate node (and the ops if you want to) - if return_ops: - return self.nodes.index(node), ops - else: - return self.nodes.index(node) - - - def split(self, node_id): - ''' - Split the node with index=node_id. - - Args: - node_id: float. The corresponding ID of the node. - ''' - # fetch the node to be split - node = self.nodes[node_id] - # the value that will be propagated to the parent is the middle one. - new_parent_value = node.values[len(node.values)//2] - if node.is_leaf: - # if the node is a leaf, the parent value should be a part of the new node (right) - # Important: in a b+tree, every value should appear in a leaf - right_values = node.values[len(node.values)//2:] - right_ptrs = node.ptrs[len(node.ptrs)//2:] - - # create the new node with the right half of the old nodes values and ptrs (including the middle ones) - right = Node(self.b, right_values, right_ptrs,\ - left_sibling=node_id, right_sibling=node.right_sibling, parent=node.parent, is_leaf=node.is_leaf) - # since the new node (right) will be the next one to be appended to the nodes list - # its index will be equal to the length of the nodes list. - # Thus we set the old nodes (now left) right sibling to the right nodes future index (len of nodes) - if node.right_sibling is not None: - self.nodes[node.right_sibling].left_sibling = len(self.nodes) - node.right_sibling = len(self.nodes) - - - else: - # if the node is not a leaf, the parent value shoudl NOT be part of the new node - right_values = node.values[len(node.values)//2+1:] - if self.b%2==1: - right_ptrs = node.ptrs[len(node.ptrs)//2:] - else: - right_ptrs = node.ptrs[len(node.ptrs)//2+1:] - - # if nonleafs should be connected change the following two lines and add siblings - right = Node(self.b, right_values, right_ptrs,\ - parent=node.parent, is_leaf=node.is_leaf) - # make sure that a non leaf node doesnt have a parent - node.right_sibling = None - # the right node's kids should have him as a parent (if not all nodes will have left as parent) - for ptr in right_ptrs: - self.nodes[ptr].parent = len(self.nodes) - - # old node (left) keeps only the first half of the values/ptrs - node.values = node.values[:len(node.values)//2] - if self.b%2==1: - node.ptrs = node.ptrs[:len(node.ptrs)//2] - else: - node.ptrs = node.ptrs[:len(node.ptrs)//2+1] - - # append the new node (right) to the nodes list - self.nodes.append(right) - - # If the new nodes have no parents (a new level needs to be added - if node.parent is None: - # its the root that is split - # new root contains the parent value and ptrs to the two recently split nodes - parent = Node(self.b, [new_parent_value], [node_id, len(self.nodes)-1]\ - ,parent=node.parent, is_leaf=False) - - # set root, and parent of split celss to the index of the new root node (len of nodes-1) - self.nodes.append(parent) - self.root = len(self.nodes)-1 - node.parent = len(self.nodes)-1 - right.parent = len(self.nodes)-1 - else: - # insert the parent value to the parent - - self.nodes[node.parent].insert(new_parent_value, len(self.nodes)-1) - # check whether the parent needs to be split - if len(self.nodes[node.parent].values)==self.b: - self.split(node.parent) - - - - - def show(self): - ''' - Show important info for each node (sort by level - root first, then left to right). - ''' - nds = [] - nds.append(self.root) - for ptr in nds: - if self.nodes[ptr].is_leaf: - continue - nds.extend(self.nodes[ptr].ptrs) - - for ptr in nds: - print(f'## {ptr} ##') - self.nodes[ptr].show() - print('----') - - - def plot(self): - ## arrange the nodes top to bottom left to right - nds = [] - nds.append(self.root) - for ptr in nds: - if self.nodes[ptr].is_leaf: - continue - nds.extend(self.nodes[ptr].ptrs) - - # add each node and each link - g = 'digraph G{\nforcelabels=true;\n' - - for i in nds: - node = self.nodes[i] - g+=f'{i} [label="{node.values}"]\n' - if node.is_leaf: - continue - # if node.left_sibling is not None: - # g+=f'"{node.values}"->"{self.nodes[node.left_sibling].values}" [color="blue" constraint=false];\n' - # if node.right_sibling is not None: - # g+=f'"{node.values}"->"{self.nodes[node.right_sibling].values}" [color="green" constraint=false];\n' - # - # g+=f'"{node.values}"->"{self.nodes[node.parent].values}" [color="red" constraint=false];\n' - else: - for child in node.ptrs: - g+=f'{child} [label="{self.nodes[child].values}"]\n' - g+=f'{i}->{child};\n' - g +="}" - - try: - from graphviz import Source - src = Source(g) - src.render('bplustree', view=True) - except ImportError: - print('"graphviz" package not found. Writing to graph.gv.') - with open('graph.gv','w') as f: - f.write(g) - - def find(self, operator, value): - ''' - Return ptrs of elements where btree_value"operator"value. - Important, the user supplied "value" is the right value of the operation. That is why the operation are reversed below. - The left value of the op is the btree value. - - Args: - operator: string. The provided evaluation operator. - value: float. The value being searched for. - ''' - results = [] - # find the index of the node that the element should exist in - leaf_idx, ops = self._search(value, True) - target_node = self.nodes[leaf_idx] - - if operator == '=': - # if the element exist, append to list, else pass and return - try: - results.append(target_node.ptrs[target_node.values.index(value)]) - # print('Found') - except: - # print('Not found') - pass - - # for all other ops, the code is the same, only the operations themselves and the sibling indexes change - # for > and >= (btree value is >/>= of user supplied value), we return all the right siblings (all values are larger than current cell) - # for < and <= (btree value is ': - for idx, node_value in enumerate(target_node.values): - ops+=1 - - if node_value > value: - results.append(target_node.ptrs[idx]) - while target_node.right_sibling is not None: - target_node = self.nodes[target_node.right_sibling] - results.extend(target_node.ptrs) - - - if operator == '>=': - for idx, node_value in enumerate(target_node.values): - ops+=1 - if node_value >= value: - results.append(target_node.ptrs[idx]) - while target_node.right_sibling is not None: - target_node = self.nodes[target_node.right_sibling] - results.extend(target_node.ptrs) - - if operator == '<': - for idx, node_value in enumerate(target_node.values): - ops+=1 - if node_value < value: - results.append(target_node.ptrs[idx]) - while target_node.left_sibling is not None: - target_node = self.nodes[target_node.left_sibling] - results.extend(target_node.ptrs) - - if operator == '<=': - for idx, node_value in enumerate(target_node.values): - ops+=1 - if node_value <= value: - results.append(target_node.ptrs[idx]) - while target_node.left_sibling is not None: - target_node = self.nodes[target_node.left_sibling] - results.extend(target_node.ptrs) - - # print the number of operations (usefull for benchamrking) - # print(f'With BTree -> {ops} comparison operations') - return results From 551bed68f79b8640e529b885c890d565b504cd7c Mon Sep 17 00:00:00 2001 From: Kapiniaris-Xrisovalantis <92021533+Kapiniaris-Xrisovalantis@users.noreply.github.com> Date: Mon, 4 Sep 2023 10:50:23 +0300 Subject: [PATCH 10/16] Delete database.py added in wrong folder --- database.py | 873 ---------------------------------------------------- 1 file changed, 873 deletions(-) delete mode 100644 database.py diff --git a/database.py b/database.py deleted file mode 100644 index 054f6cc4..00000000 --- a/database.py +++ /dev/null @@ -1,873 +0,0 @@ -from __future__ import annotations -import pickle -from time import sleep, localtime, strftime -import os,sys -import logging -import warnings -#import pyreadline -import re -from tabulate import tabulate - -sys.path.append(f'{os.path.dirname(os.path.dirname(os.path.abspath(__file__)))}/miniDB') -from miniDB import table -sys.modules['table'] = table - -from hashidx import ExtendibleHashing as hash -from joins import Inlj, Smj -from btree import Btree -from misc import split_condition -from table import Table - - -# readline.clear_history() - -class Database: - ''' - Main Database class, containing tables. - ''' - - def __init__(self, name, load=True, verbose = True): - self.tables = {} - self._name = name - self.verbose = verbose - - self.savedir = f'dbdata/{name}_db' - - if load: - try: - self.load_database() - logging.info(f'Loaded "{name}".') - return - except: - if verbose: - warnings.warn(f'Database "{name}" does not exist. Creating new.') - - # create dbdata directory if it doesnt exist - if not os.path.exists('dbdata'): - os.mkdir('dbdata') - - # create new dbs save directory - try: - os.mkdir(self.savedir) - except: - pass - - # create all the meta tables - self.create_table('meta_length', 'table_name,no_of_rows', 'str,int') - self.create_table('meta_locks', 'table_name,pid,mode', 'str,int,str') - self.create_table('meta_insert_stack', 'table_name,indexes', 'str,list') - self.create_table('meta_indexes', 'table_name,index_name,index_type,column', 'str,str,str,str') - self.save_database() - - def save_database(self): - ''' - Save database as a pkl file. This method saves the database object, including all tables and attributes. - ''' - for name, table in self.tables.items(): - with open(f'{self.savedir}/{name}.pkl', 'wb') as f: - pickle.dump(table, f) - - def _save_locks(self): - ''' - Stores the meta_locks table to file as meta_locks.pkl. - ''' - with open(f'{self.savedir}/meta_locks.pkl', 'wb') as f: - pickle.dump(self.tables['meta_locks'], f) - - def load_database(self): - ''' - Load all tables that are part of the database (indices noted here are loaded). - - Args: - path: string. Directory (path) of the database on the system. - ''' - path = f'dbdata/{self._name}_db' - for file in os.listdir(path): - - if file[-3:]!='pkl': # if used to load only pkl files - continue - f = open(path+'/'+file, 'rb') - tmp_dict = pickle.load(f) - f.close() - name = f'{file.split(".")[0]}' - self.tables.update({name: tmp_dict}) - # setattr(self, name, self.tables[name]) - - #### IO #### - - def _update(self): - ''' - Update all meta tables. - ''' - self._update_meta_length() - self._update_meta_insert_stack() - - - def create_table(self, name, column_names, column_types, primary_key=None, unique=None,load=None): - ''' - This method create a new table. This table is saved and can be accessed via db_object.tables['table_name'] or db_object.table_name - - Args: - name: string. Name of table. - column_names: list. Names of columns. - column_types: list. Types of columns. - primary_key: string. The primary key (if it exists). - load: boolean. Defines table object parameters as the name of the table and the column names. - ''' - # print('here -> ', column_names.split(',')) - self.tables.update({name: Table(name=name, column_names=column_names.split(','), column_types=column_types.split(','), primary_key=primary_key,unique=unique, load=load)}) - # self._name = Table(name=name, column_names=column_names, column_types=column_types, load=load) - # check that new dynamic var doesnt exist already - # self.no_of_tables += 1 - self._update() - self.save_database() - # (self.tables[name]) - if self.verbose: - print(f'Created table "{name}".') - - - def drop_table(self, table_name): - ''' - Drop table from current database. - - Args: - table_name: string. Name of table. - ''' - self.load_database() - self.lock_table(table_name) - - self.tables.pop(table_name) - if os.path.isfile(f'{self.savedir}/{table_name}.pkl'): - os.remove(f'{self.savedir}/{table_name}.pkl') - else: - warnings.warn(f'"{self.savedir}/{table_name}.pkl" not found.') - self.delete_from('meta_locks', f'table_name={table_name}') - self.delete_from('meta_length', f'table_name={table_name}') - self.delete_from('meta_insert_stack', f'table_name={table_name}') - - if self._has_index(table_name): - to_be_deleted = [] - for key, table in enumerate(self.tables['meta_indexes'].column_by_name('table_name')): - if table == table_name: - to_be_deleted.append(key) - - for i in reversed(to_be_deleted): - self.drop_index(self.tables['meta_indexes'].data[i][1]) - - try: - delattr(self, table_name) - except AttributeError: - pass - # self._update() - self.save_database() - - - def import_table(self, table_name, filename, column_types=None, primary_key=None, unique=None): - ''' - Creates table from CSV file. - - Args: - filename: string. CSV filename. If not specified, filename's name will be used. - column_types: list. Types of columns. If not specified, all will be set to type str. - primary_key: string. The primary key (if it exists). - ''' - file = open(filename, 'r') - - first_line=True - for line in file.readlines(): - if first_line: - colnames = line.strip('\n') - if column_types is None: - column_types = ",".join(['str' for _ in colnames.split(',')]) - self.create_table(name=table_name, column_names=colnames, column_types=column_types, primary_key=primary_key,unique=unique) - lock_ownership = self.lock_table(table_name, mode='x') - first_line = False - continue - self.tables[table_name]._insert(line.strip('\n').split(',')) - - if lock_ownership: - self.unlock_table(table_name) - self._update() - self.save_database() - - - def export(self, table_name, filename=None): - ''' - Transform table to CSV. - - Args: - table_name: string. Name of table. - filename: string. Output CSV filename. - ''' - res = '' - for row in [self.tables[table_name].column_names]+self.tables[table_name].data: - res+=str(row)[1:-1].replace('\'', '').replace('"','').replace(' ','')+'\n' - - if filename is None: - filename = f'{table_name}.csv' - - with open(filename, 'w') as file: - file.write(res) - - def table_from_object(self, new_table): - ''' - Add table object to database. - - Args: - new_table: string. Name of new table. - ''' - - self.tables.update({new_table._name: new_table}) - if new_table._name not in self.__dir__(): - setattr(self, new_table._name, new_table) - else: - raise Exception(f'"{new_table._name}" attribute already exists in class "{self.__class__.__name__}".') - self._update() - self.save_database() - - - - ##### table functions ##### - - # In every table function a load command is executed to fetch the most recent table. - # In every table function, we first check whether the table is locked. Since we have implemented - # only the X lock, if the tables is locked we always abort. - # After every table function, we update and save. Update updates all the meta tables and save saves all - # tables. - - # these function calls are named close to the ones in postgres - - def cast(self, column_name, table_name, cast_type): - ''' - Modify the type of the specified column and cast all prexisting values. - (Executes type() for every value in column and saves) - - Args: - table_name: string. Name of table (must be part of database). - column_name: string. The column that will be casted (must be part of database). - cast_type: type. Cast type (do not encapsulate in quotes). - ''' - self.load_database() - - lock_ownership = self.lock_table(table_name, mode='x') - self.tables[table_name]._cast_column(column_name, eval(cast_type)) - if lock_ownership: - self.unlock_table(table_name) - self._update() - self.save_database() - - def insert_into(self, table_name, row_str): - ''' - Inserts data to given table. - - Args: - table_name: string. Name of table (must be part of database). - row: list. A list of values to be inserted (will be casted to a predifined type automatically). - lock_load_save: boolean. If False, user needs to load, lock and save the states of the database (CAUTION). Useful for bulk-loading. - ''' - row = row_str.strip().split(',') - self.load_database() - # fetch the insert_stack. For more info on the insert_stack - # check the insert_stack meta table - - lock_ownership = self.lock_table(table_name, mode='x') - insert_stack = self._get_insert_stack_for_table(table_name) - try: - self.tables[table_name]._insert(row, insert_stack) - except Exception as e: - logging.info(e) - logging.info('ABORTED') - - self._update_meta_insert_stack_for_tb(table_name, insert_stack[:-1]) - - if lock_ownership: - self.unlock_table(table_name) - self._update() - self.save_database() - - - def update_table(self, table_name, set_args, condition): - ''' - Update the value of a column where a condition is met. - - Args: - table_name: string. Name of table (must be part of database). - set_value: string. New value of the predifined column name. - set_column: string. The column to be altered. - condition: string. A condition using the following format: - 'column[<,<=,==,>=,>]value' or - 'value[<,<=,==,>=,>]column'. - - Operatores supported: (<,<=,==,>=,>) - ''' - set_column, set_value = set_args.replace(' ','').split('=') - self.load_database() - - lock_ownership = self.lock_table(table_name, mode='x') - self.tables[table_name]._update_rows(set_value, set_column, condition) - if lock_ownership: - self.unlock_table(table_name) - self._update() - self.save_database() - - def delete_from(self, table_name, condition): - ''' - Delete rows of table where condition is met. - - Args: - table_name: string. Name of table (must be part of database). - condition: string. A condition using the following format: - 'column[<,<=,==,>=,>]value' or - 'value[<,<=,==,>=,>]column'. - - Operatores supported: (<,<=,==,>=,>) - ''' - self.load_database() - - lock_ownership = self.lock_table(table_name, mode='x') - deleted = self.tables[table_name]._delete_where(condition) - if lock_ownership: - self.unlock_table(table_name) - self._update() - self.save_database() - # we need the save above to avoid loading the old database that still contains the deleted elements - if table_name[:4]!='meta': - self._add_to_insert_stack(table_name, deleted) - self.save_database() - - def select(self, columns, table_name, condition, distinct=None, order_by=None, \ - limit=True, desc=None, save_as=None, return_object=True): - ''' - Selects and outputs a table's data where condtion is met. - - Args: - table_name: string. Name of table (must be part of database). - columns: list. The columns that will be part of the output table (use '*' to select all available columns) - condition: string. A condition using the following format: - 'column[<,<=,==,>=,>]value' or - 'value[<,<=,==,>=,>]column'. - - Operatores supported: (<,<=,==,>=,>) - order_by: string. A column name that signals that the resulting table should be ordered based on it (no order if None). - desc: boolean. If True, order_by will return results in descending order (True by default). - limit: int. An integer that defines the number of rows that will be returned (all rows if None). - save_as: string. The name that will be used to save the resulting table into the database (no save if None). - return_object: boolean. If True, the result will be a table object (useful for internal use - the result will be printed by default). - distinct: boolean. If True, the resulting table will contain only unique rows. - ''' - - # print(table_name) - self.load_database() - if isinstance(table_name,Table): - return table_name._select_where(columns, condition, distinct, order_by, desc, limit) - - condition_column="" - if condition is not None: - if "between" in condition.split(): - condition_column = condition.split()[0] - elif "and" in condition.split() : - columns2=[] - conditions=[] - conditions=condition.split(" and ") - for con in conditions: - col=self.tables[table_name]._parse_condition(con) - columns2.append(col) - for con in columns2: - if (self.tables[table_name].pk_idx is not None and con[0] == self.tables[table_name].column_names[self.tables[table_name].pk_idx]) or (self.tables[table_name].unique is not None and con[0] in self.tables[table_name].unique ):#since only pk supports index , and only one pk per table we keep the column if it is pk - condition_column = con[0] - else: - condition_column="" - - elif "or" in condition.split(): - - columns2=[] - conditions=[] - conditions=condition.split(" or ") - for con in conditions: - - col=self.tables[table_name]._parse_condition(con) - - columns2.append(col) - - for con in columns2: - - - if self.tables[table_name].pk_idx is not None and con[0] == self.tables[table_name].column_names[self.tables[table_name].pk_idx]: - condition_column = con[0] - else: - condition_column = "" - else: - - col,op=self.tables[table_name]._parse_condition(condition) - if op=="=": - condition_column=col - - - - - - - - # self.lock_table(table_name, mode='x') - if self.is_locked(table_name): - return - - if self.tables[table_name].pk_idx is not None and self._has_index(table_name,condition_column): - print("here") - index_name = self.select('*', 'meta_indexes', f'table_name={table_name}',return_object=True).column_by_name('index_name')[0] - index_type = self.select('*', 'meta_indexes', f'table_name={table_name}',return_object=True).column_by_name('index_type')[0] - if index_type=="btree": - bt = self._load_idx(index_name) - table = self.tables[table_name]._select_where_with_btree(columns, bt, condition, distinct, order_by, desc, limit) - elif index_type=="hash": - bt = self._load_idx(index_name) - table = self.tables[table_name]._select_where_with_hash(columns, bt, condition, distinct, order_by, desc, limit) - - elif self.tables[table_name].unique_idx is not None and self._has_index(table_name): - - found = False - for j in range (len(self.tables[table_name].unique)): - - if self.tables[table_name].unique[j] in condition_column: - found = True - index_name = self.select('*', 'meta_indexes', f'table_name={table_name}', return_object=True).column_by_name('index_name')[0] - print(index_name) - bt = self._load_idx(index_name) - bt.show() - table = self.tables[table_name]._select_where_with_btree(columns, bt, condition, distinct, order_by, desc, limit) - break - if not found: - - table = self.tables[table_name]._select_where(columns, condition, distinct, order_by, desc, limit) - else: - - table = self.tables[table_name]._select_where(columns, condition, distinct, order_by, desc, limit) - # self.unlock_table(table_name) - if save_as is not None: - table._name = save_as - self.table_from_object(table) - else: - if return_object: - return table - else: - return table.show() - - - def show_table(self, table_name, no_of_rows=None): - ''' - Print table in a readable tabular design (using tabulate). - - Args: - table_name: string. Name of table (must be part of database). - ''' - self.load_database() - - self.tables[table_name].show(no_of_rows, self.is_locked(table_name)) - - - def sort(self, table_name, column_name, asc=False): - ''' - Sorts a table based on a column. - - Args: - table_name: string. Name of table (must be part of database). - column_name: string. the column name that will be used to sort. - asc: If True sort will return results in ascending order (False by default). - ''' - - self.load_database() - - lock_ownership = self.lock_table(table_name, mode='x') - self.tables[table_name]._sort(column_name, asc=asc) - if lock_ownership: - self.unlock_table(table_name) - self._update() - self.save_database() - - def create_view(self, table_name, table): - ''' - Create a virtual table based on the result-set of the SQL statement provided. - - Args: - table_name: string. Name of the table that will be saved. - table: table. The table that will be saved. - ''' - table._name = table_name - self.table_from_object(table) - - def join(self, mode, left_table, right_table, condition, save_as=None, return_object=True): - ''' - Join two tables that are part of the database where condition is met. - - Args: - left_table: string. Name of the left table (must be in DB) or Table obj. - right_table: string. Name of the right table (must be in DB) or Table obj. - condition: string. A condition using the following format: - 'column[<,<=,==,>=,>]value' or - 'value[<,<=,==,>=,>]column'. - - Operators supported: (<,<=,==,>=,>) - save_as: string. The output filename that will be used to save the resulting table in the database (won't save if None). - return_object: boolean. If True, the result will be a table object (useful for internal usage - the result will be printed by default). - ''' - self.load_database() - if self.is_locked(left_table) or self.is_locked(right_table): - return - - left_table = left_table if isinstance(left_table, Table) else self.tables[left_table] - right_table = right_table if isinstance(right_table, Table) else self.tables[right_table] - - - if mode=='inner': - res = left_table._inner_join(right_table, condition) - - elif mode=='left': - res = left_table._left_join(right_table, condition) - - elif mode=='right': - res = left_table._right_join(right_table, condition) - - elif mode=='full': - res = left_table._full_join(right_table, condition) - - elif mode=='inl': - # Check if there is an index of either of the two tables available, as if there isn't we can't use inlj - leftIndexExists = self._has_index(left_table._name) - rightIndexExists = self._has_index(right_table._name) - - if not leftIndexExists and not rightIndexExists: - res = None - raise Exception('Index-nested-loop join cannot be executed. Use inner join instead.\n') - elif rightIndexExists: - index_name = self.select('*', 'meta_indexes', f'table_name={right_table._name}', return_object=True).column_by_name('index_name')[0] - res = Inlj(condition, left_table, right_table, self._load_idx(index_name), 'right').join() - elif leftIndexExists: - index_name = self.select('*', 'meta_indexes', f'table_name={left_table._name}', return_object=True).column_by_name('index_name')[0] - res = Inlj(condition, left_table, right_table, self._load_idx(index_name), 'left').join() - - elif mode=='sm': - res = Smj(condition, left_table, right_table).join() - - else: - raise NotImplementedError - - if save_as is not None: - res._name = save_as - self.table_from_object(res) - else: - if return_object: - return res - else: - res.show() - - if return_object: - return res - else: - res.show() - - def lock_table(self, table_name, mode='x'): - ''' - Locks the specified table using the exclusive lock (X). - - Args: - table_name: string. Table name (must be part of database). - ''' - if table_name[:4]=='meta' or table_name not in self.tables.keys() or isinstance(table_name,Table): - return - - with open(f'{self.savedir}/meta_locks.pkl', 'rb') as f: - self.tables.update({'meta_locks': pickle.load(f)}) - - try: - pid = self.tables['meta_locks']._select_where('pid',f'table_name={table_name}').data[0][0] - if pid!=os.getpid(): - raise Exception(f'Table "{table_name}" is locked by process with pid={pid}') - else: - return False - - except IndexError: - pass - - if mode=='x': - self.tables['meta_locks']._insert([table_name, os.getpid(), mode]) - else: - raise NotImplementedError - self._save_locks() - return True - # print(f'Locking table "{table_name}"') - - def unlock_table(self, table_name, force=False): - ''' - Unlocks the specified table that is exclusively locked (X). - - Args: - table_name: string. Table name (must be part of database). - ''' - if table_name not in self.tables.keys(): - raise Exception(f'Table "{table_name}" is not in database') - - if not force: - try: - # pid = self.select('*','meta_locks', f'table_name={table_name}', return_object=True).data[0][1] - pid = self.tables['meta_locks']._select_where('pid',f'table_name={table_name}').data[0][0] - if pid!=os.getpid(): - raise Exception(f'Table "{table_name}" is locked by the process with pid={pid}') - except IndexError: - pass - self.tables['meta_locks']._delete_where(f'table_name={table_name}') - self._save_locks() - # print(f'Unlocking table "{table_name}"') - - def is_locked(self, table_name): - ''' - Check whether the specified table is exclusively locked (X). - - Args: - table_name: string. Table name (must be part of database). - ''' - if isinstance(table_name,Table) or table_name[:4]=='meta': # meta tables will never be locked (they are internal) - return False - - with open(f'{self.savedir}/meta_locks.pkl', 'rb') as f: - self.tables.update({'meta_locks': pickle.load(f)}) - - try: - pid = self.tables['meta_locks']._select_where('pid',f'table_name={table_name}').data[0][0] - if pid!=os.getpid(): - raise Exception(f'Table "{table_name}" is locked by the process with pid={pid}') - - except IndexError: - pass - return False - - - #### META #### - - # The following functions are used to update, alter, load and save the meta tables. - # Important: Meta tables contain info regarding the NON meta tables ONLY. - # i.e. meta_length will not show the number of rows in meta_locks etc. - - def _update_meta_length(self): - ''' - Updates the meta_length table. - ''' - - for table in self.tables.values(): - - if table._name[:4]=='meta': #skip meta tables - continue - if table._name not in self.tables['meta_length'].column_by_name('table_name'): # if new table, add record with 0 no. of rows - self.tables['meta_length']._insert([table._name, 0]) - - # the result needs to represent the rows that contain data. Since we use an insert_stack - # some rows are filled with Nones. We skip these rows. - non_none_rows = len([row for row in table.data if any(row)]) - self.tables['meta_length']._update_rows(non_none_rows, 'no_of_rows', f'table_name={table._name}') - # self.update_row('meta_length', len(table.data), 'no_of_rows', 'table_name', '==', table._name) - - def _update_meta_locks(self): - ''' - Updates the meta_locks table. - ''' - for table in self.tables.values(): - if table._name[:4]=='meta': #skip meta tables - continue - if table._name not in self.tables['meta_locks'].column_by_name('table_name'): - - self.tables['meta_locks']._insert([table._name, False]) - # self.insert('meta_locks', [table._name, False]) - - def _update_meta_insert_stack(self): - ''' - Updates the meta_insert_stack table. - ''' - for table in self.tables.values(): - if table._name[:4]=='meta': #skip meta tables - continue - if table._name not in self.tables['meta_insert_stack'].column_by_name('table_name'): - self.tables['meta_insert_stack']._insert([table._name, []]) - - - def _add_to_insert_stack(self, table_name, indexes): - ''' - Adds provided indices to the insert stack of the specified table. - - Args: - table_name: string. Table name (must be part of database). - indexes: list. The list of indices that will be added to the insert stack (the indices of the newly deleted elements). - ''' - old_lst = self._get_insert_stack_for_table(table_name) - self._update_meta_insert_stack_for_tb(table_name, old_lst+indexes) - - def _get_insert_stack_for_table(self, table_name): - ''' - Returns the insert stack of the specified table. - - Args: - table_name: string. Table name (must be part of database). - ''' - return self.tables['meta_insert_stack']._select_where('*', f'table_name={table_name}').column_by_name('indexes')[0] - # res = self.select('meta_insert_stack', '*', f'table_name={table_name}', return_object=True).indexes[0] - # return res - - def _update_meta_insert_stack_for_tb(self, table_name, new_stack): - ''' - Replaces the insert stack of a table with the one supplied by the user. - - Args: - table_name: string. Table name (must be part of database). - new_stack: string. The stack that will be used to replace the existing one. - ''' - self.tables['meta_insert_stack']._update_rows(new_stack, 'indexes', f'table_name={table_name}') - - - # indexes - def create_index(self, index_name, table_name,index_type='btree' ,column='pkey'): - ''' - Creates an index on a specified table with a given name. - Important: An index can only be created on a primary key (the user does not specify the column). - - Args: - table_name: string. Table name (must be part of database). - index_name: string. Name of the created index. - ''' - #if the user didn't specify a column, make the index on primary key - if self.tables[table_name].pk_idx is not None and column=='pkey': - column = self.tables[table_name].pk - - - if self.tables[table_name].pk_idx is None and self.tables[table_name].unique is None: # if no primary key, no index - raise Exception('Cannot create index. Table has no primary key or unique columns.') - - if index_name not in self.tables['meta_indexes'].column_by_name('index_name'): - # currently only btree is supported. This can be changed by adding another if. - if index_type=='btree': - logging.info('Creating Btree index.') - print('Creating BTREE index') - # insert a record with the name of the index and the table on which it's created to the meta_indexes table - self.tables['meta_indexes']._insert([table_name, index_name,index_type,column]) - # crate the actual index - self._construct_index(table_name, index_name,column,index_type) - #self._construct_index(table_name, index_name) - self.save_database() - if index_type=='hash': - logging.info('Creating Hash index.') - print('Creating HASH index') - # insert a record with the name of the index and the table on which it's created to the meta_indexes table - self.tables['meta_indexes']._insert([table_name, index_name,index_type,column]) - # crate the actual index - self._construct_index(table_name, index_name,column,index_type) - #self._construct_index(table_name, index_name) - self.save_database() - else: - raise Exception('Cannot create index. Another index with the same name already exists.') - - def _construct_index(self, table_name, index_name,column,index_type='btree'): - ''' - Construct a btree on a table and save. - - Args: - table_name: string. Table name (must be part of database). - index_name: string. Name of the created index. - ''' - if index_type=='btree': - bt = Btree(1) # 3 is arbitrary - - # for each record in the primary key of the table, insert its value and index to the btree - if self.tables[table_name].pk is not None and column==self.tables[table_name].pk: - for idx, key in enumerate(self.tables[table_name].column_by_name(column)): - if key is None: - continue - bt.insert(key, idx) - - elif self.tables[table_name].unique is not None and column in self.tables[table_name].unique: - - for idx, key in enumerate(self.tables[table_name].column_by_name(column)): - - if key is None: - continue - bt.insert(key, idx) - - else: - raise ValueError(f'##ERROR->{column} is not primary key or unique') - # save the btree - - self._save_index(index_name, bt) - elif index_type=='hash': - hi = hash(1) - if self.tables[table_name].pk is not None and column==self.tables[table_name].pk: - for idx, key in enumerate(self.tables[table_name].column_by_name(column)): - if key is None: - - continue - - hi.insert(idx, key)#idx will be used as the hashing key, which is the order that the values are in the table - #key is the value that will be inserted - elif self.tables[table_name].unique is not None and column in self.tables[table_name].unique: - - for idx, key in enumerate(self.tables[table_name].column_by_name(column)): - - if key is None: - continue - hi.insert(idx,key) - self._save_index(index_name, hi) - hi.show() - def _has_index(self, table_name, column): - ''' - Check whether the specified table's primary key column is indexed. - - Args: - table_name: string. Table name (must be part of the database). - column: string. Column name to check for indexing. - ''' - if table_name in self.tables['meta_indexes'].column_by_name('table_name') and column in self.tables['meta_indexes'].column_by_name('column'): - return True - else: - return False - - - def _save_index(self, index_name, index): - ''' - Save the index object. - - Args: - index_name: string. Name of the created index. - index: obj. The actual index object (btree object). - ''' - try: - os.mkdir(f'{self.savedir}/indexes') - except: - pass - - with open(f'{self.savedir}/indexes/meta_{index_name}_index.pkl', 'wb') as f: - pickle.dump(index, f) - - def _load_idx(self, index_name): - ''' - Load and return the specified index. - - Args: - index_name: string. Name of created index. - ''' - f = open(f'{self.savedir}/indexes/meta_{index_name}_index.pkl', 'rb') - index = pickle.load(f) - f.close() - return index - - def drop_index(self, index_name): - ''' - Drop index from current database. - - Args: - index_name: string. Name of index. - ''' - if index_name in self.tables['meta_indexes'].column_by_name('index_name'): - self.delete_from('meta_indexes', f'index_name = {index_name}') - - if os.path.isfile(f'{self.savedir}/indexes/meta_{index_name}_index.pkl'): - os.remove(f'{self.savedir}/indexes/meta_{index_name}_index.pkl') - else: - warnings.warn(f'"{self.savedir}/indexes/meta_{index_name}_index.pkl" not found.') - - self.save_database() - \ No newline at end of file From 9688dc0135c7949c2a7d5c733ebd116c9650fb6d Mon Sep 17 00:00:00 2001 From: Kapiniaris-Xrisovalantis <92021533+Kapiniaris-Xrisovalantis@users.noreply.github.com> Date: Mon, 4 Sep 2023 10:53:47 +0300 Subject: [PATCH 11/16] where statements,unique support,hash/btree support updated where statements for select,delete,update, added unique support, added btree support for unique column, added hash index --- miniDB/btree.py | 2 + miniDB/database.py | 181 +++++++++++++++++++------ miniDB/hashidx.py | 82 ++++++++++++ miniDB/table.py | 319 +++++++++++++++++++++++++++++++++++++++++---- 4 files changed, 523 insertions(+), 61 deletions(-) create mode 100644 miniDB/hashidx.py diff --git a/miniDB/btree.py b/miniDB/btree.py index f0676209..6b3a6fdc 100644 --- a/miniDB/btree.py +++ b/miniDB/btree.py @@ -116,6 +116,7 @@ def insert(self, value, ptr, rptr=None): # insert to it self.nodes[index].insert(value,ptr) # if the node has more elements than b-1, split the node + if len(self.nodes[index].values)==self.b: self.split(index) @@ -309,6 +310,7 @@ def find(self, operator, value): if operator == '>': for idx, node_value in enumerate(target_node.values): ops+=1 + if node_value > value: results.append(target_node.ptrs[idx]) while target_node.right_sibling is not None: diff --git a/miniDB/database.py b/miniDB/database.py index 44c699e4..054f6cc4 100644 --- a/miniDB/database.py +++ b/miniDB/database.py @@ -4,7 +4,7 @@ import os,sys import logging import warnings -import pyreadline +#import pyreadline import re from tabulate import tabulate @@ -12,6 +12,7 @@ from miniDB import table sys.modules['table'] = table +from hashidx import ExtendibleHashing as hash from joins import Inlj, Smj from btree import Btree from misc import split_condition @@ -55,7 +56,7 @@ def __init__(self, name, load=True, verbose = True): self.create_table('meta_length', 'table_name,no_of_rows', 'str,int') self.create_table('meta_locks', 'table_name,pid,mode', 'str,int,str') self.create_table('meta_insert_stack', 'table_name,indexes', 'str,list') - self.create_table('meta_indexes', 'table_name,index_name', 'str,str') + self.create_table('meta_indexes', 'table_name,index_name,index_type,column', 'str,str,str,str') self.save_database() def save_database(self): @@ -102,7 +103,7 @@ def _update(self): self._update_meta_insert_stack() - def create_table(self, name, column_names, column_types, primary_key=None, load=None): + def create_table(self, name, column_names, column_types, primary_key=None, unique=None,load=None): ''' This method create a new table. This table is saved and can be accessed via db_object.tables['table_name'] or db_object.table_name @@ -114,7 +115,7 @@ def create_table(self, name, column_names, column_types, primary_key=None, load= load: boolean. Defines table object parameters as the name of the table and the column names. ''' # print('here -> ', column_names.split(',')) - self.tables.update({name: Table(name=name, column_names=column_names.split(','), column_types=column_types.split(','), primary_key=primary_key, load=load)}) + self.tables.update({name: Table(name=name, column_names=column_names.split(','), column_types=column_types.split(','), primary_key=primary_key,unique=unique, load=load)}) # self._name = Table(name=name, column_names=column_names, column_types=column_types, load=load) # check that new dynamic var doesnt exist already # self.no_of_tables += 1 @@ -161,7 +162,7 @@ def drop_table(self, table_name): self.save_database() - def import_table(self, table_name, filename, column_types=None, primary_key=None): + def import_table(self, table_name, filename, column_types=None, primary_key=None, unique=None): ''' Creates table from CSV file. @@ -178,7 +179,7 @@ def import_table(self, table_name, filename, column_types=None, primary_key=None colnames = line.strip('\n') if column_types is None: column_types = ",".join(['str' for _ in colnames.split(',')]) - self.create_table(name=table_name, column_names=colnames, column_types=column_types, primary_key=primary_key) + self.create_table(name=table_name, column_names=colnames, column_types=column_types, primary_key=primary_key,unique=unique) lock_ownership = self.lock_table(table_name, mode='x') first_line = False continue @@ -276,6 +277,7 @@ def insert_into(self, table_name, row_str): except Exception as e: logging.info(e) logging.info('ABORTED') + self._update_meta_insert_stack_for_tb(table_name, insert_stack[:-1]) if lock_ownership: @@ -359,38 +361,86 @@ def select(self, columns, table_name, condition, distinct=None, order_by=None, \ if isinstance(table_name,Table): return table_name._select_where(columns, condition, distinct, order_by, desc, limit) + condition_column="" if condition is not None: if "between" in condition.split(): condition_column = condition.split()[0] elif "and" in condition.split() : columns2=[] conditions=[] - conditions=condition.split("and") + conditions=condition.split(" and ") for con in conditions: col=self.tables[table_name]._parse_condition(con) columns2.append(col) for con in columns2: - if con[0] == self.tables[table_name].column_names[self.tables[table_name].pk_idx] :#since only pk supports index , and only one pk per table we keep the column if it is pk + if (self.tables[table_name].pk_idx is not None and con[0] == self.tables[table_name].column_names[self.tables[table_name].pk_idx]) or (self.tables[table_name].unique is not None and con[0] in self.tables[table_name].unique ):#since only pk supports index , and only one pk per table we keep the column if it is pk condition_column = con[0] - + else: + condition_column="" - elif " or " in condition.split(): - ca=[] - else: + elif "or" in condition.split(): - condition_column = self.tables[table_name]._parse_condition(condition) + columns2=[] + conditions=[] + conditions=condition.split(" or ") + for con in conditions: + + col=self.tables[table_name]._parse_condition(con) + + columns2.append(col) + + for con in columns2: + + + if self.tables[table_name].pk_idx is not None and con[0] == self.tables[table_name].column_names[self.tables[table_name].pk_idx]: + condition_column = con[0] + else: + condition_column = "" else: - condition_column = '' + + col,op=self.tables[table_name]._parse_condition(condition) + if op=="=": + condition_column=col + + + + + # self.lock_table(table_name, mode='x') if self.is_locked(table_name): return - if self._has_index(table_name) and condition_column==self.tables[table_name].column_names[self.tables[table_name].pk_idx]: - index_name = self.select('*', 'meta_indexes', f'table_name={table_name}', return_object=True).column_by_name('index_name')[0] - bt = self._load_idx(index_name) - table = self.tables[table_name]._select_where_with_btree(columns, bt, condition, distinct, order_by, desc, limit) + + if self.tables[table_name].pk_idx is not None and self._has_index(table_name,condition_column): + print("here") + index_name = self.select('*', 'meta_indexes', f'table_name={table_name}',return_object=True).column_by_name('index_name')[0] + index_type = self.select('*', 'meta_indexes', f'table_name={table_name}',return_object=True).column_by_name('index_type')[0] + if index_type=="btree": + bt = self._load_idx(index_name) + table = self.tables[table_name]._select_where_with_btree(columns, bt, condition, distinct, order_by, desc, limit) + elif index_type=="hash": + bt = self._load_idx(index_name) + table = self.tables[table_name]._select_where_with_hash(columns, bt, condition, distinct, order_by, desc, limit) + + elif self.tables[table_name].unique_idx is not None and self._has_index(table_name): + + found = False + for j in range (len(self.tables[table_name].unique)): + + if self.tables[table_name].unique[j] in condition_column: + found = True + index_name = self.select('*', 'meta_indexes', f'table_name={table_name}', return_object=True).column_by_name('index_name')[0] + print(index_name) + bt = self._load_idx(index_name) + bt.show() + table = self.tables[table_name]._select_where_with_btree(columns, bt, condition, distinct, order_by, desc, limit) + break + if not found: + + table = self.tables[table_name]._select_where(columns, condition, distinct, order_by, desc, limit) else: + table = self.tables[table_name]._select_where(columns, condition, distinct, order_by, desc, limit) # self.unlock_table(table_name) if save_as is not None: @@ -601,7 +651,9 @@ def _update_meta_length(self): ''' Updates the meta_length table. ''' + for table in self.tables.values(): + if table._name[:4]=='meta': #skip meta tables continue if table._name not in self.tables['meta_length'].column_by_name('table_name'): # if new table, add record with 0 no. of rows @@ -670,7 +722,7 @@ def _update_meta_insert_stack_for_tb(self, table_name, new_stack): # indexes - def create_index(self, index_name, table_name, index_type='btree'): + def create_index(self, index_name, table_name,index_type='btree' ,column='pkey'): ''' Creates an index on a specified table with a given name. Important: An index can only be created on a primary key (the user does not specify the column). @@ -679,21 +731,38 @@ def create_index(self, index_name, table_name, index_type='btree'): table_name: string. Table name (must be part of database). index_name: string. Name of the created index. ''' - if self.tables[table_name].pk_idx is None: # if no primary key, no index - raise Exception('Cannot create index. Table has no primary key.') + #if the user didn't specify a column, make the index on primary key + if self.tables[table_name].pk_idx is not None and column=='pkey': + column = self.tables[table_name].pk + + + if self.tables[table_name].pk_idx is None and self.tables[table_name].unique is None: # if no primary key, no index + raise Exception('Cannot create index. Table has no primary key or unique columns.') + if index_name not in self.tables['meta_indexes'].column_by_name('index_name'): # currently only btree is supported. This can be changed by adding another if. if index_type=='btree': logging.info('Creating Btree index.') + print('Creating BTREE index') + # insert a record with the name of the index and the table on which it's created to the meta_indexes table + self.tables['meta_indexes']._insert([table_name, index_name,index_type,column]) + # crate the actual index + self._construct_index(table_name, index_name,column,index_type) + #self._construct_index(table_name, index_name) + self.save_database() + if index_type=='hash': + logging.info('Creating Hash index.') + print('Creating HASH index') # insert a record with the name of the index and the table on which it's created to the meta_indexes table - self.tables['meta_indexes']._insert([table_name, index_name]) + self.tables['meta_indexes']._insert([table_name, index_name,index_type,column]) # crate the actual index - self._construct_index(table_name, index_name) + self._construct_index(table_name, index_name,column,index_type) + #self._construct_index(table_name, index_name) self.save_database() else: raise Exception('Cannot create index. Another index with the same name already exists.') - def _construct_index(self, table_name, index_name): + def _construct_index(self, table_name, index_name,column,index_type='btree'): ''' Construct a btree on a table and save. @@ -701,25 +770,61 @@ def _construct_index(self, table_name, index_name): table_name: string. Table name (must be part of database). index_name: string. Name of the created index. ''' - bt = Btree(3) # 3 is arbitrary - - # for each record in the primary key of the table, insert its value and index to the btree - for idx, key in enumerate(self.tables[table_name].column_by_name(self.tables[table_name].pk)): - if key is None: - continue - bt.insert(key, idx) - # save the btree - self._save_index(index_name, bt) - - - def _has_index(self, table_name): + if index_type=='btree': + bt = Btree(1) # 3 is arbitrary + + # for each record in the primary key of the table, insert its value and index to the btree + if self.tables[table_name].pk is not None and column==self.tables[table_name].pk: + for idx, key in enumerate(self.tables[table_name].column_by_name(column)): + if key is None: + continue + bt.insert(key, idx) + + elif self.tables[table_name].unique is not None and column in self.tables[table_name].unique: + + for idx, key in enumerate(self.tables[table_name].column_by_name(column)): + + if key is None: + continue + bt.insert(key, idx) + + else: + raise ValueError(f'##ERROR->{column} is not primary key or unique') + # save the btree + + self._save_index(index_name, bt) + elif index_type=='hash': + hi = hash(1) + if self.tables[table_name].pk is not None and column==self.tables[table_name].pk: + for idx, key in enumerate(self.tables[table_name].column_by_name(column)): + if key is None: + + continue + + hi.insert(idx, key)#idx will be used as the hashing key, which is the order that the values are in the table + #key is the value that will be inserted + elif self.tables[table_name].unique is not None and column in self.tables[table_name].unique: + + for idx, key in enumerate(self.tables[table_name].column_by_name(column)): + + if key is None: + continue + hi.insert(idx,key) + self._save_index(index_name, hi) + hi.show() + def _has_index(self, table_name, column): ''' Check whether the specified table's primary key column is indexed. Args: - table_name: string. Table name (must be part of database). + table_name: string. Table name (must be part of the database). + column: string. Column name to check for indexing. ''' - return table_name in self.tables['meta_indexes'].column_by_name('table_name') + if table_name in self.tables['meta_indexes'].column_by_name('table_name') and column in self.tables['meta_indexes'].column_by_name('column'): + return True + else: + return False + def _save_index(self, index_name, index): ''' diff --git a/miniDB/hashidx.py b/miniDB/hashidx.py new file mode 100644 index 00000000..1fa908a1 --- /dev/null +++ b/miniDB/hashidx.py @@ -0,0 +1,82 @@ +class ExtendibleHashing: + def __init__(self, global_depth): + self.global_depth = global_depth + self.directory = {} + self.bucket_size = 4 # Number of elements in each bucket + + def hash_function(self, key): + return key % (2 ** self.global_depth) + + def insert(self, key, value): + hashed_key = self.hash_function(key) + if hashed_key in self.directory: + bucket = self.directory[hashed_key] + for i, (k, v) in enumerate(bucket): + if k == key: + bucket[i] = (k, value) # Update value for existing key + return + if len(bucket) < self.bucket_size: + bucket.append((key, value)) + else: + if self.global_depth == len(bin(hashed_key)) - 2: + self.double_directory() + self.split_bucket(hashed_key) + self.insert(key, value) + else: + if self.global_depth == len(bin(hashed_key)) - 2: + self.double_directory() + self.directory[hashed_key] = [(key, value)] + + + def double_directory(self): + self.global_depth += 1 + directory_size = 2 ** (self.global_depth - 1) + for i in range(directory_size): + if i in self.directory: + self.directory[i + directory_size] = self.directory[i] + else: + self.directory[i + directory_size] = [] + + def split_bucket(self, hashed_key): + bucket = self.directory[hashed_key] + new_bucket = [] + split_index = len(bucket) // 2 + + # Split the bucket into two by creating a new bucket and updating the directory + new_bucket = bucket[split_index:] + bucket = bucket[:split_index] + self.directory[hashed_key] = bucket + + # Update the hashed keys of the new bucket and its duplicates + new_hashed_key = hashed_key + (2 ** (self.global_depth - 1)) + for key in range(new_hashed_key, new_hashed_key + (2 ** (self.global_depth - 1)), 2 ** (self.global_depth - 1)): + self.directory[key] = new_bucket.copy() + + + + + def find(self, key): + hashed_key = self.hash_function(key) + if hashed_key in self.directory: + bucket = self.directory[hashed_key] + for item in bucket: + if item[0] == key: + return item[1] + return None + + def delete(self, key): + hashed_key = self.hash_function(key) + if hashed_key in self.directory: + bucket = self.directory[hashed_key] + for i, item in enumerate(bucket): + if item[0] == key: + del bucket[i] + return True + return False + + def show(self): + for hashed_key, bucket in self.directory.items(): + print(f"Hashed Key: {hashed_key}") + for item in bucket: + print(f" Key: {item[0]}, Value: {item[1]}") + diff --git a/miniDB/table.py b/miniDB/table.py index c291ce3c..755d4585 100644 --- a/miniDB/table.py +++ b/miniDB/table.py @@ -27,9 +27,10 @@ class Table: - a dictionary that includes the appropriate info (all the attributes in __init__) ''' - def __init__(self, name=None, column_names=None, column_types=None, primary_key=None, load=None): + def __init__(self, name=None, column_names=None, column_types=None, primary_key=None,unique=None, load=None): if load is not None: + # if load is a dict, replace the object dict with it (replaces the object with the specified one) if isinstance(load, dict): self.__dict__.update(load) @@ -37,6 +38,7 @@ def __init__(self, name=None, column_names=None, column_types=None, primary_key= # if load is str, load from a file elif isinstance(load, str): self._load_from_file(load) + # if name, columns_names and column types are not none elif (name is not None) and (column_names is not None) and (column_types is not None): @@ -49,7 +51,7 @@ def __init__(self, name=None, column_names=None, column_types=None, primary_key= self.column_names = column_names self.columns = [] - + for col in self.column_names: if col not in self.__dir__(): # this is used in order to be able to call a column using its name as an attribute. @@ -65,10 +67,25 @@ def __init__(self, name=None, column_names=None, column_types=None, primary_key= # if primary key is set, keep its index as an attribute if primary_key is not None: self.pk_idx = self.column_names.index(primary_key) + else: self.pk_idx = None self.pk = primary_key + + self.unique_idx = [] + + if unique is not None: + self.unique = unique.split(',') + unique=unique.split(',') + for unq in unique: + if unq in self.column_names: + self.unique_idx.append(self.column_names.index(unq)) + else: + self.unique_idx=None + self.unique=None + + # self._update() # if any of the name, columns_names and column types are none. return an empty table object @@ -111,9 +128,10 @@ def _insert(self, row, insert_stack=[]): row: list. A list of values to be inserted (will be casted to a predifined type automatically). insert_stack: list. The insert stack (empty by default). ''' + if len(row)!=len(self.column_names): raise ValueError(f'ERROR -> Cannot insert {len(row)} values. Only {len(self.column_names)} columns exist') - + for i in range(len(row)): # for each value, cast and replace it in row. try: @@ -124,13 +142,22 @@ def _insert(self, row, insert_stack=[]): except TypeError as exc: if row[i] != None: print(exc) - + # if value is to be appended to the primary_key column, check that it doesnt alrady exist (no duplicate primary keys) if i==self.pk_idx and row[i] in self.column_by_name(self.pk): raise ValueError(f'## ERROR -> Value {row[i]} already exists in primary key column.') + elif i==self.pk_idx and row[i] is None: raise ValueError(f'ERROR -> The value of the primary key cannot be None.') - + + + if self.unique_idx is not None: + if i in self.unique_idx: + for j in range (len(self.unique)): + if row[i] in self.column_by_name(self.unique[j]): + raise ValueError(f'## ERROR -> Value {row[i]} already exists in unique column.') + + # if insert_stack is not empty, append to its last index if insert_stack != []: self.data[insert_stack[-1]] = row @@ -151,6 +178,41 @@ def _update_rows(self, set_value, set_column, condition): Operatores supported: (<,<=,=,>=,>) ''' + if condition is not None: + condition = self.replace_between(condition) + sub_conditions = condition.split(" or ") + print(sub_conditions) + rows = set() + for sub_cond in sub_conditions: + is_not = False + if sub_cond.startswith("not "): + is_not = True + sub_cond = sub_cond[4:] + sub_cond=sub_cond.replace("( ", "").replace(" )", "") + + conditions = sub_cond.split(" and ") + rows_L=[] + for condition_ in conditions: + column_name, operator, value = self._parse_condition(condition_) + column = self.column_by_name(column_name) + rows_L.append([ind for ind, x in enumerate(column) if get_op(operator, x, value)]) + + and_rows = set(rows_L[0]).intersection(*rows_L) if rows_L else set() + + if is_not: + not_rows = set(range(len(column))) - and_rows + rows.update(not_rows) + else: + rows.update(and_rows) + #rows.update(self.find_rows(sub_cond)) + + + rows_to_upd = list(rows) + set_column_idx = self.column_names.index(set_column) + for idx in rows_to_upd: + self.data[idx][set_column_idx] = set_value + + ''' if condition is not None: if re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition) or condition.startswith("not "):#simple condition or not @@ -176,7 +238,7 @@ def _update_rows(self, set_value, set_column, condition): elif re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+\s+or\s+\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition): - conditions = condition.split("or") + conditions = condition.split(" or ") set_column_idx = self.column_names.index(set_column) for condition_ in conditions: column_name, operator, value = self._parse_condition(condition_) @@ -189,7 +251,7 @@ def _update_rows(self, set_value, set_column, condition): elif re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+\s+and\s+\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition): column_name = condition.split()[0] - conditions = condition.split("and") + conditions = condition.split(" and ") set_column_idx = self.column_names.index(set_column) rows_L=[] for condition_ in conditions: @@ -198,7 +260,7 @@ def _update_rows(self, set_value, set_column, condition): rows_L.append([ind for ind, x in enumerate(column) if get_op(operator, x, value)]) - print(rows_L) + rows = set(rows_L[0]).intersection(*rows_L) @@ -209,6 +271,7 @@ def _update_rows(self, set_value, set_column, condition): raise("invalid where condition") #else: #rows = [i for i in range(len(self.data))] + ''' # parse the condition ###column_name, operator, value = self._parse_condition(condition) @@ -243,6 +306,37 @@ def _delete_where(self, condition): ''' indexes_to_del = [] + if condition is not None: + condition = self.replace_between(condition) + sub_conditions = condition.split(" or ") + print(sub_conditions) + rows = set() + for sub_cond in sub_conditions: + is_not = False + if sub_cond.startswith("not "): + is_not = True + sub_cond = sub_cond[4:] + sub_cond=sub_cond.replace("( ", "").replace(" )", "") + + conditions = sub_cond.split(" and ") + rows_L=[] + for condition_ in conditions: + column_name, operator, value = self._parse_condition(condition_) + column = self.column_by_name(column_name) + rows_L.append([ind for ind, x in enumerate(column) if get_op(operator, x, value)]) + + and_rows = set(rows_L[0]).intersection(*rows_L) if rows_L else set() + + if is_not: + not_rows = set(range(len(column))) - and_rows + rows.update(not_rows) + else: + rows.update(and_rows) + #rows.update(self.find_rows(sub_cond)) + + + indexes_to_del = list(rows) + ''' if re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition) or condition.startswith("not "):#simple condition or not column_name, operator, value = self._parse_condition(condition) column = self.column_by_name(column_name) @@ -253,13 +347,17 @@ def _delete_where(self, condition): query = condition.split() index = query.index("between") - megalutero = query[index+1] - mikrotero = query[index+3] + column_name = query[index-1] column = self.column_by_name(column_name) + for i in range (len(self.columns)): + if column_name==self.column_names[i]: + + megalutero = self.column_types[i](query[index+1]) + mikrotero = self.column_types[i](query[index+3]) for i,j in enumerate(column): - if str(j) >= megalutero and str(j) <= mikrotero: + if j >= megalutero and j <= mikrotero: indexes_to_del.append(i) elif re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+\s+or\s+\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition): @@ -287,11 +385,11 @@ def _delete_where(self, condition): indexes_to_del = set(rows_L[0]).intersection(*rows_L) - rows = list(rows) + indexes_to_del = list(indexes_to_del) else: raise("invalid where condition") - + ''' # we pop from highest to lowest index in order to avoid removing the wrong item @@ -336,7 +434,36 @@ def _select_where(self, return_columns, condition=None, distinct=False, order_by # if condition is None, return all rows # if not, return the rows with values where condition is met for value if condition is not None: + condition = self.replace_between(condition) + sub_conditions = condition.split(" or ") + print(sub_conditions) + rows = set() + for sub_cond in sub_conditions: + is_not = False + if sub_cond.startswith("not "): + is_not = True + sub_cond = sub_cond[4:] + sub_cond=sub_cond.replace("( ", "").replace(" )", "") + + conditions = sub_cond.split(" and ") + rows_L=[] + for condition_ in conditions: + column_name, operator, value = self._parse_condition(condition_) + column = self.column_by_name(column_name) + rows_L.append([ind for ind, x in enumerate(column) if get_op(operator, x, value)]) + + and_rows = set(rows_L[0]).intersection(*rows_L) if rows_L else set() + + if is_not: + not_rows = set(range(len(column))) - and_rows + rows.update(not_rows) + else: + rows.update(and_rows) + #rows.update(self.find_rows(sub_cond)) + + rows = list(rows) + ''' if re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition) or condition.startswith("not "):#simple condition or not column_name, operator, value = self._parse_condition(condition) column = self.column_by_name(column_name) @@ -345,10 +472,20 @@ def _select_where(self, return_columns, condition=None, distinct=False, order_by query = condition.split() index = query.index("between") - megalutero = query[index+1] - mikrotero = query[index+3] + + #megalutero = query[index+1] + #mikrotero = query[index+3] column_name = query[index-1] + + + column = self.column_by_name(column_name) + for i in range (len(self.columns)): + if column_name==self.column_names[i]: + + megalutero = self.column_types[i](query[index+1]) + mikrotero = self.column_types[i](query[index+3]) + rows = [] for i,j in enumerate(column): if j >= megalutero and j <= mikrotero: @@ -356,9 +493,10 @@ def _select_where(self, return_columns, condition=None, distinct=False, order_by elif re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+\s+or\s+\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition): column_name = condition.split()[0] - conditions = condition.split("or") + conditions = condition.split(" or ") rows_L=[] for condition_ in conditions: + column_name, operator, value = self._parse_condition(condition_) column = self.column_by_name(column_name) rows_L.append([ind for ind, x in enumerate(column) if get_op(operator, x, value)]) @@ -369,7 +507,7 @@ def _select_where(self, return_columns, condition=None, distinct=False, order_by rows.append(row) elif re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+\s+and\s+\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition): column_name = condition.split()[0] - conditions = condition.split("and") + conditions = condition.split(" and ") rows_L=[] for condition_ in conditions: column_name, operator, value = self._parse_condition(condition_) @@ -382,6 +520,7 @@ def _select_where(self, return_columns, condition=None, distinct=False, order_by else: raise("invalid where condition") + ''' else: rows = [i for i in range(len(self.data))] @@ -416,8 +555,36 @@ def _select_where(self, return_columns, condition=None, distinct=False, order_by return s_table - def _select_where_with_btree(self, return_columns, bt, condition, distinct=False, order_by=None, desc=True, limit=None): + def replace_between(self,condition): + + query = condition.split() + try: + index = query.index("between") + except: + return condition + + + + + megalutero = query[index+1] + mikrotero = query[index+3] + column_name = query[index-1] + between_condition = str(column_name) + ">=" + str(megalutero) + " and " + str(column_name) + "<=" + str(mikrotero) + + del query[index-1:index+4] + query.insert(index-1, between_condition) + blank =" " + new_condition = blank.join(query) + + new_condition = self.replace_between(new_condition) + return new_condition + + + + + def _select_where_with_btree(self, return_columns, bt, condition, distinct=False, order_by=None, desc=True, limit=None): + print("select from btree") # if * return all columns, else find the column indexes for the columns specified if return_columns == '*': return_cols = [i for i in range(len(self.column_names))] @@ -425,14 +592,115 @@ def _select_where_with_btree(self, return_columns, bt, condition, distinct=False return_cols = [self.column_names.index(colname) for colname in return_columns] - column_name, operator, value = self._parse_condition(condition) + #column_name, operator, value = self._parse_condition(condition) + if condition is not None: + condition = self.replace_between(condition) + sub_conditions = condition.split(" or ") + print(sub_conditions) + rows = set() + for sub_cond in sub_conditions: + is_not = False + if sub_cond.startswith("not "): + is_not = True + sub_cond = sub_cond[4:] + sub_cond=sub_cond.replace("( ", "").replace(" )", "") + + conditions = sub_cond.split(" and ") + rows_L=[] + for condition_ in conditions: + column_name, operator, value = self._parse_condition(condition_) + column = self.column_by_name(column_name) + rows_L.append(bt.find(operator, value)) + + + and_rows = set(rows_L[0]).intersection(*rows_L) if rows_L else set() + + if is_not: + not_rows = set(range(len(column))) - and_rows + rows.update(not_rows) + else: + rows.update(and_rows) + + + + rows = list(rows) + + + + '''if re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition) or condition.startswith("not "):#simple condition or not + column_name, operator, value = self._parse_condition(condition) + column = self.column_by_name(column_name) + if (self.pk_idx is not None and column_name != self.column_names[self.pk_idx]) and (self.unique is not None and column_name not in self.unique): + #print('Column is not PK. Aborting') + raise ValueError('Column is not PK or UNIQUE') + rows1 = [] + opsseq = 0 + for ind, x in enumerate(column): + opsseq+=1 + if get_op(operator, x, value): + rows1.append(ind) + rows = bt.find(operator, value) + + + elif re.match(r"^\w+\s+between\s+\w+\s+and\s+\w+$", condition): + + query = condition.split() + index = query.index("between") + + column_name = query[index-1] + if (self.pk_idx is not None and column_name != self.column_names[self.pk_idx]) and (self.unique is not None and column_name not in self.unique): + #print('Column is not PK. Aborting') + raise ValueError('Column is not PK or UNIQUE') + column = self.column_by_name(column_name) + for i in range (len(self.columns)): + if column_name==self.column_names[i]: + + megalutero = self.column_types[i](query[index+1]) + mikrotero = self.column_types[i](query[index+3]) + rows_greater = bt.find('>=',str(megalutero)) + rows_less = bt.find('<=',str(mikrotero)) + rows = set(rows_greater).intersection(rows_less) + rows = list(rows) + + elif re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+\s+or\s+\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition): + + print(condition) + conditions = condition.split(" or ") + rows_L=[] + for condition_ in conditions: + + column_name, operator, value = self._parse_condition(condition_) + if (self.pk_idx is not None and column_name != self.column_names[self.pk_idx]) and (self.unique is not None and column_name not in self.unique): + #print('Column is not PK. Aborting') + raise ValueError('Column is not PK or UNIQUE') + + rows_L.append(bt.find(operator, value)) + rows=[] + print(rows_L) + for rlist in rows_L: + for row in rlist: + rows.append(row) + elif re.match(r"^\w+\s*(=|<=|>=|<|>|!=)\s*\w+\s+and\s+\w+\s*(=|<=|>=|<|>|!=)\s*\w+$", condition): + + conditions = condition.split(" and ") + rows_L=[] + for condition_ in conditions: + column_name, operator, value = self._parse_condition(condition_) + if (self.pk_idx is not None and column_name != self.column_names[self.pk_idx]) and (self.unique is not None and column_name not in self.unique): + #print('Column is not PK. Aborting') + continue + column = self.column_by_name(column_name) + rows_L.append(bt.find(operator, value)) + + rows = set(rows_L[0]).intersection(*rows_L) + rows = list(rows)''' # if the column in condition is not a primary key, abort the select - if column_name != self.column_names[self.pk_idx]: - print('Column is not PK. Aborting') + # here we run the same select twice, sequentially and using the btree. # we then check the results match and compare performance (number of operation) + ''' column = self.column_by_name(column_name) # sequential @@ -445,13 +713,14 @@ def _select_where_with_btree(self, return_columns, bt, condition, distinct=False # btree find rows = bt.find(operator, value) - + ''' try: k = int(limit) except TypeError: k = None # same as simple select from now on rows = rows[:k] + # TODO: this needs to be dumbed down dict = {(key):([[self.data[i][j] for j in return_cols] for i in rows] if key=="data" else value) for key,value in self.__dict__.items()} @@ -467,7 +736,7 @@ def _select_where_with_btree(self, return_columns, bt, condition, distinct=False if isinstance(limit,str): s_table.data = [row for row in s_table.data if row is not None][:int(limit)] - + return s_table def order_by(self, column_name, desc=True): @@ -679,6 +948,10 @@ def show(self, no_of_rows=None, is_locked=False): if self.pk_idx is not None: # table has a primary key, add PK next to the appropriate column headers[self.pk_idx] = headers[self.pk_idx]+' #PK#' + + if self.unique_idx is not None: + for unq in self.unique_idx: + headers[unq]=headers[unq]+' #UNQ#' # detect the rows that are no tfull of nones (these rows have been deleted) # if we dont skip these rows, the returning table has empty rows at the deleted positions non_none_rows = [row for row in self.data if any(row)] From 9027040f26b30248f4464688b1a534ff877ed50f Mon Sep 17 00:00:00 2001 From: Kapiniaris-Xrisovalantis <92021533+Kapiniaris-Xrisovalantis@users.noreply.github.com> Date: Mon, 4 Sep 2023 16:09:06 +0300 Subject: [PATCH 12/16] finished hash support --- miniDB/database.py | 26 ++++++++++++++--------- miniDB/hashidx.py | 15 ++++++++----- miniDB/table.py | 52 ++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 74 insertions(+), 19 deletions(-) diff --git a/miniDB/database.py b/miniDB/database.py index 054f6cc4..4e47efb6 100644 --- a/miniDB/database.py +++ b/miniDB/database.py @@ -362,6 +362,7 @@ def select(self, columns, table_name, condition, distinct=None, order_by=None, \ return table_name._select_where(columns, condition, distinct, order_by, desc, limit) condition_column="" + op="" if condition is not None: if "between" in condition.split(): condition_column = condition.split()[0] @@ -396,10 +397,9 @@ def select(self, columns, table_name, condition, distinct=None, order_by=None, \ condition_column = con[0] else: condition_column = "" - else: - - col,op=self.tables[table_name]._parse_condition(condition) - if op=="=": + else: + + col,op,_= self.tables[table_name]._parse_condition(condition) condition_column=col @@ -412,18 +412,24 @@ def select(self, columns, table_name, condition, distinct=None, order_by=None, \ if self.is_locked(table_name): return - if self.tables[table_name].pk_idx is not None and self._has_index(table_name,condition_column): - print("here") + + if self.tables[table_name].pk_idx is not None and self.tables[table_name].pk == condition_column and self._has_index(table_name,condition_column): + index_name = self.select('*', 'meta_indexes', f'table_name={table_name}',return_object=True).column_by_name('index_name')[0] index_type = self.select('*', 'meta_indexes', f'table_name={table_name}',return_object=True).column_by_name('index_type')[0] + if index_type=="btree": + bt = self._load_idx(index_name) table = self.tables[table_name]._select_where_with_btree(columns, bt, condition, distinct, order_by, desc, limit) elif index_type=="hash": - bt = self._load_idx(index_name) - table = self.tables[table_name]._select_where_with_hash(columns, bt, condition, distinct, order_by, desc, limit) - - elif self.tables[table_name].unique_idx is not None and self._has_index(table_name): + print("hash") + if op== "=": + bt = self._load_idx(index_name) + table = self.tables[table_name]._select_where_with_hash(columns, bt, condition, distinct, order_by, desc, limit) + else: + table = self.tables[table_name]._select_where(columns, condition, distinct, order_by, desc, limit) + elif self.tables[table_name].unique_idx is not None and self._has_index(table_name,condition_column): found = False for j in range (len(self.tables[table_name].unique)): diff --git a/miniDB/hashidx.py b/miniDB/hashidx.py index 1fa908a1..49d71f04 100644 --- a/miniDB/hashidx.py +++ b/miniDB/hashidx.py @@ -5,14 +5,18 @@ def __init__(self, global_depth): self.bucket_size = 4 # Number of elements in each bucket def hash_function(self, key): - return key % (2 ** self.global_depth) + + hash_value = 0 + for char in key: + hash_value = (hash_value * 31 + ord(char)) % (2 ** self.global_depth) + return hash_value def insert(self, key, value): - hashed_key = self.hash_function(key) + hashed_key = self.hash_function(value) if hashed_key in self.directory: bucket = self.directory[hashed_key] for i, (k, v) in enumerate(bucket): - if k == key: + if k == value: bucket[i] = (k, value) # Update value for existing key return if len(bucket) < self.bucket_size: @@ -56,12 +60,13 @@ def split_bucket(self, hashed_key): def find(self, key): + self.global_depth=1 hashed_key = self.hash_function(key) if hashed_key in self.directory: bucket = self.directory[hashed_key] for item in bucket: - if item[0] == key: - return item[1] + if item[1] == key: + return item[0] return None def delete(self, key): diff --git a/miniDB/table.py b/miniDB/table.py index 755d4585..4ad5c75e 100644 --- a/miniDB/table.py +++ b/miniDB/table.py @@ -181,7 +181,7 @@ def _update_rows(self, set_value, set_column, condition): if condition is not None: condition = self.replace_between(condition) sub_conditions = condition.split(" or ") - print(sub_conditions) + rows = set() for sub_cond in sub_conditions: is_not = False @@ -309,7 +309,7 @@ def _delete_where(self, condition): if condition is not None: condition = self.replace_between(condition) sub_conditions = condition.split(" or ") - print(sub_conditions) + rows = set() for sub_cond in sub_conditions: is_not = False @@ -436,7 +436,7 @@ def _select_where(self, return_columns, condition=None, distinct=False, order_by if condition is not None: condition = self.replace_between(condition) sub_conditions = condition.split(" or ") - print(sub_conditions) + rows = set() for sub_cond in sub_conditions: is_not = False @@ -596,7 +596,7 @@ def _select_where_with_btree(self, return_columns, bt, condition, distinct=False if condition is not None: condition = self.replace_between(condition) sub_conditions = condition.split(" or ") - print(sub_conditions) + rows = set() for sub_cond in sub_conditions: is_not = False @@ -739,6 +739,50 @@ def _select_where_with_btree(self, return_columns, bt, condition, distinct=False return s_table + def _select_where_with_hash(self, return_columns, bt, condition, distinct=False, order_by=None, desc=True, limit=None): + print("select with hash") + # if * return all columns, else find the column indexes for the columns specified + if return_columns == '*': + return_cols = [i for i in range(len(self.column_names))] + else: + return_cols = [self.column_names.index(colname) for colname in return_columns] + + + column_name, operator, value = self._parse_condition(condition) + rows=[] + rows.append(bt.find(value)) + + + + + try: + k = int(limit) + except TypeError: + k = None + # same as simple select from now on + rows = rows[:k] + rows=list(rows) + print(rows) + + # TODO: this needs to be dumbed down + dict = {(key):([[self.data[i][j] for j in return_cols] for i in rows] if key=="data" else value) for key,value in self.__dict__.items()} + + dict['column_names'] = [self.column_names[i] for i in return_cols] + dict['column_types'] = [self.column_types[i] for i in return_cols] + + s_table = Table(load=dict) + + s_table.data = list(set(map(lambda x: tuple(x), s_table.data))) if distinct else s_table.data + + if order_by: + s_table.order_by(order_by, desc) + + if isinstance(limit,str): + s_table.data = [row for row in s_table.data if row is not None][:int(limit)] + + return s_table + + def order_by(self, column_name, desc=True): ''' Order table based on column. From 9e40aa25be673782b257863f783f9c706a7a5813 Mon Sep 17 00:00:00 2001 From: UndedBan <79803875+UndedBan@users.noreply.github.com> Date: Mon, 4 Sep 2023 16:36:12 +0300 Subject: [PATCH 13/16] Add files via upload --- equivalent_ra.py | 136 +++++++++++++++++++++++++++++++++++++++++++++++ mdb.py | 10 ++-- 2 files changed, 143 insertions(+), 3 deletions(-) create mode 100644 equivalent_ra.py diff --git a/equivalent_ra.py b/equivalent_ra.py new file mode 100644 index 00000000..438e2d34 --- /dev/null +++ b/equivalent_ra.py @@ -0,0 +1,136 @@ +def ra_eq1(query): + ''' + σθ1∧θ2(E) = σθ1(σθ2(E)) + ''' + #Έλεγχος για το αν υπάρχει condition στο where και αν υπαρχει και and + if query['where'] is not None and query['where'].find('and') != -1: + #Split το condition σε 2 μέρη και ανακατασκευή. + condition1, condition2 = query['where'].split('and') + equiv_query = query.copy() + equiv_query['where'] = condition1 + equiv_query['from'] = {'select': '*', 'from': equiv_query['from'], 'where': condition2,'distinct': None, 'orderby': None,'limit': None,'desc': None} + return equiv_query + return None + +def ra_eq2(query): + ''' + E1 inner join θ E2 ---> E2 inner join θ + ''' + if 'from' in query and 'join' in query['from']: + left = query['from']['left'] + right = query['from']['right'] + equiv_query = query.copy() + equiv_query['from']['left'] = right + equiv_query['from']['right'] = left + return equiv_query + else: + return query + +def ra_eq3(query): + ''' + select θ (Ε1 Inner Join θ1 Ε2) ----> (select θ Ε1) Inner Join θ1 Ε2 + ''' + if 'from' in query and 'join' in query['from'] and query['from']['join'] == 'inner': + left_table = query['from']['left'] + right_table = query['from']['right'] + on_condition = query['from']['on'] + where_condition = query['where'] + + #Ελεγχος για το table στο οποίο ανήκει το condition (π.χ classroom.capacity το condition ανήκει στο table classroom) + if where_condition and where_condition.startswith(f"{left_table}."): + nested_left = { + 'select': query['select'], + 'from': left_table, + 'where': where_condition + } + nested_right = right_table + elif where_condition and where_condition.startswith(f"{right_table}."): + nested_left = left_table + nested_right = { + 'select': query['select'], + 'from': right_table, + 'where': where_condition + } + else: + nested_left = left_table + nested_right = right_table + + transformed_query = { + 'select': query['select'], + 'from': { + 'join': 'inner', + 'left': nested_left, + 'right': nested_right, + 'on': on_condition + }, + 'where': None, # Αφου το where condition έχει ήδη γίνει apply πιο πάνω. + 'distinct': query['distinct'], + 'order by': query['order by'], + 'limit': query['limit'] + } + print("OEOOOO") + print(transformed_query) + return transformed_query + else: + return query + +def ra_eq4(query): + ''' + Given a query, apply the transformation rule + select θ1 AND θ2 (Ε1 inner join Ε2) -> (select θ1 Ε1) INNER JOIN (select θ2 Ε2) + ''' + if 'from' in query and 'join' in query['from'] and query['from']['join'] == 'inner': + left_table = query['from']['left'] + right_table = query['from']['right'] + on_condition = query['from']['on'] + where_condition = query['where'] + + theta1, theta2 = where_condition.split('and') + + transformed_query_left = { + 'select': query['select'], + 'from': left_table, + 'where': "", + 'distinct': query['distinct'], + 'order by': query['order by'], + 'limit': query['limit'] + } + + transformed_query_right = { + 'select': query['select'], + 'from': right_table, + 'where': "", + 'distinct': query['distinct'], + 'order by': query['order by'], + 'limit': query['limit'] + } + + #Λοοπ μέσα από όλα τα conditions και κάνει assign το κάθε where condition στο αντίστοιχο table + for condition in (theta1, theta2): + condition = condition.strip() + if left_table in condition: + transformed_query_left['where'] += f"{condition}" + elif right_table in condition: + transformed_query_right['where'] += f"{condition}" + + transformed_query_left['where'] = transformed_query_left['where'].rstrip('and') + transformed_query_right['where'] = transformed_query_right['where'].rstrip('and') + + + final_transformed_query = { + 'select': '*', + 'from': { + 'join': 'inner', + 'left': transformed_query_left, + 'right': transformed_query_right, + 'on': on_condition + }, + 'where': None, + 'distinct': query['distinct'], + 'order by': query['order by'], + 'limit': query['limit'] + } + + return final_transformed_query + + return query \ No newline at end of file diff --git a/mdb.py b/mdb.py index bb9323cc..69b32de6 100644 --- a/mdb.py +++ b/mdb.py @@ -7,6 +7,11 @@ import shutil sys.path.append('miniDB') +from equivalent_ra import ra_eq1 +from equivalent_ra import ra_eq2 +from equivalent_ra import ra_eq3 +from equivalent_ra import ra_eq4 + from database import Database from table import Table # art font is "big" @@ -89,10 +94,8 @@ def create_query_plan(query, keywords, action): else: dic['desc'] = False dic['order by'] = dic['order by'].removesuffix(' asc').removesuffix(' desc') - else: dic['desc'] = None - if action=='create table': args = dic['create table'][dic['create table'].index('('):dic['create table'].index(')')+1] dic['create table'] = dic['create table'].removesuffix(args).strip() @@ -197,7 +200,8 @@ def evaluate_from_clause(dic): join_dic['right'] = interpret(join_dic['right'][1:-1].strip()) dic['from'] = join_dic - + print(ra_eq4(dic)) + print(dic) return dic def interpret(query): From 3aab368181d34717c2d8b34ebaabe884dcaf142d Mon Sep 17 00:00:00 2001 From: Kapiniaris-Xrisovalantis <92021533+Kapiniaris-Xrisovalantis@users.noreply.github.com> Date: Mon, 4 Sep 2023 17:55:10 +0300 Subject: [PATCH 14/16] unique index support From 659017e09488a7adf0d3b13ebf67976c9db9a26b Mon Sep 17 00:00:00 2001 From: Kapiniaris-Xrisovalantis <92021533+Kapiniaris-Xrisovalantis@users.noreply.github.com> Date: Mon, 4 Sep 2023 17:56:54 +0300 Subject: [PATCH 15/16] Add files via upload From 428f4e68ba19535c91a199542144a4068c757558 Mon Sep 17 00:00:00 2001 From: Kapiniaris-Xrisovalantis <92021533+Kapiniaris-Xrisovalantis@users.noreply.github.com> Date: Mon, 4 Sep 2023 17:58:04 +0300 Subject: [PATCH 16/16] Add files via upload --- miniDB/database.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/miniDB/database.py b/miniDB/database.py index 4e47efb6..cc26aa63 100644 --- a/miniDB/database.py +++ b/miniDB/database.py @@ -436,12 +436,21 @@ def select(self, columns, table_name, condition, distinct=None, order_by=None, \ if self.tables[table_name].unique[j] in condition_column: found = True - index_name = self.select('*', 'meta_indexes', f'table_name={table_name}', return_object=True).column_by_name('index_name')[0] - print(index_name) - bt = self._load_idx(index_name) - bt.show() - table = self.tables[table_name]._select_where_with_btree(columns, bt, condition, distinct, order_by, desc, limit) - break + index_name = self.select('*', 'meta_indexes', f'table_name={table_name}',return_object=True).column_by_name('index_name')[0] + index_type = self.select('*', 'meta_indexes', f'table_name={table_name}',return_object=True).column_by_name('index_type')[0] + + if index_type=="btree": + + bt = self._load_idx(index_name) + table = self.tables[table_name]._select_where_with_btree(columns, bt, condition, distinct, order_by, desc, limit) + elif index_type=="hash": + print("hash") + if op== "=": + bt = self._load_idx(index_name) + table = self.tables[table_name]._select_where_with_hash(columns, bt, condition, distinct, order_by, desc, limit) + else: + table = self.tables[table_name]._select_where(columns, condition, distinct, order_by, desc, limit) + break if not found: table = self.tables[table_name]._select_where(columns, condition, distinct, order_by, desc, limit)