diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..f84a5f5e --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "python.analysis.extraPaths": [ + "./miniDB" + ] +} \ No newline at end of file diff --git a/Documentation.pdf b/Documentation.pdf new file mode 100644 index 00000000..08a0ff1c Binary files /dev/null and b/Documentation.pdf differ diff --git a/mdb.py b/mdb.py index a981e5be..65be0f8f 100644 --- a/mdb.py +++ b/mdb.py @@ -40,7 +40,7 @@ def in_paren(qsplit, ind): def create_query_plan(query, keywords, action): ''' - Given a query, the set of keywords that we expect to pe present and the overall action, return the query plan for this query. + Given a query, the set of keywords that we expect to be present and the overall action, return the query plan for this query. This can and will be used recursively ''' @@ -71,7 +71,7 @@ def create_query_plan(query, keywords, action): for i in range(len(kw_in_query)-1): dic[kw_in_query[i]] = ' '.join(ql[kw_positions[i]+1:kw_positions[i+1]]) - + if action == 'create view': dic['as'] = interpret(dic['as']) @@ -83,16 +83,15 @@ def create_query_plan(query, keywords, action): dic['distinct'] = True if dic['order by'] is not None: - dic['from'] = dic['from'] + dic['from'] = dic['from'] # ? if 'desc' in dic['order by']: dic['desc'] = True else: dic['desc'] = False dic['order by'] = dic['order by'].removesuffix(' asc').removesuffix(' desc') - else: dic['desc'] = None - + if action=='create table': args = dic['create table'][dic['create table'].index('('):dic['create table'].index(')')+1] dic['create table'] = dic['create table'].removesuffix(args).strip() @@ -100,6 +99,7 @@ def create_query_plan(query, keywords, action): arglist = [val.strip().split(' ') for val in arg_nopk.split(',')] dic['column_names'] = ','.join([val[0] for val in arglist]) dic['column_types'] = ','.join([val[1] for val in arglist]) + dic['unique column_names'] = ','.join([val[0] for val in arglist if len(val) == 3]) if 'primary key' in args: arglist = args[1:-1].split(' ') dic['primary key'] = arglist[arglist.index('primary')-2] @@ -120,7 +120,13 @@ def create_query_plan(query, keywords, action): dic['force'] = True else: dic['force'] = False - + + if action == 'create index': + tmp = dic['on'].split('.') + dic['column'] = tmp[1] + dic['on'] = tmp[0] + + print(f'Dict in create query: {dic}') return dic @@ -157,7 +163,6 @@ def evaluate_from_clause(dic): join_dic['right'] = interpret(join_dic['right'][1:-1].strip()) dic['from'] = join_dic - return dic def interpret(query): diff --git a/miniDB/btree.py b/miniDB/btree.py index f0676209..ee044fe5 100644 --- a/miniDB/btree.py +++ b/miniDB/btree.py @@ -297,9 +297,9 @@ def find(self, operator, value): # if the element exist, append to list, else pass and return try: results.append(target_node.ptrs[target_node.values.index(value)]) - # print('Found') + print('Found') except: - # print('Not found') + print('Not found') pass # for all other ops, the code is the same, only the operations themselves and the sibling indexes change diff --git a/miniDB/condtion_handler.py b/miniDB/condtion_handler.py new file mode 100644 index 00000000..d2ef58a3 --- /dev/null +++ b/miniDB/condtion_handler.py @@ -0,0 +1,192 @@ +import operator + +''' + Returns true if the query condition could result in multiple records, + false if the result is at most one record. + For example: + id > 12 -> true + id = 12 -> false + (assuming column id is unique) + + Args: + condition_tokens: the list of tokens that the condition consists of + condition_tokens: [ 'column_name', 'operator', 'value' ] +''' +def is_range_query(condition_tokens): + if len(condition_tokens) > 1: + return condition_tokens[1] != '=' # if the operator is anything other than '=' return true + + +''' + Creates and returns a condition plan (dictionary) based on the given condition, + by recursively expanding a dictionary + + Args: + condition: a string (the part of the query to the right of keyword 'where') + + Examples: + example0: + condition: 'id > 3 and name = john' + result: { 'id > 3: [ id, '>', 3, 'and' ], { 'name = john': [ name, '=', 'john' , ''] } } + + example1: + condition: 5 > 3 and 50 between 40 and 60 or 20 between 80 and 100 + result: { '5 > 3' : [ '5', '>', '3', 'and', { '50 between 40 and 60' : [ '50', 'between', '40', 'and', '60', 'or', { '20 between 80 and 100': [ '20', 'between', '80', 'and', '100', '' ] } ] } ] } + + The condition_plan consists of: + { query: [ column_name/value, operator, column_name/value, keyword_operator(and/or/ect) ], empty string or another dictionary of the same form } + where keyword_operator is the keyword operator with which the left condtion connects with the right condition +''' +def get_condition_plan(condition, join=False): + if join: + return get_condition_plan(condition) + + kw_ops = [ 'between', 'and', 'or', 'not' ] + l_substr, op, r_substr = '', '', '' + + condition = condition.replace('\"', '') + condition = condition.strip() + + for word in condition.split(' '): + if word in kw_ops: + indx = condition.find(' ' + word + ' ') + if indx != -1: + if word == 'between': # case where word is 'between' must be handled differently + tmp = condition[ indx + len('between') + 1 :].split(' ') + l_substr = condition[:indx] + ' between' + ' '.join(tmp[:4]) + l_substr = l_substr.strip(' ') + if len(tmp) > 4: + r_substr = ' '.join(tmp[5:]) + op = tmp[4] + else: + l_substr = condition[:indx] + r_substr = condition[indx + len(word) + 1 :] + op = word + + if r_substr != '': + if word == 'not': + r_substr = condition[:indx] + r_substr + if r_substr[0] == '(' and r_substr[-1] == ')': + r_substr = r_substr.strip('()') + break + dic = dict() + l_substr = l_substr.strip(" '\"''\t") + + if l_substr == '': + l_substr = condition + tokens = l_substr.split(' ') + dic[l_substr] = [ tokens[i] for i in range(len(tokens)) ] + + if op != 'between': + dic[l_substr].append(op) + + if ' ' in r_substr: + r_substr = r_substr.strip(" '\"''\t") + dic[l_substr].append(get_condition_plan(r_substr)) + return dic + + + +''' + Recursively scans given condition_plan and returns all the column names in it as a set of strings + + Args: + condition_plan: a condition plan generated from the get_condition_plan function above + column_names: the set of unique column names that the condition_plan contains +''' +def get_column_names_from_condition_plan(condition_plan, column_names=set()): + kw_ops = set(['between', 'and', 'or', 'not']) + arithmetic_ops = set(['>=', '<=', '=', '!=', '>', '<']) + + k = list(condition_plan.keys())[0] + for word in k.split(): + if word not in kw_ops and word not in arithmetic_ops and not word.isnumeric(): # if word is not an operator or a value its a column name, add it to column_names set + column_names.add(word) + if isinstance(condition_plan[k][-1], dict): # if the last element in the list is a dictionary call get_column_names_from_condition_plan with the subdictionary as condition_plan + return get_column_names_from_condition_plan(condition_plan[k][-1], column_names) + return column_names # if the last element in the list is not a dictionary the scan is over, return found column_names + + +''' + Returns True if x is in the range [low, high], false otherwise + + Args: + x: value in question, could be an arithmetic value or a string + low: lower limit, could be an arithmetic value or a string + high: upper limit, could be an arithmetic value or a string + +''' +def between(x, low, high): + return eval(x + '>=' + low) and eval(x + '<=' + high) # operator.ge(x, low) and operator.le(x, high) + + + +''' + Evaluates given condition, returns True or False + + Args: + condtion: a condition plan like the one get_condition_plan function returns (type dictionary) +''' +def evaluate_condition(condition): + ''' + Evaluates a basic condition + Args: + condition: a list or a tuple consisting of at most 5 elements + condition: + [ operand, operator, operand ] or + [ operand, 'between', lower_limit, 'and', upper_limit] if the keyword operator is 'between' + ''' + def evaluate_basic_condition(condition): + ops = { + '>=': operator.ge, + '<=': operator.le, + '=': operator.eq, + '!=': operator.ne, + '>': operator.gt, + '<': operator.lt, + 'between': between + } + + if isinstance(condition, (list, tuple)): + if len(condition) == 3: + if not condition[0].isnumeric(): + return ops[condition[1]](condition[0], condition[2]) + if condition[1] == '=': + condition[1] = '==' + return eval(condition[0] + condition[1] + condition[2]) + elif len(condition) == 5: # has between + return between(condition[0], condition[2], condition[4]) + + k = list(condition.keys())[0] # condition has only one key + l = condition[k] + + if isinstance(l[-1], dict): + right_part_result = evaluate_condition(l[-1]) + + if l[-2] == 'and' and not right_part_result: + return False + + left_part_result = evaluate_basic_condition(l[:-2]) + + if l[-2] == 'or': + return left_part_result or right_part_result + elif l[-2] == 'and': + return left_part_result and right_part_result + elif l[-2] == 'not': + return not right_part_result + else: + return evaluate_basic_condition(l[:-1]) + + +#print(get_condition_plan('tot_cred >= 30 and tot_cred <= 60 or tot_cred >= 80 and tot_cred <= 100 and dept_name = "history"')) +#print(get_condition_plan('tot_cred > 80')) +#print(get_condition_plan('not id > 3')) +#print(get_condition_plan('id > 3 and (tot_cred between 40 and 60 or tot_cred between 80 and 100)')) +#condition_plan = get_condition_plan('4 > 3 and 12 not between 40 and 60 or 12 between 20 and 100') +#print(condition_plan) +#print(evaluate_condition(condition_plan)) +#column_names_in_condition = get_column_names_from_condition_plan(condition_plan) +#print(column_names_in_condition) +#print(get_condition_plan('tot_cred between 40 and 60 or tot_cred between 80 and 100')) +#print( evaluate_condition( { '5 > 3' : [ '5', '>', '3', 'and', { '50 between 40 and 60' : [ '50', 'between', '40', 'and', '60', 'or', { '20 between 80 and 100': [ '20', 'between', '80', 'and', '100', '' ] } ] } ] } ) ) +#print(get_condition_plan('word > a or id > 3')) diff --git a/miniDB/database.py b/miniDB/database.py index a3ac6be7..99f3b7c1 100644 --- a/miniDB/database.py +++ b/miniDB/database.py @@ -6,6 +6,7 @@ import warnings import readline from tabulate import tabulate +import condtion_handler sys.path.append(f'{os.path.dirname(os.path.dirname(os.path.abspath(__file__)))}/miniDB') from miniDB import table @@ -13,6 +14,7 @@ from joins import Inlj, Smj from btree import Btree +from extendible_hash import ExtendibleHash from misc import split_condition from table import Table @@ -101,7 +103,7 @@ def _update(self): self._update_meta_insert_stack() - def create_table(self, name, column_names, column_types, primary_key=None, load=None): + def create_table(self, name, column_names, column_types, unique_column_names=None, primary_key=None, load=None): ''' This method create a new table. This table is saved and can be accessed via db_object.tables['table_name'] or db_object.table_name @@ -113,7 +115,7 @@ def create_table(self, name, column_names, column_types, primary_key=None, load= load: boolean. Defines table object parameters as the name of the table and the column names. ''' # print('here -> ', column_names.split(',')) - self.tables.update({name: Table(name=name, column_names=column_names.split(','), column_types=column_types.split(','), primary_key=primary_key, load=load)}) + self.tables.update({name: Table(name=name, column_names=column_names.split(','), column_types=column_types.split(','), unique_column_names=unique_column_names.split(','), primary_key=primary_key, load=load)}) # self._name = Table(name=name, column_names=column_names, column_types=column_types, load=load) # check that new dynamic var doesnt exist already # self.no_of_tables += 1 @@ -267,7 +269,7 @@ def insert_into(self, table_name, row_str): self.load_database() # fetch the insert_stack. For more info on the insert_stack # check the insert_stack meta table - lock_ownership = self.lock_table(table_name, mode='x') + #lock_ownership = self.lock_table(table_name, mode='x') # uncomment this later insert_stack = self._get_insert_stack_for_table(table_name) try: self.tables[table_name]._insert(row, insert_stack) @@ -276,8 +278,8 @@ def insert_into(self, table_name, row_str): logging.info('ABORTED') self._update_meta_insert_stack_for_tb(table_name, insert_stack[:-1]) - if lock_ownership: - self.unlock_table(table_name) + #if lock_ownership: + # self.unlock_table(table_name) self._update() self.save_database() @@ -351,28 +353,58 @@ def select(self, columns, table_name, condition, distinct=None, order_by=None, \ return_object: boolean. If True, the result will be a table object (useful for internal use - the result will be printed by default). distinct: boolean. If True, the resulting table will contain only unique rows. ''' - - # print(table_name) self.load_database() if isinstance(table_name,Table): return table_name._select_where(columns, condition, distinct, order_by, desc, limit) - + + range_query = False if condition is not None: - condition_column = split_condition(condition)[0] + # we need to store the first subcondition of the full condition + # if condition: id > 3 and name = john, the generated condition_plan is: { 'id > 3': [ 'name', '=', 'john', '' ] } + # we want to get the first part of the condition, meaning -> 'id > 3' then split it into a list + tokens = list(condtion_handler.get_condition_plan(condition).keys())[0].split() + print(f'tokens: {tokens}, {condtion_handler.get_condition_plan(condition).keys()}') + condition_column = tokens[0] + range_query = condtion_handler.is_range_query(tokens) # we need to know where or not the query is a range query in case we're using EHash Index + #condition_column = split_condition(condition)[0] else: condition_column = '' - # self.lock_table(table_name, mode='x') if self.is_locked(table_name): return - if self._has_index(table_name) and condition_column==self.tables[table_name].column_names[self.tables[table_name].pk_idx]: - index_name = self.select('*', 'meta_indexes', f'table_name={table_name}', return_object=True).column_by_name('index_name')[0] - bt = self._load_idx(index_name) - table = self.tables[table_name]._select_where_with_btree(columns, bt, condition, distinct, order_by, desc, limit) - else: - table = self.tables[table_name]._select_where(columns, condition, distinct, order_by, desc, limit) - # self.unlock_table(table_name) + + this_table = self.tables[table_name] + + if self._has_index(table_name): + index_name = self.select('*', 'meta_indexes', f'table_name={table_name}', return_object=True).column_by_name('index_name')[0] # the name of the index + index_info = self._load_idx(index_name) # a list: [index object, indexed_column name] + index = index_info[0] # the actual index onject (either a Btree object or an ExtendibleHash object) + indexed_column = index_info[1] # the name of the column that is indexed + + table = None + + if indexed_column == condition_column: # if the indexed_column name is the same with the condition_column name then we check if we can use our index + if isinstance(index, Btree): + print('USING BTREE') + table = this_table._select_where_with_btree(columns, index, condition, distinct, order_by, desc, limit) + elif isinstance(index, ExtendibleHash): + if not range_query: # ExtendibleHash support only non-range based queries + print('USING EHASH') + table = this_table._select_where_with_extendible_hash(columns, index, condition, distinct, order_by, desc, limit) + else: # if it is a range query then we cant use Hash index so we go with linear search + print(f'USING LINEAR -> indexed_column: {indexed_column}, condition_column: {condition_column}') + table = this_table._select_where(columns, condition, distinct, order_by, desc, limit) + else: # if the indexed_column name is not the same with the condition_column name we use linear search + print(f'USING LINEAR -> indexed_column: {indexed_column}, condition_column: {condition_column}') + table = this_table._select_where(columns, condition, distinct, order_by, desc, limit) + else: # if there is no index we use linear search + print('USING LINEAR') + table = this_table._select_where(columns, condition, distinct, order_by, desc, limit) + + ##### self.unlock_table(table_name) + self.unlock_table(table_name) + if save_as is not None: table._name = save_as self.table_from_object(table) @@ -514,7 +546,6 @@ def lock_table(self, table_name, mode='x'): raise Exception(f'Table "{table_name}" is locked by process with pid={pid}') else: return False - except IndexError: pass @@ -546,7 +577,6 @@ def unlock_table(self, table_name, force=False): pass self.tables['meta_locks']._delete_where(f'table_name={table_name}') self._save_locks() - # print(f'Unlocking table "{table_name}"') def is_locked(self, table_name): ''' @@ -565,7 +595,6 @@ def is_locked(self, table_name): pid = self.tables['meta_locks']._select_where('pid',f'table_name={table_name}').data[0][0] if pid!=os.getpid(): raise Exception(f'Table "{table_name}" is locked by the process with pid={pid}') - except IndexError: pass return False @@ -650,46 +679,66 @@ def _update_meta_insert_stack_for_tb(self, table_name, new_stack): # indexes - def create_index(self, index_name, table_name, index_type='btree'): + def create_index(self, index_name, table_name, index_type, column_name): ''' Creates an index on a specified table with a given name. - Important: An index can only be created on a primary key (the user does not specify the column). + Important: An index can only be created on a primary key or on a unique column (the user specifies the column like this: create index index_name on table_name.column_name using index_type). Args: table_name: string. Table name (must be part of database). index_name: string. Name of the created index. + index_type: string. Type of index, either btree or ehash + column_name: string. The indexed column ''' - if self.tables[table_name].pk_idx is None: # if no primary key, no index - raise Exception('Cannot create index. Table has no primary key.') + if column_name != self.tables[table_name].pk and column_name not in self.tables[table_name].unique_column_names: + raise Exception('Cannot create index. Given column is not unique or primary key.') + if index_name not in self.tables['meta_indexes'].column_by_name('index_name'): - # currently only btree is supported. This can be changed by adding another if. - if index_type=='btree': + if index_type == 'btree': logging.info('Creating Btree index.') - # insert a record with the name of the index and the table on which it's created to the meta_indexes table - self.tables['meta_indexes']._insert([table_name, index_name]) - # crate the actual index - self._construct_index(table_name, index_name) - self.save_database() + elif index_type == 'ehash': + logging.info('Creating ExtendibleHash index.') + # insert a record with the name of the index and the table on which it's created to the meta_indexes table + self.tables['meta_indexes']._insert([table_name, index_name]) + # crate the actual index + print(f'Creating index: {index_name}, on: {table_name}.{column_name}') + self._construct_index(table_name, index_name, index_type, column_name) + self.save_database() else: raise Exception('Cannot create index. Another index with the same name already exists.') - def _construct_index(self, table_name, index_name): + def _construct_index(self, table_name, index_name, index_type, column_name): ''' Construct a btree on a table and save. Args: table_name: string. Table name (must be part of database). index_name: string. Name of the created index. - ''' - bt = Btree(3) # 3 is arbitrary - - # for each record in the primary key of the table, insert its value and index to the btree - for idx, key in enumerate(self.tables[table_name].column_by_name(self.tables[table_name].pk)): - if key is None: - continue - bt.insert(key, idx) - # save the btree - self._save_index(index_name, bt) + index_type: string. The type of the created index, either btree or ehash + column_name: string. The name of the indexed column + ''' + index = None + if index_type == 'btree': + index = Btree(3) # 3 is arbitrary + elif index_type == 'ehash': + index = ExtendibleHash(method='lsb') + + #if on_pk[0]: + # # for each record in the primary key of the table, insert its value and index to the btree + # for idx, key in enumerate(self.tables[table_name].column_by_name(self.tables[table_name].pk)): + # if key is None: + # continue + # index.insert(key, idx) + #else: + # for idx, key in enumerate(self.tables[table_name].column_by_name(on_pk[1])): + # if key is None: + # continue + # index.insert(key, idx) + for idx, key in enumerate(self.tables[table_name].column_by_name(column_name)): + if key is None: + continue + index.insert(key, idx) + self._save_index(index_name, column_name, index) # saving : name of the index, name of index column, index object def _has_index(self, table_name): @@ -700,8 +749,8 @@ def _has_index(self, table_name): table_name: string. Table name (must be part of database). ''' return table_name in self.tables['meta_indexes'].column_by_name('table_name') - - def _save_index(self, index_name, index): + + def _save_index(self, index_name, column_name, index): ''' Save the index object. @@ -715,7 +764,8 @@ def _save_index(self, index_name, index): pass with open(f'{self.savedir}/indexes/meta_{index_name}_index.pkl', 'wb') as f: - pickle.dump(index, f) + print(f'Saving index: {index_name}, indexed column: {column_name}') + pickle.dump([index, column_name], f) # saves index object and column_name (the column that the index is on) def _load_idx(self, index_name): ''' @@ -727,7 +777,8 @@ def _load_idx(self, index_name): f = open(f'{self.savedir}/indexes/meta_{index_name}_index.pkl', 'rb') index = pickle.load(f) f.close() - return index + print(f'Loading index: {index_name}, indexed_column: {index[1]}') + return index # returns index onject and the column name the index is on def drop_index(self, index_name): ''' @@ -738,9 +789,10 @@ def drop_index(self, index_name): ''' if index_name in self.tables['meta_indexes'].column_by_name('index_name'): self.delete_from('meta_indexes', f'index_name = {index_name}') - + print(f'Index name: {index_name} is in meta_indexes') if os.path.isfile(f'{self.savedir}/indexes/meta_{index_name}_index.pkl'): os.remove(f'{self.savedir}/indexes/meta_{index_name}_index.pkl') + print(f'Index name: {index_name} removed from meta_indexes') else: warnings.warn(f'"{self.savedir}/indexes/meta_{index_name}_index.pkl" not found.') diff --git a/miniDB/extendible_hash.py b/miniDB/extendible_hash.py new file mode 100644 index 00000000..7ec29e6a --- /dev/null +++ b/miniDB/extendible_hash.py @@ -0,0 +1,173 @@ +import numbers +#from faker import Faker + + +class ExtendibleHash: + ''' + Creates an ExtendibleHash object. + By default it uses LSB method. + This can be changed by passing string 'msb' into the constructor. + + Args: + method: a string, either 'lsb', or 'msb' options are available + bucket_depth: an int, the number of values a bucket can consist of before splitting + ''' + def __init__(self, method='lsb', bucket_depth=3): + method = method.lower() + + self.__no_of_bits = 1 # number of bits used as keys (potentialy changes during insertions) + self.__bucket_depth = bucket_depth # maximum number of values a bucket can hold (this doens't change during insertions) + self.__method = method # method used to hash values in combination with self.__hash__ + + if self.__method not in ['lsb', 'msb']: + raise ValueError('method must be either lsb or msb') + + self.data = dict() + self.data[format(0, 'b')] = [] + self.data[format(1, 'b')] = [] + + + ''' + Gets called after each insert operation. + Readjusts the self.data if the number of values in a bucket exceeds self.__bucket_depth. + - Copies the data into a list. + - Increases the number of bits used as keys by 1. + - Clears the self.data obj. + - Creates the new keys. + - Rehashes the old values into adjusted self.data. + + Args: + bucket_let: an int, the number of values stored in the bucket, + if it exceeds bucket_depth we need to split, otherwise nothing happens + ''' + def __readjust(self, bucket_len): + if bucket_len > self.__bucket_depth: + old_data = list(self.data.values()) + self.__no_of_bits += 1 + + self.data.clear() + + for i in range(2 ** self.__no_of_bits): + id = self.__binary_value(i) + self.data[id] = [] + + for bucket in old_data: + for l in bucket: + self.insert(l[0], l[1]) + + + ''' + Returns the binary representation of given value. + Args: + value: any basic numeric type(int, float) or a string. + ''' + def __binary_value(self, value): + rv = '' + + if isinstance(value, numbers.Number): + rv = format(value, 'b') + elif isinstance(value, str): + rv = ''.join(format(ord(x), 'b') for x in value) + else: + raise TypeError + + # if number of bits used as keys is greater than the number of bits of given value: + # add '0's (diff zeros) at the front + diff = self.__no_of_bits - len(rv) + if diff > 0: rv = '0' * diff + rv + return rv + + ''' + Hashes the given value + + Args: + value: any basic numeric type(int, float) or a string. + ''' + def __hash_value(self, value): + if isinstance(value, str): + return (int(''.join(format(ord(x), 'b') for x in value), 2)) % 64 # string -> binary string -> int -> mod 64 + return value % 64 + + + ''' + Inserts value into self.data , value consists of [actual_value, row_index] actual_value must be either numeric or of type str. + - Converts given value into binary. + - Finds which bucket it must be inserted to using the last self.__number_of_bits bits as key. + - Calls self.__readjust__ to check for overflows and readjust the data if need be. + + Args: + value: any basic numeric type(int, float) or a string. + idx: an int, the index of the value(record index) + ''' + def insert(self, value, idx): + hashed_value = self.__hash_value(value) + bin_value = self.__binary_value(hashed_value) + value_id = bin_value[-self.__no_of_bits:] + self.data[value_id].append([value, idx]) + self.__readjust(len(self.data[value_id])) + + + ''' + Finds the bucket that given value is stored in, value must be either numberic or of type str. + If the value is indeed stored in self.data, returns tuple (True, the bucket that its stored in(list of values within bucket)). + If the value is not stored in self.data, returns tuple (False, None). + ''' + def contains(self, value): + hashed_value = self.__hash_value(value) + bin_value = self.__binary_value(hashed_value) + value_id = None + + if self.__method == 'lsb': + value_id = bin_value[-self.__no_of_bits:] + else: + value_id = bin_value[:self.__no_of_bits] + + if value_id in list(self.data.keys()): + return True, self.data[value_id] + return False, None + + + ''' + Finds the bucket that given value is stored in, value must be either numberic or of type str. + If the value is indeed stored in self.data, returns a list withe the indexes that was given when the value was inserted + If the value is not stored in self.data, returns empty list. + If get_duplicates==False and the value is within self.data the returned list will consist of a single element + + Args: + value: any basic numeric type(int, float) or a string. + get_duplicates: a boolean, set to True if duplicates are allowed, default is False + ''' + def find(self, value, get_duplicates=False): + hashed_value = self.__hash_value(value) + bin_value = self.__binary_value(hashed_value) + value_id = None + + if self.__method == 'lsb': + value_id = bin_value[-self.__no_of_bits:] + else: + value_id = bin_value[:self.__no_of_bits] + + rows = [] + if value_id in list(self.data.keys()): + bucket = self.data[value_id] + for l in bucket: + if l[0] == value: + if not get_duplicates: + return [l[1]] + else: + rows.append(l[1]) + return rows + + +##faker = Faker() +##names = [faker.unique.first_name() for _ in range(10)] +##for name in names: +## hash.insert(name) +#l = [28, 4, 19, 1, 22, 16, 12, 0, 5, 7] +#for i in range(len(l)): +# hash.insert(l[i], i) + +#print(hash.data) + +#for value in l: +# print(value, hash.find(value)) \ No newline at end of file diff --git a/miniDB/misc.py b/miniDB/misc.py index aefada74..d2bd4fee 100644 --- a/miniDB/misc.py +++ b/miniDB/misc.py @@ -19,9 +19,10 @@ def split_condition(condition): ops = {'>=': operator.ge, '<=': operator.le, '=': operator.eq, + '!=' : operator.ne, '>': operator.gt, '<': operator.lt} - + for op_key in ops.keys(): splt=condition.split(op_key) if len(splt)>1: diff --git a/miniDB/optimizer.py b/miniDB/optimizer.py new file mode 100644 index 00000000..66d2943d --- /dev/null +++ b/miniDB/optimizer.py @@ -0,0 +1,39 @@ + +class Optimizer: + def __init__(self, query): + self.input_query_list = query.split() + self.kw_ops = set(['and', 'or']) + + ''' + Swaps condition tokens for possible index usage, example: + for the condition: id between 30 and 50 or name = hello + suppose there is a supported index on column 'name' but not on 'id' + it would be better if we swapped the tokens as such: + name = hello or id between 30 and 50 + this way we would make use of the index on column name resulting in faster execution time + ''' + def swap_condition_tokens_of_where(self): + for kw_op in self.kw_ops: + try: + kw_op_idx = self.input_query_list.index(kw_op) + if self.input_query_list[kw_op_idx - 2] == 'between': + l_sub_list = self.input_query_list[:kw_op_idx + 2] + r_sub_list = self.input_query_list[kw_op_idx + 3:] + alt_query = ' '.join(r_sub_list) + ' ' + self.input_query_list[kw_op_idx + 2] + ' ' + ' '.join(l_sub_list) + else: + l_sub_list = self.input_query_list[:kw_op_idx] + r_sub_list = self.input_query_list[kw_op_idx + 1:] + alt_query = ' '.join(r_sub_list) + ' ' + kw_op + ' ' + ' '.join(l_sub_list) + return alt_query + except ValueError: + pass + + def gen_query_alts(self): + alts = [] + alts.append(self.swap_condition_tokens_of_where()) + return alts + + + +opt = Optimizer('id between 30 and 50 or name = hello') +print(opt.gen_query_alts()) diff --git a/miniDB/table.py b/miniDB/table.py index 0ab15686..4b701e8c 100644 --- a/miniDB/table.py +++ b/miniDB/table.py @@ -3,6 +3,7 @@ import pickle import os import sys +import condtion_handler sys.path.append(f'{os.path.dirname(os.path.dirname(os.path.abspath(__file__)))}/miniDB') @@ -26,8 +27,8 @@ class Table: - a dictionary that includes the appropriate info (all the attributes in __init__) ''' - def __init__(self, name=None, column_names=None, column_types=None, primary_key=None, load=None): - + def __init__(self, name=None, column_names=None, column_types=None, unique_column_names=None, primary_key=None, load=None): + if load is not None: # if load is a dict, replace the object dict with it (replaces the object with the specified one) if isinstance(load, dict): @@ -37,6 +38,8 @@ def __init__(self, name=None, column_names=None, column_types=None, primary_key= elif isinstance(load, str): self._load_from_file(load) + self.unique_column_names = unique_column_names + # if name, columns_names and column types are not none elif (name is not None) and (column_names is not None) and (column_types is not None): @@ -60,7 +63,10 @@ def __init__(self, name=None, column_names=None, column_types=None, primary_key= self.column_types = [eval(ct) if not isinstance(ct, type) else ct for ct in column_types] self.data = [] # data is a list of lists, a list of rows that is. - + + self.unique_column_names = unique_column_names + print(f'Tables unique columns: {self.unique_column_names}') + # if primary key is set, keep its index as an attribute if primary_key is not None: self.pk_idx = self.column_names.index(primary_key) @@ -75,7 +81,6 @@ def __init__(self, name=None, column_names=None, column_types=None, primary_key= def column_by_name(self, column_name): return [row[self.column_names.index(column_name)] for row in self.data] - def _update(self): ''' Update all the available columns with the appended rows. @@ -129,6 +134,17 @@ def _insert(self, row, insert_stack=[]): raise ValueError(f'## ERROR -> Value {row[i]} already exists in primary key column.') elif i==self.pk_idx and row[i] is None: raise ValueError(f'ERROR -> The value of the primary key cannot be None.') + + try: + for name in self.unique_column_names: + print(row[i], self.column_by_name(name)) + if row[i] in self.column_by_name(name): + raise ValueError(f'## ERROR -> Value {row[i]} already exists in unique column {name}') + except ValueError as e: # case where the user tried to insert an a record that contains an already existing unique value + print(e) + return + except AttributeError: # the meta_indexes table was saved in a previus version where it didn't have attribute unique_column_names, in this case we just conitinue + pass # if insert_stack is not empty, append to its last index if insert_stack != []: @@ -137,7 +153,7 @@ def _insert(self, row, insert_stack=[]): self.data.append(row) # self._update() - def _update_rows(self, set_value, set_column, condition): + def _update_rows(self, set_value, set_column, condition): ''' Update where Condition is met. @@ -149,21 +165,31 @@ def _update_rows(self, set_value, set_column, condition): 'value[<,<=,=,>=,>]column'. Operatores supported: (<,<=,=,>=,>) - ''' + ''' # parse the condition - column_name, operator, value = self._parse_condition(condition) + #column_name, operator, value = self._parse_condition(condition) # get the condition and the set column - column = self.column_by_name(column_name) + #column = self.column_by_name(column_name) set_column_idx = self.column_names.index(set_column) # set_columns_indx = [self.column_names.index(set_column_name) for set_column_name in set_column_names] # for each value in column, if condition, replace it with set_value - for row_ind, column_value in enumerate(column): - if get_op(operator, column_value, value): - self.data[row_ind][set_column_idx] = set_value + #for row_ind, column_value in enumerate(column): + # if get_op(operator, column_value, value): + # self.data[row_ind][set_column_idx] = set_value + for i in range(len(self.data)): + cond_cpy = condition + for column_name in self.column_names: + if column_name in cond_cpy.split(): + cond_cpy = cond_cpy.replace(column_name, str(self.data[i][self.column_names.index(column_name)])) + condition_plan = condtion_handler.get_condition_plan(cond_cpy) + result = condtion_handler.evaluate_condition(condition_plan) + if result: + self.data[i][set_column_idx] = set_value + # self._update() # print(f"Updated {len(indexes_to_del)} rows") @@ -182,14 +208,24 @@ def _delete_where(self, condition): Operatores supported: (<,<=,==,>=,>) ''' - column_name, operator, value = self._parse_condition(condition) + #column_name, operator, value = self._parse_condition(condition) indexes_to_del = [] - - column = self.column_by_name(column_name) - for index, row_value in enumerate(column): - if get_op(operator, row_value, value): - indexes_to_del.append(index) + + for i in range(len(self.data)): + cond_cpy = condition + for column_name in self.column_names: + if column_name in cond_cpy.split(): + cond_cpy = cond_cpy.replace(column_name, str(self.data[i][self.column_names.index(column_name)])) + condition_plan = condtion_handler.get_condition_plan(cond_cpy) + result = condtion_handler.evaluate_condition(condition_plan) + if result: + indexes_to_del.append(i) + + #column = self.column_by_name(column_name) + #for index, row_value in enumerate(column): + # if get_op(operator, row_value, value): + # indexes_to_del.append(index) # we pop from highest to lowest index in order to avoid removing the wrong item # since we dont delete, we dont have to to pop in that order, but since delete is used @@ -232,13 +268,27 @@ def _select_where(self, return_columns, condition=None, distinct=False, order_by # if condition is None, return all rows # if not, return the rows with values where condition is met for value + rows = [] if condition is not None: - column_name, operator, value = self._parse_condition(condition) - column = self.column_by_name(column_name) - rows = [ind for ind, x in enumerate(column) if get_op(operator, x, value)] + if 'table_name=' not in condition: # if table_name= is in condition then this is meta_indexes table related, otherwise do as follows: + for i in range(len(self.data)): + cond_cpy = condition + for column_name in self.column_names: + if column_name in cond_cpy.split(): + cond_cpy = cond_cpy.replace(column_name, str(self.data[i][self.column_names.index(column_name)])) + condition_plan = condtion_handler.get_condition_plan(cond_cpy) + print(condition_plan) + result = condtion_handler.evaluate_condition(condition_plan) + print(condition_plan, result) + if result: + rows.append(i) + else: + column_name, operator, value = self._parse_condition(condition) + column = self.column_by_name(column_name) + rows = [ind for ind, x in enumerate(column) if get_op(operator, x, value)] else: rows = [i for i in range(len(self.data))] - + # copy the old dict, but only the rows and columns of data with index in rows/columns (the indexes that we want returned) dict = {(key):([[self.data[i][j] for j in return_cols] for i in rows] if key=="data" else value) for key,value in self.__dict__.items()} @@ -254,16 +304,6 @@ def _select_where(self, return_columns, condition=None, distinct=False, order_by if order_by: s_table.order_by(order_by, desc) - # if isinstance(limit, str): - # try: - # k = int(limit) - # except ValueError: - # raise Exception("The value following 'top' in the query should be a number.") - - # # Remove from the table's data all the None-filled rows, as they are not shown by default - # # Then, show the first k rows - # s_table.data.remove(len(s_table.column_names) * [None]) - # s_table.data = s_table.data[:k] if isinstance(limit,str): s_table.data = [row for row in s_table.data if any(row)][:int(limit)] @@ -278,36 +318,117 @@ def _select_where_with_btree(self, return_columns, bt, condition, distinct=False else: return_cols = [self.column_names.index(colname) for colname in return_columns] - - column_name, operator, value = self._parse_condition(condition) + # we want to find the records using btree based on the first subcondition of the full condition + # then we will scan the returned records do see if the rest of the condition also applies, if yes we add this to the returned result + # for example if condition is: id > 3 and name = john + # the generated condition plan is: { 'id > 3': ['id', '>', '3', 'and'], { 'name = john': ['name', '=', 'john', ''] } } + # we want to search based on the first part, meaning 'id > 3' using btree and then we'll check about the second part, 'and name = john' + first_condition_tokens = list(condtion_handler.get_condition_plan(condition).keys())[0].split() + column_name = first_condition_tokens[0] + operator = first_condition_tokens[1] + value = first_condition_tokens[2] + + # remove this if need be + #column_name, operator, value = self._parse_condition(condition) # if the column in condition is not a primary key, abort the select - if column_name != self.column_names[self.pk_idx]: - print('Column is not PK. Aborting') + #if self.pk_idx is not None: + # if column_name != self.column_names[self.pk_idx]: + # print('Column is not PK. Aborting') # here we run the same select twice, sequentially and using the btree. # we then check the results match and compare performance (number of operation) - column = self.column_by_name(column_name) - + #column = self.column_by_name(column_name) # sequential - rows1 = [] - opsseq = 0 - for ind, x in enumerate(column): - opsseq+=1 - if get_op(operator, x, value): - rows1.append(ind) + #rows1 = [] + #opsseq = 0 + #for ind, x in enumerate(column): + # opsseq+=1 + # #print(f'operator:{operator}, x:{x}, value:{value}') + # if get_op(operator, x, value): + # rows1.append(ind) # btree find + bt.show() rows = bt.find(operator, value) + #print(f'rows1: {rows1}, rows: {rows}') + try: + k = int(limit) + except TypeError: + k = None + # same as simple select from now on + rows = rows[:k] + final_rows = [] + for i in range(len(self.data)): + cond_cpy = condition + for column_name in self.column_names: + if column_name in cond_cpy.split(): + cond_cpy = cond_cpy.replace(column_name, str(self.data[i][self.column_names.index(column_name)])) + condition_plan = condtion_handler.get_condition_plan(cond_cpy) + result = condtion_handler.evaluate_condition(condition_plan) + print(condition_plan, result) + if result: + final_rows.append(i) + + + print(f'Rows from btree find: {rows}, Final rows: {final_rows}') + # TODO: this needs to be dumbed down + dict = {(key):([[self.data[i][j] for j in return_cols] for i in final_rows] if key=="data" else value) for key,value in self.__dict__.items()} + + dict['column_names'] = [self.column_names[i] for i in return_cols] + dict['column_types'] = [self.column_types[i] for i in return_cols] + + s_table = Table(load=dict) + + s_table.data = list(set(map(lambda x: tuple(x), s_table.data))) if distinct else s_table.data + + if order_by: + s_table.order_by(order_by, desc) + + if isinstance(limit,str): + s_table.data = [row for row in s_table.data if row is not None][:int(limit)] + + return s_table + + def _select_where_with_extendible_hash(self, return_columns, ehash, condition, distinct=False, order_by=None, desc=True, limit=None): + if return_columns == '*': + return_cols = [i for i in range(len(self.column_names))] + else: + return_cols = [self.column_names.index(colname) for colname in return_columns] + + #column_name, operator, value = self._parse_condition(condition) + + first_condition_tokens = list(condtion_handler.get_condition_plan(condition).keys())[0].split() + column_name = first_condition_tokens[0] + value = first_condition_tokens[2] + + + # extendible hash find + rows = ehash.find(value, distinct) try: k = int(limit) except TypeError: k = None # same as simple select from now on rows = rows[:k] + + final_rows = [] + for i in range(len(self.data)): + cond_cpy = condition + for column_name in self.column_names: + if column_name in cond_cpy.split(): + cond_cpy = cond_cpy.replace(column_name, str(self.data[i][self.column_names.index(column_name)])) + condition_plan = condtion_handler.get_condition_plan(cond_cpy) + result = condtion_handler.evaluate_condition(condition_plan) + print(f'condition_plan: {condition_plan}, result: {result}') + if result: + final_rows.append(i) + + print(f'Rows from ehash find: {rows}, final_rows: {final_rows}') + # TODO: this needs to be dumbed down - dict = {(key):([[self.data[i][j] for j in return_cols] for i in rows] if key=="data" else value) for key,value in self.__dict__.items()} + dict = {(key):([[self.data[i][j] for j in return_cols] for i in final_rows] if key=="data" else value) for key,value in self.__dict__.items()} dict['column_names'] = [self.column_names[i] for i in return_cols] dict['column_types'] = [self.column_types[i] for i in return_cols] @@ -324,6 +445,7 @@ def _select_where_with_btree(self, return_columns, bt, condition, distinct=False return s_table + def order_by(self, column_name, desc=True): ''' Order table based on column. @@ -549,7 +671,7 @@ def _parse_condition(self, condition, join=False): 'column[<,<=,==,>=,>]value' or 'value[<,<=,==,>=,>]column'. - Operatores supported: (<,<=,==,>=,>) + Operators supported: (<,<=,==,>=,>) join: boolean. Whether to join or not (False by default). ''' # if both_columns (used by the join function) return the names of the names of the columns (left first)