fasttext_classifications/6_fasttext_without_tunning.py at master · isultane/fasttext_classifications · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
# Author: Sultan S. Alqahtani
# Date: 05/04/2022

import time
from operator import index, mod
import os
import fasttext
import numpy as np
from numpy.lib.function_base import average
import pandas as pd
import re
import csv
import glob

from utilities import read_training_data
from utilities import parse_labels
from utilities import predict_labels
from utilities import calc_accurecy
from utilities import calc_precision_recall_f1


from scipy.sparse.sputils import matrix
from sklearn import metrics
from sklearn.model_selection import train_test_split
from sklearn.model_selection import KFold
#from sklearn.metrics import accuracy_score, f1_score
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import precision_recall_fscore_support as score
from sklearn.metrics import accuracy_score


bugreports_source_dataset = glob.glob(
    "./data/temp/*.txt")  # source datasets path (bug reports after extracted in fasttext format)

class fasttextModelWithoutTunning(object):
    def split_train_test(self, datasetFile, training_path, testing_path, testing_size=0.2):
        # split dataset into train and test, then save them (training 80% and testing 20%), same as it is followed by [REF]
        with open(datasetFile, "r") as reader:
            data = reader.readlines()
            x_train, x_test = train_test_split(data, test_size=testing_size)
            # save training data
            with open("./data/bug_reports/results/"+training_path, 'w') as trainFile:
                trainFile.writelines(x_train)
            # save testing data
            with open("./data/bug_reports/results/"+testing_path, 'w') as testFile:
                testFile.writelines(x_test)
    # this code for testing the approach of k-fold validation
    def fasttext_kfold_model(self, df, k, project_name):
        start_time = time.time()
        print("Processing project" + project_name)
        # record results
        models = []

        train_fold = ""
        test_fold = ""

        # split the dataset into k folds
        kf = KFold(n_splits=k, shuffle=True)
        fold_counter = 0
        # for each fold
        for train_index, test_index in kf.split(df['label'], df['text']):
            fold_counter += 1
            #print("Processing fold: ", fold_counter)
            train_fold = './data/bug_reports/results/ft_k' + \
                str(fold_counter)+"_"+str(project_name)+'.train'
            test_fold = './data/bug_reports/results/ft_k' + \
                str(fold_counter)+"_"+str(project_name)+'.valid'
            # save training subset for each fold
            df[['label', 'text']].iloc[train_index].to_csv(
                train_fold, index=None, header=None, sep=' ')

            # save tesing subset for each trained fold
            df[['label', 'text']].iloc[test_index].to_csv(
                test_fold, index=None, header=None, sep=' ')


            # fit model for this set of parameter values
            model = fasttext.train_supervised(train_fold, epoch=40, lr=0.5, loss='ova')
            test_labels = parse_labels(test_fold)
            pred_labels = predict_labels(test_fold, model)
            precision_score, recall_socre, f1_score, pf, g_score = calc_accurecy(test_labels, pred_labels)

            # hard coded fucntion to calcualte P, R, F1, pf, and G
            #precision_score, recall_socre, f1_score, pf, g_score = calc_precision_recall_f1(test_labels, pred_labels)
            #print("Precision: ",precision_score , " Recall: ",recall_socre," F1_score: ",f1_score, "prob. false alarm: ", pf, "g_score", g_score)

            '''
            print('mean train scores: ', np.mean(train_scores))
            print('mean val scores: ', np.mean(val_scores))
            '''
            self.write_kfold_results(precision_score, recall_socre, f1_score, pf, g_score, fold_counter,project_name)

            # to get the best k-fold model results and save it to be used later
            #model.save_model("./data/bug_reports/results/k" + str(fold_counter)+"_"+str(project_name)+"_model.bin")

            print("Deleting tmp files: train and test files")
            if os.path.exists(train_fold):
                os.remove(train_fold)
            if os.path.exists(test_fold):
                os.remove(test_fold)

           #print("best values: ", best_results["f_score"], best_results["p_score"], best_results["r_score"])
           #print("************************************ FOLD DONE ************************************")
        train_time = time.time()
        #print('Train & tesing time: {:.2f}s'.format(train_time - start_time))
        total_traning_testing_time = 'Train & tesing time: {:.2f}s'.format(train_time - start_time)
        self.write_training_time(total_traning_testing_time, project_name)

        print("Deleting tmp files")
        if os.path.exists(train_fold) and os.path.exists(test_fold):
            os.remove(train_fold)
            os.remove(test_fold)

        return models

    # write kfold results to CSV file
    def write_kfold_results(self, p_score, r_socre, f1_score, pf, g_score,fold_counter, pname):
        with open('./data/bug_reports/results/kfold_'+str(pname)+'_results.csv', 'a') as results:
            write = csv.writer(results)
            data = [p_score, r_socre, f1_score, pf, g_score,fold_counter, pname]
            write.writerow(data)

    # write training time results to CSV file
    def write_training_time(self, training_time, pname):
        with open('./data/bug_reports/results/total_training_time_results.csv', 'a') as tresults:
            write = csv.writer(tresults)
            data = [training_time, pname]
            write.writerow(data)

if __name__ == '__main__':
    fasttext_model_object = fasttextModelWithoutTunning()
    '''
    1- for bug reeport project: reading bug reprts for projects Ambari, Camel, Derby, Wicket and Chromium after they pre-processed and splited into
    .train and .valid format of fasttext.
    '''
    for project_data in bugreports_source_dataset:
        project_name = os.path.basename(project_data)
        #print("processing project: " + project_name)

        # prepare .train and .valid files names
        training_file = project_name.split('.')[0]
        training_file = training_file + '.train'
        #print(training_file)

        testing_file = project_name.split('.')[0]
        testing_file = testing_file + '.valid'
        #print(testing_file)
        # split the data set into training and validating sets
        fasttext_model_object.split_train_test(
            project_data, training_file, testing_file)

        df = read_training_data("./data/bug_reports/results/"+training_file)

        models = fasttext_model_object.fasttext_kfold_model(df, k=10, project_name=project_name.split('.')[0])