AI-OpenMic/Model.py at main · cfiltnlp/AI-OpenMic · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
# -*- coding: utf-8 -*-
"""Model

Automatically generated by Colaboratory.

"""

import numpy as np
import tensorflow as tf
import keras
import numpy as np
import torch
import os, glob
import pickle as pkl
from keras import layers
from keras.utils import to_categorical
from keras.models import Sequential
from keras.layers import Dense, Flatten, Dropout, Input, Bidirectional, LSTM, Activation, TimeDistributed, BatchNormalization
from sklearn.model_selection import train_test_split
from keras.utils.vis_utils import plot_model
import tensorflow.keras.backend as K
import pandas as pd
from sklearn.metrics import confusion_matrix
import seaborn as sns

from google.colab import drive
drive.mount('/content/drive')

#loading inputs and outputs
path = '/content/drive/Shareddrives/NLP_group/Project/Datasetv3/ImportantObjects/'
bert_embeddings = np.load(path + 'BertBase.pkl', allow_pickle=True) #loading the pickle file containing all bert embeddings
audio_embeddings = np.load(path + "audioembed.npy")              #loading the numpy file containing all audioembedding
y = np.load(path + "score.npy")                                  #loading the numpy file containing all scores

#Neural Network
#Creating the network with 2 input layers
inputs1 = keras.Input(shape=(33, 8000,), name = "Audio Features")
inputs2 = keras.Input(shape=(512, 768,), name = "Text Embeddings")
# x1 = layers.Bidirectional(layers.LSTM(256, return_sequences=True))(inputs1)
x1 = layers.Bidirectional(layers.LSTM(256, return_sequences=True), name = "Audio_Layer")(inputs1)
# x2 = layers.Bidirectional(layers.LSTM(256, return_sequences=True))(inputs2)
x2 = layers.Bidirectional(layers.LSTM(256, return_sequences=True), name = "Text_Layer")(inputs2)
a1 = layers.Bidirectional(layers.LSTM(128),name = "Audio_2_Layer")(x1)
a2 = layers.Bidirectional(layers.LSTM(128), name = "Text_2_Layer")(x2)
c1 = layers.Dense(100, activation="sigmoid", name = "Audio")(a1)
c2 = layers.Dense(100, activation="sigmoid", name = "Text")(a2)

c3 = layers.Concatenate(axis=1, name="Combined")([c1, c2])   #concatenating the 2 outputs and passing through a classification layer
outputs = layers.Dense(5, activation="softmax", name = "Output")(c3)

model2 = keras.Model([inputs1,inputs2], outputs)
model2.summary()
# model2.compile("adam","mean_squared_error" , metrics=["accuracy"]) #can use categorical cross entropy as loss too
model2.compile("adam","categorical_crossentropy" , metrics=["accuracy"])

plot_model(model2, to_file='modelplot.png', show_shapes=True, show_layer_names=True)

#train, test split
xtrain_aud, xtest_aud, xtrain_bert, xtest_bert, ytrain, ytest= train_test_split(audio_embeddings, bert_embeddings, y, train_size=0.9, random_state=10)

#Converting to Keras tensors
X_train_aud = K.constant(xtrain_aud)
X_train_bert = K.constant(xtrain_bert)
Y_train = K.constant(ytrain)
X_test_aud = K.constant(xtest_aud)
X_test_bert = K.constant(xtest_bert)
Y_test = K.constant(ytest)

#checking the shapes of input
print(np.shape(X_train_aud))
print(np.shape(X_train_bert))

#train
model2.fit([X_train_aud, X_train_bert],Y_train, epochs= 12)

#testing on test set
model2.evaluate([X_test_aud, X_test_bert],Y_test)

#printing the actual and predicted scores
predictedY = np.argmax(model2.predict([X_test_aud, X_test_bert]), axis=-1)
actualY = np.argmax(Y_test, axis=-1)
print(predictedY, actualY)
# predictedY = anot3
# actualY = model


#generating the confusion matrix
C = confusion_matrix(actualY, predictedY)
# print(C)
#generating the heatmap
sns.heatmap(C, annot=True, cmap='Blues')

#Computing QWK
w = np.zeros((5,5))
for i in range(len(w)):
    for j in range(len(w)):
        w[i][j] = float(((i-j)**2)/16)

# print(w)
N=5
act_hist = np.zeros([N])
pred_hist = np.zeros([N])
for item in actualY:
    act_hist[item]+=1
for item in predictedY:
    pred_hist[item]+=1
# print(act_hist)
# print(pred_hist)
E = np.outer(act_hist, pred_hist)
# print(E)
E = E/E.sum()
C = C/C.sum()
# print(E,C)

num=0
den=0
for i in range(len(w)):
    for j in range(len(w)):
        num+=w[i][j]*C[i][j]
        den+=w[i][j]*E[i][j]

weighted_kappa = (1 - (num/den))
print(weighted_kappa)