Skip to content

Commit 6ba3cfa

Browse files
vijayg15vijayg15
authored
project_report
1 parent 3e63aef commit 6ba3cfa

17 files changed

Lines changed: 1414 additions & 0 deletions

‎preprocessed_test_images/test_preproc_CNN.npy‎

22.5 MB
Binary file not shown.

‎preprocessed_test_images/test_preproc_resnet.npy‎

22.5 MB
Binary file not shown.

‎preprocessed_test_images/test_preproc_vgg16.npy‎

22.5 MB
Binary file not shown.

‎preprocessed_test_images/test_preproc_vgg19.npy‎

22.5 MB
Binary file not shown.

‎scripts/wp_CNN_test_preprocess.py‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
# In[1]:
2+
import numpy as np
3+
import pandas as pd
4+
5+
# In[2]:
6+
from keras.preprocessing.image import load_img, img_to_array
7+
8+
# In[3]:
9+
10+
img_width, img_height = 256, 256
11+
12+
# In[4]:
13+
def preprocess_image(path):
14+
img = load_img(path, target_size = (img_height, img_width))
15+
a = img_to_array(img)
16+
a = np.expand_dims(a, axis = 0)
17+
a /= 255.
18+
return a
19+
20+
# In[5]:
21+
test_images_dir = '../dataset/alien_test/'
22+
23+
# In[6]:
24+
test_df = pd.read_csv('../dataset/test.csv')
25+
26+
test_dfToList = test_df['Image_id'].tolist()
27+
test_ids = [str(item) for item in test_dfToList]
28+
29+
# In[7]:
30+
31+
test_images = [test_images_dir+item for item in test_ids]
32+
test_preprocessed_images = np.vstack([preprocess_image(fn) for fn in test_images])
33+
34+
# In[8]:
35+
np.save('../test_preproc_CNN.npy', test_preprocessed_images)
36+
37+

‎scripts/wp_EDA_preprocessing.py‎

Lines changed: 206 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,206 @@
1+
#!/usr/bin/env python
2+
# coding: utf-8
3+
4+
# In[1]:
5+
6+
7+
import os
8+
import random
9+
from shutil import copyfile
10+
11+
12+
# In[2]:
13+
14+
15+
import numpy as np
16+
import pandas as pd
17+
18+
19+
# In[3]:
20+
21+
22+
import matplotlib.pyplot as plt
23+
import seaborn as sns
24+
from matplotlib.image import imread
25+
import pathlib
26+
27+
28+
# In[4]:
29+
30+
31+
image_folder = ['cloudy', 'foggy', 'rainy', 'shine', 'sunrise']
32+
nimgs = {}
33+
for i in image_folder:
34+
nimages = len(os.listdir('../dataset/'+i+'/'))
35+
nimgs[i]=nimages
36+
plt.figure(figsize=(10, 8))
37+
plt.bar(range(len(nimgs)), list(nimgs.values()), align='center')
38+
plt.xticks(range(len(nimgs)), list(nimgs.keys()))
39+
plt.title('Distribution of different classes of Dataset')
40+
plt.show()
41+
42+
43+
# In[5]:
44+
45+
46+
image_folder = ['cloudy', 'foggy', 'rainy', 'shine', 'sunrise']
47+
48+
for i in image_folder:
49+
sample_images = list(pathlib.Path('../dataset/'+i+'/').rglob('*/'))
50+
np.random.seed(42)
51+
rand_imgs = np.random.choice(sample_images, size=10*10)
52+
53+
shapes = []
54+
for img in rand_imgs:
55+
shapes.append(imread(str(img)).shape)
56+
57+
shapes = pd.DataFrame().assign(X=pd.Series(shapes).map(lambda s: s[0]), Y=pd.Series(shapes).map(lambda s: s[1]))
58+
59+
plt.figure(figsize=(12, 8))
60+
sns.set_context("notebook", font_scale=1.5)
61+
sns.kdeplot(shapes['X'], bw=75)
62+
sns.kdeplot(shapes['Y'], bw=75)
63+
plt.title('Distribution of {}_image Sizes'.format(i))
64+
ax = plt.gca()
65+
ax.set_xlim(0, ax.get_xlim()[1])
66+
67+
68+
# In[ ]:
69+
70+
71+
try:
72+
os.mkdir('../weather_pred/Data')
73+
os.mkdir('../weather_pred/Data/training')
74+
os.mkdir('..weather_pred/Data/validation')
75+
os.mkdir('../weather_pred/Data/training/cloudy')
76+
os.mkdir('../weather_pred/Data/training/foggy')
77+
os.mkdir('../weather_pred/Data/training/rainy')
78+
os.mkdir('../weather_pred/Data/training/shine')
79+
os.mkdir('../weather_pred/Data/training/sunrise')
80+
os.mkdir('../weather_pred/Data/validation/cloudy')
81+
os.mkdir('../weather_pred/Data/validation/foggy')
82+
os.mkdir('../weather_pred/Data/validation/rainy')
83+
os.mkdir('../weather_pred/Data/validation/shine')
84+
os.mkdir('../weather_pred/Data/validation/sunrise')
85+
except OSError:
86+
pass
87+
88+
89+
# In[ ]:
90+
91+
92+
def split_data(SOURCE, TRAINING, VALIDATION, SPLIT_SIZE):
93+
files = []
94+
for filename in os.listdir(SOURCE):
95+
file = SOURCE + filename
96+
if os.path.getsize(file) > 0:
97+
files.append(filename)
98+
else:
99+
print(filename + " is zero length, so ignoring.")
100+
101+
training_length = int(len(files) * SPLIT_SIZE)
102+
valid_length = int(len(files) - training_length)
103+
shuffled_set = random.sample(files, len(files))
104+
training_set = shuffled_set[0:training_length]
105+
valid_set = shuffled_set[training_length:]
106+
107+
for filename in training_set:
108+
this_file = SOURCE + filename
109+
destination = TRAINING + filename
110+
copyfile(this_file, destination)
111+
112+
for filename in valid_set:
113+
this_file = SOURCE + filename
114+
destination = VALIDATION + filename
115+
copyfile(this_file, destination)
116+
117+
118+
# In[ ]:
119+
120+
121+
CLOUDY_SOURCE_DIR = '.../dataset/cloudy/'
122+
TRAINING_CLOUDY_DIR = '.../weather_pred/Data/training/cloudy/'
123+
VALID_CLOUDY_DIR = '.../weather_pred/Data/validation/cloudy/'
124+
125+
FOGGY_SOURCE_DIR = '.../dataset/foggy/'
126+
TRAINING_FOGGY_DIR = '../weather_pred/Data/training/foggy/'
127+
VALID_FOGGY_DIR = '.../weather_pred/Data/validation/foggy/'
128+
129+
RAINY_SOURCE_DIR = '.../dataset/rainy/'
130+
TRAINING_RAINY_DIR = '.../weather_pred/Data/training/rainy/'
131+
VALID_RAINY_DIR = '.../weather_pred/Data/validation/rainy/'
132+
133+
SHINE_SOURCE_DIR = '.../dataset/shine/'
134+
TRAINING_SHINE_DIR = '.../weather_pred/Data/training/shine/'
135+
VALID_SHINE_DIR = '.../weather_pred/Data/validation/shine/'
136+
137+
SUNRISE_SOURCE_DIR = '.../dataset/sunrise/'
138+
TRAINING_SUNRISE_DIR = '.../weather_pred/Data/training/sunrise/'
139+
VALID_SUNRISE_DIR = '.../weather_pred/Data/validation/sunrise/'
140+
141+
142+
# In[ ]:
143+
144+
145+
split_size = .85
146+
147+
148+
# In[ ]:
149+
150+
151+
split_data(CLOUDY_SOURCE_DIR, TRAINING_CLOUDY_DIR, VALID_CLOUDY_DIR, split_size)
152+
split_data(FOGGY_SOURCE_DIR, TRAINING_FOGGY_DIR, VALID_FOGGY_DIR, split_size)
153+
split_data(RAINY_SOURCE_DIR, TRAINING_RAINY_DIR, VALID_RAINY_DIR, split_size)
154+
split_data(SHINE_SOURCE_DIR, TRAINING_SHINE_DIR, VALID_SHINE_DIR, split_size)
155+
split_data(SUNRISE_SOURCE_DIR, TRAINING_SUNRISE_DIR, VALID_SUNRISE_DIR, split_size)
156+
157+
158+
# In[6]:
159+
160+
161+
image_folder = ['cloudy', 'foggy', 'rainy', 'shine', 'sunrise']
162+
nimgs = {}
163+
for i in image_folder:
164+
nimages = len(os.listdir('../weather_pred/Data/training/'+i+'/'))
165+
nimgs[i]=nimages
166+
plt.figure(figsize=(9, 6))
167+
plt.bar(range(len(nimgs)), list(nimgs.values()), align='center')
168+
plt.xticks(range(len(nimgs)), list(nimgs.keys()))
169+
plt.title('Distribution of different classes in Training Dataset')
170+
plt.show()
171+
172+
173+
# In[7]:
174+
175+
176+
for i in ['cloudy', 'foggy', 'rainy', 'shine', 'sunrise']:
177+
print('Training {} images are: '.format(i)+str(len(os.listdir('../weather_pred/Data/training/'+i+'/'))))
178+
179+
180+
# In[8]:
181+
182+
183+
image_folder = ['cloudy', 'foggy', 'rainy', 'shine', 'sunrise']
184+
nimgs = {}
185+
for i in image_folder:
186+
nimages = len(os.listdir('../weather_pred/Data/validation/'+i+'/'))
187+
nimgs[i]=nimages
188+
plt.figure(figsize=(9, 6))
189+
plt.bar(range(len(nimgs)), list(nimgs.values()), align='center')
190+
plt.xticks(range(len(nimgs)), list(nimgs.keys()))
191+
plt.title('Distribution of different classes in Validation Dataset')
192+
plt.show()
193+
194+
195+
# In[9]:
196+
197+
198+
for i in ['cloudy', 'foggy', 'rainy', 'shine', 'sunrise']:
199+
print('Valid {} images are: '.format(i)+str(len(os.listdir('../weather_pred/Data/validation/'+i+'/'))))
200+
201+
202+
# In[ ]:
203+
204+
205+
206+

‎scripts/wp_confusion_matrix.py‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
# In[1]:
2+
3+
import os
4+
import numpy as np
5+
import pandas as pd
6+
import matplotlib.pyplot as plt
7+
8+
# In[2]:
9+
10+
from tensorflow.keras.models import load_model
11+
12+
# In[3]:
13+
14+
test_preprocessed_images = np.load('D../test_preproc_resnet.npy')
15+
16+
# In[4]:
17+
model_path = '../resnet101_drop_batch_best_weights_256.h5'
18+
19+
#Load the pre-trained models
20+
model = load_model(model_path)
21+
22+
# In[5]:
23+
#Prediction Function
24+
array = model.predict(test_preprocessed_images, batch_size=1, verbose=1)
25+
y_pred = np.argmax(array, axis=1)
26+
27+
# In[6]:
28+
test_df = pd.read_csv('../dataset/test.csv')
29+
y_true = test_df['labels']
30+
31+
# In[7]:
32+
from sklearn.metrics import confusion_matrix
33+
conf_mat = confusion_matrix(y_true, y_pred)
34+
35+
# In[8]:
36+
train_dir = '../weather_pred/Data/training/'
37+
classes = os.listdir(train_dir)
38+
39+
# In[9]:
40+
41+
import itertools
42+
def plot_confusion_matrix(cm, classes,
43+
normalize=False,
44+
title='Confusion matrix',
45+
cmap=plt.cm.Reds):
46+
"""
47+
This function prints and plots the confusion matrix.
48+
Normalization can be applied by setting `normalize=True`.
49+
"""
50+
plt.imshow(cm, interpolation='nearest', cmap=cmap)
51+
plt.title(title)
52+
plt.colorbar()
53+
tick_marks = np.arange(len(classes))
54+
plt.xticks(tick_marks, classes, rotation=45)
55+
plt.yticks(tick_marks, classes)
56+
57+
if normalize:
58+
cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
59+
cm = cm.round(2)
60+
print("Normalized confusion matrix")
61+
else:
62+
print('Confusion matrix, without normalization')
63+
64+
print(cm)
65+
66+
thresh = cm.max() / 2.
67+
for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
68+
plt.text(j, i, cm[i, j],
69+
horizontalalignment="center",
70+
color="white" if cm[i, j] > thresh else "black")
71+
72+
plt.tight_layout()
73+
plt.ylabel('True label')
74+
plt.xlabel('Predicted label')
75+
76+
# In[10]:
77+
np.set_printoptions(precision=2)
78+
79+
fig1 = plt.figure(figsize=(7,6))
80+
plot_confusion_matrix(conf_mat, classes=classes, title='Confusion matrix, without normalization')
81+
fig1.savefig('../cm_wo_norm.jpg')
82+
plt.show()
83+
84+
# In[11]:
85+
np.set_printoptions(precision=2)
86+
87+
fig2 = plt.figure(figsize=(7,6))
88+
plot_confusion_matrix(conf_mat, classes=classes, normalize = True, title='Normalized Confusion matrix')
89+
fig2.savefig('../cm_norm.jpg')
90+
plt.show()
91+
92+
93+

‎scripts/wp_prediction_and_log_loss.py‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
# In[1]:
2+
import numpy as np
3+
import pandas as pd
4+
5+
# In[2]:
6+
7+
from tensorflow.keras.models import load_model
8+
9+
# In[3]:
10+
img_width, img_height = 256, 256
11+
12+
# In[4]:
13+
test_preprocessed_images = np.load('../test_preproc_CNN.npy')
14+
15+
# In[5]:
16+
#Define Path
17+
model_path = '../CNN_best_weights_256.h5'
18+
#model_path = '../CNN_augmentation_best_weights_256.h5'
19+
#model_path = '../vgg16_best_weights_256.h5'
20+
#model_path = '../vgg16_drop_batch_best_weights_256.h5'
21+
#model_path = '../vgg19_drop_batch_best_weights_256.h5'
22+
#model_path = '../resnet101_drop_batch_best_weights_256.h5'
23+
24+
#Load the pre-trained models
25+
model = load_model(model_path)
26+
27+
28+
# In[6]:
29+
#Prediction Function
30+
array = model.predict(test_preprocessed_images, batch_size=1, verbose=1)
31+
answer = np.argmax(array, axis=1)
32+
33+
# In[7]:
34+
test_df = pd.read_csv('../dataset/test.csv')
35+
y_true = test_df['labels']
36+
y_pred = array
37+
38+
# In[8]:
39+
from sklearn.metrics import log_loss
40+
loss = log_loss(y_true, y_pred, eps=1e-15, normalize=True, sample_weight=None, labels=None)
41+
42+

0 commit comments

Comments
 (0)