crowdsourced_mapping_notebook.py

# -*- coding: utf-8 -*-
"""Crowdsourced Mapping - Notebook.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1FvnrHDYAazsf6mbTvKp0xq-mzB90Wdl4

### **Crowdsourced Mapping** - (Mulivariate Classification)
"""

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import plotly.express as px
from imblearn.over_sampling import SMOTE
from imblearn.under_sampling import RandomUnderSampler
from imblearn.pipeline import Pipeline
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split, KFold, cross_val_score
from sklearn.manifold import TSNE
from sklearn.metrics import confusion_matrix, accuracy_score, recall_score
import tensorflow as tf
from tensorflow.keras import layers, models
from tensorflow.keras.layers import Dense, Conv2D, MaxPooling2D, Activation
from tensorflow.keras.losses import SparseCategoricalCrossentropy
import timeit

"""Loading the dataset:-"""

train_data = pd.read_csv('training.csv')
test_data = pd.read_csv('testing.csv')

"""# 1. Data Exploration"""

train_data.info()

train_data.describe(include="all")

unitr = train_data['class'].unique()
print(f"Total number of unique in class '",len(unitr), "' and they are \n", unitr)

# Calculate the mean for each class
class_means = train_data.groupby('class').mean()

# Display the mean values for each class
class_means

# Plot line plot
plt.figure(figsize=(12, 8))
for class_label, values in class_means.iterrows():
    plt.plot(class_means.columns, values, label=class_label)

plt.title('Distribution of Max_ndvi Over Time for Different Classes')
plt.xlabel('Date')
plt.ylabel('Max_Ndvi Values')
plt.legend(title='Class Lables', bbox_to_anchor=(1.05, 1), loc='upper right')

plt.xticks(rotation=45, ha="right")
plt.show()

df_no_class = class_means

# Plot the distribution of each row
plt.figure(figsize=(13, 6))
sns.set(style="whitegrid")

for index, row in df_no_class.iterrows():
    sns.kdeplot(row, label=index, fill=True)

plt.title('Distribution of Each Row')
plt.xlabel('Feature Values')
plt.ylabel('Density')
plt.legend()
plt.show()

df = pd.DataFrame(class_means, index=['farm', 'forest', 'grass', 'impervious', 'orchard', 'water'])
df = df.drop('max_ndvi', axis=1)

# Plot the distribution of each row
for index, row in df.iterrows():
    plt.figure(figsize=(8, 6))
    plt.bar(row.index, row.values, color='skyblue')
    plt.title(f'Distribution for {index}')
    plt.xlabel('Date')
    plt.ylabel('Value')
    plt.xticks(rotation=90, ha="right")
    plt.show()

trainx, y_train = train_data.iloc[:, 1:], train_data.iloc[:, 0]
x_test, y_test = test_data.iloc[:, 1:], test_data.iloc[:, 0]

"""Applying min-max scaling"""

# Function to perform min-max scaling
def min_max_scaling(data):
    min_val = np.min(data, axis=0)
    max_val = np.max(data, axis=0)
    scaled_data = (data - min_val) / (max_val - min_val)
    return scaled_data

# Fit and transform trainx
min_val_train = np.min(trainx, axis=0)
max_val_train = np.max(trainx, axis=0)
trainx = (trainx - min_val_train) / (max_val_train - min_val_train)

# Transform x_test using the min and max values from trainx
x_test = (x_test - min_val_train) / (max_val_train - min_val_train)

"""Distribution of different classes with in the training dataset:-"""

# To understand the distribution among the different classes in training data.
train_data['class'].value_counts().plot(kind='bar')

train_data['class'].value_counts()

"""Regarding the distribution, it's evident that nearly 75% of the data points are attributed to a single class, highlighting a pronounced imbalance in the dataset. Some classes have only 205 and 53 instances, respectively, out of the total 10,545.

To tackle the data imbalance challenge, we implemented SMOTE oversampling, resulting in an expanded dataset of nearly 45,000 data points—three times the size of the original set. Instead of using a widespread oversampling method that could overshadow the original data, we chose to focus on boosting the representation of minority classes and also preserve the original dataset(means:- oversampling data should not dominate the original data). This specific oversampling approach contributed to better generalization and enhanced prediction capabilities.
"""

target_column = 'class'
X = trainx
y = train_data['class']

# Oversample the minority class
oversample = SMOTE(sampling_strategy={'water': 2000, 'farm': 2000, 'orchard': 2000, 'grass': 2000, 'impervious': 2000})

# Undersample the majority class
undersample = RandomUnderSampler(sampling_strategy={'forest': 7000})

# Create a pipeline that first oversamples, then undersamples
pipeline = Pipeline([('o', oversample), ('u', undersample)])

# Apply the pipeline to your data
X_resampled, y_resampled = pipeline.fit_resample(X, y)

# The resampled feature set and target
resampled_data = pd.DataFrame(X_resampled, columns=train_data.drop(target_column, axis=1).columns)
resampled_data['class'] = y_resampled

# Now, 'resampled_data' is the balanced dataset
resampled_data

resampled_data['class'].value_counts().plot(kind='bar')

"""Correlation analysis:-"""

corr_matrix = X_resampled.corr()

# Heatmap using seaborn
plt.figure(figsize=(20,20))
sns.heatmap(corr_matrix, annot=True, cmap='coolwarm', fmt='.2f', linewidths=0.25)
plt.title('Plot for Correlation Matrix')
plt.show()

"""Apart from one specific pair of columns in the dataset, we did not identify any other significant correlations throughout the entire dataset.

# 2. Dimensionality Reduction

We are planning to deploy two dimensionality reduction techniques:

a.   Principal Component Analysis (PCA)

b.   t-Distributed Stochastic Neighbor Embedding (t-SNE)

Principal Component Analysis
"""

class PCA:

    def __init__(self, desired_components=None):
        self.desired_components = desired_components

    def fit_transform(self, X):

        self.mean_ = np.mean(X, axis=0)
        X_centered = X - self.mean_
        cov_matrix = np.cov(X_centered.T)

        eigenvalues, eigenvectors = np.linalg.eig(cov_matrix)
        eigen_indices = np.argsort(eigenvalues)[::-1]  # Sort in descending order

        self.eigenvalues_ = eigenvalues[eigen_indices]
        self.eigenvectors_ = eigenvectors[:, eigen_indices]

        if self.desired_components is not None:
            self.eigenvectors_ = self.eigenvectors_[:, :self.desired_components]

        X_transformed = np.dot(X_centered, self.eigenvectors_)

        return X_transformed

    def transform(self, X):

        # Transforms data to the principal component space.

        # Check if already fit

        if not hasattr(self, "mean_"):
            raise ValueError("Model not fitted yet! Call `fit` or `fit_transform` first.")

        X_centered = X - self.mean_

        return np.dot(X_centered, self.eigenvectors_)

    def inverse_transform(self, X):
        # Transforms data back to the original space.

        # Check if already fit

        if not hasattr(self, "mean_"):
            raise ValueError("Model not fitted yet! Call `fit` or `fit_transform` first.")

        return np.dot(X, self.eigenvectors_.T) + self.mean_

    def explained_variance_ratio_(self):

        if not hasattr(self, "eigenvalues_"):
            raise ValueError("Model not fitted yet! Call `fit` or `fit_transform` first.")

        return self.eigenvalues_ / np.sum(self.eigenvalues_)

    def variance_captured(self):

        if not hasattr(self, "desired_components"):
            raise ValueError("Number of components not specified.")

        if not hasattr(self, "eigenvalues_"):
            raise ValueError("Model not fitted yet! Call `fit` or `fit_transform` first.")

        captured_variance = np.sum(self.eigenvalues_[: self.desired_components]) / np.sum(self.eigenvalues_)
        return captured_variance

features = X_resampled
desired_components=28
# Performing PCA
pca = PCA(desired_components)
pca_results = pca.fit_transform(features)

print("The Variance Captured by", desired_components, "components:", round(pca.variance_captured(), 2) * 100, "%", '\n')

""" t - Distributed Stochastic Neighbor Embedding"""

Labels = resampled_data['class']

# For plotting, we create a color map
unique_labels = np.unique(Labels)
colours = plt.cm.rainbow(np.linspace(0, 1, len(unique_labels)))
color_map = dict(zip(unique_labels, colours))

# Executing t-SNE on curated data
features = X_resampled
tsne = TSNE(n_components=3, perplexity=150, verbose=1, n_iter=500)
tsne_results = tsne.fit_transform(features)

tsne_results.shape

Labels = y_resampled
df = pd.DataFrame({'Y1': tsne_results[:, 0], 'Y2': tsne_results[:, 1],'Y3': tsne_results[:, 2], 'label': Labels})

plt.figure(figsize=(10, 8))
sns.scatterplot(data=df, x='Y1', y='Y2', hue='label')

# Setting plot title and labels
plt.title('t-SNE Visualization')
plt.xlabel('Y1')
plt.ylabel('Y2')

"""
We opt for incorporating elevated perplexity values as an alternative to expanding the iteration count, aiming to streamline the execution time."""

# Convert the results to a DataFrame
df_tsne = pd.DataFrame(tsne_results, columns=['Y1', 'Y2','Y3'])

# Display the first few rows
df_head = df_tsne.head()
df_head

df = pd.DataFrame({
    'Y1': tsne_results[:, 0],
    'Y2': tsne_results[:, 1],
    'Y3': tsne_results[:, 2],
    'label': Labels
})

# For plotting, create a color palette
palette = sns.color_palette("hsv", len(df['label'].unique()))
# Create a 3D scatter plot using plotly
fig = px.scatter_3d(df, x='Y1', y='Y2', z='Y3', color='label',
                    color_discrete_sequence=['#D53E4F', '#FC8D59', '#FEE08B', '#FFFFBF', '#E6F598', '#99D594'])

# Update layout for axes titles
fig.update_layout(scene=dict(
                    xaxis_title='Y1',
                    yaxis_title='Y2',
                    zaxis_title='Y3'))

# Show the plot
fig.show()

"""We're using something called t-SNE to simplify complex patterns in data. But, unfortunately, it's tough to clearly see the different groups in the data. We've been tweaking some hyperparameters to improve this, but there are still a lot of areas where the groups overlap.

# 3. Models and Performance evaluation

Classification models used for the project:

1. **Logistic Regression:**
   
   Simple and efficient for binary classification.
   
   Assumes a linear relationship between the features and the log-odds of the response.

4. **Neural Networks:**

   Deep learning models with multiple layers of neurons.
   
   Powerful for complex tasks but may require a large amount of data.

Logistic regression
"""

# Creating the LabelEncoder object
encoder = LabelEncoder()

# Fitting the encoder to the training data and transforming both train and test labels
#y_train_encoded = encoder.fit_transform(y_resampled)
y_test_encoded = encoder.fit_transform(y_test)
y_train_encoded = encoder.fit_transform(y_train)

from sklearn.model_selection import train_test_split
class LogisticRegression:
    def __init__(self, X_train, y_train, X_test, y_test, learningRate, tolerance, maxIteration = 1000, indexes=[]):
        self.learningRate = learningRate
        self.tolerance = tolerance
        self.maxIteration = maxIteration
        self.indexes = indexes
        self.X_train = X_train
        self.y_train = y_train
        self.X_test = X_test
        self.y_test = y_test
    '''
    def datasetReader(self, indexes):
      X_train = x_train
      y_train = y_train_encoded
      X_test = x_test
      y_test = y_test_encoded
      return X_train, y_train, X_test, y_test
    '''

    def addX0(self, X):
        return np.column_stack([np.ones([X.shape[0], 1]), X])

    def sigmoid(self, z_value):
        sig_value = 1/(1 + np.exp(-z_value))
        return sig_value

    def costFunction(self, X, y):
        pred_value_ = np.log(np.ones(X.shape[0]) + np.exp(X.dot(self.w))) - X.dot(self.w)*y
        cost_value = pred_value_.sum()
        return cost_value

    def gradient(self, X, y):
        sig_value = self.sigmoid(X.dot(self.w))
        gradient_value = (sig_value - y).dot(X)
        return gradient_value

    def gradientDescent(self, X, y):
        cost_sequences = []
        last_cost = float('inf')
        for i in tqdm(range(self.maxIteration)):
            self.w = self.w - self.learningRate * self.gradient(X, y)
            cur_cost = self.costFunction(X, y)

            diff = last_cost - cur_cost
            last_cost = cur_cost
            cost_sequences.append(cur_cost)
            if diff < self.tolerance:
                print('The model Converged')
                break

        self.plotCost(cost_sequences)
        return

    def plotCost(self, error_sequences):
        s = np.array(error_sequences)
        t = np.arange(s.size)
        fig, ax = plt.subplots()
        ax.plot(t, s)
        ax.set(xlabel='Iteration', ylabel='Error')

    def plot(self):
        plt.figure(figsize=(12, 8))
        ax = plt.axes(projection='3d')

        ax.scatter3D(self.X_train[:, 0], self.X_train[:, 1],
                     self.sigmoid(self.X_train.dot(self.w)),
                     c = self.y_train[:], cmap='viridis', s=100)

        ax.set_xlim3d(55, 80)
        ax.set_ylim3d(80, 240)
        plt.ylabel('$x_2$ feature', fontsize=15)
        plt.xlabel('$x_1$ feature', fontsize=15)
        ax.set_zlabel('$P(Y = 1|x_1, x_2)$', fontsize=15, rotation = 0)

    def scatterPlt(self):
        x_min, x_max = 55, 80
        y_min, y_max = 80, 240

        xx, yy = np.meshgrid(np.linspace(x_min, x_max, 250),
                             np.linspace(y_min, y_max, 250))
        grid = np.c_[xx.ravel(), yy.ravel()]
        probs = grid.dot(self.w).reshape(xx.shape)

        f, ax = plt.subplots(figsize=(14,12))

        ax.contour(xx, yy, probs, levels=[0.5], cmap="Greys", vmin=0, vmax=.6)

        ax.scatter(self.X_train[:, 0], self.X_train[:, 1],
                   c=self.y_train[:], s=50,
                   cmap="RdBu", vmin=-.2, vmax=1.2,
                   edgecolor="white", linewidth=1)

        plt.xlabel('x1 feature')
        plt.ylabel('x2 feature')

    def plot3D(self):
        x_min, x_max = 55, 80
        y_min, y_max = 80, 240

        xx, yy = np.meshgrid(np.linspace(x_min, x_max, 250),
                             np.linspace(y_min, y_max, 250))

        grid = np.c_[xx.ravel(), yy.ravel()]
        probs = grid.dot(self.w).reshape(xx.shape)
        fig = plt.figure(figsize=(14,12))
        ax = plt.axes(projection='3d')
        ax.contour3D(xx, yy, probs, 50, cmap='binary')

        ax.scatter3D(self.X_train[:, 0], self.X_train[:, 1],
                   c=self.y_train[:], s=50,
                   cmap="RdBu", vmin=-.2, vmax=1.2,
                   edgecolor="white", linewidth=1)

        ax.set_xlabel('x1')
        ax.set_ylabel('x2')
        ax.set_zlabel('probs')
        ax.set_title('3D contour')
        plt.show()

    def predict(self, X):
        sigmoid_value = self.sigmoid(X.dot(self.w))
        return np.around(sigmoid_value)

    def evaluate(self, y, y_hat):
        y = (y == 1)
        y_hat = (y_hat == 1)
        accuracy = (y == y_hat).sum()/y.size
        precision = (y & y_hat).sum()/y_hat.sum()
        recall = (y & y_hat).sum()/y.sum()
        return accuracy, precision, recall

    def runModel(self):

        self.w = np.ones(self.X_train.shape[1], dtype = np.float64) * 0
        self.gradientDescent(self.X_train, self.y_train)
        y_hat_train = self.predict(self.X_train)
        accuracy, precision, recall = self.evaluate(self.y_train, y_hat_train)

        print('\nAccuracy :', round(accuracy,3)*100)
        print('Precision :', round(precision,3))
        print('Recall :', round(recall,3))

# Creating an instance
from tqdm import tqdm
start = timeit.default_timer()
lr = LogisticRegression(trainx, y_train_encoded,x_test, y_test_encoded,tolerance=0.0001, learningRate=0.1e-5, maxIteration=10000)
lr.runModel()
stop = timeit.default_timer()
print('Time: ', round((stop - start),2), 'seconds')

"""Neural networks"""

import tensorflow as tf

model = tf.keras.Sequential()
model.add(tf.keras.layers.Dense(28, input_dim=28, activation='relu'))
model.add(tf.keras.layers.Dense(512, activation='relu'))
model.add(tf.keras.layers.Dense(256, activation='relu'))
model.add(tf.keras.layers.Dense(128, activation='relu'))
model.add(tf.keras.layers.Dense(6, activation='softmax'))

model.summary()

model.compile(optimizer=tf.keras.optimizers.Adam(0.00001),
              loss=SparseCategoricalCrossentropy(from_logits=True),
              metrics=[tf.keras.metrics.SparseCategoricalAccuracy()])

hist=model.fit(
    trainx,
    y_train_encoded,
    epochs=200,
    validation_split=0.15, verbose=1, batch_size=128)

y_pred = model.predict(x_test)
y_pred

lst=[]
for i in range(0,len(y_pred)):
     k=np.argmax(y_pred[i]) #it gives index value of the highest probability for each iteration
     lst.append(k)
y_pred_label=np.array(lst)

matr = confusion_matrix(y_test_encoded, y_pred_label)
sns.heatmap(matr, square=True, annot=True, cbar=False)
plt.xlabel('Predicted value')
plt.ylabel('True value');

y_pred_label = np.argmax(y_pred, axis=1)
# Now calculate the accuracy and recall
print('Accuracy: %.3f' % accuracy_score(y_true=y_test_encoded, y_pred=y_pred_label))
print('Recall (macro average): %.3f' % recall_score(y_true=y_test_encoded, y_pred=y_pred_label, average='macro'))
print('Recall (weighted average): %.3f' % recall_score(y_true=y_test_encoded, y_pred=y_pred_label, average='weighted'))

loss=hist.history['loss']
def plot(loss):
        axis=list(range(0, len(loss),1))
        fig, ax = plt.subplots()
        ax.plot(axis, loss)
        ax.set_xlabel('epoch')
        ax.set_ylabel('loss')
        ax.grid()
        plt.show()
plot(loss)



"""#Neural Network V2"""

class CustomNeuralNetwork:

    def __init__(self, input_size, hidden_size, output_size, learning_rate=0.01, n_iters=100):
        self.input_size = input_size
        self.hidden_size = hidden_size
        self.output_size = output_size
        self.learning_rate = learning_rate
        self.n_iters = n_iters

        # Initialize weights and biases
        self.weights_input_hidden = np.random.rand(self.input_size, self.hidden_size)
        self.bias_hidden = np.zeros((1, self.hidden_size))
        self.weights_hidden_output = np.random.rand(self.hidden_size, self.output_size)
        self.bias_output = np.zeros((1, self.output_size))

    def sigmoid(self, x):
        return 1 / (1 + np.exp(-x))

    def sigmoid_derivative(self, x):
        return x * (1 - x)

    def forward(self, X):
        # Input to hidden layer
        self.hidden_layer_input = np.dot(X, self.weights_input_hidden) + self.bias_hidden
        self.hidden_layer_output = self.sigmoid(self.hidden_layer_input)

        # Hidden to output layer
        self.output_layer_input = np.dot(self.hidden_layer_output, self.weights_hidden_output) + self.bias_output
        self.output_layer_output = self.sigmoid(self.output_layer_input)

        return self.output_layer_output

    def train(self, X, y):
        for i in range(self.n_iters):
            # Forward pass
            output = self.forward(X)

            # Convert y to one-hot encoding
            y_int = y.astype(int)  # Convert to integers
            y_one_hot = np.eye(self.output_size)[y_int]

            # Backward pass
            self.backward(X, y_one_hot, output)

    def backward(self, X, y, output):
        # Calculate output layer error
        output_error = output - y
        output_delta = output_error * self.sigmoid_derivative(output)

        # Calculate hidden layer error
        hidden_layer_error = output_delta.dot(self.weights_hidden_output.T)
        hidden_layer_delta = hidden_layer_error * self.sigmoid_derivative(self.hidden_layer_output)

        # Update weights and biases
        self.weights_hidden_output += self.learning_rate * self.hidden_layer_output.T.dot(output_delta)
        self.bias_output += self.learning_rate * np.sum(output_delta, axis=0, keepdims=True)
        self.weights_input_hidden += self.learning_rate * X.T.dot(hidden_layer_delta)
        self.bias_hidden += self.learning_rate * np.sum(hidden_layer_delta, axis=0, keepdims=True)


    def evaluate(self, y, y_hat):
        acc_value = np.mean(y == y_hat)
        prec_value = np.sum((y == 1) & (y_hat == 1)) / np.sum(y_hat == 1) if np.sum(y_hat == 1) != 0 else 0
        rec_value = np.sum((y == 1) & (y_hat == 1)) / np.sum(y == 1) if np.sum(y == 1) != 0 else 0
        return acc_value, prec_value, rec_value

    def predict(self, X, Y):
        output = self.forward(X)
        predictions = np.round(output)
        acc_value, prec_value, rec_value = self.evaluate(Y, predictions)
        print("The Accuracy value is:", np.round(acc_value, 2) * 100, "%")
        print("The Precision value is:", np.round(prec_value, 2))
        print("The Recall value is:", np.round(rec_value, 2))

nn = CustomNeuralNetwork(input_size=28, hidden_size=4, output_size=6, learning_rate=0.1, n_iters=10000)
nn.train(trainx, y_train_encoded)

from sklearn.preprocessing import OneHotEncoder
encoder = OneHotEncoder(sparse=True)
y_test_onehot = encoder.fit_transform(np.array(y_train_encoded).reshape(-1, 1))

'nn.predict(x_test, y_test_onehot)'

"""We attempted to build a neural network using equations and implemented the program. However, we encountered issues, especially when predicting with the model. To address this, we turned to standard libraries for a more reliable solution."""