softmax.py

from builtins import range
from matplotlib import axis
import numpy as np
from random import shuffle


def softmax_loss_naive(W, X, y, reg):

    loss = 0.0
    dW = np.zeros_like(W)

    num_classes = W.shape[1]
    num_train = X.shape[0]

    for i in range(num_train):
        scores = X[i].dot(W)

        # Numerical stability: subtract the maximum score to prevent overflow
        scores -= np.max(scores)
        p = np.exp(scores)
        p /= p.sum()
        logp = np.log(p)

        loss -= logp[y[i]]

        for j in range(num_classes):
            if j == y[i]:
                dW[:, j] -= (1 - p[j]) * X[i]
            else:
                dW[:, j] += p[j] * X[i]
                
    loss = loss / num_train + reg * np.sum(W * W)
    dW = dW / num_train + 2 * reg * W  # Fixed: was / num_classes

    return loss, dW


def softmax_loss_vectorized(W, X, y, reg):
    loss = 0.0
    dW = np.zeros_like(W)
    
    N = X.shape[0]
    scores = X.dot(W)
    
    # Numerical stability: subtract the maximum score to prevent overflow
    scores -= scores.max(axis=1, keepdims=True)
    exp_scores = np.exp(scores)
    probs = exp_scores / exp_scores.sum(axis=1, keepdims=True)
    
    
    correct_logprobs = -np.log(probs[np.arange(N), y])
    data_loss = correct_logprobs.mean()
    reg_loss = reg * np.sum(W * W)  
    loss = data_loss + reg_loss
    
    dscores = probs
    dscores[np.arange(N), y] -= 1
    dW = X.T.dot(dscores) / N
    dW += 2 * reg * W
    
    return loss, dW