accuracy.py

# =================================================================================================================================== #
# ----------------------------------------------------------- DESCRIPTION ----------------------------------------------------------- #
# Contains functions to determine accuracy, focused on the ROC (Receiver-Operating Characteristic) curve and AUC (Area Under the      #
# Curve).                                                                                                                             #
# =================================================================================================================================== #


# =================================================================================================================================== #
# --------------------------------------------------------- EXTERNAL IMPORTS -------------------------------------------------------- #
import pandas as pd # Data manipulation and analysis.                                                                                 #
import numpy as np # Mathematical functions.                                                                                          #
import os # Operating system dependent functionality.                                                                                 #
import re # Regular expression operations.                                                                                            #
import src.gaussian_mixture_models as gmm # Gaussian Mixture Models.                                                                  #
import src.gaussian_mixture_models_3D as gmm3D # Gaussian Mixture Models in 3D.                                                       #
import src.data_management as dm # Data management.                                                                                   #
from matplotlib.colors import ListedColormap  # Colormap for plotting.                                                                #
from sklearn.metrics import accuracy_score, confusion_matrix, roc_auc_score, roc_curve, precision_recall_curve # Accuracy metrics.    #
import matplotlib.pyplot as plt # Plotting.                                                                                           #
import scienceplots # Custom plotting styles for matplotlib.                                                                          #
import seaborn as sns # Data visualization library based on matplotlib.                                                               #
import matplotlib.patches as mpatches # For creating custom legends.                                                                  #
# =================================================================================================================================== #

 
# =================================================================================================================================== #
# ---------------------------------------------------------- PLOTTING SETTINGS ------------------------------------------------------ #
plt.style.use(["science", "no-latex"])                                                                                                #
plt.rcParams.update({                                                                                                                 #
    "font.size": 8,               # Base font size for all text in the plot.                                                          #
    "axes.titlesize": 9,          # Title size for axes.                                                                              #
    "axes.labelsize": 9,          # Axis labels size.                                                                                 #
    "xtick.labelsize": 8,         # X-axis tick labels size.                                                                          #
    "ytick.labelsize": 8,         # Y-axis tick labels size.                                                                          #
    "legend.fontsize": 8,         # Legend font size for all text in the legend.                                                      #
    "figure.titlesize": 9,        # Overall figure title size for all text in the figure.                                             #
})                                                                                                                                    #
sns.set_theme(style="whitegrid", context="paper") # Set seaborn theme for additional aesthetics and context.                          #                                                                             #
plt.rcParams["figure.figsize"] = (6, 4)  # Set default figure size for all plots to 6x4.                                              #
# =================================================================================================================================== #


# =================================================================================================================================== #
# ------------------------------------------------------------ FUNCTIONS ------------------------------------------------------------ #
def plot_roc_curve(true_y, y_prob, case, power_tx, frequency, distance_start, distance_end):
    """
    Plots the ROC curve for the given true labels and predicted probabilities.

    Parameters:
        - true_y (np.array): The true labels.
        - y_prob (np.array): The predicted probabilities.

    Returns:
        - None.
    """

    # Create a new figure for the ROC curve
    plt.figure()

    # Use LaTeX for plot typography
    plt.rc('text', usetex=True)
    plt.rc('font', family='serif')

    # Compute the ROC curve
    fpr, tpr, thresholds = roc_curve(true_y, y_prob)

    # Plot the ROC curve
    plt.plot(fpr, tpr, color='red', label='ROC', linewidth=4)
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plot_title = f"ROC Curve (Case {case}) - {power_tx} dBm ({round(dm.dBm_to_W(power_tx), 1)}W) @ {frequency} MHz ({round(distance_start, 2)}m - {round(distance_end, 2)}m)"
    plt.title(plot_title)
    plt.legend()

    # Show the plot
    plt.show()

def plot_accuracy_vs_threshold(true_y, y_prob, case, power_tx, frequency, distance_start, distance_end):
    """
    Plots accuracy as a function of the decision threshold.

    Parameters:
        - true_y (np.array): The true labels.
        - y_prob (np.array): The predicted probabilities.

    Returns:
        - None.
    """
    thresholds = np.arange(0.0, 1.01, 0.01)
    accuracies = []

    for threshold in thresholds:
        y_pred = (y_prob >= threshold).astype(int)
        accuracies.append(accuracy_score(true_y, y_pred))

    plt.figure()
    plt.plot(thresholds, accuracies, color='green', label='Accuracy vs. Threshold', linewidth=4)
    plt.xlabel('Threshold')
    plt.ylabel('Accuracy')
    plot_title = f"Accuracy vs. Threshold (Case {case}) - {power_tx} dBm ({round(dm.dBm_to_W(power_tx), 1)}W) @ {frequency} MHz ({round(distance_start, 2)}m - {round(distance_end, 2)}m)"
    plt.title(plot_title)
    plt.legend()
    plt.show()

def plot_precision_recall_curve(true_y, y_prob, case, power_tx, frequency, distance_start, distance_end):
    """
    Plots the Precision-Recall curve for the given true labels and predicted probabilities.

    Parameters:
        - true_y (np.array): The true labels.
        - y_prob (np.array): The predicted probabilities.

    Returns:
        - None.
    """
    precision, recall, thresholds = precision_recall_curve(true_y, y_prob)

    plt.figure()
    plt.plot(recall, precision, color='blue', label='Precision-Recall Curve', linewidth=4)
    plt.xlabel('Recall')
    plt.ylabel('Precision')
    plot_title = f"Precision-Recall Curve (Case {case}) - {power_tx} dBm ({round(dm.dBm_to_W(power_tx), 1)}W) @ {frequency} MHz ({round(distance_start, 2)}m - {round(distance_end, 2)}m)"
    plt.title(plot_title)
    plt.legend()
    plt.show()

def plot_combined_analysis(true_y, y_prob, case, power_tx, frequency, distance_start, distance_end):
    """
    Creates a combined plot with three subplots: ROC Curve, Precision-Recall Curve, and Accuracy vs. Threshold.

    Parameters:
        - true_y (np.array): The true labels.
        - y_prob (np.array): The predicted probabilities.

    Returns:
        - None.
    """
    # Compute metrics
    fpr, tpr, _ = roc_curve(true_y, y_prob)
    precision, recall, _ = precision_recall_curve(true_y, y_prob)
    thresholds = np.arange(0.0, 1.01, 0.01)
    accuracies = [(accuracy_score(true_y, (y_prob >= t).astype(int))) for t in thresholds]

    # Create subplots
    fig, axes = plt.subplots(1, 3, figsize=(18, 6))  # 1 row, 3 columns

    # ROC Curve
    axes[0].plot(fpr, tpr, color='red', linewidth=2, label='ROC Curve')
    axes[0].set_xlabel('False Positive Rate')
    axes[0].set_ylabel('True Positive Rate')
    axes[0].set_title(f"ROC Curve (Case {case})")
    axes[0].legend()

    # Precision-Recall Curve
    axes[1].plot(recall, precision, color='blue', linewidth=2, label='Precision-Recall Curve')
    axes[1].set_xlabel('Recall')
    axes[1].set_ylabel('Precision')
    axes[1].set_title(f"Precision-Recall Curve (Case {case})")
    axes[1].legend()

    # Accuracy vs. Threshold
    axes[2].plot(thresholds, accuracies, color='green', linewidth=2, label='Accuracy vs. Threshold')
    axes[2].set_xlabel('Threshold')
    axes[2].set_ylabel('Accuracy')
    axes[2].set_title(f"Accuracy vs. Threshold (Case {case})")
    axes[2].legend()

    # Adjust layout and show the plot
    plt.tight_layout()
    plt.show()

def full_roc_auc_analysis_original(tag, tag_name, filepath_cluster0, cluster0, cluster0_name, filepath_cluster1, cluster1, cluster1_name, transmission_power, start_distance, end_distance, case):
    """
    Performs a full ROC AUC analysis on the given data using the original approach (100% of data for training and testing).
    
    Parameters:
        - tag (str): Tag identifier for the measurements.
        - tag_name (str): Descriptive name for the tag.
        - filepath_cluster0 (str): File path to the first cluster data.
        - cluster0 (str): Identifier for the first cluster.
        - cluster0_name (str): Descriptive name for the first cluster.
        - filepath_cluster1 (str): File path to the second cluster data.
        - cluster1 (str): Identifier for the second cluster.
        - cluster1_name (str): Descriptive name for the second cluster.
        - transmission_power (float): Transmission power in dBm.
        - start_distance (float): Starting distance for measurements in meters.
        - end_distance (float): Ending distance for measurements in meters.
        - case (int): Case identifier for the analysis.
    
    Returns:
        - None. Generates plots and saves analysis results to a text file.
    """
    # Physical parameters
    frequency, transmission_power, antenna_gain, tag_gain, expected_distances = dm.parameter_setup(transmission_power=transmission_power, start_distance=start_distance, end_distance=end_distance, step_distance=0.0001)

    # Obtain cluster data.
    cluster_list = [cluster0, cluster1]
    cluster_list_name = [cluster0_name, cluster1_name]
    cluster_list_filepath = [filepath_cluster0, filepath_cluster1]

    # Obtain the data
    file_list = []
    for i in range(len(cluster_list_filepath)):
        files = dm.get_measurements_power_distance(tag, cluster_list_filepath[i], cluster_list[i], transmission_power, start_distance, end_distance)
        file_list.append(files)

    # GMM 
    X, y, xx, yy = dm.create_real_dataset_rssi_phase(expected_distances, transmission_power, frequency, cluster_list_name, file_list)

    gaussian_mixture_model = gmm.GaussianMixtureModel(2)

    gaussian_mixture_model.fit(X, max_iter=600, tol=1e-4, display_message=False, random_state=0)

    gmm.plot_result(gaussian_mixture_model, xx, yy, X, y)

    # Accuracy Analysis
    thresholds = np.arange(0.0, 1.0, 0.01)
    
    # From the files, obtain all rssi and phase measurements.
    rssi_measurements = []
    phase_measurements = []
    ground_truth = []

    for i in range(len(file_list)):
        for file in file_list[i]:
            df = pd.read_csv(file)
            start_time = df['time_reader'].iloc[0]
            df = df[df['time_reader'] - start_time > 0.5]
            rssi_values = df['peakRssi'].values
            phase_values = df['phase'].values
            for rssi in rssi_values:
                rssi_measurements.append(rssi)
            for phase in phase_values:
                phase_measurements.append(phase)
            for j in range(len(rssi_values)):
                ground_truth.append(int(i))

    ground_truth = np.array(ground_truth)

    measurements = []
    # List of phase and rssi tuples.
    for i in range(len(rssi_measurements)):
        meas_tuple = (phase_measurements[i], rssi_measurements[i])
        measurements.append(meas_tuple)

    # Probability of being in cluster 1 (cluster 1 = humid)
    probability = []
    
    for point in measurements:
        probabilities = gaussian_mixture_model.predict_probability(np.array([point]))
        probability.append(probabilities[0][1])

    y_proba = np.array(probability)
    
    # Plot ROC Curve
    plot_roc_curve(ground_truth, probability, case, transmission_power, frequency, start_distance, end_distance)

    # Plot Precision-Recall Curve
    plot_precision_recall_curve(ground_truth, probability, case, transmission_power, frequency, start_distance, end_distance)

    # Plot Accuracy vs. Threshold
    plot_accuracy_vs_threshold(ground_truth, y_proba, case, transmission_power, frequency, start_distance, end_distance)

    # Plot Combined Analysis
    plot_combined_analysis(ground_truth, y_proba, case, transmission_power, frequency, start_distance, end_distance)

    # Save the results to a text file
    output_file = f"case{case}_roc_auc_analysis.txt"
    with open(output_file, "w") as f:
        # Write a header for the file
        f.write("=" * 80 + "\n")
        f.write(f"ROC AUC Analysis for Case {case}\n")
        f.write("=" * 80 + "\n\n")

        # Amount of data
        f.write(f"Ammount of data\n\n")
        #print the number of 0 and the number or 1s in the ground truth
        f.write(f"Number of 0s in ground truth: {np.sum(ground_truth == 0)}\n")
        f.write(f"Number of 1s in ground truth: {np.sum(ground_truth == 1)}\n")
        f.write(f"Total Data Points: {len(ground_truth)}\n")
        f.write("\n\n")

        # ROC AUC Score
        roc_auc = roc_auc_score(ground_truth, y_proba)
        f.write(f"ROC AUC Score: {roc_auc:.4f}\n\n")

        f.write("=" * 80 + "\n")
        f.write("Threshold Analysis\n")
        f.write("=" * 80 + "\n")

        # Threshold Analysis
        f.write(f"{'Threshold':<12}{'Accuracy':<12}{'TP':<8}{'FP':<8}{'TN':<8}{'FN':<8}\n")
        f.write("-" * 80 + "\n")
        for threshold in thresholds:
            y_pred = (y_proba >= threshold).astype(int)
            # Accuracy Score
            accuracy = accuracy_score(ground_truth, y_pred)
            # Confusion Matrix
            cf_mat = confusion_matrix(ground_truth, y_pred)
            tn, fp, fn, tp = cf_mat.ravel() if cf_mat.size == 4 else (0, 0, 0, 0)
            f.write(f"{threshold:<12.2f}{accuracy:<12.4f}{tp:<8}{fp:<8}{tn:<8}{fn:<8}\n")

        f.write("=" * 80 + "\n")
        f.write("End of Analysis\n")
        f.write("=" * 80 + "\n")

def full_roc_auc_analysis(tag, tag_name, cluster0_name, cluster1_name, transmission_power, start_distance, end_distance, case, train_filepath, test_filepath):
    """
    Performs a full ROC AUC analysis on the given data using different training and test datasets.
    
    Parameters:
        - tag (str): Tag identifier for the measurements.
        - tag_name (str): Descriptive name for the tag.
        - cluster0_name (str): Descriptive name for the first cluster.
        - cluster1_name (str): Descriptive name for the second cluster.
        - transmission_power (float): Transmission power in dBm.
        - start_distance (float): Starting distance for measurements in meters.
        - end_distance (float): Ending distance for measurements in meters.
        - case (int): Case identifier for the analysis.
        - train_filepath (str): Path to the training dataset.
        - test_filepath (str): Path to the test dataset.
    
    Returns:
        - None. Generates plots and saves analysis results to a text file.
    """
    # Physical parameters
    frequency, transmission_power, antenna_gain, tag_gain, expected_distances = dm.parameter_setup(transmission_power=transmission_power, start_distance=start_distance, end_distance=end_distance, step_distance=0.0001)

    # Obtain cluster data.
    cluster_list_name = [cluster0_name, cluster1_name]

    # GMM 
    X, y, xx, yy = dm.create_real_dataset_rssi_phase_new(expected_distances, transmission_power, frequency, cluster_list_name, train_filepath)

    gaussian_mixture_model = gmm.GaussianMixtureModel(2)

    gaussian_mixture_model.fit(X, max_iter=600, tol=1e-4, display_message=False, random_state=0)

    gmm.plot_result(gaussian_mixture_model, xx, yy, X, y)

    # Accuracy Analysis
    thresholds = np.arange(0.0, 1.0, 0.01)
    
    # From the files, obtain all rssi and phase measurements.
    rssi_measurements = []
    phase_measurements = []
    ground_truth = []

    df = pd.read_csv(test_filepath)
    # For each line in the csv file, if the distance is within the range, and the moisture level is a certain value, append the data
    for index, row in df.iterrows():
        distance = row['pos_y']
        mois = row['moist']
        if (start_distance <= distance <= end_distance):
            rssi_measurements.append(row['peakRssi'])
            phase_measurements.append(row['phase'])
            if (mois == 0.01):
                ground_truth.append(0)
            elif (mois == 0.18):
                ground_truth.append(1)
            else:
                ground_truth.append(-1)

    ground_truth = np.array(ground_truth)

    measurements = []
    # List of phase and rssi tuples.
    for i in range(len(rssi_measurements)):
        meas_tuple = (phase_measurements[i], rssi_measurements[i])
        measurements.append(meas_tuple)

    # Probability of being in cluster 1 (cluster 1 = humid)
    probability = []
    
    for point in measurements:
        probabilities = gaussian_mixture_model.predict_probability(np.array([point]))
        probability.append(probabilities[0][1])

    y_proba = np.array(probability)
    
    # Plot ROC Curve
    plot_roc_curve(ground_truth, probability, case, transmission_power, frequency, start_distance, end_distance)

    # Plot Precision-Recall Curve
    plot_precision_recall_curve(ground_truth, probability, case, transmission_power, frequency, start_distance, end_distance)

    # Plot Accuracy vs. Threshold
    plot_accuracy_vs_threshold(ground_truth, y_proba, case, transmission_power, frequency, start_distance, end_distance)

    # Plot Combined Analysis
    plot_combined_analysis(ground_truth, y_proba, case, transmission_power, frequency, start_distance, end_distance)

    # Save the results to a text file
    output_file = f"case{case}_roc_auc_analysis.txt"
    with open(output_file, "w") as f:
        # Write a header for the file
        f.write("=" * 80 + "\n")
        f.write(f"ROC AUC Analysis for Case {case}\n")
        f.write("=" * 80 + "\n\n")

        # Amount of data
        f.write(f"Ammount of data\n\n")
        #print the number of 0 and the number or 1s in the ground truth
        f.write(f"Number of 0s in ground truth: {np.sum(ground_truth == 0)}\n")
        f.write(f"Number of 1s in ground truth: {np.sum(ground_truth == 1)}\n")
        f.write(f"Total Data Points: {len(ground_truth)}\n")
        f.write("\n\n")

        # ROC AUC Score
        roc_auc = roc_auc_score(ground_truth, y_proba)
        f.write(f"ROC AUC Score: {roc_auc:.4f}\n\n")

        f.write("=" * 80 + "\n")
        f.write("Threshold Analysis\n")
        f.write("=" * 80 + "\n")

        # Threshold Analysis
        f.write(f"{'Threshold':<12}{'Accuracy':<12}{'TP':<8}{'FP':<8}{'TN':<8}{'FN':<8}\n")
        f.write("-" * 80 + "\n")
        for threshold in thresholds:
            y_pred = (y_proba >= threshold).astype(int)
            # Accuracy Score
            accuracy = accuracy_score(ground_truth, y_pred)
            # Confusion Matrix
            cf_mat = confusion_matrix(ground_truth, y_pred)
            tn, fp, fn, tp = cf_mat.ravel() if cf_mat.size == 4 else (0, 0, 0, 0)
            f.write(f"{threshold:<12.2f}{accuracy:<12.4f}{tp:<8}{fp:<8}{tn:<8}{fn:<8}\n")

        f.write("=" * 80 + "\n")
        f.write("End of Analysis\n")
        f.write("=" * 80 + "\n")

def full_roc_auc_analysis_3D(tag, tag_name, filepath_cluster0, cluster0, cluster0_name, filepath_cluster1, cluster1, cluster1_name, transmission_power, start_distance, end_distance, case):
    """
    Performs a full ROC AUC analysis on the given data using the original approach (100% of data for training and testing).
    
    Parameters:
        - tag (str): Tag identifier for the measurements.
        - tag_name (str): Descriptive name for the tag.
        - filepath_cluster0 (str): File path to the first cluster data.
        - cluster0 (str): Identifier for the first cluster.
        - cluster0_name (str): Descriptive name for the first cluster.
        - filepath_cluster1 (str): File path to the second cluster data.
        - cluster1 (str): Identifier for the second cluster.
        - cluster1_name (str): Descriptive name for the second cluster.
        - transmission_power (float): Transmission power in dBm.
        - start_distance (float): Starting distance for measurements in meters.
        - end_distance (float): Ending distance for measurements in meters.
        - case (int): Case identifier for the analysis.
    
    Returns:
        - None. Generates plots and saves analysis results to a text file.
    """
    # Physical parameters
    frequency, transmission_power, antenna_gain, tag_gain, expected_distances = dm.parameter_setup(
        transmission_power=transmission_power, 
        start_distance=start_distance, 
        end_distance=end_distance, 
        step_distance=0.0001
    )

    # Obtain cluster data
    cluster_list = [cluster0, cluster1]
    cluster_list_name = [cluster0_name, cluster1_name]
    cluster_list_filepath = [filepath_cluster0, filepath_cluster1]

    # Obtain the data
    file_list = []
    for i in range(len(cluster_list_filepath)):
        files = dm.get_measurements_power_distance(
            tag, cluster_list_filepath[i], cluster_list[i], 
            transmission_power, start_distance, end_distance
        )
        file_list.append(files)

    # GMM 
    X, y, xx, yy, zz = dm.create_real_dataset_rssi_phase_3D(
        expected_distances, transmission_power, frequency, 
        cluster_list_name, file_list
    )

    gaussian_mixture_model = gmm3D.GaussianMixtureModel(2)
    gaussian_mixture_model.fit(X, max_iter=600, tol=1e-4, display_message=False, random_state=0)

    #gmm.plot_result(gaussian_mixture_model, xx, yy, X, y)

    # Accuracy Analysis
    thresholds = np.arange(0.0, 1.0, 0.01)
    
    # From the files, obtain all rssi and phase measurements
    rssi_measurements = []
    phase_values = []
    ground_truth = []

    for i in range(len(file_list)):
        for file in file_list[i]:
            df = pd.read_csv(file)
            start_time = df['time_reader'].iloc[0]
            df = df[df['time_reader'] - start_time > 0.5]
            rssi_values = df['peakRssi'].values
            phases = df['phase'].values
            
            for rssi in rssi_values:
                rssi_measurements.append(rssi)
                
            for phase in phases:
                phase_values.append(phase)
                
            for j in range(len(rssi_values)):
                ground_truth.append(int(i))

    ground_truth = np.array(ground_truth)

    # Convert phase values to circular representation (cos and sin components)
    phase_rad = np.deg2rad(phase_values)
    cos_phase = np.cos(phase_rad)
    sin_phase = np.sin(phase_rad)

    # Create 3D measurements (cos_phase, sin_phase, rssi)
    measurements = []
    for i in range(len(rssi_measurements)):
        meas_tuple = (cos_phase[i], sin_phase[i], rssi_measurements[i])
        measurements.append(meas_tuple)

    # Calculate probabilities for each point
    probability = []
    for point in measurements:
        probabilities = gaussian_mixture_model.predict_probability(np.array([point]))
        probability.append(probabilities[0][1])

    y_proba = np.array(probability)
    
    # Plot ROC Curve
    plot_roc_curve(ground_truth, probability, case, transmission_power, frequency, start_distance, end_distance)

    # Plot Precision-Recall Curve
    plot_precision_recall_curve(ground_truth, probability, case, transmission_power, frequency, start_distance, end_distance)

    # Plot Accuracy vs. Threshold
    plot_accuracy_vs_threshold(ground_truth, y_proba, case, transmission_power, frequency, start_distance, end_distance)

    # Plot Combined Analysis
    plot_combined_analysis(ground_truth, y_proba, case, transmission_power, frequency, start_distance, end_distance)

    # Save the results to a text file
    output_file = f"case{case}_roc_auc_analysis.txt"
    with open(output_file, "w") as f:
        # Write a header for the file
        f.write("=" * 80 + "\n")
        f.write(f"ROC AUC Analysis for Case {case}\n")
        f.write("=" * 80 + "\n\n")

        # Amount of data
        f.write(f"Ammount of data\n\n")
        #print the number of 0 and the number or 1s in the ground truth
        f.write(f"Number of 0s in ground truth: {np.sum(ground_truth == 0)}\n")
        f.write(f"Number of 1s in ground truth: {np.sum(ground_truth == 1)}\n")
        f.write(f"Total Data Points: {len(ground_truth)}\n")
        f.write("\n\n")

        # ROC AUC Score
        roc_auc = roc_auc_score(ground_truth, y_proba)
        f.write(f"ROC AUC Score: {roc_auc:.4f}\n\n")

        f.write("=" * 80 + "\n")
        f.write("Threshold Analysis\n")
        f.write("=" * 80 + "\n")

        # Threshold Analysis
        f.write(f"{'Threshold':<12}{'Accuracy':<12}{'TP':<8}{'FP':<8}{'TN':<8}{'FN':<8}\n")
        f.write("-" * 80 + "\n")
        for threshold in thresholds:
            y_pred = (y_proba >= threshold).astype(int)
            # Accuracy Score
            accuracy = accuracy_score(ground_truth, y_pred)
            # Confusion Matrix
            cf_mat = confusion_matrix(ground_truth, y_pred)
            tn, fp, fn, tp = cf_mat.ravel() if cf_mat.size == 4 else (0, 0, 0, 0)
            f.write(f"{threshold:<12.2f}{accuracy:<12.4f}{tp:<8}{fp:<8}{tn:<8}{fn:<8}\n")

        f.write("=" * 80 + "\n")
        f.write("End of Analysis\n")
        f.write("=" * 80 + "\n")
# =================================================================================================================================== #