ceemdantnccatboost.py

# -*- coding: utf-8 -*-
"""ceemdantnccatboost

Automatically generated by Colab.

Original file is located at
    https://colab.research.google.com/drive/1cD2MZQ_dB4BFwJWdcycLDiIT37VjColo
"""

# Data Loading

!pip install keras-tcn==3.4.0
!pip3 install PyEMD
!pip3 install EMD-signal
!pip install tensorflow keras keras-tuner
!pip install pandas numpy matplotlib scikit-learn tensorflow PyEMD EMD-signal
!pip install catboost

from google.colab import drive
drive.mount('/content/drive')

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from PyEMD import CEEMDAN
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import GRU, Dense
from tcn import compiled_tcn
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import MinMaxScaler
from sklearn.metrics import mean_squared_error
from sklearn.impute import KNNImputer
from sklearn.ensemble import IsolationForest
from sklearn.ensemble import RandomForestRegressor
import keras_tuner as kt
from tensorflow.keras.optimizers import Adam
import os

def setup_directories(save_dir):
    if not os.path.exists(save_dir):
        os.makedirs(save_dir)
    print(f"Directory {save_dir} is set up.")

def load_data(data_path):
    print("Loading data...")
    data = pd.read_csv(data_path, parse_dates=['sent_date'], index_col='sent_date')
    print("Data loaded successfully.")
    return data

def filter_data_by_id(data, device_ids):
    print(f"Filtering data for device_ids: {device_ids}")
    return data[data['device_id'].isin(device_ids)]

def main():
    save_dir = '/content/drive/MyDrive/Thesis1/'
    data_path = '/content/drive/MyDrive/data_inverter_tn.csv'

    setup_directories(save_dir)

    data = load_data(data_path)

    device_ids = [155, 156]
    filtered_data = filter_data_by_id(data, device_ids)
    print(f"Filtered data contains {filtered_data.shape[0]} rows.")

    return filtered_data, save_dir

filtered_data, save_dir = main()

# Print
print(filtered_data.head())

# Data Processing

import pandas as pd
import os

def load_data(data_path):
    print("Loading data...")
    data = pd.read_csv(data_path, parse_dates=['sent_date'], index_col='sent_date')
    print("Data loaded successfully.")
    return data

def sample_device_data(data, device_id, frac=0.25, random_state=42):
    print(f"Sampling {frac*100}% of data for device_id: {device_id}")
    device_data = data[data['device_id'] == device_id]
    sampled_data = device_data.sample(frac=frac, random_state=random_state)
    print(f"Sampled data contains {sampled_data.shape[0]} rows for device_id {device_id}.")
    return sampled_data

data_path = '/content/drive/MyDrive/data_inverter_tn.csv'
data = load_data(data_path)

sampled_data_155 = sample_device_data(data, 155)
sampled_data_156 = sample_device_data(data, 156)

combined_sampled_data = pd.concat([sampled_data_155, sampled_data_156])

save_path = '/content/drive/MyDrive/Thesis1/combined_sampled_data.csv'
combined_sampled_data.to_csv(save_path)
print(f"Combined sampled data saved to {save_path}")

import pandas as pd
import matplotlib.pyplot as plt

data_path = '/content/drive/MyDrive/data_inverter_tn.csv'
data = pd.read_csv(data_path, parse_dates=['sent_date'], index_col='sent_date')

def plot_power_output(data, device_id, ax, title):
    device_data = data[data['device_id'] == device_id]
    ax.plot(device_data.index, device_data['W'], label=f'Device {device_id} Power (W)', color='blue', linewidth=0.5)
    ax.set_xlabel('Time (30min)')
    ax.set_ylabel('Power (W)')
    ax.set_title(title)
    ax.grid(True)

fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(10, 12))

plot_power_output(data, 155, ax1, '(a)')

plot_power_output(data, 156, ax2, '(b)')

plt.tight_layout()

plt.show()

import pandas as pd
import numpy as np
from sklearn.impute import KNNImputer
from sklearn.ensemble import IsolationForest
from sklearn.neighbors import LocalOutlierFactor
from sklearn.preprocessing import MinMaxScaler
import matplotlib.pyplot as plt


data_path = '/content/drive/MyDrive/data_inverter_tn.csv'
data = pd.read_csv(data_path, parse_dates=['sent_date'], index_col='sent_date')

data = data.replace({',': ''}, regex=True)
data = data.apply(pd.to_numeric, errors='coerce')

print("Data after converting strings to numeric values:")
print(data.head())

print(f"Total data before sampling: {data.shape[0]} rows")

def sample_device_data(data, device_id, frac=0.05):
    print(f"Sampling {frac*100}% of data for device_id: {device_id}")
    device_data = data[data['device_id'] == device_id]
    sampled_data = device_data.sample(frac=frac, random_state=42)
    print(f"Sampled data contains {sampled_data.shape[0]} rows for device_id {device_id}.")
    return sampled_data

sampled_data_155 = sample_device_data(data, 155, frac=0.20)
sampled_data_156 = sample_device_data(data, 156, frac=0.20)

combined_sampled_data = pd.concat([sampled_data_155, sampled_data_156])

print(f"Total combined sampled data: {combined_sampled_data.shape[0]} rows")

scaler = MinMaxScaler()
scaled_data = pd.DataFrame(scaler.fit_transform(combined_sampled_data), columns=combined_sampled_data.columns, index=combined_sampled_data.index)

def handle_missing_values(data):
    print("Handling missing values...")
    imputer = KNNImputer(n_neighbors=5)
    imputed_data = pd.DataFrame(imputer.fit_transform(data), columns=data.columns, index=data.index)
    print(f"Data after handling missing values: {imputed_data.shape[0]} rows")
    return imputed_data

def remove_outliers_isolation_forest(data):
    print("Detecting and removing outliers using Isolation Forest...")
    iso_forest = IsolationForest(contamination=0.01, random_state=42)
    predictions = iso_forest.fit_predict(data)
    mask = predictions != -1
    clean_data = data[mask]
    print(f"Data after removing outliers (Isolation Forest): {clean_data.shape[0]} rows")
    return clean_data

data_cleaned = handle_missing_values(scaled_data)

data_no_outliers = remove_outliers_isolation_forest(data_cleaned)

def remove_outliers_lof(data):
    print("Further detecting and removing outliers using Local Outlier Factor (LOF)...")
    lof = LocalOutlierFactor(n_neighbors=20, contamination=0.01)
    predictions = lof.fit_predict(data)
    mask = predictions != -1
    clean_data = data[mask]
    print(f"Data after removing outliers (LOF): {clean_data.shape[0]} rows")
    return clean_data

data_no_outliers_lof = remove_outliers_lof(data_no_outliers)

resampled_data = data_no_outliers_lof.resample('D').mean()
print(f"Data after resampling: {resampled_data.shape[0]} rows")
plt.figure(figsize=(10, 6))
plt.plot(resampled_data.index, resampled_data['W'], label='Power (W)', color='blue')
plt.xlabel('Time (Days)')
plt.ylabel('Power (W)')
plt.title('Power Output After Advanced Preprocessing')
plt.grid(True)
plt.show()

cleaned_data_path = '/content/drive/MyDrive/Thesis1/cleaned_data.csv'
resampled_data.to_csv(cleaned_data_path)
print(f"Cleaned data saved to {cleaned_data_path}")

from scipy.stats import zscore

z_scores = zscore(data['W'])
outliers_z = data[(z_scores > 3) | (z_scores < -3)]
Q1 = data['W'].quantile(0.25)
Q3 = data['W'].quantile(0.75)
IQR = Q3 - Q1

outliers_iqr = data[(data['W'] < (Q1 - 1.5 * IQR)) | (data['W'] > (Q3 + 1.5 * IQR))]
contamination = len(outliers_iqr) / len(data)
print(f"Estimated contamination: {contamination:.2%}")

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from PyEMD import CEEMDAN
import os
import time
from sklearn.impute import KNNImputer
from sklearn.ensemble import IsolationForest
from sklearn.preprocessing import MinMaxScaler

# Part 1
def load_data(data_path, sample_frac=0.20):

    data = pd.read_csv(data_path, parse_dates=['sent_date'], index_col='sent_date')
    data = data.replace({',': ''}, regex=True)
    data = data.apply(pd.to_numeric, errors='coerce')

    print(f"Total data before sampling: {data.shape[0]} rows")
    sampled_data = data.sample(frac=sample_frac, random_state=42)
    print(f"Sampled data contains {sampled_data.shape[0]} rows.")

    return sampled_data

# Part 2: Preprocess Data
def preprocess_data(data):
    print("Preprocessing data...")
    scaler = MinMaxScaler(feature_range=(0, 1))
    if 'W' in data.columns:
        data['W'] = scaler.fit_transform(data[['W']].values.reshape(-1, 1)).flatten()
        print("Data preprocessing completed.")
    else:
        raise ValueError("Column 'W' is missing in the data.")
    return data, scaler

# Part 3: Detect and Remove Outliers
def detect_and_remove_outliers(data, feature='W'):
    print("Detecting and removing outliers...")
    iso_forest = IsolationForest(contamination=0.21, random_state=42, n_jobs=-1)
    iso_forest.fit(data[[feature]])
    outliers = iso_forest.predict(data[[feature]])
    data['outlier'] = outliers
    clean_data = data[data['outlier'] == 1].drop('outlier', axis=1)
    print("Outliers removed.")
    return clean_data

# Part 4: Impute Missing Values
def impute_missing_values(data):
    print("Imputing missing values...")
    imputer = KNNImputer(n_neighbors=5)
    data_filled = pd.DataFrame(imputer.fit_transform(data), columns=data.columns, index=data.index)
    print("Missing values imputed.")
    return data_filled

# Part 5: Decompose Series using CEEMDAN
def decompose_series(data, save_dir, title='Decomposed IMFs', num_imfs=8):
    print("Decomposing time series...")
    ceemdan = CEEMDAN()

    imfs = ceemdan(data)

    imfs = imfs[:num_imfs]

    residual = data - np.sum(imfs, axis=0)

    np.save(os.path.join(save_dir, f'{title}_imfs.npy'), imfs)
    np.save(os.path.join(save_dir, f'{title}_residual.npy'), residual)

    plot_imfs_and_residual(imfs, residual, title, save_dir)
    print("Time series decomposed, IMFs and residual saved.")
    return imfs, residual

# Part 6: Plot IMFs and Residual
def plot_imfs_and_residual(imfs, residual, title, save_dir):
    plt.figure(figsize=(15, 12))
    num_imfs = len(imfs)

    for i, imf in enumerate(imfs):
        plt.subplot(num_imfs + 1, 1, i + 1)
        plt.plot(imf)
        plt.title(f'IMF {i + 1}')

    plt.subplot(num_imfs + 1, 1, num_imfs + 1)
    plt.plot(residual)
    plt.title('Residual')

    plt.tight_layout()
    plt.savefig(os.path.join(save_dir, f'{title}.png'))
    plt.close()
    print(f"IMFs and residual plotted and saved as {title}.png.")

# Part 7: Preprocess and Decompose Data for One Device
def preprocess_and_decompose_one_device(filtered_data, save_dir, device_id, batch_size=5000, num_imfs=8):
    print(f"Processing device_id: {device_id}")
    device_data = filtered_data[filtered_data['device_id'] == device_id]

    total_rows = device_data.shape[0]
    print(f"Total rows to process for device_id {device_id}: {total_rows}")

    imfs_list = []
    residuals_list = []
    for start in range(0, total_rows, batch_size):
        end = min(start + batch_size, total_rows)
        batch_data = device_data.iloc[start:end]

        print(f"Processing batch from row {start} to {end}...")

        start_time = time.time()

        batch_data, scaler = preprocess_data(batch_data)
        batch_data = detect_and_remove_outliers(batch_data)
        batch_data = impute_missing_values(batch_data)

        imfs, residual = decompose_series(batch_data['W'], save_dir, title=f'Decomposed IMFs for Device {device_id} Batch {start}-{end}', num_imfs=num_imfs)
        imfs_list.append(imfs)
        residuals_list.append(residual)

        end_time = time.time()
        print(f"Batch processed in {end_time - start_time:.2f} seconds.")

    combined_imfs = np.concatenate(imfs_list, axis=1)
    combined_residual = np.concatenate(residuals_list, axis=0)


    np.save(os.path.join(save_dir, f'combined_imfs_{device_id}.npy'), combined_imfs)
    np.save(os.path.join(save_dir, f'combined_residual_{device_id}.npy'), combined_residual)

    return combined_imfs, combined_residual

data_path = '/content/drive/MyDrive/data_inverter_tn.csv'
sampled_data = load_data(data_path, sample_frac=0.10)

save_dir = '/content/drive/MyDrive/Thesis1/Decomposed_IMFs'

device_id = 155
combined_imfs_155, combined_residual_155 = preprocess_and_decompose_one_device(sampled_data, save_dir, device_id, batch_size=5000, num_imfs=8)

device_id = 156
combined_imfs_156, combined_residual_156 = preprocess_and_decompose_one_device(sampled_data, save_dir, device_id, batch_size=5000, num_imfs=8)

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.impute import KNNImputer
from sklearn.ensemble import IsolationForest
from sklearn.preprocessing import MinMaxScaler
import os


def load_and_preprocess_original_data(data_path, save_dir, device_id, sample_frac=0.20):

    data = pd.read_csv(data_path, parse_dates=['sent_date'], index_col='sent_date')

    data = data.replace({',': ''}, regex=True)
    data = data.apply(pd.to_numeric, errors='coerce')

    device_data = data[data['device_id'] == device_id].sample(frac=sample_frac, random_state=42)

    print("Preprocessing data...")
    scaler = MinMaxScaler(feature_range=(0, 1))
    if 'W' in device_data.columns:
        device_data['W'] = scaler.fit_transform(device_data[['W']].values.reshape(-1, 1)).flatten()
    else:
        raise ValueError("Column 'W' is missing in the data.")

    print("Detecting and removing outliers...")
    iso_forest = IsolationForest(contamination=0.21, random_state=42, n_jobs=-1)
    iso_forest.fit(device_data[['W']])
    outliers = iso_forest.predict(device_data[['W']])
    device_data['outlier'] = outliers
    device_data = device_data[device_data['outlier'] == 1].drop('outlier', axis=1)

    print("Imputing missing values...")
    imputer = KNNImputer(n_neighbors=5)
    device_data_imputed = pd.DataFrame(imputer.fit_transform(device_data), columns=device_data.columns, index=device_data.index)

    original_data_path = os.path.join(save_dir, f'device_{device_id}_original_data.npy')
    np.save(original_data_path, device_data_imputed['W'].values)

    print(f"Original preprocessed data for device {device_id} saved at: {original_data_path}")

    return device_data_imputed['W'].values

data_path = '/content/drive/MyDrive/data_inverter_tn.csv'
save_dir = '/content/drive/MyDrive/Thesis1/Decomposed_IMFs'

device_id = 155
original_data_155 = load_and_preprocess_original_data(data_path, save_dir, device_id, sample_frac=0.10)

device_id = 156
original_data_156 = load_and_preprocess_original_data(data_path, save_dir, device_id, sample_frac=0.10)

print("Checking for NaN values in original data...")
print(original_data_156.isna().sum())

print("First few values of the original data:")
print(original_data_156.head())

print("First few values of the first IMF:")
print(combined_imfs_156[0][:10])

print(f"Length of original data: {len(original_data_156)}")
print(f"Length of first IMF: {len(combined_imfs_156[0])}")

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import os

def load_original_data(device_id, save_dir):
    original_data_path = os.path.join(save_dir, f'device_{device_id}_original_data.npy')
    if not os.path.exists(original_data_path):
        raise FileNotFoundError(f"Original data file for device {device_id} not found at {original_data_path}.")
    original_data = np.load(original_data_path)
    return original_data

def load_imfs_and_residual(device_id, save_dir):
    imfs_path = os.path.join(save_dir, f'combined_imfs_{device_id}.npy')
    if not os.path.exists(imfs_path):
        raise FileNotFoundError(f"IMFs file for device {device_id} not found at {imfs_path}.")
    imfs = np.load(imfs_path)


    residual_path = os.path.join(save_dir, f'combined_residual_{device_id}.npy')
    if not os.path.exists(residual_path):
        raise FileNotFoundError(f"Residual file for device {device_id} not found at {residual_path}.")
    residual = np.load(residual_path)

    return imfs, residual

def compare_imfs_with_original(original_data, imfs, residual, device_id, save_dir):
    plt.figure(figsize=(15, 18))
    num_imfs = imfs.shape[0]

    min_length = min(len(original_data), imfs.shape[1])
    original_data = original_data[:min_length]
    imfs = imfs[:, :min_length]
    residual = residual[:min_length]

    plt.subplot(num_imfs + 2, 1, 1)
    plt.plot(original_data, label='Original Data')
    plt.title(f'Device {device_id} - Original Data')
    plt.legend()

    correlations = {}
    for i in range(num_imfs):
        plt.subplot(num_imfs + 2, 1, i + 2)
        plt.plot(imfs[i], label=f'IMF {i + 1}')
        plt.title(f'Device {device_id} - IMF {i + 1}')
        plt.legend()

        correlation = np.corrcoef(original_data, imfs[i])[0, 1]
        correlations[f'IMF {i + 1}'] = correlation

    plt.subplot(num_imfs + 2, 1, num_imfs + 2)
    plt.plot(residual, label='Residual')
    plt.title(f'Device {device_id} - Residual')
    plt.legend()

    plt.tight_layout()
    plt.savefig(os.path.join(save_dir, f'device_{device_id}_comparison.png'))
    plt.show()

    print(f"Correlations between Original Data and IMFs for Device {device_id}:")
    for imf_name, correlation in correlations.items():
        print(f"{imf_name}: {correlation:.4f}")

    return correlations

save_dir = '/content/drive/MyDrive/Thesis1/Decomposed_IMFs'

try:
    original_data_155 = load_original_data(device_id=155, save_dir=save_dir)
    combined_imfs_155, combined_residual_155 = load_imfs_and_residual(device_id=155, save_dir=save_dir)
    correlations_155 = compare_imfs_with_original(original_data_155, combined_imfs_155, combined_residual_155, device_id=155, save_dir=save_dir)
except FileNotFoundError as e:
    print(f"Error for Device 155: {e}")

try:
    original_data_156 = load_original_data(device_id=156, save_dir=save_dir)
    combined_imfs_156, combined_residual_156 = load_imfs_and_residual(device_id=156, save_dir=save_dir)
    correlations_156 = compare_imfs_with_original(original_data_156, combined_imfs_156, combined_residual_156, device_id=156, save_dir=save_dir)
except FileNotFoundError as e:
    print(f"Error for Device 156: {e}")

import numpy as np
import os
from catboost import CatBoostRegressor
from tcn import compiled_tcn
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error, mean_absolute_error

def create_dataset_from_imfs(imfs, selected_imfs, look_back=10):
    X, y = [], []
    selected_data = imfs[selected_imfs, :]
    for i in range(selected_data.shape[1] - look_back - 1):
        X.append(selected_data[:, i:(i + look_back)].T)
        y.append(selected_data[0, i + look_back])
    return np.array(X), np.array(y)

def prepare_data_for_catboost(imfs, selected_imfs):
    selected_data = imfs[selected_imfs, :].T
    return selected_data

def prepare_data_for_tcn(imfs, selected_imfs, look_back=10):
    X, y = create_dataset_from_imfs(imfs, selected_imfs, look_back)
    return X, y

import numpy as np

def synchronize_lengths(original_data, imfs):
    min_length = min(len(original_data), imfs.shape[1])
    original_data = original_data[:min_length]
    imfs = imfs[:, :min_length]
    return original_data, imfs

original_data_155, imfs_155 = synchronize_lengths(original_data_155, imfs_155)
original_data_156, imfs_156 = synchronize_lengths(original_data_156, imfs_156)
def create_dataset_from_imfs(imfs, selected_imfs, look_back=10):
    X, y = [], []
    selected_data = imfs[selected_imfs, :]
    for i in range(selected_data.shape[1] - look_back - 1):
        X.append(selected_data[:, i:(i + look_back)].T)
        y.append(selected_data[0, i + look_back])
    return np.array(X), np.array(y)

def prepare_data_for_tcn(imfs, selected_imfs, look_back=10):
    X, y = create_dataset_from_imfs(imfs, selected_imfs, look_back)
    return X, y

from catboost import CatBoostRegressor
from sklearn.metrics import mean_squared_error, mean_absolute_error

def train_evaluate_catboost(imfs, selected_imfs, original_data, save_dir, device_id):
    print(f"Training CatBoost for device_id: {device_id}")

    X = prepare_data_for_catboost(imfs, selected_imfs)
    y = original_data[:len(X)]

    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

    catboost_model = CatBoostRegressor()
    catboost_model.fit(X_train, y_train)

    predictions = catboost_model.predict(X_test)

    rmse = np.sqrt(mean_squared_error(y_test, predictions))
    mae = mean_absolute_error(y_test, predictions)
    nrmse = rmse / (np.max(y_test) - np.min(y_test))

    print(f"CatBoost Model RMSE for device_id {device_id}: {rmse}")
    print(f"CatBoost Model MAE for device_id {device_id}: {mae}")
    print(f"CatBoost Model n-RMSE for device_id {device_id}: {nrmse}")

    return catboost_model

catboost_model_155 = train_evaluate_catboost(imfs_155, selected_imfs_155_catboost, original_data_155, save_dir, 155)

catboost_model_156 = train_evaluate_catboost(imfs_156, selected_imfs_156_catboost, original_data_156, save_dir, 156)

import numpy as np
import os
from keras.models import Sequential
from keras.layers import Conv1D, Dense, Flatten, BatchNormalization, Dropout
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error, mean_absolute_error

def prepare_data_for_tcn(imfs, selected_imfs):
    selected_data = np.stack([imfs[i] for i in selected_imfs], axis=1)
    return selected_data

def train_tcn(X_train, y_train, X_test, input_shape):
    model = Sequential()
    model.add(Conv1D(64, 2, padding='causal', activation='relu', input_shape=input_shape))
    model.add(BatchNormalization())
    model.add(Dropout(0.2))
    model.add(Conv1D(64, 2, padding='causal', activation='relu'))
    model.add(BatchNormalization())
    model.add(Dropout(0.2))
    model.add(Flatten())
    model.add(Dense(50, activation='relu'))
    model.add(Dense(1))
    model.compile(optimizer='adam', loss='mse')

    model.fit(X_train, y_train, epochs=50, batch_size=32, verbose=1)
    predictions = model.predict(X_test).flatten()
    return predictions, model

imfs_155 = np.load('/content/drive/MyDrive/Thesis1/Decomposed_IMFs/combined_imfs_155.npy')
imfs_156 = np.load('/content/drive/MyDrive/Thesis1/Decomposed_IMFs/combined_imfs_156.npy')

selected_imfs_155 = [2, 3, 4]
selected_imfs_156 = [3, 2, 4]

X_155 = prepare_data_for_tcn(imfs_155, selected_imfs_155)
X_156 = prepare_data_for_tcn(imfs_156, selected_imfs_156)

y_155 = X_155[:, 0]
y_156 = X_156[:, 0]

X_train_155, X_test_155, y_train_155, y_test_155 = train_test_split(X_155, y_155, test_size=0.2, random_state=42)
X_train_156, X_test_156, y_train_156, y_test_156 = train_test_split(X_156, y_156, test_size=0.2, random_state=42)

X_train_155_tcn = X_train_155.reshape(X_train_155.shape[0], X_train_155.shape[1], 1)
X_test_155_tcn = X_test_155.reshape(X_test_155.shape[0], X_test_155.shape[1], 1)
X_train_156_tcn = X_train_156.reshape(X_train_156.shape[0], X_train_156.shape[1], 1)
X_test_156_tcn = X_test_156.reshape(X_test_156.shape[0], X_test_156.shape[1], 1)
tcn_pred_155, tcn_model_155 = train_tcn(X_train_155_tcn, y_train_155, X_test_155_tcn, (X_train_155_tcn.shape[1], 1))
tcn_pred_156, tcn_model_156 = train_tcn(X_train_156_tcn, y_train_156, X_test_156_tcn, (X_train_156_tcn.shape[1], 1))
mae_155 = mean_absolute_error(y_test_155, tcn_pred_155)
rmse_155 = np.sqrt(mean_squared_error(y_test_155, tcn_pred_155))
n_rmse_155 = (rmse_155 / (y_test_155.max() - y_test_155.min())) * 100

mae_156 = mean_absolute_error(y_test_156, tcn_pred_156)
rmse_156 = np.sqrt(mean_squared_error(y_test_156, tcn_pred_156))
n_rmse_156 = (rmse_156 / (y_test_156.max() - y_test_156.min())) * 100

print(f"TCN Model Results for Device 155:")
print(f"MAE: {mae_155:.2f} W")
print(f"RMSE: {rmse_155:.2f} W")
print(f"n-RMSE: {n_rmse_155:.2f} %")

print(f"TCN Model Results for Device 156:")
print(f"MAE: {mae_156:.2f} W")
print(f"RMSE: {rmse_156:.2f} W")
print(f"n-RMSE: {n_rmse_156:.2f} %")

import numpy as np
from keras.models import Sequential
from keras.layers import Conv1D, Dense, Flatten, BatchNormalization, Dropout
from sklearn.model_selection import KFold
from sklearn.metrics import mean_squared_error, mean_absolute_error

def prepare_data_for_tcn(imfs, selected_imfs):
    selected_data = np.stack([imfs[i] for i in selected_imfs], axis=1)
    return selected_data

def train_tcn(X_train, y_train, X_test, input_shape):
    model = Sequential()
    model.add(Conv1D(64, 2, padding='causal', activation='relu', input_shape=input_shape))
    model.add(BatchNormalization())
    model.add(Dropout(0.2))
    model.add(Conv1D(64, 2, padding='causal', activation='relu'))
    model.add(BatchNormalization())
    model.add(Dropout(0.2))
    model.add(Flatten())
    model.add(Dense(50, activation='relu'))
    model.add(Dense(1))
    model.compile(optimizer='adam', loss='mse')

    model.fit(X_train, y_train, epochs=50, batch_size=32, verbose=0)
    predictions = model.predict(X_test).flatten()
    return predictions, model

imfs_155 = np.load('/content/drive/MyDrive/Thesis1/Decomposed_IMFs/combined_imfs_155.npy')
imfs_156 = np.load('/content/drive/MyDrive/Thesis1/Decomposed_IMFs/combined_imfs_156.npy')

selected_imfs_155 = [2, 3, 4]
selected_imfs_156 = [3, 2, 4]

X_155 = prepare_data_for_tcn(imfs_155, selected_imfs_155)
X_156 = prepare_data_for_tcn(imfs_156, selected_imfs_156)

y_155 = X_155[:, 0]
y_156 = X_156[:, 0]

X_155_tcn = X_155.reshape(X_155.shape[0], X_155.shape[1], 1)
X_156_tcn = X_156.reshape(X_156.shape[0], X_156.shape[1], 1)

kf = KFold(n_splits=5, shuffle=True, random_state=42)

mae_155_list, rmse_155_list, n_rmse_155_list = [], [], []
mae_156_list, rmse_156_list, n_rmse_156_list = [], [], []

for train_index, test_index in kf.split(X_155_tcn):
    X_train_155, X_test_155 = X_155_tcn[train_index], X_155_tcn[test_index]
    y_train_155, y_test_155 = y_155[train_index], y_155[test_index]

    tcn_pred_155, _ = train_tcn(X_train_155, y_train_155, X_test_155, (X_train_155.shape[1], 1))
    mae_155 = mean_absolute_error(y_test_155, tcn_pred_155)
    rmse_155 = np.sqrt(mean_squared_error(y_test_155, tcn_pred_155))
    n_rmse_155 = (rmse_155 / (y_test_155.max() - y_test_155.min())) * 100

    mae_155_list.append(mae_155)
    rmse_155_list.append(rmse_155)
    n_rmse_155_list.append(n_rmse_155)

for train_index, test_index in kf.split(X_156_tcn):
    X_train_156, X_test_156 = X_156_tcn[train_index], X_156_tcn[test_index]
    y_train_156, y_test_156 = y_156[train_index], y_156[test_index]

    tcn_pred_156, _ = train_tcn(X_train_156, y_train_156, X_test_156, (X_train_156.shape[1], 1))
    mae_156 = mean_absolute_error(y_test_156, tcn_pred_156)
    rmse_156 = np.sqrt(mean_squared_error(y_test_156, tcn_pred_156))
    n_rmse_156 = (rmse_156 / (y_test_156.max() - y_test_156.min())) * 100

    mae_156_list.append(mae_156)
    rmse_156_list.append(rmse_156)
    n_rmse_156_list.append(n_rmse_156)

avg_mae_155 = np.mean(mae_155_list)
avg_rmse_155 = np.mean(rmse_155_list)
avg_n_rmse_155 = np.mean(n_rmse_155_list)

avg_mae_156 = np.mean(mae_156_list)
avg_rmse_156 = np.mean(rmse_156_list)
avg_n_rmse_156 = np.mean(n_rmse_156_list)

print(f"Cross-Validation Results for Device 155:")
print(f"Average MAE: {avg_mae_155:.2f} W")
print(f"Average RMSE: {avg_rmse_155:.2f} W")
print(f"Average n-RMSE: {avg_n_rmse_155:.2f} %")

print(f"Cross-Validation Results for Device 156:")
print(f"Average MAE: {avg_mae_156:.2f} W")
print(f"Average RMSE: {avg_rmse_156:.2f} W")
print(f"Average n-RMSE: {avg_n_rmse_156:.2f} %")

import numpy as np
from keras.models import Sequential
from keras.layers import Conv1D, Dense, Flatten, BatchNormalization, Dropout
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error, mean_absolute_error

def prepare_data_for_tcn(imfs, selected_imfs):
    selected_data = np.stack([imfs[i] for i in selected_imfs], axis=1)
    return selected_data

def train_tcn(X_train, y_train, X_test, input_shape):
    model = Sequential()
    model.add(Conv1D(64, 2, padding='causal', activation='relu', input_shape=input_shape))
    model.add(BatchNormalization())
    model.add(Dropout(0.2))
    model.add(Conv1D(64, 2, padding='causal', activation='relu'))
    model.add(BatchNormalization())
    model.add(Dropout(0.2))
    model.add(Flatten())
    model.add(Dense(50, activation='relu'))
    model.add(Dense(1))
    model.compile(optimizer='adam', loss='mse')

    model.fit(X_train, y_train, epochs=50, batch_size=32, verbose=1)
    predictions = model.predict(X_test).flatten()
    return predictions, model

imfs_155 = np.load('/content/drive/MyDrive/Thesis1/Decomposed_IMFs/combined_imfs_155.npy')
imfs_156 = np.load('/content/drive/MyDrive/Thesis1/Decomposed_IMFs/combined_imfs_156.npy')

selected_imfs_155 = [2, 3, 4]
selected_imfs_156 = [3, 2, 4]

X_155 = prepare_data_for_tcn(imfs_155, selected_imfs_155)
X_156 = prepare_data_for_tcn(imfs_156, selected_imfs_156)

y_155 = X_155[:, 0]
y_156 = X_156[:, 0]

X_train_155, X_test_155, y_train_155, y_test_155 = train_test_split(X_155, y_155, test_size=0.2, random_state=42)
X_train_156, X_test_156, y_train_156, y_test_156 = train_test_split(X_156, y_156, test_size=0.2, random_state=42)

X_train_155_tcn = X_train_155.reshape(X_train_155.shape[0], X_train_155.shape[1], 1)
X_test_155_tcn = X_test_155.reshape(X_test_155.shape[0], X_test_155.shape[1], 1)
X_train_156_tcn = X_train_156.reshape(X_train_156.shape[0], X_train_156.shape[1], 1)
X_test_156_tcn = X_test_156.reshape(X_test_156.shape[0], X_test_156.shape[1], 1)

tcn_pred_155, tcn_model_155 = train_tcn(X_train_155_tcn, y_train_155, X_test_155_tcn, (X_train_155_tcn.shape[1], 1))

tcn_pred_156, tcn_model_156 = train_tcn(X_train_156_tcn, y_train_156, X_test_156_tcn, (X_train_156_tcn.shape[1], 1))

if 'tcn_model_155' not in globals() or 'tcn_model_156' not in globals():
    print("Models are not defined. Please check the training process.")

predicted_w_155 = tcn_model_155.predict(X_test_155_tcn).flatten()

predicted_w_156 = tcn_model_156.predict(X_test_156_tcn).flatten()

mae_155 = mean_absolute_error(y_test_155, predicted_w_155)
rmse_155 = np.sqrt(mean_squared_error(y_test_155, predicted_w_155))
n_rmse_155 = (rmse_155 / (y_test_155.max() - y_test_155.min())) * 100

mae_156 = mean_absolute_error(y_test_156, predicted_w_156)
rmse_156 = np.sqrt(mean_squared_error(y_test_156, predicted_w_156))
n_rmse_156 = (rmse_156 / (y_test_156.max() - y_test_156.min())) * 100

print(f"Predicted Power Output for Device 155:")
print(f"Predicted W: {predicted_w_155}")
print(f"MAE: {mae_155:.2f} W")
print(f"RMSE: {rmse_155:.2f} W")
print(f"n-RMSE: {n_rmse_155:.2f} %")

print(f"Predicted Power Output for Device 156:")
print(f"Predicted W: {predicted_w_156}")
print(f"MAE: {mae_156:.2f} W")
print(f"RMSE: {rmse_156:.2f} W")
print(f"n-RMSE: {n_rmse_156:.2f} %")

from sklearn.metrics import r2_score

r2_155 = r2_score(y_test_155, predicted_w_155)
r2_156 = r2_score(y_test_156, predicted_w_156)

print(f"R² for Device 155: {r2_155 * 100:.2f}%")
print(f"R² for Device 156: {r2_156 * 100:.2f}%")

min_length = min(len(predicted_w_155), len(predicted_w_156))

predicted_w_155 = predicted_w_155[:min_length]
predicted_w_156 = predicted_w_156[:min_length]

predicted_w_total = predicted_w_155 + predicted_w_156

mae_total = mean_absolute_error(y_test_155[:min_length], predicted_w_total)
rmse_total = np.sqrt(mean_squared_error(y_test_155[:min_length], predicted_w_total))
n_rmse_total = (rmse_total / (y_test_155[:min_length].max() - y_test_155[:min_length].min())) * 100

print(f"Overall Predicted Power Output for Devices 155 and 156:")
print(f"Predicted Total W: {predicted_w_total}")
print(f"MAE: {mae_total:.2f} W")
print(f"RMSE: {rmse_total:.2f} W")
print(f"n-RMSE: {n_rmse_total:.2f} %")

from sklearn.metrics import r2_score

r2_155 = r2_score(y_test_155, predicted_w_155)
r2_156 = r2_score(y_test_156, predicted_w_156)
total_r2_percentage = (r2_155 + r2_156) * 100

print(f"Overall R² for Devices 155 and 156: {total_r2_percentage:.2f}%")