Data Ingestion part completed

Author: GoJo-Rika

.gitignore

@@ -16,4 +16,7 @@ wheels/
 .DS_Store
 
 # Log files
-logs/
+logs/
+
+# Artifacts
+Artifacts/

network_security/components/data_ingestion.py

@@ -0,0 +1,106 @@
+import os
+import sys
+from pathlib import Path
+from urllib.parse import quote_plus
+
+import numpy as np
+import pandas as pd
+import pymongo
+from dotenv import load_dotenv
+from sklearn.model_selection import train_test_split
+
+from network_security.entity.artifact_entity import DataIngestionArtifact
+
+## Configuration of the Data Ingestion Config
+from network_security.entity.config_entity import DataIngestionConfig
+from network_security.exception.exception import NetworkSecurityException
+from network_security.logging.logger import logging
+
+load_dotenv()
+
+username = os.getenv("MONGO_DB_USERNAME")
+password = os.getenv("MONGO_DB_PASSWORD")
+
+username = quote_plus(username)
+password = quote_plus(password)
+
+MONGO_DB_URL: str = f"mongodb+srv://{username}:{password}@cluster0.l5ee6dv.mongodb.net/?retryWrites=true&w=majority&appName=Cluster0"
+
+
+class DataIngestion:
+    def __init__(self, data_ingestion_config: DataIngestionConfig) -> None:
+        try:
+            self.data_ingestion_config = data_ingestion_config
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+
+    def export_collection_as_dataframe(self) -> pd.DataFrame:
+        """Read data from mongodb."""
+        try:
+            database_name = self.data_ingestion_config.database_name
+            collection_name = self.data_ingestion_config.collection_name
+            self.mongo_client = pymongo.MongoClient(MONGO_DB_URL)
+            collection = self.mongo_client[database_name][collection_name]
+
+            df = pd.DataFrame(list(collection.find()))
+            if "_id" in df.columns.to_list():
+                df = df.drop(columns=["_id"], axis=1)
+
+            df.replace({"na": np.nan}, inplace=True)
+            return df
+        except Exception as e:
+            raise NetworkSecurityException
+
+    def export_data_into_feature_store(self, dataframe: pd.DataFrame) -> pd.DataFrame:
+        try:
+            feature_store_file_path = self.data_ingestion_config.feature_store_file_path
+            dir_path = Path(feature_store_file_path).parent
+            Path(dir_path).mkdir(parents=True, exist_ok=True)
+            dataframe.to_csv(feature_store_file_path, index=False, header=True)
+            return dataframe
+
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+
+    def split_data_as_train_test(self, dataframe: pd.DataFrame) -> None:
+        try:
+            train_set, test_set = train_test_split(
+                dataframe, test_size=self.data_ingestion_config.train_test_split_ratio,
+            )
+            logging.info("Performed train test split on the dataframe")
+
+            logging.info(
+                "Exited split_data_as_train_test method of Data_Ingestion class",
+            )
+            dir_path = Path(self.data_ingestion_config.training_file_path).parent
+
+            Path(dir_path).mkdir(parents=True, exist_ok=True)
+            Path(dir_path).mkdir(parents=True, exist_ok=True)
+
+            logging.info("Exporting train and test file path.")
+
+            train_set.to_csv(
+                self.data_ingestion_config.training_file_path, index=False, header=True,
+            )
+
+            test_set.to_csv(
+                self.data_ingestion_config.testing_file_path, index=False, header=True,
+            )
+            logging.info("Exported train and test file path.")
+
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+
+    def initiate_data_ingestion(self) -> DataIngestionArtifact:
+        try:
+            dataframe = self.export_collection_as_dataframe()
+            dataframe = self.export_data_into_feature_store(dataframe)
+            self.split_data_as_train_test(dataframe)
+            dataingestionartifact = DataIngestionArtifact(
+                trained_file_path=self.data_ingestion_config.training_file_path,
+                test_file_path=self.data_ingestion_config.testing_file_path,
+            )
+            return dataingestionartifact
+
+        except Exception as e:
+            raise NetworkSecurityException

network_security/constant/training_pipeline/__init__.py

@@ -0,0 +1,31 @@
+import os
+import sys
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+
+"""
+defining common constant variable for training pipeline
+"""
+TARGET_COLUMN = "Result"
+PIPELINE_NAME: str = "NetworkSecurity"
+ARTIFACT_DIR: str = "Artifacts"
+FILE_NAME: str = "phisingData.csv"
+
+TRAIN_FILE_NAME: str = "train.csv"
+TEST_FILE_NAME: str = "test.csv"
+SCHEMA_FILE_PATH = Path("data_schema") / "schema.yaml"
+SAVED_MODEL_DIR = Path("saved_models")
+MODEL_FILE_NAME = "model.pkl"
+
+
+"""
+Data Ingestion related constant start with DATA_INGESTION VAR NAME
+"""
+DATA_INGESTION_COLLECTION_NAME: str = "NetworkData"
+DATA_INGESTION_DATABASE_NAME: str = "TEST_DB"
+DATA_INGESTION_DIR_NAME: str = "data_ingestion"
+DATA_INGESTION_FEATURE_STORE_DIR: str = "feature_store"
+DATA_INGESTION_INGESTED_DIR: str = "ingested"
+DATA_INGESTION_TRAIN_TEST_SPLIT_RATION: float = 0.2

network_security/entity/artifact_entity.py

@@ -0,0 +1,7 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class DataIngestionArtifact:
+    trained_file_path: str
+    test_file_path: str

network_security/entity/config_entity.py

@@ -0,0 +1,47 @@
+from datetime import datetime
+from pathlib import Path
+
+from network_security.constant import training_pipeline
+
+print(training_pipeline.PIPELINE_NAME)
+print(training_pipeline.ARTIFACT_DIR)
+
+
+class TrainingPipelineConfig:
+    def __init__(self, timestamp: datetime = None) -> None:
+        if timestamp is None:
+            timestamp = datetime.now().astimezone()
+        timestamp_str = timestamp.strftime("%m_%d_%Y_%H_%M_%S")
+        self.pipeline_name = training_pipeline.PIPELINE_NAME
+        self.artifact_name = training_pipeline.ARTIFACT_DIR
+        self.artifact_dir = Path(self.artifact_name) / timestamp_str
+        self.model_dir = Path("final_model")
+        self.timestamp: str = timestamp_str
+
+
+class DataIngestionConfig:
+    def __init__(self, training_pipeline_config: TrainingPipelineConfig) -> None:
+        self.data_ingestion_dir: Path = (
+            Path(training_pipeline_config.artifact_dir)
+            / training_pipeline.DATA_INGESTION_DIR_NAME
+        )
+        self.feature_store_file_path: Path = (
+            self.data_ingestion_dir
+            / training_pipeline.DATA_INGESTION_FEATURE_STORE_DIR
+            / training_pipeline.FILE_NAME
+        )
+        self.training_file_path: Path = (
+            self.data_ingestion_dir
+            / training_pipeline.DATA_INGESTION_INGESTED_DIR
+            / training_pipeline.TRAIN_FILE_NAME
+        )
+        self.testing_file_path: Path = (
+            self.data_ingestion_dir
+            / training_pipeline.DATA_INGESTION_INGESTED_DIR
+            / training_pipeline.TEST_FILE_NAME
+        )
+        self.train_test_split_ratio: float = (
+            training_pipeline.DATA_INGESTION_TRAIN_TEST_SPLIT_RATION
+        )
+        self.collection_name: str = training_pipeline.DATA_INGESTION_COLLECTION_NAME
+        self.database_name: str = training_pipeline.DATA_INGESTION_DATABASE_NAME

pyproject.toml

@@ -8,7 +8,8 @@ dependencies = [
     "certifi>=2025.6.15",
     "numpy>=2.3.0",
     "pandas>=2.3.0",
-    "pymongo[srv]>=3.12.0",
+    "pymongo[srv]==3.12",
     "python-dotenv>=1.1.0",
+    "scikit-learn>=1.7.0",
     "setuptools>=80.9.0",
 ]

requirements.txt

@@ -4,5 +4,6 @@ numpy
 pymongo
 certifi
 pymongo[srv]==3.12
+scikit-learn
 
 # -e .