finished training pipeline, will build app.py to trigger it

Author: pycoder49

app.py

@@ -77,6 +77,9 @@ def initiate_data_transformation(self) -> DataTransformationArtifact:
             save_numpy_array(self.data_transformation_config.transformed_test_file_path, array=test_nparray)
             save_object(self.data_transformation_config.transformed_object_file_path, obj=knn_processor_obj)
 
+            # saving the final transformation object in comomn directory with model.pkl
+            save_object("final_model/transformer.pkl", obj=knn_processor_obj)
+
             # preparing artifacts
             data_transformation_artifact = DataTransformationArtifact(
                 transformation_object_path = self.data_transformation_config.transformed_object_file_path,

final_model/model.pkl

@@ -9,7 +9,8 @@
 )
 from network_security.entity.artifact_entity import (
     DataTransformationArtifact,
-    ModelTrainerArtifact
+    ModelTrainerArtifact,
+    ClassificationMetricArtifact
 )
 from network_security.utils.main_utils.utils import (
     save_object, 
@@ -28,11 +29,16 @@
 )
 from xgboost import XGBClassifier
 from sklearn.metrics import r2_score
+import mlflow
+import dagshub
 
 import pandas as pd
 import numpy as np
 import os, sys
 
+import dagshub
+dagshub.init(repo_owner='pycoder49', repo_name='networkSecuritySystem', mlflow=True)
+
 
 class ModelTrainer:
     def __init__(self, model_trainer_config: ModelTrainerConfig,
@@ -43,7 +49,24 @@ def __init__(self, model_trainer_config: ModelTrainerConfig,
         except Exception as e:
             raise NetworkSecurityException(e, sys)
 
+    def track_mlflow(self, model, 
+                     train_metric: ClassificationMetricArtifact
+                     ):
+        try:
+            with mlflow.start_run():
+                f1_score = train_metric.f1_score
+                precision = train_metric.precision_score
+                recall = train_metric.recall_score
+
+                mlflow.log_metric("F1_Score", f1_score)
+                mlflow.log_metric("Precision", precision)
+                mlflow.log_metric("Recall", recall)
 
+                # Use artifact_path for DagHub compatibility
+                mlflow.sklearn.log_model(model, "model")
+
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
 
     def train_model(self, X_train, y_train, X_test, y_test) -> object:
         try:
@@ -124,15 +147,15 @@ def train_model(self, X_train, y_train, X_test, y_test) -> object:
             best_model_score = max(sorted(model_report.values()))
             best_model = models[best_model_name]
 
+            # getting the train classification metrics and tracking using MLFlow
             y_train_pred = best_model.predict(X_train)
             train_classification_metric = get_classification_score(y_true=y_train, y_pred=y_train_pred)
+            self.track_mlflow(best_model, train_classification_metric)
 
-            # tracking the MLFlow
-
-
-            # getting the test classification metrics
+            # getting the test classification metrics and tracking using MLFlow
             y_test_pred = best_model.predict(X_test)
             test_classification_metric = get_classification_score(y_true=y_test, y_pred=y_test_pred)
+            self.track_mlflow(best_model, test_classification_metric)
 
             # loading the object, saving it
             preprocessor = load_object(file_path=self.data_transformation_artifact.transformation_object_path)
@@ -146,6 +169,9 @@ def train_model(self, X_train, y_train, X_test, y_test) -> object:
                 obj=network_model
             )
 
+            # saving the final best model in a common directory with preprocessor.pkl
+            save_object("final_model/model.pkl", best_model)
+
             # saving the model trainer artifact
             model_trainer_artifact = ModelTrainerArtifact(
                 trained_model_file_path=self.model_trainer_config.trained_model_file_path,

final_model/transformer.pkl

@@ -0,0 +1,99 @@
+from network_security.components.data_ingestion import DataIngestion
+from network_security.components.data_validation import DataValidation
+from network_security.components.data_transformation import DataTransformation
+from network_security.components.model_trainer import ModelTrainer
+
+from network_security.entity.config_entity import (
+    TrainingPipelineConfig,
+    DataIngestionConfig,
+    DataValidationConfig,
+    DataTransformationConfig,
+    ModelTrainerConfig
+)
+
+from network_security.entity.artifact_entity import (
+    DataIngestionArtifact,
+    DataValidationArtifact,
+    DataTransformationArtifact,
+    ModelTrainerArtifact
+)
+
+from network_security.exceptions.exception import NetworkSecurityException
+from network_security.logging.logger import logging
+
+import os, sys
+
+class TrainingPipeline:
+    def __init__(self):
+        self.training_pipeline_config = TrainingPipelineConfig()
+    
+    def start_data_ingestion(self) -> DataIngestionArtifact:
+        try:
+            self.data_ingestion_config = DataIngestionConfig(self.training_pipeline_config)
+            logging.info("Starting data ingestion")
+
+            data_ingestion = DataIngestion(data_ingestion_config=self.data_ingestion_config)
+            data_ingestion_artifact = data_ingestion.initiate_data_ingestion()
+            logging.info("Data ingestion completed")
+
+            return data_ingestion_artifact
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+
+    def start_data_validation(self, data_ingestion_artifact: DataIngestionArtifact) -> DataValidationArtifact:
+        try:
+            self.data_validation_config = DataValidationConfig(self.training_pipeline_config)
+            logging.info("Starting data validation")
+
+            data_validation = DataValidation(
+                data_ingestion_artifact=data_ingestion_artifact,
+                data_validation_config=self.data_validation_config
+            )
+            data_validation_artifact = data_validation.initiate_data_validation()
+            logging.info("Data validation completed")
+
+            return data_validation_artifact
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+        
+    def start_data_transformation(self, data_validation_artifact: DataValidationArtifact) -> DataTransformationArtifact:
+        try:
+            self.data_transofmation_config = DataTransformationConfig(self.training_pipeline_config)
+            logging.info("Starting data transformation")
+
+            data_transformation = DataTransformation(
+                data_validation_artifact=data_validation_artifact,
+                data_transformation_config=self.data_transformation_config
+            )
+            data_transformation_artifact = data_transformation.initiate_data_transformation()
+            logging.info("Data transformation completed")
+
+            return data_transformation_artifact
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+        
+    def start_model_trainer(self, data_transformation_artifact: DataTransformationArtifact) -> ModelTrainerArtifact:
+        try:
+            self.model_trainer_config = ModelTrainerConfig(self.training_pipeline_config)
+            logging.info("Starting model trainer")
+
+            model_trainer = ModelTrainer(
+                model_trainer_config=self.model_trainer_config,
+                data_transformation_artifact=data_transformation_artifact
+            )
+            model_trainer_artifact = model_trainer.initiate_model_trainer()
+            logging.info("Model trainer completed")
+
+            return model_trainer_artifact
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)
+        
+    def run_pipeline(self):
+        try:
+            data_ingestion_artifact = self.start_data_ingestion()
+            data_validation_artifact = self.start_data_validation(data_ingestion_artifact=data_ingestion_artifact)
+            data_transformation_artifact = self.start_data_transformation(data_validation_artifact=data_validation_artifact)
+            model_trainer_artifact = self.start_model_trainer(data_transformation_artifact=data_transformation_artifact)
+            
+        except Exception as e:
+            raise NetworkSecurityException(e, sys)

network_security/components/data_transformation.py

@@ -1,11 +1,13 @@
 python-dotenv
 pandas
-numpy
-pymongo
-certifi
+numpy>=1.24.0
 pymongo[srv]==3.12
-pymongo
+certifi
 scikit-learn
+mlflow<2.16
+dagshub
 dill
 pyaml
+xgboost
+scipy
 # -e .