Evaluating Machine Learning Algorithms

"""
Entry point for running simulation on classification with class imbalance.

Overall goal of simulation: evaluate effect of various techniques for dealing
with unbalanced classes in binary classification problems.

The code compares correction techniques in terms of precision, recall, and the
F_1 score. Techniques considered:
    - Not doing anything.
    - SMOTE (synthetic oversampling).
    - Introducing class weights in the criterion function.

Usage:
    python -X gil=0 main.py

Output:
    Console printout of precision, recall, and F1 scores
"""

import pandas as pd

from simulation_infrastructure.orchestrator import SimulationOrchestrator
from simulation_infrastructure.scenarios import scenarios


def main():
    # Create and run the orchestrator
    orchestrator = SimulationOrchestrator(scenarios, n_workers=4)
    orchestrator.run_all()
    combined_results = pd.concat(orchestrator.summary_results.values())

    # Print key results as a markdown table
    print(
        combined_results.groupby(by=["algorithm", "n_training", "first_class_weight"])[
            ["precision_1", "recall_1", "f1_1"]
        ]
        .mean()
        .round(3)
        .to_markdown()
    )


if __name__ == "__main__":
    main()

"""
Simulation runner for evaluating prediction algorithms via Monte Carlo.

Classes:
    - SimulationRunner: executes simulation scenario, potentially in parallel.
"""

from concurrent.futures import ThreadPoolExecutor
from typing import Any, Type

import pandas as pd
from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score
from tqdm import tqdm

from simulation_infrastructure.protocols import AlgorithmProtocol, DGPProtocol


class SimulationRunner:
    """Runs Monte Carlo simulations for a given DGP and list of algorithms.

    Attributes:
        dgp_type (Type[DGPProtocol]): class of the Data Generating Process.
        dgp_kwargs (dict[str, Any]): keyword arguments for initializing the DGP.
        algorithm_types (list[Type[AlgorithmProtocol]]): list of algorithm classes.
        algorithm_kwargs_list list[dict[str, Any]]): list of keyword argument
            dictionaries for initializing each algorithm, ordered in the same
            order as algorithm_types
        n_simulations (int): number of Monte Carlo simulations. Defaults to 1000.
        n_workers (int): number of threads for parallel execution. Defaults to 1.
    """

    def __init__(
        self,
        dgp_type: Type[DGPProtocol],
        dgp_kwargs: dict[str, Any],
        algorithm_types: list[Type[AlgorithmProtocol]],
        algorithm_kwargs_list: list[dict[str, Any]],
        n_simulations: int = 1000,
        n_workers: int = 1,
    ):
        self.dgp_type = dgp_type
        self.dgp_kwargs = dgp_kwargs
        self.algorithm_types = algorithm_types
        self.algorithm_kwargs_list = algorithm_kwargs_list
        self.n_simulations = n_simulations
        self.n_workers = n_workers
        self.results = []

    def _run_single_simulation(self, seed: int) -> dict[str, Any]:
        """Run a single Monte Carlo simulation for all algorithms.

        Args:
            seed (int): seed for data sampling
        """
        # Initialize DGP
        dgp = self.dgp_type(**self.dgp_kwargs)
        X_train, X_test, y_train, y_test = dgp.sample(seed=seed)

        sim_results = {}

        # Initialize and fit each algorithm
        for algo_type, algo_kwargs in zip(
            self.algorithm_types, self.algorithm_kwargs_list
        ):
            algo = algo_type(**algo_kwargs)
            algo.fit(X_train, y_train)
            y_pred = algo.predict(X_test)

            accuracy = accuracy_score(y_test, y_pred)
            precision_0 = precision_score(y_test, y_pred, pos_label=0, zero_division=0)
            recall_0 = recall_score(y_test, y_pred, pos_label=0, zero_division=0)
            precision_1 = precision_score(y_test, y_pred, pos_label=1, zero_division=0)
            recall_1 = recall_score(y_test, y_pred, pos_label=1, zero_division=0)
            f1_0 = f1_score(y_test, y_pred, pos_label=0, zero_division=0)
            f1_1 = f1_score(y_test, y_pred, pos_label=1, zero_division=0)

            result_key = dgp.name + " + " + algo.name
            sim_results[result_key] = {
                "n_training": dgp.n_train_samples,
                "first_class_weight": dgp.weights[0],
                "accuracy": accuracy,
                "precision_0": precision_0,
                "recall_0": recall_0,
                "precision_1": precision_1,
                "recall_1": recall_1,
                "f1_0": f1_0,
                "f1_1": f1_1,
            }

        return sim_results

    def run_all(self) -> pd.DataFrame:
        """Run all simulations in parallel and return aggregated results.

        Returns:
            pd.DataFrame: DataFrame with simulation results.
        """
        with ThreadPoolExecutor(max_workers=self.n_workers) as executor:
            futures = [
                executor.submit(self._run_single_simulation, seed)
                for seed in range(self.n_simulations)
            ]
            for future in tqdm(
                futures, total=self.n_simulations, desc="Running simulations"
            ):
                self.results.append(future.result())

        # Aggregate results into a DataFrame
        df_list = []
        for sim_result in self.results:
            for algo_name, metrics in sim_result.items():
                row = {"algorithm": algo_name, **metrics}
                df_list.append(row)
        return pd.DataFrame(df_list)

"""
Algorithms based on logistic regression, with and without class proportion
corrections.

Classes in this module:
    - LogisticRegressionAlgorithm: vanilla scikit-learn regression
    - LogisticRegressionSMOTE: logistic regression with SMOTE
"""

import numpy as np
from imblearn.over_sampling import SMOTE
from imblearn.pipeline import Pipeline as ImbPipeline
from sklearn.linear_model import LogisticRegression


class LogisticRegressionSK:
    """Logistic regression classifier with optional class weighting.

    Attributes:
        name (str): name for reporting purposes.
        class_weight (str | None): optional class weights for imbalanced datasets.
            Defaults to None (no weighting).
        model (sklearn.linear_model.LogisticRegression): an sklearn logistic
            regression model.
    """

    def __init__(
        self,
        class_weight: str | None = None,
        random_state: int | None = None,
    ) -> None:
        self.class_weight = class_weight
        self.model = LogisticRegression(
            class_weight=class_weight, random_state=random_state
        )
        self.name = f"LogisticRegression (class_weight={self.class_weight})"

    def fit(self, X: np.ndarray, y: np.ndarray) -> None:
        """Fit the logistic regression model to the training data.

        Args:
            X (np.ndarray): training features.
            y (np.ndarray): training labels.
        """
        self.model.fit(X, y)

    def predict(self, X: np.ndarray) -> np.ndarray:
        """Predict labels for the test data.

        Args:
            X (np.ndarray): test features.

        Returns:
            np.ndarray: predicted labels.
        """
        return self.model.predict(X)


class LogisticRegressionSMOTE:
    """Logistic regression classifier with SMOTE oversampling.

    Attributes:
        name (str): name for reporting purposes.
        model (imblearn.pipeline.ImbPipeline): pipeline combining SMOTE 
            oversampling and logistic regression.
    """

    def __init__(
        self,
        random_state: int | None = None,
    ) -> None:
        self.name = "Logistic Regression with SMOTE"
        self.model = ImbPipeline(
            [
                ("smote", SMOTE(random_state=random_state)),
                ("logistic", LogisticRegression(random_state=random_state)),
            ]
        )

    def fit(self, X: np.ndarray, y: np.ndarray) -> None:
        """Fit the logistic regression model with SMOTE to the training data.

        Args:
            X (np.ndarray): training features.
            y (np.ndarray): training labels.
        """
        self.model.fit(X, y)

    def predict(self, X: np.ndarray) -> np.ndarray:
        """Predict labels for the test data.

        Args:
            X (np.ndarray): test features.

        Returns:
            np.ndarray: tredicted labels.
        """
        return self.model.predict(X)