test_repl.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-05T14:53:03.310260Z",
     "start_time": "2024-07-05T14:53:01.693813Z"
    }
   },
   "outputs": [],
   "source": [
    "from py_pkg.core.core import get_human_time, read_dataset_and_split, get_test_dataset\n",
    "from py_pkg.algos.algo import Algo\n",
    "from py_pkg.entities.entities import TestDataType, AttackType, AlgoToPredict\n",
    "from py_pkg.testing.testing import Testing\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn import preprocessing\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-05T14:53:09.142380Z",
     "start_time": "2024-07-05T14:53:03.311952Z"
    }
   },
   "outputs": [],
   "source": [
    "# import datasets\n",
    "training_dataset_dir = './datasets/cantrainandtest/can-train-and-test/set_01/train_01/'\n",
    "testing_dataset_dir_1 = './datasets/cantrainandtest/can-train-and-test/set_01/test_01_known_vehicle_known_attack/'\n",
    "testing_dataset_dir_2 = './datasets/cantrainandtest/can-train-and-test/set_01/test_02_unknown_vehicle_known_attack/'\n",
    "\n",
    "# import kvka test datasets\n",
    "DoS_3_kvka = pd.read_csv(testing_dataset_dir_1 + \"DoS-3.csv\")\n",
    "DoS_4_kvka = pd.read_csv(testing_dataset_dir_1 + \"DoS-4.csv\")\n",
    "\n",
    "force_neutral_3_kvka = pd.read_csv(testing_dataset_dir_1 + \"force-neutral-3.csv\")\n",
    "force_neutral_4_kvka = pd.read_csv(testing_dataset_dir_1 + \"force-neutral-4.csv\")\n",
    "\n",
    "# import uvka test datasets\n",
    "DoS_3_uvka = pd.read_csv(testing_dataset_dir_2 + \"DoS-3.csv\")\n",
    "DoS_4_uvka = pd.read_csv(testing_dataset_dir_2 + \"DoS-4.csv\")\n",
    "\n",
    "force_neutral_3_uvka = pd.read_csv(testing_dataset_dir_2 + \"force-neutral-3.csv\")\n",
    "force_neutral_4_uvka = pd.read_csv(testing_dataset_dir_2 + \"force-neutral-4.csv\")\n",
    "\n",
    "# import train dataset\n",
    "attack_free_1 = pd.read_csv(training_dataset_dir + \"attack-free-1.csv\")\n",
    "attack_free_2 = pd.read_csv(training_dataset_dir + \"attack-free-2.csv\")\n",
    "\n",
    "DoS_1 = pd.read_csv(training_dataset_dir + \"DoS-1.csv\")\n",
    "DoS_2 = pd.read_csv(training_dataset_dir + \"DoS-2.csv\")\n",
    "\n",
    "force_neutral_1 = pd.read_csv(training_dataset_dir + \"force-neutral-1.csv\")\n",
    "force_neutral_2 = pd.read_csv(training_dataset_dir + \"force-neutral-2.csv\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-05T14:53:09.655339Z",
     "start_time": "2024-07-05T14:53:09.143549Z"
    }
   },
   "outputs": [],
   "source": [
    "# concatenate test dataset kvka\n",
    "DoS_kvka = pd.concat([DoS_3_kvka, DoS_4_kvka])\n",
    "force_neutral_kvka = pd.concat([force_neutral_3_kvka, force_neutral_4_kvka])\n",
    "\n",
    "# concatenate test dataset uvka\n",
    "DoS_uvka = pd.concat([DoS_3_uvka, DoS_4_uvka])\n",
    "force_neutral_uvka = pd.concat([force_neutral_3_uvka, force_neutral_4_uvka])\n",
    "\n",
    "# concatenate train datasets\n",
    "attack_free = pd.concat([attack_free_1, attack_free_2])\n",
    "DoS = pd.concat([DoS_1, DoS_2])\n",
    "force_neutral = pd.concat([force_neutral_1, force_neutral_2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-05T14:53:17.502778Z",
     "start_time": "2024-07-05T14:53:09.656475Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>timestamp</th>\n",
       "      <th>arbitration_id</th>\n",
       "      <th>data_field</th>\n",
       "      <th>attack</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.672531e+09</td>\n",
       "      <td>409</td>\n",
       "      <td>10376029658402455807</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.672531e+09</td>\n",
       "      <td>388</td>\n",
       "      <td>12884901888</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.672531e+09</td>\n",
       "      <td>413</td>\n",
       "      <td>4611756382877457663</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.672531e+09</td>\n",
       "      <td>417</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.672531e+09</td>\n",
       "      <td>455</td>\n",
       "      <td>1868041717547071</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>839868</th>\n",
       "      <td>1.672532e+09</td>\n",
       "      <td>481</td>\n",
       "      <td>4279566368</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>839869</th>\n",
       "      <td>1.672532e+09</td>\n",
       "      <td>193</td>\n",
       "      <td>240048196083575707</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>839870</th>\n",
       "      <td>1.672532e+09</td>\n",
       "      <td>197</td>\n",
       "      <td>229227940826484838</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>839871</th>\n",
       "      <td>1.672532e+09</td>\n",
       "      <td>485</td>\n",
       "      <td>5115999566495240705</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>839872</th>\n",
       "      <td>1.672532e+09</td>\n",
       "      <td>499</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1744018 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           timestamp  arbitration_id            data_field  attack\n",
       "0       1.672531e+09             409  10376029658402455807       0\n",
       "1       1.672531e+09             388           12884901888       0\n",
       "2       1.672531e+09             413   4611756382877457663       0\n",
       "3       1.672531e+09             417                     0       0\n",
       "4       1.672531e+09             455      1868041717547071       0\n",
       "...              ...             ...                   ...     ...\n",
       "839868  1.672532e+09             481            4279566368       0\n",
       "839869  1.672532e+09             193    240048196083575707       0\n",
       "839870  1.672532e+09             197    229227940826484838       0\n",
       "839871  1.672532e+09             485   5115999566495240705       0\n",
       "839872  1.672532e+09             499                     0       0\n",
       "\n",
       "[1744018 rows x 4 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def hex_to_decimal(hex_str):\n",
    "    return int(hex_str, 16)\n",
    "\n",
    "def decimal_to_hex(dec):\n",
    "    return hex(dec)\n",
    "\n",
    "# update column datatype for train data\n",
    "DoS['data_field'] = DoS['data_field'].apply(hex_to_decimal)\n",
    "DoS['arbitration_id'] = DoS['arbitration_id'].apply(hex_to_decimal)\n",
    "\n",
    "force_neutral['data_field'] = force_neutral['data_field'].apply(hex_to_decimal)\n",
    "force_neutral['arbitration_id'] = force_neutral['arbitration_id'].apply(hex_to_decimal)\n",
    "\n",
    "# update column datatype for test data kvka\n",
    "DoS_kvka['data_field'] = DoS_kvka['data_field'].apply(hex_to_decimal)\n",
    "DoS_kvka['arbitration_id'] = DoS_kvka['arbitration_id'].apply(hex_to_decimal)\n",
    "\n",
    "force_neutral_kvka['data_field'] = force_neutral_kvka['data_field'].apply(hex_to_decimal)\n",
    "force_neutral_kvka['arbitration_id'] = force_neutral_kvka['arbitration_id'].apply(hex_to_decimal)\n",
    "\n",
    "# update column datatype for test data uvka\n",
    "DoS_uvka['data_field'] = DoS_uvka['data_field'].apply(hex_to_decimal)\n",
    "DoS_uvka['arbitration_id'] = DoS_uvka['arbitration_id'].apply(hex_to_decimal)\n",
    "\n",
    "force_neutral_uvka['data_field'] = force_neutral_uvka['data_field'].apply(hex_to_decimal)\n",
    "force_neutral_uvka['arbitration_id'] = force_neutral_uvka['arbitration_id'].apply(hex_to_decimal)\n",
    "\n",
    "display(force_neutral_kvka)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-05T14:53:36.562921Z",
     "start_time": "2024-07-05T14:53:17.504522Z"
    }
   },
   "outputs": [],
   "source": [
    "# save training data\n",
    "DoS.to_csv(\"./datasets/clean-data-2/set-1/training-data/dos/dos.csv\", sep=',', index=False, encoding='utf-8')\n",
    "force_neutral.to_csv(\"./datasets/clean-data-2/set-1/training-data/force_neutral/force_neutral.csv\", sep=',', index=False, encoding='utf-8')\n",
    "\n",
    "# save testing data kvka\n",
    "DoS_kvka.to_csv(\"./datasets/clean-data-2/set-1/testing-data/kv-ka/dos/dos.csv\", sep=',', index=False, encoding='utf-8')\n",
    "force_neutral_kvka.to_csv(\"./datasets/clean-data-2/set-1/testing-data/kv-ka/force_neutral/force_neutral.csv\", sep=',', index=False, encoding='utf-8')\n",
    "\n",
    "# save testing data uvka\n",
    "DoS_uvka.to_csv(\"./datasets/clean-data-2/set-1/testing-data/uv-ka/dos/dos.csv\", sep=',', index=False, encoding='utf-8')\n",
    "force_neutral_uvka.to_csv(\"./datasets/clean-data-2/set-1/testing-data/uv-ka/force_neutral/force_neutral.csv\", sep=',', index=False, encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-05T14:54:18.468028Z",
     "start_time": "2024-07-05T14:53:36.563909Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==> ml-algo [Initializing]\n",
      "==> ml-algo [KMeans Implemented in 00:01]\n",
      "==> ml-algo [KMeans Implemented in 00:07]\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "dos_dir = \"./datasets/clean-data-2/set-1/training-data/dos/dos.csv\"\n",
    "fn_dir = \"./datasets/clean-data-2/set-1/training-data/force_neutral/force_neutral.csv\"\n",
    "\n",
    "start_time = time.time()\n",
    "\n",
    "dos_dataset = read_dataset_and_split(dos_dir)\n",
    "fn_dataset = read_dataset_and_split(fn_dir)\n",
    "\n",
    "# initialize algo\n",
    "algo_1 = Algo(dos_dataset)\n",
    "algo_2 = Algo(fn_dataset)\n",
    "\n",
    "print(\"==> ml-algo [Initializing]\")\n",
    "\n",
    "# implement individual algo\n",
    "# algo_1.impl_random_forests()\n",
    "# algo_1.impl_xgboost()\n",
    "algo_1.impl_kmeans()\n",
    "\n",
    "# algo_2.impl_random_forests()\n",
    "# algo_2.impl_xgboost()\n",
    "algo_2.impl_kmeans()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-05T15:14:14.287831Z",
     "start_time": "2024-07-05T15:14:14.278770Z"
    }
   },
   "outputs": [],
   "source": [
    "from py_pkg.testing.testing import Testing\n",
    "\n",
    "tests = Testing()\n",
    "\n",
    "def run_all_tests(algo, title):\n",
    "    # # Random Forests\n",
    "    # true_values_pred_values_rf_kv_ka = tests.test_for_2(TestDataType.kv_ka, algo, AlgoToPredict.random_forest)\n",
    "    # test_metrics_rf_kv_ka = tests.generate_all_test_metrics(true_values_pred_values_rf_kv_ka)\n",
    "    \n",
    "    # true_values_pred_values_rf_uv_ka = tests.test_for_2(TestDataType.uv_ka, algo, AlgoToPredict.random_forest)\n",
    "    # test_metrics_rf_uv_ka = tests.generate_all_test_metrics(true_values_pred_values_rf_uv_ka)\n",
    "    \n",
    "    # # save results\n",
    "    # tests.save_testing_results_2(test_metrics_rf_kv_ka, f\"[{title}] Random Forests - KV-KA\")\n",
    "    # tests.save_testing_results_2(test_metrics_rf_uv_ka, f\"[{title}] Random Forests - UV-KA\")\n",
    "    \n",
    "    \n",
    "    # # Extreme Gradient Boosting\n",
    "    # true_values_pred_values_xg_kv_ka = tests.test_for_2(TestDataType.kv_ka, algo, AlgoToPredict.xgboost)\n",
    "    # test_metrics_xg_kv_ka = tests.generate_all_test_metrics(true_values_pred_values_xg_kv_ka)\n",
    "    \n",
    "    # true_values_pred_values_xg_uv_ka = tests.test_for_2(TestDataType.uv_ka, algo, AlgoToPredict.xgboost)\n",
    "    # test_metrics_xg_uv_ka = tests.generate_all_test_metrics(true_values_pred_values_xg_uv_ka)\n",
    "    \n",
    "    # # save results\n",
    "    # tests.save_testing_results_2(test_metrics_xg_kv_ka, f\"[{title}] Extreme Gradient Boosting - KV-KA\")\n",
    "    # tests.save_testing_results_2(test_metrics_xg_uv_ka, f\"[{title}] Extreme Gradient Boosting - UV-KA\")\n",
    "    \n",
    "    # K-Means Clustering\n",
    "    true_values_pred_values_kmeans_kv_ka = tests.test_for_2(TestDataType.kv_ka, algo, AlgoToPredict.k_means)\n",
    "    test_metrics_kmeans_kv_ka = tests.generate_all_test_metrics(true_values_pred_values_kmeans_kv_ka)\n",
    "    \n",
    "    true_values_pred_values_kmeans_uv_ka = tests.test_for_2(TestDataType.uv_ka, algo, AlgoToPredict.k_means)\n",
    "    test_metrics_kmeans_uv_ka = tests.generate_all_test_metrics(true_values_pred_values_kmeans_uv_ka)\n",
    "    \n",
    "    # save results\n",
    "    tests.save_testing_results_2(test_metrics_kmeans_kv_ka, f\"[{title}] K-Means - KV-KA\")\n",
    "    tests.save_testing_results_2(test_metrics_kmeans_uv_ka, f\"[{title}] K-Means - UV-KA\")\n",
    "    \n",
    "    close_time = time.time()\n",
    "    \n",
    "    print(f\"\\n==> ml-algo [Model training for {title} and Testing Completed in {get_human_time(close_time, start_time)}]\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2024-07-05T15:14:17.291108Z"
    },
    "jupyter": {
     "is_executing": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==> ml-algo [Predicting for KMeans - DoS]\n",
      "==> ml-algo [Predicting for KMeans - Force Neutral]\n",
      "\n",
      "==> ml-algo [k_means Predicted in 00:00 for kv_ka test set] \n",
      "\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Target is multiclass but average='binary'. Please choose another average setting, one of [None, 'micro', 'macro', 'weighted'].",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[8], line 4\u001b[0m\n\u001b[1;32m      1\u001b[0m algos \u001b[38;5;241m=\u001b[39m [(algo_1, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mDoS\u001b[39m\u001b[38;5;124m'\u001b[39m), (algo_2, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mForce Neutral\u001b[39m\u001b[38;5;124m'\u001b[39m)]\n\u001b[1;32m      3\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m algo, title \u001b[38;5;129;01min\u001b[39;00m algos:\n\u001b[0;32m----> 4\u001b[0m     \u001b[43mrun_all_tests\u001b[49m\u001b[43m(\u001b[49m\u001b[43malgo\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtitle\u001b[49m\u001b[43m)\u001b[49m\n",
      "Cell \u001b[0;32mIn[7], line 31\u001b[0m, in \u001b[0;36mrun_all_tests\u001b[0;34m(algo, title)\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mrun_all_tests\u001b[39m(algo, title):\n\u001b[1;32m      6\u001b[0m     \u001b[38;5;66;03m# # Random Forests\u001b[39;00m\n\u001b[1;32m      7\u001b[0m     \u001b[38;5;66;03m# true_values_pred_values_rf_kv_ka = tests.test_for_2(TestDataType.kv_ka, algo, AlgoToPredict.random_forest)\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     28\u001b[0m     \n\u001b[1;32m     29\u001b[0m     \u001b[38;5;66;03m# K-Means Clustering\u001b[39;00m\n\u001b[1;32m     30\u001b[0m     true_values_pred_values_kmeans_kv_ka \u001b[38;5;241m=\u001b[39m tests\u001b[38;5;241m.\u001b[39mtest_for_2(TestDataType\u001b[38;5;241m.\u001b[39mkv_ka, algo, AlgoToPredict\u001b[38;5;241m.\u001b[39mk_means)\n\u001b[0;32m---> 31\u001b[0m     test_metrics_kmeans_kv_ka \u001b[38;5;241m=\u001b[39m \u001b[43mtests\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgenerate_all_test_metrics\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtrue_values_pred_values_kmeans_kv_ka\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     33\u001b[0m     true_values_pred_values_kmeans_uv_ka \u001b[38;5;241m=\u001b[39m tests\u001b[38;5;241m.\u001b[39mtest_for_2(TestDataType\u001b[38;5;241m.\u001b[39muv_ka, algo, AlgoToPredict\u001b[38;5;241m.\u001b[39mk_means)\n\u001b[1;32m     34\u001b[0m     test_metrics_kmeans_uv_ka \u001b[38;5;241m=\u001b[39m tests\u001b[38;5;241m.\u001b[39mgenerate_all_test_metrics(true_values_pred_values_kmeans_uv_ka)\n",
      "File \u001b[0;32m~/Automotive-IDS-ML/py_pkg/testing/testing.py:103\u001b[0m, in \u001b[0;36mTesting.generate_all_test_metrics\u001b[0;34m(self, true_values_and_predicted_values)\u001b[0m\n\u001b[1;32m    100\u001b[0m     y_true \u001b[38;5;241m=\u001b[39m true_values[i]\n\u001b[1;32m    101\u001b[0m     y_pred \u001b[38;5;241m=\u001b[39m predicted_values[i]\n\u001b[0;32m--> 103\u001b[0m     class_report, df, cm \u001b[38;5;241m=\u001b[39m \u001b[43mgenerate_metrics_report\u001b[49m\u001b[43m(\u001b[49m\u001b[43my_true\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43my_pred\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    104\u001b[0m     resulting_metrics\u001b[38;5;241m.\u001b[39mappend((class_report, df, cm))\n\u001b[1;32m    106\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m resulting_metrics\n",
      "File \u001b[0;32m~/Automotive-IDS-ML/py_pkg/core/core.py:37\u001b[0m, in \u001b[0;36mgenerate_metrics_report\u001b[0;34m(y_true, y_pred)\u001b[0m\n\u001b[1;32m     35\u001b[0m accuracy \u001b[38;5;241m=\u001b[39m accuracy_score(y_true, y_pred)\n\u001b[1;32m     36\u001b[0m cm \u001b[38;5;241m=\u001b[39m confusion_matrix(y_true, y_pred)\n\u001b[0;32m---> 37\u001b[0m precision \u001b[38;5;241m=\u001b[39m \u001b[43mprecision_score\u001b[49m\u001b[43m(\u001b[49m\u001b[43my_true\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43my_pred\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     38\u001b[0m recall \u001b[38;5;241m=\u001b[39m recall_score(y_true, y_pred)\n\u001b[1;32m     39\u001b[0m f1 \u001b[38;5;241m=\u001b[39m f1_score(y_true, y_pred)\n",
      "File \u001b[0;32m/home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/sklearn/utils/_param_validation.py:214\u001b[0m, in \u001b[0;36mvalidate_params.<locals>.decorator.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    208\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m    209\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m config_context(\n\u001b[1;32m    210\u001b[0m         skip_parameter_validation\u001b[38;5;241m=\u001b[39m(\n\u001b[1;32m    211\u001b[0m             prefer_skip_nested_validation \u001b[38;5;129;01mor\u001b[39;00m global_skip_validation\n\u001b[1;32m    212\u001b[0m         )\n\u001b[1;32m    213\u001b[0m     ):\n\u001b[0;32m--> 214\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    215\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m InvalidParameterError \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m    216\u001b[0m     \u001b[38;5;66;03m# When the function is just a wrapper around an estimator, we allow\u001b[39;00m\n\u001b[1;32m    217\u001b[0m     \u001b[38;5;66;03m# the function to delegate validation to the estimator, but we replace\u001b[39;00m\n\u001b[1;32m    218\u001b[0m     \u001b[38;5;66;03m# the name of the estimator by the name of the function in the error\u001b[39;00m\n\u001b[1;32m    219\u001b[0m     \u001b[38;5;66;03m# message to avoid confusion.\u001b[39;00m\n\u001b[1;32m    220\u001b[0m     msg \u001b[38;5;241m=\u001b[39m re\u001b[38;5;241m.\u001b[39msub(\n\u001b[1;32m    221\u001b[0m         \u001b[38;5;124mr\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mparameter of \u001b[39m\u001b[38;5;124m\\\u001b[39m\u001b[38;5;124mw+ must be\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    222\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mparameter of \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mfunc\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__qualname__\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m must be\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    223\u001b[0m         \u001b[38;5;28mstr\u001b[39m(e),\n\u001b[1;32m    224\u001b[0m     )\n",
      "File \u001b[0;32m/home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/sklearn/metrics/_classification.py:2131\u001b[0m, in \u001b[0;36mprecision_score\u001b[0;34m(y_true, y_pred, labels, pos_label, average, sample_weight, zero_division)\u001b[0m\n\u001b[1;32m   1973\u001b[0m \u001b[38;5;129m@validate_params\u001b[39m(\n\u001b[1;32m   1974\u001b[0m     {\n\u001b[1;32m   1975\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124my_true\u001b[39m\u001b[38;5;124m\"\u001b[39m: [\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124marray-like\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msparse matrix\u001b[39m\u001b[38;5;124m\"\u001b[39m],\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   2000\u001b[0m     zero_division\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mwarn\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m   2001\u001b[0m ):\n\u001b[1;32m   2002\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Compute the precision.\u001b[39;00m\n\u001b[1;32m   2003\u001b[0m \n\u001b[1;32m   2004\u001b[0m \u001b[38;5;124;03m    The precision is the ratio ``tp / (tp + fp)`` where ``tp`` is the number of\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   2129\u001b[0m \u001b[38;5;124;03m    array([0.5, 1. , 1. ])\u001b[39;00m\n\u001b[1;32m   2130\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m-> 2131\u001b[0m     p, _, _, _ \u001b[38;5;241m=\u001b[39m \u001b[43mprecision_recall_fscore_support\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   2132\u001b[0m \u001b[43m        \u001b[49m\u001b[43my_true\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2133\u001b[0m \u001b[43m        \u001b[49m\u001b[43my_pred\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2134\u001b[0m \u001b[43m        \u001b[49m\u001b[43mlabels\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlabels\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2135\u001b[0m \u001b[43m        \u001b[49m\u001b[43mpos_label\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpos_label\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2136\u001b[0m \u001b[43m        \u001b[49m\u001b[43maverage\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43maverage\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2137\u001b[0m \u001b[43m        \u001b[49m\u001b[43mwarn_for\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mprecision\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2138\u001b[0m \u001b[43m        \u001b[49m\u001b[43msample_weight\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msample_weight\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2139\u001b[0m \u001b[43m        \u001b[49m\u001b[43mzero_division\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mzero_division\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2140\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   2141\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m p\n",
      "File \u001b[0;32m/home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/sklearn/utils/_param_validation.py:187\u001b[0m, in \u001b[0;36mvalidate_params.<locals>.decorator.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    185\u001b[0m global_skip_validation \u001b[38;5;241m=\u001b[39m get_config()[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mskip_parameter_validation\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[1;32m    186\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m global_skip_validation:\n\u001b[0;32m--> 187\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    189\u001b[0m func_sig \u001b[38;5;241m=\u001b[39m signature(func)\n\u001b[1;32m    191\u001b[0m \u001b[38;5;66;03m# Map *args/**kwargs to the function signature\u001b[39;00m\n",
      "File \u001b[0;32m/home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1724\u001b[0m, in \u001b[0;36mprecision_recall_fscore_support\u001b[0;34m(y_true, y_pred, beta, labels, pos_label, average, warn_for, sample_weight, zero_division)\u001b[0m\n\u001b[1;32m   1566\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Compute precision, recall, F-measure and support for each class.\u001b[39;00m\n\u001b[1;32m   1567\u001b[0m \n\u001b[1;32m   1568\u001b[0m \u001b[38;5;124;03mThe precision is the ratio ``tp / (tp + fp)`` where ``tp`` is the number of\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1721\u001b[0m \u001b[38;5;124;03m array([2, 2, 2]))\u001b[39;00m\n\u001b[1;32m   1722\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m   1723\u001b[0m zero_division_value \u001b[38;5;241m=\u001b[39m _check_zero_division(zero_division)\n\u001b[0;32m-> 1724\u001b[0m labels \u001b[38;5;241m=\u001b[39m \u001b[43m_check_set_wise_labels\u001b[49m\u001b[43m(\u001b[49m\u001b[43my_true\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43my_pred\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43maverage\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlabels\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpos_label\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1726\u001b[0m \u001b[38;5;66;03m# Calculate tp_sum, pred_sum, true_sum ###\u001b[39;00m\n\u001b[1;32m   1727\u001b[0m samplewise \u001b[38;5;241m=\u001b[39m average \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msamples\u001b[39m\u001b[38;5;124m\"\u001b[39m\n",
      "File \u001b[0;32m/home/zeus/miniconda3/envs/cloudspace/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1518\u001b[0m, in \u001b[0;36m_check_set_wise_labels\u001b[0;34m(y_true, y_pred, average, labels, pos_label)\u001b[0m\n\u001b[1;32m   1516\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m y_type \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmulticlass\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m   1517\u001b[0m             average_options\u001b[38;5;241m.\u001b[39mremove(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msamples\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m-> 1518\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m   1519\u001b[0m             \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mTarget is \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m but average=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mbinary\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m. Please \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m   1520\u001b[0m             \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mchoose another average setting, one of \u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m%\u001b[39m (y_type, average_options)\n\u001b[1;32m   1521\u001b[0m         )\n\u001b[1;32m   1522\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m pos_label \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m (\u001b[38;5;28;01mNone\u001b[39;00m, \u001b[38;5;241m1\u001b[39m):\n\u001b[1;32m   1523\u001b[0m     warnings\u001b[38;5;241m.\u001b[39mwarn(\n\u001b[1;32m   1524\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mNote that pos_label (set to \u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[38;5;124m) is ignored when \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m   1525\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124maverage != \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mbinary\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m (got \u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[38;5;124m). You may use \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1528\u001b[0m         \u001b[38;5;167;01mUserWarning\u001b[39;00m,\n\u001b[1;32m   1529\u001b[0m     )\n",
      "\u001b[0;31mValueError\u001b[0m: Target is multiclass but average='binary'. Please choose another average setting, one of [None, 'micro', 'macro', 'weighted']."
     ]
    }
   ],
   "source": [
    "algos = [(algo_1, 'DoS'), (algo_2, 'Force Neutral')]\n",
    "\n",
    "for algo, title in algos:\n",
    "    run_all_tests(algo, title)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}