DataProcessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from scipy.io import arff\n",
    "\n",
    "# Load ARFF file\n",
    "data, meta = arff.loadarff('Data\\Training Dataset.arff')\n",
    "\n",
    "# Convert to DataFrame\n",
    "df = pd.DataFrame(data)\n",
    "\n",
    "# Save DataFrame to CSV\n",
    "df.to_csv('Data\\PhishingWebsites.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\anany\\AppData\\Local\\Temp\\ipykernel_15588\\1264249751.py:2: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n",
      "  df = df.applymap(lambda x: x.decode('utf-8') if isinstance(x, bytes) else x)\n"
     ]
    }
   ],
   "source": [
    "# Remove the 'b' prefix from the byte format\n",
    "df = df.applymap(lambda x: x.decode('utf-8') if isinstance(x, bytes) else x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert columns to numeric where applicable\n",
    "df = df.apply(pd.to_numeric, errors='coerce')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop irrelevant or less useful columns\n",
    "columns_to_drop = ['port', 'Favicon', 'on_mouseover', 'RightClick', 'popUpWidnow', 'Iframe', 'Redirect']\n",
    "df_cleaned = df.drop(columns=columns_to_drop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Missing values in each column:\n",
      " having_IP_Address              0\n",
      "URL_Length                     0\n",
      "Shortining_Service             0\n",
      "having_At_Symbol               0\n",
      "double_slash_redirecting       0\n",
      "Prefix_Suffix                  0\n",
      "having_Sub_Domain              0\n",
      "SSLfinal_State                 0\n",
      "Domain_registeration_length    0\n",
      "HTTPS_token                    0\n",
      "Request_URL                    0\n",
      "URL_of_Anchor                  0\n",
      "Links_in_tags                  0\n",
      "SFH                            0\n",
      "Submitting_to_email            0\n",
      "Abnormal_URL                   0\n",
      "age_of_domain                  0\n",
      "DNSRecord                      0\n",
      "web_traffic                    0\n",
      "Page_Rank                      0\n",
      "Google_Index                   0\n",
      "Links_pointing_to_page         0\n",
      "Statistical_report             0\n",
      "Result                         0\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# Check for missing values in the entire DataFrame\n",
    "missing_values = df_cleaned.isnull().sum()\n",
    "# Print columns with missing values and the count of missing values\n",
    "print(\"Missing values in each column:\\n\", missing_values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data types of each column:\n",
      " having_IP_Address              int64\n",
      "URL_Length                     int64\n",
      "Shortining_Service             int64\n",
      "having_At_Symbol               int64\n",
      "double_slash_redirecting       int64\n",
      "Prefix_Suffix                  int64\n",
      "having_Sub_Domain              int64\n",
      "SSLfinal_State                 int64\n",
      "Domain_registeration_length    int64\n",
      "HTTPS_token                    int64\n",
      "Request_URL                    int64\n",
      "URL_of_Anchor                  int64\n",
      "Links_in_tags                  int64\n",
      "SFH                            int64\n",
      "Submitting_to_email            int64\n",
      "Abnormal_URL                   int64\n",
      "age_of_domain                  int64\n",
      "DNSRecord                      int64\n",
      "web_traffic                    int64\n",
      "Page_Rank                      int64\n",
      "Google_Index                   int64\n",
      "Links_pointing_to_page         int64\n",
      "Statistical_report             int64\n",
      "Result                         int64\n",
      "dtype: object\n"
     ]
    }
   ],
   "source": [
    "# Check the data types of each column\n",
    "print(\"Data types of each column:\\n\", df_cleaned.dtypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Column 'having_IP_Address' unique values: [-1  1]\n",
      "Column 'URL_Length' unique values: [ 1  0 -1]\n",
      "Column 'Shortining_Service' unique values: [ 1 -1]\n",
      "Column 'having_At_Symbol' unique values: [ 1 -1]\n",
      "Column 'double_slash_redirecting' unique values: [-1  1]\n",
      "Column 'Prefix_Suffix' unique values: [-1  1]\n",
      "Column 'having_Sub_Domain' unique values: [-1  0  1]\n",
      "Column 'SSLfinal_State' unique values: [-1  1  0]\n",
      "Column 'Domain_registeration_length' unique values: [-1  1]\n",
      "Column 'HTTPS_token' unique values: [-1  1]\n",
      "Column 'Request_URL' unique values: [ 1 -1]\n",
      "Column 'URL_of_Anchor' unique values: [-1  0  1]\n",
      "Column 'Links_in_tags' unique values: [ 1 -1  0]\n",
      "Column 'SFH' unique values: [-1  1  0]\n",
      "Column 'Submitting_to_email' unique values: [-1  1]\n",
      "Column 'Abnormal_URL' unique values: [-1  1]\n",
      "Column 'age_of_domain' unique values: [-1  1]\n",
      "Column 'DNSRecord' unique values: [-1  1]\n",
      "Column 'web_traffic' unique values: [-1  0  1]\n",
      "Column 'Page_Rank' unique values: [-1  1]\n",
      "Column 'Google_Index' unique values: [ 1 -1]\n",
      "Column 'Links_pointing_to_page' unique values: [ 1  0 -1]\n",
      "Column 'Statistical_report' unique values: [-1  1]\n",
      "Column 'Result' unique values: [-1  1]\n"
     ]
    }
   ],
   "source": [
    "# Check unique values for each column\n",
    "for col in df_cleaned.columns:\n",
    "    unique_values = df_cleaned[col].unique()\n",
    "    print(f\"Column '{col}' unique values: {unique_values}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   having_IP_Address  Shortining_Service  having_At_Symbol  \\\n",
      "0                  0                   1                 1   \n",
      "1                  1                   1                 1   \n",
      "2                  1                   1                 1   \n",
      "3                  1                   1                 1   \n",
      "4                  1                   0                 1   \n",
      "\n",
      "   double_slash_redirecting  Prefix_Suffix  Domain_registeration_length  \\\n",
      "0                         0              0                            0   \n",
      "1                         1              0                            0   \n",
      "2                         1              0                            0   \n",
      "3                         1              0                            1   \n",
      "4                         1              0                            0   \n",
      "\n",
      "   Favicon  port  HTTPS_token  Request_URL  ...  having_Sub_Domain_0  \\\n",
      "0        1     1            0            1  ...                False   \n",
      "1        1     1            0            1  ...                 True   \n",
      "2        1     1            0            1  ...                False   \n",
      "3        1     1            0            0  ...                False   \n",
      "4        1     1            1            1  ...                False   \n",
      "\n",
      "   having_Sub_Domain_1  SSLfinal_State_0  SSLfinal_State_1  URL_of_Anchor_0  \\\n",
      "0                False             False             False            False   \n",
      "1                False             False              True             True   \n",
      "2                False             False             False             True   \n",
      "3                False             False             False             True   \n",
      "4                 True             False              True             True   \n",
      "\n",
      "   URL_of_Anchor_1  Links_in_tags_0  Links_in_tags_1  web_traffic_0  \\\n",
      "0            False            False             True          False   \n",
      "1            False            False            False           True   \n",
      "2            False            False            False          False   \n",
      "3            False             True            False          False   \n",
      "4            False             True            False           True   \n",
      "\n",
      "   web_traffic_1  \n",
      "0          False  \n",
      "1          False  \n",
      "2           True  \n",
      "3           True  \n",
      "4          False  \n",
      "\n",
      "[5 rows x 37 columns]\n"
     ]
    }
   ],
   "source": [
    "# Step 1: Identify binary columns (those that only have -1 and 1)\n",
    "binary_columns = [col for col in df.columns if df[col].nunique() == 2]\n",
    "\n",
    "# Step 2: Encode binary columns\n",
    "for col in binary_columns:\n",
    "    df[col] = df[col].replace({-1: 0, 1: 1})\n",
    "\n",
    "# Step 3: Identify multi-class columns (if needed, for example)\n",
    "# Here we assume 'URL_Length' has multiple values which should be one-hot encoded.\n",
    "multi_class_columns = ['URL_Length', 'having_Sub_Domain', 'SSLfinal_State', 'URL_of_Anchor', 'Links_in_tags', 'web_traffic']\n",
    "\n",
    "# Step 4: Apply one-hot encoding for multi-class columns\n",
    "df = pd.get_dummies(df, columns=multi_class_columns, drop_first=True)\n",
    "\n",
    "# Display the updated DataFrame\n",
    "print(df.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Column Names:\n",
      "having_IP_Address\n",
      "Shortining_Service\n",
      "having_At_Symbol\n",
      "double_slash_redirecting\n",
      "Prefix_Suffix\n",
      "Domain_registeration_length\n",
      "Favicon\n",
      "port\n",
      "HTTPS_token\n",
      "Request_URL\n",
      "SFH\n",
      "Submitting_to_email\n",
      "Abnormal_URL\n",
      "Redirect\n",
      "on_mouseover\n",
      "RightClick\n",
      "popUpWidnow\n",
      "Iframe\n",
      "age_of_domain\n",
      "DNSRecord\n",
      "Page_Rank\n",
      "Google_Index\n",
      "Links_pointing_to_page\n",
      "Statistical_report\n",
      "Result\n",
      "URL_Length_0\n",
      "URL_Length_1\n",
      "having_Sub_Domain_0\n",
      "having_Sub_Domain_1\n",
      "SSLfinal_State_0\n",
      "SSLfinal_State_1\n",
      "URL_of_Anchor_0\n",
      "URL_of_Anchor_1\n",
      "Links_in_tags_0\n",
      "Links_in_tags_1\n",
      "web_traffic_0\n",
      "web_traffic_1\n"
     ]
    }
   ],
   "source": [
    "# Assuming your DataFrame is named 'df'\n",
    "column_names = df.columns.tolist()  # Convert to a list for better readability\n",
    "print(\"Column Names:\")\n",
    "for name in column_names:\n",
    "    print(name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Column Names:\n",
      "['having_IP_Address', 'Shortining_Service', 'having_At_Symbol', 'double_slash_redirecting', 'Prefix_Suffix', 'Domain_registeration_length', 'Favicon', 'port', 'HTTPS_token', 'Request_URL', 'SFH', 'Submitting_to_email', 'Abnormal_URL', 'Redirect', 'on_mouseover', 'RightClick', 'popUpWidnow', 'Iframe', 'age_of_domain', 'DNSRecord', 'Page_Rank', 'Google_Index', 'Links_pointing_to_page', 'Statistical_report', 'Result', 'URL_Length_0', 'URL_Length_1', 'having_Sub_Domain_0', 'having_Sub_Domain_1', 'SSLfinal_State_0', 'SSLfinal_State_1', 'URL_of_Anchor_0', 'URL_of_Anchor_1', 'Links_in_tags_0', 'Links_in_tags_1', 'web_traffic_0', 'web_traffic_1']\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "# Set pandas options to display all columns\n",
    "pd.set_option('display.max_columns', None)  # Set to None to display all columns\n",
    "\n",
    "# Now print all column names\n",
    "print(\"Column Names:\")\n",
    "print(df.columns.tolist())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save the cleaned DataFrame to a new CSV file\n",
    "cleaned_file_path = 'cleaned_Phishing.csv'  # Specify the path and filename\n",
    "df.to_csv(cleaned_file_path, index=False)  # Set index=False to avoid writing row indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}