data_preparation.ipynb

{
    "cells": [
        {
            "cell_type": "markdown",
            "metadata": {
                "collapsed": true
            },
            "source": [
                "## Data Preparation Notebook"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Imports"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "from sklearn.feature_selection import SelectKBest\n",
                "from sklearn.feature_selection import chi2\n",
                "from sklearn.preprocessing import OrdinalEncoder,OneHotEncoder,LabelEncoder,MinMaxScaler\n",
                "from sklearn.feature_selection import mutual_info_classif\n",
                "from sklearn.compose import ColumnTransformer\n",
                "from sklearn.pipeline import Pipeline\n",
                "from botocore.client import Config\n",
                "from ibm_watson_studio_pipelines import WSPipelines\n",
                "import matplotlib.pyplot as plt\n",
                "import seaborn as sns\n",
                "import heapq\n",
                "import pickle\n",
                "import os, types\n",
                "import pandas as pd\n",
                "import ibm_boto3"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "### Load the validated training and test data from IBM COS \n",
                "\n",
                "```\n",
                "## PROJECT COS \n",
                "AUTH_ENDPOINT = \"https://iam.cloud.ibm.com/oidc/token\"\n",
                "ENDPOINT_URL = \"https://s3.private.us.cloud-object-storage.appdomain.cloud\"\n",
                "API_KEY_COS = \"xxx\"\n",
                "BUCKET_PROJECT_COS = \"mlops-donotdelete-pr-qxxcecxi1dtw94\"\n",
                "\n",
                "\n",
                "##MLOPS COS\n",
                "ENDPOINT_URL_MLOPS = \"https://s3.jp-tok.cloud-object-storage.appdomain.cloud\"\n",
                "API_KEY_MLOPS = \"xxx\"\n",
                "CRN_MLOPS = \"xxx\"\n",
                "BUCKET_MLOPS  = \"mlops-asset\"\n",
                "```"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Pipeline Params"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "CLOUD_API_KEY = os.getenv(\"cloud_api_key\")"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "## Retrieve cos credentials from global pipeline parameters\n",
                "import json\n",
                "# Get json from environment and convert to string\n",
                "project_cos_credentials = json.loads(os.getenv('project_cos_credentials'))\n",
                "mlops_cos_credentials = json.loads(os.getenv('mlops_cos_credentials'))\n",
                "\n",
                "## PROJECT COS \n",
                "AUTH_ENDPOINT = project_cos_credentials['AUTH_ENDPOINT']\n",
                "ENDPOINT_URL = project_cos_credentials['ENDPOINT_URL']\n",
                "API_KEY_COS = project_cos_credentials['API_KEY']\n",
                "BUCKET_PROJECT_COS = project_cos_credentials['BUCKET']\n",
                "\n",
                "## MLOPS COS\n",
                "ENDPOINT_URL_MLOPS = mlops_cos_credentials['ENDPOINT_URL']\n",
                "API_KEY_MLOPS = mlops_cos_credentials['API_KEY']\n",
                "CRN_MLOPS = mlops_cos_credentials['CRN']\n",
                "BUCKET_MLOPS  = mlops_cos_credentials['BUCKET']"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "###  Read and Write Utility"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "def read_data_from_mlops_cos(key):\n",
                "    def __iter__(self): return 0\n",
                "    MLOPS_DATA_STORE_client = ibm_boto3.client(\n",
                "        service_name='s3',\n",
                "        ibm_api_key_id=API_KEY_MLOPS,\n",
                "        ibm_service_instance_id=CRN_MLOPS,\n",
                "        ibm_auth_endpoint=AUTH_ENDPOINT,\n",
                "        config=Config(signature_version='oauth'),\n",
                "        endpoint_url=ENDPOINT_URL_MLOPS)\n",
                "\n",
                "    body = MLOPS_DATA_STORE_client.get_object(Bucket=BUCKET_MLOPS, Key=key)['Body']\n",
                "    # add missing __iter__ method, so pandas accepts body as file-like object\n",
                "    if not hasattr(body, \"__iter__\"): body.__iter__ = types.MethodType( __iter__, body )\n",
                "\n",
                "    gcf_df = pd.read_csv(body)\n",
                "    return gcf_df\n",
                "\n",
                "def save_data_in_cos(df,filename,key,credentials,pipe=False):\n",
                "    \"\"\"\n",
                "\n",
                "    Save Data in IBM Cloud Object Storage\n",
                "\n",
                "    \"\"\"\n",
                "    try:\n",
                "        \n",
                "        if pipe:\n",
                "            with open (filename,'wb') as f:\n",
                "                pickle.dump(pipeline,f)\n",
                "        else:\n",
                "            df.to_csv(filename,index=False)\n",
                "        mlops_res = ibm_boto3.resource(\n",
                "            service_name='s3',\n",
                "            ibm_api_key_id=credentials['API_KEY'],\n",
                "            ibm_auth_endpoint=credentials['AUTH_ENDPOINT'],\n",
                "            config=Config(signature_version='oauth'),\n",
                "            endpoint_url=credentials['ENDPOINT_URL'])\n",
                "\n",
                "        mlops_res.Bucket(credentials['BUCKET']).upload_file(filename,key)\n",
                "        print(f\"File {filename} uploaded successfully\")\n",
                "    except Exception as e:\n",
                "        print(e)\n",
                "        print(\"File upload for {filename} failed\")\n",
                "\n",
                "        \n",
                "def check_if_file_exists(filename):\n",
                "    mlops_client = ibm_boto3.client(\n",
                "        service_name='s3',\n",
                "        ibm_api_key_id=API_KEY_MLOPS,\n",
                "        ibm_service_instance_id=CRN_MLOPS,\n",
                "        ibm_auth_endpoint=AUTH_ENDPOINT,\n",
                "        config=Config(signature_version='oauth'),\n",
                "        endpoint_url=ENDPOINT_URL_MLOPS)\n",
                "    \n",
                "    for key in mlops_client.list_objects(Bucket=BUCKET_MLOPS)['Contents']:\n",
                "        files = key['Key']\n",
                "        if files == filename:\n",
                "            return True\n",
                "    return False\n"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Train_Data "
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "train_data = read_data_from_mlops_cos('train_gcr.csv')\n",
                "train_data.head()"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "train_data.info()"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "train_data.describe()"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "object_df = train_data.select_dtypes('O')\n",
                "object_df.head()"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "object_cols = list(set(object_df.columns.tolist()) - set(['Risk']))\n",
                "object_cols"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "numerical_columns = [col for col in train_data.columns.tolist() if col not in object_cols and col!='Risk']"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Load Test Data "
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "test_data = read_data_from_mlops_cos('test_gcr.csv')\n",
                "test_data.head()"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Split X and Y "
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "y_train = train_data['Risk']\n",
                "X_train = train_data.drop(\"Risk\",axis=1)\n",
                "\n",
                "\n",
                "y_test = test_data['Risk']\n",
                "X_test = test_data.drop(\"Risk\",axis=1)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Categorcial Feature Analysis "
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "def prepare_input_data(X_train, X_test):\n",
                "    oe = OrdinalEncoder()\n",
                "    oe.fit(X_train)\n",
                "    X_train_enc = oe.transform(X_train)\n",
                "    X_test_enc = oe.transform(X_test)\n",
                "    return X_train_enc, X_test_enc\n",
                "\n",
                "\n",
                "def prepare_output_data(y_train, y_test):\n",
                "    le = LabelEncoder()\n",
                "    le.fit(y_train)\n",
                "    y_train_enc = le.transform(y_train)\n",
                "    y_test_enc = le.transform(y_test)\n",
                "    return y_train_enc, y_test_enc\n",
                "\n",
                "\n",
                "def select_best_chi2_features(X_train, y_train, X_test,score_func=chi2):\n",
                "    featureselector = SelectKBest(score_func=chi2, k='all')\n",
                "    featureselector.fit(X_train, y_train)\n",
                "    X_train_best_feat = featureselector.transform(X_train)\n",
                "    X_test_best_feat= featureselector.transform(X_test)\n",
                "    return X_train_best_feat, X_test_best_feat, featureselector\n",
                "\n",
                "\n",
                "def select_best_mutualinf_features(X_train, y_train, X_test,k=5):\n",
                "    featureselector = SelectKBest(score_func=mutual_info_classif, k=k)\n",
                "    featureselector.fit(X_train, y_train)\n",
                "    X_train_best_feat = fs.transform(X_train)\n",
                "    X_test_best_feat= fs.transform(X_test)\n",
                "    return X_train_best_feat, X_test_best_feat, featureselector\n",
                "\n",
                "def plot_scores():\n",
                "    plt.figure(figsize=(14, 12))\n",
                "    plt.subplot(221)\n",
                "\n",
                "    ax1 = sns.barplot([i for i in range(len(fs.scores_))], fs.scores_)\n",
                "    ax1.set_title(\"Chi2 Importance Scores\", fontsize=20)\n",
                "    ax1.set_xlabel(\"Features\",fontsize=15)\n",
                "    ax1.set_ylabel(\"Chi2 Scores\",fontsize=15)\n",
                "    \n",
                "    \n",
                "def get_top_k_catgeorical(fs,train_cat,k=10):\n",
                "    fs_score_map = {}\n",
                "    for i in range(len(fs.scores_)):\n",
                "        #print(f\"Feature {train_cat.columns.tolist()[i]} {fs.scores_[i]}\")\n",
                "        fs_score_map[train_cat.columns.tolist()[i]] = fs.scores_[i]\n",
                "        \n",
                "    k_keys_sorted_by_values = heapq.nlargest(k, fs_score_map, key=fs_score_map.get)\n",
                "    \n",
                "    return k_keys_sorted_by_values"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Encode and shape the Variables "
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "X_train_enc, X_test_enc = prepare_input_data(X_train[object_cols], X_test[object_cols])\n",
                "\n",
                "y_train_enc, y_test_enc = prepare_output_data(y_train, y_test)\n",
                "\n",
                "X_train_fs, X_test_fs, fs = select_best_chi2_features(X_train_enc, y_train_enc, X_test_enc)\n",
                "\n",
                "plot_scores()\n"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Top K Categorical Features  based on Chi2"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "top_k_cat = get_top_k_catgeorical(fs,X_train[object_cols])\n",
                "top_k_cat"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Top K Categorical Features  based on Mutual Information Feature Selection"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "X_train_enc_mf, X_test_enc_mf = prepare_input_data(X_train[object_cols], X_test[object_cols])\n",
                "\n",
                "y_train_enc_mf, y_test_enc_mf = prepare_output_data(y_train, y_test)\n",
                "\n",
                "X_train_fs_mf, X_test_fs_mf, fs_mf = select_best_chi2_features(X_train_enc_mf, y_train_enc_mf, X_test_enc_mf)\n",
                "\n",
                "plot_scores()"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "top_k_cat_mf = get_top_k_catgeorical(fs_mf,X_train[object_cols])\n",
                "top_k_cat_mf"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "union_features = list(set(top_k_cat+top_k_cat_mf))\n",
                "if \"Sex\" not in union_features:\n",
                "    union_features.append(\"Sex\")\n",
                "union_features"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Filter the Top K Categorical features and Merge to Original Train and Test Dataframes"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "X_train_object_filtered = X_train[union_features]\n",
                "X_test_object_filtered = X_test[union_features]\n",
                "\n",
                "X_train_final = pd.concat([X_train[numerical_columns],X_train_object_filtered],axis=1)\n",
                "\n",
                "X_test_final = pd.concat([X_test[numerical_columns],X_test_object_filtered],axis=1)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Use Column Transformer and Pipelines to encode the Input and Output Variables . Scale the Numerical columns using MinMaxScaler."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "numerical_ix = X_train_final.select_dtypes(include=['int64', 'float64']).columns\n",
                "categorical_ix = X_train_final.select_dtypes(include=['object', 'bool']).columns"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "encoding_steps = [('cat', OrdinalEncoder(), categorical_ix), ('num', MinMaxScaler(), numerical_ix)]\n",
                "col_transform = ColumnTransformer(transformers=encoding_steps)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "pipeline = Pipeline(steps=[('prep',col_transform)])"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "train_final = pd.concat([X_train_final,y_train],axis=1)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "test_final = pd.concat([X_test_final,y_test],axis=1)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "#encoded_train = pd.DataFrame(pipeline.fit_transform(X_train_final),columns=X_train_final.columns)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "#encoded_test = pd.DataFrame(pipeline.transform(X_test_final),columns=X_test_final.columns)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Save the Prepared Data to IBM COS"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "save_data_in_cos(train_final,\"train_tfr.csv\",\"train_tfr.csv\",mlops_cos_credentials)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "save_data_in_cos(test_final,\"test_tfr.csv\",\"test_tfr.csv\",mlops_cos_credentials)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "save_data_in_cos(pipeline,\"feature_encode.pickle\",\"feature_encode.pickle\",mlops_cos_credentials,pipe=True)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Check if files have been copied "
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "data_prep_done = check_if_file_exists(\"train_tfr.csv\") and check_if_file_exists(\"test_tfr.csv\") and check_if_file_exists(\"feature_encode.pickle\")\n",
                "data_prep_done"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Store Params in WS Pipelines"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "preparation_params = {}\n",
                "preparation_params['data_prep_done'] = data_prep_done\n",
                "\n",
                "pipelines_client = WSPipelines.from_apikey(apikey=CLOUD_API_KEY)\n",
                "pipelines_client.store_results(preparation_params)"
            ]
        }
    ],
    "metadata": {
        "kernelspec": {
            "display_name": "Python 3",
            "language": "python",
            "name": "python3"
        },
        "language_info": {
            "codemirror_mode": {
                "name": "ipython",
                "version": 3
            },
            "file_extension": ".py",
            "mimetype": "text/x-python",
            "name": "python",
            "nbconvert_exporter": "python",
            "pygments_lexer": "ipython3",
            "version": "3.10.10"
        },
        "vscode": {
            "interpreter": {
                "hash": "bd385fe162c5ca0c84973b7dd5c518456272446b2b64e67c2a69f949ca7a1754"
            }
        }
    },
    "nbformat": 4,
    "nbformat_minor": 1
}