ST_Dialogue_Analysis_Set_Up.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "514c041c",
   "metadata": {},
   "source": [
    "## Star Trek Dialogue Analysis - Set Up"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "345b0691",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import pandas as pd\n",
    "import os\n",
    "from itertools import chain\n",
    "# import re\n",
    "# import numpy as np\n",
    "# import logging\n",
    "# from collections import Counter\n",
    "\n",
    "# # Spacy\n",
    "# import spacy\n",
    "\n",
    "# # NLTK\n",
    "# import nltk\n",
    "# from nltk.tokenize import word_tokenize\n",
    "# from nltk.corpus import stopwords\n",
    "# from nltk.stem import WordNetLemmatizer\n",
    "\n",
    "# # Gensim\n",
    "# import gensim\n",
    "# from gensim.utils import simple_preprocess\n",
    "# from gensim import corpora\n",
    "# from gensim.models import LdaModel\n",
    "# from pprint import pprint\n",
    "# from gensim.models import CoherenceModel\n",
    "\n",
    "# # Plotting\n",
    "# import seaborn as sns\n",
    "# import pyLDAvis\n",
    "# import pyLDAvis.gensim_models\n",
    "# import matplotlib.pyplot as plt\n",
    "# %matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9fb89f9d",
   "metadata": {},
   "outputs": [],
   "source": [
    "path = r'C:\\Users\\anon\\Documents\\CareerFoundry\\Data Analytics Immersion\\6\\Data\\Raw Data'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "372821cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reading in the whole data set as dictionary\n",
    "\n",
    "with open(os.path.join(path, 'StarTrekDialogue.json'), 'r') as read_file:\n",
    "    all_series = json.load(read_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f43b12bf",
   "metadata": {},
   "source": [
    "### Wrangling & Set Up"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c90527c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define series variables and additional data sources for gender, etc. to be mapped, merged, and used to create series specific dataframes\n",
    "\n",
    "series_dict = {\n",
    "    'tos': {'data': all_series['TOS'], 'data_file': os.path.join(path, 'tos_data.csv'), 'gender_file': os.path.join(path, 'tos_gender.csv')},\n",
    "    'tas': {'data': all_series['TAS'], 'data_file': os.path.join(path, 'tas_data.csv'), 'gender_file': os.path.join(path, 'tas_gender.csv')},\n",
    "    'tng': {'data': all_series['TNG'], 'data_file': os.path.join(path, 'tng_data.csv'), 'gender_file': os.path.join(path, 'tng_gender.csv')},\n",
    "    'ds9': {'data': all_series['DS9'], 'data_file': os.path.join(path, 'ds9_data.csv'), 'gender_file': os.path.join(path, 'ds9_gender.csv')},\n",
    "    'voy': {'data': all_series['VOY'], 'data_file': os.path.join(path, 'voy_data.csv'), 'gender_file': os.path.join(path, 'voy_gender.csv')},\n",
    "    'ent': {'data': all_series['ENT'], 'data_file': os.path.join(path, 'ent_data.csv'), 'gender_file': os.path.join(path, 'ent_gender.csv')},\n",
    "    'dis': {'data': all_series['DIS'], 'data_file': os.path.join(path, 'dis_data.csv'), 'gender_file': os.path.join(path, 'dis_gender.csv')},\n",
    "    'pic': {'data': all_series['PIC'], 'data_file': os.path.join(path, 'pic_data.csv'), 'gender_file': os.path.join(path, 'pic_gender.csv')},\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "49d37e21",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define main cast variables\n",
    "\n",
    "main_cast = {\n",
    "    'tos': ['KIRK', 'SPOCK', 'UHURA', 'CHEKOV', 'SULU', 'CHAPEL', 'COMPUTER', 'MCCOY', 'SCOTT'],\n",
    "    'tas': ['KIRK', 'SPOCK', 'UHURA', 'CHEKOV', 'SULU', 'CHAPEL', 'COMPUTER', 'MCCOY', 'SCOTT'],\n",
    "    'tng': ['PICARD', 'RIKER', 'WORF', 'DATA', 'TROI', 'CRUSHER', 'TASHA', 'CHIEF', \"O'BRIEN\", 'GUINAN', 'LAFORGE', 'PULASKI', 'WESLEY'],\n",
    "    'ds9': ['SISKO', 'ODO', 'KIRA', 'JAKE', 'QUARK', 'DAX', \"O'BRIEN\", 'BASHIR', 'WORF', 'EZRI'],\n",
    "    'voy': ['JANEWAY', 'CHAKOTAY', 'TUVOK', 'PARIS', 'TORRES', 'KIM', 'EMH', 'NEELIX', 'KES', 'SEVEN', 'ICHEB', 'SESKA'],\n",
    "    'ent': ['ARCHER', 'DEGRA', 'HOSHI', 'PHLOX', 'REED', 'SHRAN', \"T'POL\", 'TRAVIS', 'TUCKER'],\n",
    "    'dis': ['BURNHAM', 'SARU', 'VOQ', 'TYLER', 'STAMETS', 'TILLY', 'LORCA', 'CULBER', 'PIKE', 'BOOK', 'NHAN', 'ADIRA', 'GRAY', 'GEORGIOU', 'DETMER', 'OWOSEKUN', \"L'RELL\", 'SAREK', 'CORNWELL', 'AIRIAM', 'SPOCK'],\n",
    "    'pic': ['PICARD', 'AGNES', 'DAHJ', 'DATA', 'ELNOR', 'HUGH', 'SOJI', 'RAFFI', 'RIOS', 'NAREK', 'SEVEN', 'RIZZO']\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d875ebb7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Helper function to split Dialogue into a list of lists\n",
    "\n",
    "# def transform_dialogue(dialogue):\n",
    "#     return [item.split(', ') for item in dialogue]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "bb0550f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Helper function to split Dialogue into a string\n",
    "\n",
    "# def join_dialogue(dialogue):\n",
    "# #     return [item.concat(', ') for item in dialogue]\n",
    "#     thingy = ''\n",
    "#     for item in dialogue: \n",
    "#         thingy = thingy + ' ' + item\n",
    "#     return thingy\n",
    "\n",
    "def join_dialogue(dialogue):\n",
    "    return ' '.join(dialogue)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "48220346",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Helper function to filter out dialogue spoken by characters other than the main cast\n",
    "\n",
    "def remove_secondary_cast_dialogue(df, main_cast_list):\n",
    "    return df[df['Character'].isin(main_cast_list)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ea7d4a50",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Helper function to merge in data (gender, etc.) from additional csv files\n",
    "\n",
    "def merge_additional_data(df, data_file, gender_file):\n",
    "    additional_data = pd.read_csv(os.path.join(path, data_file), index_col=0, delimiter=';', encoding='latin1')\n",
    "    df = df.merge(additional_data, left_on='Episode', right_index=True)\n",
    "\n",
    "    gender_mapping = pd.read_csv(os.path.join(path, gender_file), header=None, index_col=0, delimiter=';').squeeze(\"columns\").to_dict()\n",
    "\n",
    "    df['Gender'] = df['Character'].map(gender_mapping)\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "1405fdf3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a dataframe for each series\n",
    "\n",
    "series_dataframes = {}\n",
    "\n",
    "def create_series_df(series_name, series_data, main_cast_list, data_file, gender_file):\n",
    "    if series_name not in series_dict:\n",
    "        print('Series not recognized')\n",
    "        return\n",
    "\n",
    "    # Transform into dataframe:\n",
    "    series_df = pd.concat({k: pd.Series(v) for k, v in series_data.items()}).reset_index()\n",
    "    series_df.columns = ['Episode', 'Character', 'Dialogue']\n",
    "    \n",
    "    # Remove line break characters from Dialogue    series_df['Dialogue'] = series_df['Dialogue'].apply(lambda dialogues: [line.replace('\\r', '') for line in dialogues])\n",
    "    series_df['Dialogue'] = series_df['Dialogue'].apply(lambda dialogues: [line.replace('\\r', '') for line in dialogues])\n",
    "\n",
    "    # Get length of dialogue\n",
    "    series_df['Dialogue Length'] = series_df['Dialogue'].str.len()\n",
    "    \n",
    "    # Transform 'Dialogue' into a single string\n",
    "    series_df['Flattened Dialogue'] = series_df['Dialogue'].apply(join_dialogue)\n",
    "\n",
    "    # Drop dialogue not spoken by main cast\n",
    "    series_df = remove_secondary_cast_dialogue(series_df, main_cast_list)\n",
    "\n",
    "    # Merge columns regarding gender, etc. from additional data sources\n",
    "    series_df = merge_additional_data(series_df, data_file, gender_file)\n",
    "\n",
    "    series_df = series_df[['Episode', 'Season', 'Year', 'Title', 'Character', 'Gender', 'Dialogue', 'Flattened Dialogue', 'Dialogue Length']]\n",
    "\n",
    "    series_dataframes[series_name] = series_df  # Store the DataFrame in the dictionary\n",
    "    \n",
    "    \n",
    "    series_df.to_csv(r'C:\\Users\\anon\\Documents\\CareerFoundry\\Data Analytics Immersion\\6\\Data\\Altered Data\\{}_df_cleaned.csv'.format(series_name), index=False)\n",
    "\n",
    "    return series_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "757a10e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a dataframe for each series\n",
    "\n",
    "for series_name, series_info in series_dict.items():\n",
    "    create_series_df(series_name, series_info['data'], main_cast[series_name],\n",
    "                     series_info['data_file'], series_info['gender_file'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "18a9a89b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save each series into it's own dataframe\n",
    "\n",
    "tos_dataframe = series_dataframes['tos']\n",
    "tas_dataframe = series_dataframes['tas']\n",
    "tng_dataframe = series_dataframes['tng']\n",
    "ds9_dataframe = series_dataframes['ds9']\n",
    "ent_dataframe = series_dataframes['ent']\n",
    "dis_dataframe = series_dataframes['dis']\n",
    "pic_dataframe = series_dataframes['pic']\n",
    "voy_dataframe = series_dataframes['voy']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "e599f603",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Episode</th>\n",
       "      <th>Season</th>\n",
       "      <th>Year</th>\n",
       "      <th>Title</th>\n",
       "      <th>Character</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Dialogue</th>\n",
       "      <th>Flattened Dialogue</th>\n",
       "      <th>Dialogue Length</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>tos_000</td>\n",
       "      <td>tos_s1</td>\n",
       "      <td>1966</td>\n",
       "      <td>The Cage</td>\n",
       "      <td>SPOCK</td>\n",
       "      <td>m</td>\n",
       "      <td>[Check the circuit., It can't be the screen th...</td>\n",
       "      <td>Check the circuit. It can't be the screen then...</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>tos_001</td>\n",
       "      <td>tos_s1</td>\n",
       "      <td>1966</td>\n",
       "      <td>The Man Trap</td>\n",
       "      <td>KIRK</td>\n",
       "      <td>m</td>\n",
       "      <td>[Shall we pick some flowers, Doctor? When a ma...</td>\n",
       "      <td>Shall we pick some flowers, Doctor? When a man...</td>\n",
       "      <td>115</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Episode  Season  Year         Title Character Gender  \\\n",
       "0   tos_000  tos_s1  1966      The Cage     SPOCK      m   \n",
       "21  tos_001  tos_s1  1966  The Man Trap      KIRK      m   \n",
       "\n",
       "                                             Dialogue  \\\n",
       "0   [Check the circuit., It can't be the screen th...   \n",
       "21  [Shall we pick some flowers, Doctor? When a ma...   \n",
       "\n",
       "                                   Flattened Dialogue  Dialogue Length  \n",
       "0   Check the circuit. It can't be the screen then...               27  \n",
       "21  Shall we pick some flowers, Doctor? When a man...              115  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tos_dataframe.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "23396862",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"Check the circuit. It can't be the screen then. Definitely something out there, Captain, headed this way. Their call letters check with a survey expedition. SS Columbia. It disappeared in that region approximately eighteen years ago. Records show the Talos group has never been explored. Solar system similar to Earth, eleven planets. Number four seems to be Class M, oxygen atmosphere. We aren't going to go, to be certain? Mister Spock here. We're intercepting a follow-up message, sir. There are crash survivors on Talos. Preliminary lab survey ready, sir. Yes, sir. Spock here. There is no survivors' encampment, Number One. This is all some sort of trap. We've lost the Captain. Do you read? The inhabitants of this planet must live deep underground, and probably manufacture food and other needs down there. Our tests indicate the planet surface, without considerably more vegetation or some animals, simply too barren to support life. Exactly. An illusion placed in our minds by this planet's inhabitants. They may simply be studying the Captain, to find out how Earth people are put together. Or it could be something more. Look. Brains three times the size of ours. If we start buzzing about down there, we're liable to find their mental power is so great they could reach out and swat this ship as though it were a fly. Standing by, Number One. Ten, nine, eight, seven, six, five, four, three, two, one. Our circuits are beginning to heat. We'll have to cease power. We've located a magnetic field that seems to come from their underground generator. If our measurements and readings are an illusion also, one could find oneself materialised inside solid rock. The women! Address intercraft. This is the acting captain speaking. We have no choice now but to consider the safety of this vessel and the remainder of the crew. We're leaving. All decks prepare for hyperdrive. Time warp factor. Engine room! Mister Spock here. Switch to rockets. We're blasting out. Nothing. But for the batteries we'd lose gravitation and oxygen. Could be we've waited too long. It's collecting all the information stored in this fly. They've decided to swat us. Mister Spock here.\""
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tos_dataframe['Flattened Dialogue'][0]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}