UnifiedStatistics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 311,
   "id": "6593df06-3fcf-4990-ac76-90500540b33d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: ankipandas in /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages (0.3.15)\n",
      "Requirement already satisfied: numpy in /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages (2.2.1)\n",
      "Requirement already satisfied: plotly in /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages (6.0.0)\n",
      "Requirement already satisfied: kaleido in /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages (0.2.1)\n",
      "Requirement already satisfied: pandas in /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages (from ankipandas) (2.2.3)\n",
      "Requirement already satisfied: colorlog in /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages (from ankipandas) (6.9.0)\n",
      "Requirement already satisfied: randomfiletree in /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages (from ankipandas) (1.2.0)\n",
      "Requirement already satisfied: narwhals>=1.15.1 in /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages (from plotly) (1.26.0)\n",
      "Requirement already satisfied: packaging in /Users/bwerness/Library/Python/3.12/lib/python/site-packages (from plotly) (24.2)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /Users/bwerness/Library/Python/3.12/lib/python/site-packages (from pandas->ankipandas) (2.9.0.post0)\n",
      "Requirement already satisfied: pytz>=2020.1 in /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages (from pandas->ankipandas) (2025.1)\n",
      "Requirement already satisfied: tzdata>=2022.7 in /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages (from pandas->ankipandas) (2025.1)\n",
      "Requirement already satisfied: six>=1.5 in /Users/bwerness/Library/Python/3.12/lib/python/site-packages (from python-dateutil>=2.8.2->pandas->ankipandas) (1.16.0)\n",
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m25.0.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip3.12 install --upgrade pip\u001b[0m\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "%pip install ankipandas numpy plotly kaleido"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 312,
   "id": "91c918be",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: nbformat in /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages (5.10.4)\n",
      "Requirement already satisfied: fastjsonschema>=2.15 in /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages (from nbformat) (2.21.1)\n",
      "Requirement already satisfied: jsonschema>=2.6 in /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages (from nbformat) (4.23.0)\n",
      "Requirement already satisfied: jupyter-core!=5.0.*,>=4.12 in /Users/bwerness/Library/Python/3.12/lib/python/site-packages (from nbformat) (5.7.2)\n",
      "Requirement already satisfied: traitlets>=5.1 in /Users/bwerness/Library/Python/3.12/lib/python/site-packages (from nbformat) (5.14.3)\n",
      "Requirement already satisfied: attrs>=22.2.0 in /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages (from jsonschema>=2.6->nbformat) (25.1.0)\n",
      "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages (from jsonschema>=2.6->nbformat) (2024.10.1)\n",
      "Requirement already satisfied: referencing>=0.28.4 in /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages (from jsonschema>=2.6->nbformat) (0.36.2)\n",
      "Requirement already satisfied: rpds-py>=0.7.1 in /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages (from jsonschema>=2.6->nbformat) (0.22.3)\n",
      "Requirement already satisfied: platformdirs>=2.5 in /Users/bwerness/Library/Python/3.12/lib/python/site-packages (from jupyter-core!=5.0.*,>=4.12->nbformat) (4.3.6)\n",
      "Requirement already satisfied: typing-extensions>=4.4.0 in /Library/Frameworks/Python.framework/Versions/3.12/lib/python3.12/site-packages (from referencing>=0.28.4->jsonschema>=2.6->nbformat) (4.12.2)\n",
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m25.0.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip3.12 install --upgrade pip\u001b[0m\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "%pip install --upgrade nbformat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 313,
   "id": "17cb1b50-9559-44a3-b621-3fc4a4815079",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Imports\n",
    "import pandas as pd\n",
    "import re\n",
    "import random\n",
    "import copy\n",
    "import math\n",
    "import json\n",
    "import plotly.graph_objects as go\n",
    "import numpy as np\n",
    "import ankipandas\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "81e43089-c538-46ff-88e0-8236ca39a010",
   "metadata": {},
   "source": [
    "# Load Shared Datasets\n",
    "\n",
    "This section of code loads the [Google N-Grams Data](https://github.com/orgtre/google-books-ngram-frequency/blob/main/ngrams/1grams_english.csv), along with `ipadict` for [phonetic information](https://github.com/open-dict-data/ipa-dict/blob/master/data/en_US.txt)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 314,
   "id": "ad8f5083-13f4-4175-a750-07d0499c0be1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load in the one-gram information (word frequencies)\n",
    "one_grams = pd.read_csv(\"1grams_english.csv\")\n",
    "one_grams_dict = {row[1]['ngram'].lower(): row[1]['freq'] for row in one_grams.iterrows()}\n",
    "\n",
    "# Scoring methods designed to select one of the options from the IPA dictionary.\n",
    "# It selects the shortest words with the most schwas (laziest pronunciation).\n",
    "\n",
    "\n",
    "def score(word):\n",
    "    word.replace(\" \", \"\")\n",
    "    return (len(word), -word.count(\"ə\"))\n",
    "\n",
    "\n",
    "def get_laziest(ipa_list):\n",
    "    return list(sorted(ipa_list, key=score))[0]\n",
    "\n",
    "\n",
    "# Loading the dictionary and parsing out the best scoring pronunciation\n",
    "ipa = pd.read_csv(\"en_US.txt\", sep='\\t', names=[\"eng\", \"ipa\"])\n",
    "ipa_opt_dict = {row[1]['eng']: row[1]['ipa'].split(', ') for row in ipa.iterrows()}\n",
    "ipa_dict = {row[1]['eng']: get_laziest(row[1]['ipa'].split(', ')) for row in ipa.iterrows()}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "954376ff-bf9c-4941-928f-3e3054191e88",
   "metadata": {},
   "source": [
    "# Write Common Code\n",
    "\n",
    "This is code that takes in:\n",
    "\n",
    "1. A list of words,\n",
    "2. A dictionary of word transformations,\n",
    "3. An optional function that computes lengths of words (defaults to `len`),\n",
    "4. An optional function that transforms a transformed word into the constituent \"letters\".\n",
    "\n",
    "It then returns the various statistics of that model as a dictionary:\n",
    "1. The mean length normalized to assume a binary alphabet (multiplied by the $log_2$ of the alphabet size),\n",
    "2. The reconstruction entropy $H(X|f(X))$,\n",
    "3. The reconstruction error $\\mathbf{P}\\{X = \\mathrm{argmax}_{y : f(y) =  f(X)} p(y)\\}$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 315,
   "id": "df856977-76f1-4ebc-8567-42aa471c4c4c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def entropy(counts):\n",
    "    entropy = 0\n",
    "    total = 0\n",
    "    for x in counts:\n",
    "        total += counts[x]\n",
    "    for x in counts:\n",
    "        entropy -= (counts[x]/total)*math.log2(counts[x]/total)\n",
    "    \n",
    "    return entropy\n",
    "\n",
    "def transmission_cost(letters,suprisal):\n",
    "    entropy = 0\n",
    "    for l in letters:\n",
    "        entropy += suprisal[l]\n",
    "    \n",
    "    return entropy\n",
    "\n",
    "def entropy_wl(word_list):\n",
    "    countlist = {w:one_grams_dict[w] for w in word_list}\n",
    "    return entropy(countlist)\n",
    "\n",
    "def code_statistics(word_list, word_mapping, length_function = lambda x: len(x), letter_making = lambda x: x, delim = None):\n",
    "    # Data to track mean outline length\n",
    "    denom = 0\n",
    "    total_length = 0\n",
    "\n",
    "    # Data to track alphabet size\n",
    "    letter_counts = Counter()\n",
    "\n",
    "    # Data to track for computing reconstruction entropy and reconstruction probability\n",
    "    forward = {}\n",
    "    inverse = {\"\": {\"\": float('inf')}}\n",
    "    most_probable = {}\n",
    "    totals = {\"\": float('inf')}\n",
    "\n",
    "    # iterate over word list keeping track of stats for mean length and the inverse mapping\n",
    "    for word in word_list:\n",
    "        count = one_grams_dict[word]\n",
    "\n",
    "        image = word_mapping[word]\n",
    "        if delim:\n",
    "            image += delim\n",
    "\n",
    "        word_length = length_function(image) * count\n",
    "        total_length += word_length\n",
    "        denom += count\n",
    "\n",
    "        forward[word] = image\n",
    "        if image not in inverse:\n",
    "            inverse[image] = {}\n",
    "            most_probable[image] = word\n",
    "        inverse[image][word] = count+1\n",
    "        totals[image] = totals.get(image, 1) + count\n",
    "\n",
    "        current_letters = letter_making(image)\n",
    "        for l in current_letters:\n",
    "            letter_counts[l]+= count\n",
    "\n",
    "    letter_total = letter_counts.total()\n",
    "    suprisal = {letter: -math.log2(letter_counts[letter]/letter_total) for letter in letter_counts}\n",
    "\n",
    "    # compute the reconstruction entropy and reconstruction probability\n",
    "    total_reconstruction_entropy = 0\n",
    "    total_reconstruction_probability = 0\n",
    "    total_transmission_cost = 0\n",
    "    for word in word_list:\n",
    "        count = one_grams_dict[word]\n",
    "        total_reconstruction_entropy -= float('-inf') if inverse[forward[word]][word]/totals[forward[word]] == 0 else count * math.log2(inverse[forward[word]][word]/totals[forward[word]])\n",
    "        total_reconstruction_probability += count if word == most_probable[forward[word]] else 0\n",
    "        total_transmission_cost += count*transmission_cost(letter_making(forward[word]), suprisal)\n",
    "\n",
    "    running_lengths = 0\n",
    "    running_count = 1\n",
    "\n",
    "    for w in one_grams_dict:\n",
    "        if w not in word_list:\n",
    "            continue\n",
    "        prob = one_grams_dict[w]/denom\n",
    "        running_count += 1\n",
    "        running_lengths += prob*math.log2(running_count)\n",
    "\n",
    "    return {\"mean_binary_length\": math.log2(len(letter_counts))*total_length / denom,\n",
    "            \"mean_binary_excess\": math.log2(len(letter_counts))*total_length / denom - running_lengths,\n",
    "            \"mean_entropy_shift\": math.log2(len(letter_counts))*total_length / denom - entropy_wl(word_list),\n",
    "            \"mean_transmission_cost\": total_transmission_cost / denom,\n",
    "            \"mean_transmission_excess\": total_transmission_cost / denom - running_lengths,\n",
    "            \"mean_transmission_shift\": total_transmission_cost / denom - entropy_wl(word_list),\n",
    "            \"reconstruction_entropy\": total_reconstruction_entropy / denom,\n",
    "            \"reconstruction_error\": (1-total_reconstruction_probability / denom),\n",
    "            \"alphabet\": \" \".join([x for x, _ in letter_counts.most_common()])}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "83390dca-c3f7-4125-8d59-bc24b31a55a5",
   "metadata": {},
   "source": [
    "# Pure Spelling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 316,
   "id": "c25b3770-6d83-43a6-b0a7-2867709e9ade",
   "metadata": {},
   "outputs": [],
   "source": [
    "spelling = {x: x for x in one_grams_dict}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 317,
   "id": "f95aadba-bdb7-4d79-9c9d-be09891f9567",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mean_binary_length': 21.740674602773286,\n",
       " 'mean_binary_excess': 15.093721783000312,\n",
       " 'mean_entropy_shift': 11.973528232304314,\n",
       " 'mean_transmission_cost': 18.799831537970253,\n",
       " 'mean_transmission_excess': 12.152878718197279,\n",
       " 'mean_transmission_shift': 9.032685167501281,\n",
       " 'reconstruction_entropy': 0.0,\n",
       " 'reconstruction_error': 0.0,\n",
       " 'alphabet': \"e t a o i n s r h d l c u m f p w g y b v k x j q z ' é\"}"
      ]
     },
     "execution_count": 317,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "code_statistics([x for x in one_grams_dict if x in spelling], spelling)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d34d0be3-e9b3-4351-b6ba-a00e5b318022",
   "metadata": {},
   "source": [
    "# Pure IPA representation\n",
    "This removes some of the noise from the IPA representation to allow it to be tested as an encoding."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 318,
   "id": "43f450ff-0c4c-438d-a57d-d15cc009ca9a",
   "metadata": {},
   "outputs": [],
   "source": [
    "reduced_ipa = {x: ipa_dict[x].replace(\"/\",\"\").replace(\"ˌ\",\"\").replace(\"ˈ\",\"\") for x in ipa_dict}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 319,
   "id": "722644e2-cd4d-4bc3-b0cc-46317e93a8d0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mean_binary_length': 20.47519327609292,\n",
       " 'mean_binary_excess': 13.829919120320977,\n",
       " 'mean_entropy_shift': 10.710090184552453,\n",
       " 'mean_transmission_cost': 18.7229977522716,\n",
       " 'mean_transmission_excess': 12.077723596499656,\n",
       " 'mean_transmission_shift': 8.957894660731132,\n",
       " 'reconstruction_entropy': 0.042481294979739674,\n",
       " 'reconstruction_error': 0.014930232236564533,\n",
       " 'alphabet': 'ə ɪ n t d s ɫ ɹ i k ð ɝ z m ɛ v p a ʊ æ w f b ʃ e ɑ h ɔ u o ŋ ɡ j ʒ θ'}"
      ]
     },
     "execution_count": 319,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "code_statistics([x for x in one_grams_dict if x in reduced_ipa], reduced_ipa)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4f5316ed-df1a-4ef0-b185-d2e819c91f28",
   "metadata": {},
   "source": [
    "# Load Carter Briefwords\n",
    "The dictionary is available [here](https://www.reddit.com/r/shorthand/comments/xg7k10/a_briefhand_resource/)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 320,
   "id": "c949594e-62ff-40ca-9217-a9961f90dedc",
   "metadata": {},
   "outputs": [],
   "source": [
    "briefhand_raw = pd.read_csv(\"dict.csv\")\n",
    "\n",
    "briefhand = {}\n",
    "\n",
    "for _,line in briefhand_raw.iterrows():\n",
    "    briefhand[line['word']] = line['brief']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 321,
   "id": "cec0137a-74ea-43b9-9956-90883345d385",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mean_binary_length': 8.65318533787457,\n",
       " 'mean_binary_excess': 3.69443099723304,\n",
       " 'mean_entropy_shift': 0.9394725601340319,\n",
       " 'mean_transmission_cost': 7.373228049268073,\n",
       " 'mean_transmission_excess': 2.4144737086265433,\n",
       " 'mean_transmission_shift': -0.3404847284724646,\n",
       " 'reconstruction_entropy': 1.0613532563363244,\n",
       " 'reconstruction_error': 0.3048400643277738,\n",
       " 'alphabet': \"t a n s e r o l d i m f c b w p g z u h v k y 1 x j q 2 3 0 5 % 7 '\"}"
      ]
     },
     "execution_count": 321,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "code_statistics([x for x in one_grams_dict if x in briefhand], briefhand)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b195bbd2-0656-4905-be2e-434515022558",
   "metadata": {},
   "source": [
    "# Dutton Speedwords\n",
    "The dictionary is available [here](http://www2.cmp.uea.ac.uk/~jrk/conlang.dir/Speedwords.dict)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 322,
   "id": "9c048445-7aec-4865-bdf7-a22eb2054975",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"Speedwords.dict\", \"r\") as file:\n",
    "    speedwords_raw = file.readlines()\n",
    "\n",
    "speedwords = {}\n",
    "\n",
    "for line in speedwords_raw[89:]:\n",
    "    if line == '\\n':\n",
    "        continue\n",
    "\n",
    "    splits = line.replace(\"\\n\",\"\").split(\"\\t\")\n",
    "    if len(splits) != 2:\n",
    "        continue\n",
    "\n",
    "    rep = splits[0]\n",
    "    words = splits[1]\n",
    "    word_list = words.split(\",\")\n",
    "    for word in word_list:\n",
    "        speedwords[re.sub(r'[\\(\\[{].*?[\\)\\]}]', '', word)] = re.sub(r'[\\(\\[{].*?[\\)\\]}]', '', rep.replace(\"-\",\"\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 323,
   "id": "cf3e10f6-dc81-43c2-ac5d-c0b0500ee18c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mean_binary_length': 9.329723457179032,\n",
       " 'mean_binary_excess': 4.124412180082545,\n",
       " 'mean_entropy_shift': 1.2843567331541461,\n",
       " 'mean_transmission_cost': 8.54975320966042,\n",
       " 'mean_transmission_excess': 3.3444419325639325,\n",
       " 'mean_transmission_shift': 0.5043864856355338,\n",
       " 'reconstruction_entropy': 0.38624091801605515,\n",
       " 'reconstruction_error': 0.11017558339765876,\n",
       " 'alphabet': 'i e u l a o s d r n t y v m p & g b q f h k z x c j w O K'}"
      ]
     },
     "execution_count": 323,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "code_statistics([x for x in one_grams_dict if x in speedwords], speedwords)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "25ab721f-dfe1-42d7-af11-d0a2cf1bdbe8",
   "metadata": {},
   "source": [
    "# Gregg Anniversary\n",
    "\n",
    "This loads in Gregg Anniversary from [here](https://github.com/grascii/dictionaries).  The key here is that blends should be counted as single alphabet entries, and all is separated by hyphens, so we need custom length and letter functions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 324,
   "id": "bbf2b39d-fb36-436d-bbfb-aaa8631d4a71",
   "metadata": {},
   "outputs": [],
   "source": [
    "gregg_anniversary_raw = pd.read_csv(\"anniversary_core.csv\")\n",
    "gregg_anniversary_raw_2= pd.read_csv(\"anniversary_supplement.csv\")\n",
    "\n",
    "gregg_anniversary = {}\n",
    "\n",
    "for _,line in gregg_anniversary_raw.iterrows():\n",
    "    gregg_anniversary[str(line['word']).lower()] = re.sub(\"^/^-\",\"\",line['form'])\n",
    "\n",
    "for _,line in gregg_anniversary_raw_2.iterrows():\n",
    "    gregg_anniversary[str(line['word']).lower()] = re.sub(\"^/^-\",\"\",line['form'])\n",
    "\n",
    "gregg_length = lambda x: x.count(\"-\")+1\n",
    "gregg_letters = lambda x: x.split(\"-\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 325,
   "id": "dd06f003-9c64-49fc-9a23-78fd6a3150c0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mean_binary_length': 15.425163710803988,\n",
       " 'mean_binary_excess': 9.260242654630629,\n",
       " 'mean_entropy_shift': 6.228619368943665,\n",
       " 'mean_transmission_cost': 11.212263985017435,\n",
       " 'mean_transmission_excess': 5.047342928844076,\n",
       " 'mean_transmission_shift': 2.0157196431571123,\n",
       " 'reconstruction_entropy': 0.25155755894450227,\n",
       " 'reconstruction_error': 0.07505138574335446,\n",
       " 'alphabet': 'e o a t s n th u r k m s2 h d l f nt \\\\ b p sh v i ^ / g pr ch j e2 tn mn th2 td ths pl tm ss ng df a2 o2 ld bl br fr ya ea nk s2s x fl dfl jnt ss2 w ye i2 ia mt ye2 kp c bld y s2h z us as2 s2s2 ya2 >  vr E ot s22 u\\\\ nd q ea2 ao dm mm < xs \\\\v oe se rd o1 re dya om s2o es pld nu ge ta rdf'}"
      ]
     },
     "execution_count": 325,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "code_statistics([x for x in one_grams_dict if x in gregg_anniversary], gregg_anniversary, length_function = gregg_length, letter_making = gregg_letters)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ab7f005a-7d40-4757-aa59-36ce7bf74e6b",
   "metadata": {},
   "source": [
    "# Gregg Notehand\n",
    "\n",
    "This loads in Gregg Notehand from [here](https://gregg-shorthand.com/2015/09/05/notehand-dictionary/).  The key here is that blends should be counted as single alphabet entries, and all is separated by hyphens, so we need custom length and letter functions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 326,
   "id": "1dde41dd-2fb7-4f6c-86a6-fa78f733a3cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "gregg_notehand_raw = pd.read_csv(\"Notehand.csv\")\n",
    "\n",
    "gregg_notehand = {}\n",
    "\n",
    "for _,line in gregg_notehand_raw.iterrows():\n",
    "    gregg_notehand[str(line['word']).lower()] = str(line['form'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 327,
   "id": "90f08ff6-f879-458d-9bb0-9551917897c9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mean_binary_length': 15.343301937676637,\n",
       " 'mean_binary_excess': 9.916704355750431,\n",
       " 'mean_entropy_shift': 7.050406666597668,\n",
       " 'mean_transmission_cost': 11.665776329835321,\n",
       " 'mean_transmission_excess': 6.239178747909115,\n",
       " 'mean_transmission_shift': 3.3728810587563522,\n",
       " 'reconstruction_entropy': 0.11181820004163713,\n",
       " 'reconstruction_error': 0.033954582356027774,\n",
       " 'alphabet': 'e a o t r s over_th n oo l left_s m k d f nd p b h I v sh under_th g ing ch th nt j  nan ng ld ten rd ted men oi ye ia ngk _ Ia ea md c . mt I_rd over_the x e_p r–s left d–s ya e_d over nk i er a_ ee + e_t a_sh'}"
      ]
     },
     "execution_count": 327,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "code_statistics([x for x in one_grams_dict if x in gregg_notehand], gregg_notehand, length_function = gregg_length, letter_making = gregg_letters)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "24df0399-bcea-4f06-b740-fa86f0f6e0cf",
   "metadata": {},
   "source": [
    "# Gregg Simplified\n",
    "\n",
    "This is a different encoding of Gregg simplified, from [here](https://www.reddit.com/r/shorthand/comments/1e7ie7g/gregg_simplified_computerreadable_dictionary/)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 328,
   "id": "4450e3f2-8df0-4d99-98fa-820d83307cff",
   "metadata": {},
   "outputs": [],
   "source": [
    "gregg_simplified_raw = pd.read_csv(\"LEGS.csv\",names=['word','form'],header = None)\n",
    "\n",
    "gregg_simplified = {}\n",
    "\n",
    "for _,line in gregg_simplified_raw.iterrows():\n",
    "    gregg_simplified[str(line['word']).lower()] = str(line['form'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 329,
   "id": "25bcdac7-348f-49af-8a97-871bc7c34f68",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mean_binary_length': 11.769702509512225,\n",
       " 'mean_binary_excess': 7.344900086927768,\n",
       " 'mean_entropy_shift': 4.669998288575941,\n",
       " 'mean_transmission_cost': 9.674913725740863,\n",
       " 'mean_transmission_excess': 5.250111303156406,\n",
       " 'mean_transmission_shift': 2.5752095048045787,\n",
       " 'reconstruction_entropy': 0.2091399622585796,\n",
       " 'reconstruction_error': 0.06600137338137413,\n",
       " 'alphabet': '( e t u n a s N T r o l m A b k d p f z v i h $ D g j ) M : c % G 5 q V L ^ y R w X K 3 ~ J x P *'}"
      ]
     },
     "execution_count": 329,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "code_statistics([x for x in one_grams_dict if x in gregg_simplified], gregg_simplified, letter_making = lambda x: x)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "77c2acef-f437-44fc-ae63-f2bdc2b2b84a",
   "metadata": {},
   "source": [
    "# bref\n",
    "\n",
    "This loads the bref dictionary from [here](https://www.reddit.com/r/shorthand/comments/esjhdk/bref_shorthand/)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 330,
   "id": "e80c28b0-1abb-41b3-ab54-8e66c1b448a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"11661 WORDS FORWARD.txt\", \"r\") as file:\n",
    "    bref_raw = file.readlines()\n",
    "\n",
    "bref = {}\n",
    "\n",
    "for line in bref_raw:\n",
    "    splits = line.split(\" = \")\n",
    "    if len(splits) != 2:\n",
    "        continue\n",
    "    rep = splits[1].replace(\"\\n\",\"\")\n",
    "    word = splits[0]\n",
    "    bref[word] = rep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 331,
   "id": "d6e25a7c-126c-48a8-9359-975400db43be",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mean_binary_length': 12.321204615957537,\n",
       " 'mean_binary_excess': 5.900270582891406,\n",
       " 'mean_entropy_shift': 2.8401015434801753,\n",
       " 'mean_transmission_cost': 11.28052963889117,\n",
       " 'mean_transmission_excess': 4.85959560582504,\n",
       " 'mean_transmission_shift': 1.7994265664138087,\n",
       " 'reconstruction_entropy': 0.0,\n",
       " 'reconstruction_error': 0.0,\n",
       " 'alphabet': 't s r n e o l d c a m p i f w g b u h z x v k j y q'}"
      ]
     },
     "execution_count": 331,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "code_statistics([x for x in one_grams_dict if x in bref], bref)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b1e7c34e-8f36-4949-bccb-48721382620b",
   "metadata": {},
   "source": [
    "# Yublin\n",
    "\n",
    "This loads the Yublin dictionary from [here](http://jonathanaquino.com/yublin.csv)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 332,
   "id": "247d715c-6a9d-4c68-961a-612a612a3929",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"yublin.csv\", \"r\") as file:\n",
    "    raw_yublin = file.readlines()\n",
    "\n",
    "yublin = {}\n",
    "\n",
    "for line in raw_yublin:\n",
    "    splits = line.replace(\"\\n\",\"\").split(\",\")\n",
    "    if len(splits) != 2:\n",
    "        continue\n",
    "    rep = splits[1]\n",
    "    word = splits[0]\n",
    "\n",
    "    yublin[word] = rep\n",
    "\n",
    "for x in one_grams_dict:\n",
    "    if x not in yublin:\n",
    "        yublin[x] = x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 333,
   "id": "84500265-4824-46ac-b706-bdb4898e1236",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mean_binary_length': 16.559197673153815,\n",
       " 'mean_binary_excess': 9.912244853380841,\n",
       " 'mean_entropy_shift': 6.792051302684843,\n",
       " 'mean_transmission_cost': 14.715842759284635,\n",
       " 'mean_transmission_excess': 8.068889939511662,\n",
       " 'mean_transmission_shift': 4.948696388815664,\n",
       " 'reconstruction_entropy': 0.0,\n",
       " 'reconstruction_error': 0.0,\n",
       " 'alphabet': \"e t i o n a s r l c d f h u m p g y b w v k j x q z ' é\"}"
      ]
     },
     "execution_count": 333,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "code_statistics([x for x in one_grams_dict if x in yublin], yublin)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "12164f1e-f8d6-4b3b-88c2-01b84c4c463b",
   "metadata": {},
   "source": [
    "# Cut Spelng\n",
    "\n",
    "This loads the Cut Spelling dictionary form [here](https://github.com/DanielTillett/CutSpel/blob/master/cutspel.csv)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 334,
   "id": "4e2e20bf-97d8-4ac0-912a-616e84da21cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "cut_spelng_raw = pd.read_csv(\"cutspel.csv\",names = [\"word\",\"brief\"], header = None)\n",
    "\n",
    "cut_spelng = {}\n",
    "\n",
    "for _,line in cut_spelng_raw.iterrows():\n",
    "    cut_spelng[line['word']] = line['brief'].replace(\" \",\"\")\n",
    "cut_spelng['muscle'] = 'musl'\n",
    "\n",
    "for x in one_grams_dict:\n",
    "    if x not in cut_spelng:\n",
    "        cut_spelng[x] = x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 335,
   "id": "79ac4b4b-32f6-4060-9e3a-98dbea1e8f94",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mean_binary_length': 19.852082039357892,\n",
       " 'mean_binary_excess': 13.205129219584919,\n",
       " 'mean_entropy_shift': 10.084935668888921,\n",
       " 'mean_transmission_cost': 17.34277231881932,\n",
       " 'mean_transmission_excess': 10.695819499046348,\n",
       " 'mean_transmission_shift': 7.57562594835035,\n",
       " 'reconstruction_entropy': 0.015095746507235148,\n",
       " 'reconstruction_error': 0.005325956756667138,\n",
       " 'alphabet': \"t e a o n i s r h d l c u m f p y w g b v k j x q z é '\"}"
      ]
     },
     "execution_count": 335,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "code_statistics([x for x in one_grams_dict if x in cut_spelng], cut_spelng)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0de132f2-681c-421c-8fdb-2d86c4033533",
   "metadata": {},
   "source": [
    "# Taylor\n",
    "\n",
    "This attempts to construct an allignment of the spelling and of the IPA sounds then construct the Taylor representation of that word."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 336,
   "id": "60d945b2-9007-44fb-97f3-ae13a5b38689",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Improved drop vowels function to handle stress and long vowels and ensure proper vowel removal\n",
    "def drop_vowels(word):\n",
    "    vowels = \"aeiouAEIOUɑæɪɔʊəɜɛːɒʌ\"\n",
    "    punctuation_and_spaces = \"ˈˌ. ,-'\\\";:!?()[]{}<>/@#%^&*~`\"\n",
    "    return ''.join([char for char in word if char not in vowels and char not in punctuation_and_spaces])\n",
    "\n",
    "# Recursive alignment function with debug statements including match identification\n",
    "def find_allowable_pairs_recursive(str1, str2, allowable_pairs, debug = False):\n",
    "    def helper(s1, s2, pairs, path):\n",
    "        if not s1 and not s2:\n",
    "            return path\n",
    "        for pair in pairs:\n",
    "            l1, l2 = len(pair[0]), len(pair[1])\n",
    "            if s1[:l1] == pair[0] and s2[:l2] == pair[1]:\n",
    "                if debug:\n",
    "                    print(f\"Matching: {pair} with {s1[:l1]} and {s2[:l2]}\")\n",
    "                result = helper(s1[l1:], s2[l2:], pairs, path + [pair])\n",
    "                if result is not None:\n",
    "                    return result\n",
    "        if debug:\n",
    "            print(f\"Failed to match: {s1} with {s2}\")\n",
    "        return None\n",
    "\n",
    "    return helper(str1, str2, allowable_pairs, [])\n",
    "\n",
    "# Function to process the word and find alignment with debug statements\n",
    "def process_word(word, ipa_dict, consonant_pairs, debug = False):\n",
    "    if word not in ipa_dict:\n",
    "        return f\"Word '{word}' not found in IPA dictionary.\"\n",
    "    \n",
    "    ipa_representation = ipa_dict[word]\n",
    "    word_consonants = drop_vowels(word)\n",
    "    ipa_consonants = drop_vowels(ipa_representation)\n",
    "    \n",
    "    if debug:\n",
    "        print(f\"Processing '{word}' -> '{word_consonants}' with IPA '{ipa_representation}' -> '{ipa_consonants}'\")\n",
    "    \n",
    "    alignment = find_allowable_pairs_recursive(word_consonants, ipa_consonants, consonant_pairs, debug = debug)\n",
    "    \n",
    "    if alignment is None:\n",
    "        return f\"No valid alignment found for '{word}'.\"\n",
    "    return alignment\n",
    "\n",
    "# Reordered consonant_pairs to prioritize more common matches first and adding back the \"f\" matching to itself\n",
    "consonant_pairs = [\n",
    "    (\"c\", \"k\"),  # voiceless velar plosive, usually in \"cat\"\n",
    "    (\"t\", \"t\"),  # voiceless alveolar plosive\n",
    "    (\"d\", \"d\"),  # voiced alveolar plosive\n",
    "    (\"d\", \"t\"),  # voiced alveolar plosive\n",
    "    (\"t\", \"d\"),  # voiced alveolar plosive\n",
    "    (\"n\", \"n\"),  # alveolar nasal\n",
    "    (\"s\", \"s\"),  # voiceless alveolar fricative\n",
    "    (\"l\", \"l\"),  # alveolar lateral approximant\n",
    "    (\"l\", \"ɫ\"),\n",
    "    (\"r\", \"r\"),  # alveolar approximant matching itself\n",
    "    (\"r\", \"ɹ\"),  # alveolar approximant\n",
    "    (\"r\", \"ɝ\"),  # alveolar approximant\n",
    "    (\"m\", \"m\"),  # bilabial nasal\n",
    "    (\"b\", \"b\"),  # voiced bilabial plosive\n",
    "    (\"k\", \"k\"),  # voiceless velar plosive\n",
    "    (\"f\", \"f\"),  # voiceless labiodental fricative\n",
    "    (\"g\", \"g\"),  # voiced velar plosive\n",
    "    (\"g\", \"ɡ\"),\n",
    "    (\"v\", \"v\"),  # voiced labiodental fricative\n",
    "    (\"x\", \"ks\"), # voiceless velar-alveolar fricative, \"taxi\"\n",
    "    (\"z\", \"z\"),  # voiced alveolar fricative\n",
    "    (\"p\", \"p\"),  # voiceless bilabial plosive\n",
    "    (\"h\", \"h\"),  # voiceless glottal fricative\n",
    "    (\"w\", \"w\"),  # voiced labial-velar approximant\n",
    "    (\"y\", \"j\"),  # voiced palatal approximant\n",
    "    (\"j\", \"dʒ\"), # voiced palato-alveolar affricate, often \"j\" sound\n",
    "    (\"j\", \"ʤ\"), # voiced palato-alveolar affricate, often \"j\" sound\n",
    "    (\"j\", \"ʒ\"), # voiced palato-alveolar affricate, often \"j\" sound\n",
    "    (\"ng\", \"ŋ\"), # voiced velar nasal\n",
    "    (\"n\", \"ŋ\"), # voiced velar nasal\n",
    "    (\"ch\", \"ʧ\"), # voiceless palato-alveolar affricate\n",
    "    (\"ch\", \"tʃ\"), # voiceless palato-alveolar fricative\n",
    "    (\"sh\", \"tʃ\"), # voiceless palato-alveolar fricative\n",
    "    (\"sh\", \"ʧ\"), # voiceless palato-alveolar affricate\n",
    "    (\"sh\", \"ʃ\"), # voiceless palato-alveolar fricative\n",
    "    (\"ch\", \"ʃ\"), # voiceless palato-alveolar fricative\n",
    "    (\"th\", \"θ\"), # voiceless dental fricative\n",
    "    (\"th\", \"ð\"), # voiced dental fricative, as in \"this\"\n",
    "    (\"t\", \"tʃ\"), # voiceless palato-alveolar fricative\n",
    "    (\"g\", \"ʒ\"),  # voiced palato-alveolar fricative\n",
    "    (\"s\", \"z\"),  # voiced alveolar fricative, usually in \"dogs\"\n",
    "    (\"s\", \"ʒ\"),  # voiced palato-alveolar fricative, as in \"measure\"\n",
    "    (\"s\", \"ʃ\"),  # voiceless palato-alveolar fricative, as in \"champagne\"\n",
    "    (\"z\", \"ʒ\"),  # voiced palato-alveolar fricative, as in \"measure\"\n",
    "    (\"z\", \"ʃ\"),  # voiceless palato-alveolar fricative, as in \"champagne\"\n",
    "    (\"c\", \"ʃ\"),  # voiceless palato-alveolar fricative, as in \"champagne\"\n",
    "    (\"ck\", \"k\"), # voiceless velar plosive, as in \"back\"\n",
    "    (\"ps\", \"s\"), # \"ps\" matching \"s\"\n",
    "    (\"q\", \"kw\"), # voiceless velar and labial-velar approximant, usually in \"queen\"\n",
    "    (\"b\", \"\"),   # silent b\n",
    "    (\"h\", \"\"),   # silent h\n",
    "    (\"w\", \"\"),   # silent w\n",
    "    (\"y\", \"\"),   # sometimes silent y\n",
    "    (\"f\", \"v\"),  # voiced labiodental fricative\n",
    "    (\"ph\", \"f\"), # voiceless labiodental fricative, usually in \"phone\"\n",
    "    (\"gh\", \"f\"), # voiceless labiodental fricative, as in \"laugh\"\n",
    "    (\"gh\", \"\"),  # silent \"gh\", as in \"though\"\n",
    "    (\"kn\", \"n\"), # silent \"k\", as in \"knee\"\n",
    "    (\"gn\", \"n\"), # silent \"g\", as in \"gnome\"\n",
    "    (\"mn\", \"m\"), # \"mn\" matching \"m\"\n",
    "    (\"mn\", \"n\"), # \"mn\" matching \"n\"\n",
    "    (\"wr\", \"r\"), # silent \"w\", as in \"write\" with matching \"r\"\n",
    "    (\"wh\", \"w\"), # voiced labial-velar approximant, as in \"what\"\n",
    "    (\"sc\", \"s\"), # \"sc\" matching \"s\"\n",
    "    (\"c\", \"s\"),  # voiceless alveolar fricative, usually in \"cent\"\n",
    "    (\"ld\", \"d\"), # \"dd\" matching \"d\"\n",
    "    (\"dd\", \"d\"), # \"dd\" matching \"d\"\n",
    "    (\"ll\", \"l\"), # \"ll\" matching \"l\"\n",
    "    (\"ll\", \"ɫ\"),\n",
    "    (\"mm\", \"m\"), # \"mm\" matching \"m\"\n",
    "    (\"nn\", \"n\"), # \"nn\" matching \"n\"\n",
    "    (\"rr\", \"r\"), # \"nn\" matching \"n\"\n",
    "    (\"pp\", \"p\"), # \"pp\" matching \"p\"\n",
    "    (\"ff\", \"f\"), # \"pp\" matching \"p\"\n",
    "    (\"tt\", \"t\"), # \"tt\" matching \"t\"\n",
    "    (\"ss\", \"s\"), # \"ss\" matching \"s\"\n",
    "    (\"ss\", \"z\"), # \"ss\" matching \"z\"\n",
    "    (\"ss\", \"ʒ\"), # \"ss\" matching \"ʒ\"\n",
    "    (\"ss\", \"ʃ\"), # \"ss\" matching \"ʃ\"\n",
    "    (\"t\", \"ʒ\"),  # \"t\" matching \"ʒ\"\n",
    "    (\"t\", \"ʃ\"),  # \"t\" matching \"ʃ\"\n",
    "    (\"x\", \"gz\"), # voiced velar-alveolar fricative, \"exam\"\n",
    "    (\"x\", \"z\"),  # voiced alveolar fricative, \"xylophone\"\n",
    "    (\"zz\", \"ts\"),# \"zz\" matching \"ts\"\n",
    "    (\"z\", \"ts\"), # \"z\" matching \"ts\"\n",
    "    (\"g\", \"dʒ\"), # voiced palato-alveolar affricate\n",
    "    (\"g\", \"ʤ\"), # voiced palato-alveolar affricate\n",
    "    (\"gg\", \"g\"), # \"gg\" matching voiced velar plosive\n",
    "    (\"gg\", \"ʒ\"), # \"gg\" matching voiced palato-alveolar fricative\n",
    "    (\"gg\", \"dʒ\"),# \"gg\" matching voiced palato-alveolar affricate\n",
    "    (\"gg\", \"ʤ\"),# \"gg\" matching voiced palato-alveolar affricate\n",
    "    (\"ch\", \"k\"), # voiceless velar plosive, as in \"school\"\n",
    "    (\"cc\", \"k\"),  # voiceless velar plosive, usually in \"cat\"\n",
    "    (\"\", \"j\"),   # silent matching \"j\"\n",
    "    (\"p\",\"\"),    # silent \"p\" in words like \"pnumonia\" or \"coup\"\n",
    "    (\"t\",\"\"),    # silent \"p\" in words like \"pnumonia\" or \"coup\"\n",
    "    (\"sl\",\"l\"),  # silent \"s\" like in \"island\"\n",
    "    (\"ch\",\"\"),   # to handle \"yacht\"\n",
    "    (\"q\",\"k\"),   # to handle \"queue\"\n",
    "    (\"l\",\"r\"),   # to handle \"colonel\"\n",
    "    (\"s\",\"\"),    # silent \"s\" in words like \"debris\"\n",
    "    (\"ct\",\"t\"),  # to handle \"indict\"\n",
    "    (\"l\",\"\"),    # silent \"l\" in words like \"salmon\"\n",
    "    (\"dn\",\"n\"),  # to handle \"wednesday\"\n",
    "    (\"z\",\"\"),    # to handle \"rendezvous\"\n",
    "    (\"r\",\"\"),    # to handle \"dossier\"\n",
    "    (\"ç\",\"s\"),   # to handle \"façade\"\n",
    "    (\"gm\",\"m\"),  # to handle \"rendezvous\"\n",
    "    (\"\",\"w\"),    # to handle \"bourgeois\"\n",
    "    (\"sc\",\"ʃ\"),  # \"concience\"\n",
    "    (\"d\",\"\"),    # \"handkerchief\"\n",
    "    ('x', 'ɡz'),\n",
    "    ('gg', 'ɡ'),\n",
    "    (\"cq\",\"kw\"),\n",
    "    (\"ng\",\"n\"),\n",
    "    (\"kd\",\"t\"),\n",
    "    (\"nm\",\"m\"),\n",
    "    (\"d\",\"dʒ\"),\n",
    "    (\"x\",\"kʃ\"),\n",
    "    (\"x\",\"k\"),\n",
    "    (\"x\",\"ʃ\"),\n",
    "    (\"l\",\"ɝ\"),\n",
    "    (\"x\",\"ɡʒ\"),\n",
    "    (\"s\",\"tʃ\")\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 337,
   "id": "c8d7e23f-a79e-4ad1-b9d7-1c2ed46e2e03",
   "metadata": {},
   "outputs": [],
   "source": [
    "taylor_convert = {\n",
    "    (\"c\", \"k\"):\"k\",  # voiceless velar plosive, usually in \"cat\"\n",
    "    (\"t\", \"t\"):\"t\",  # voiceless alveolar plosive\n",
    "    (\"d\", \"d\"):\"d\",  # voiced alveolar plosive\n",
    "    (\"d\", \"t\"):\"t\",  # voiced alveolar plosive\n",
    "    (\"t\", \"d\"):\"t\",  # voiced alveolar plosive\n",
    "    (\"n\", \"n\"):\"n\",  # alveolar nasal\n",
    "    (\"s\", \"s\"):\"s\",  # voiceless alveolar fricative\n",
    "    (\"l\", \"l\"):\"l\",  # alveolar lateral approximant\n",
    "    (\"l\", \"ɫ\"):\"l\",\n",
    "    (\"r\", \"r\"):\"r\",  # alveolar approximant matching itself\n",
    "    (\"r\", \"ɹ\"):\"r\",  # alveolar approximant\n",
    "    (\"r\", \"ɝ\"):\"r\",  # alveolar approximant\n",
    "    (\"m\", \"m\"):\"m\",  # bilabial nasal\n",
    "    (\"b\", \"b\"):\"b\",  # voiced bilabial plosive\n",
    "    (\"k\", \"k\"):\"k\",  # voiceless velar plosive\n",
    "    (\"f\", \"f\"):\"f\",  # voiceless labiodental fricative\n",
    "    (\"g\", \"g\"):\"g\",  # voiced velar plosive\n",
    "    (\"g\", \"ɡ\"):\"g\",\n",
    "    (\"v\", \"v\"):\"f\",  # voiced labiodental fricative\n",
    "    (\"x\", \"ks\"):\"x\", # voiceless velar-alveolar fricative, \"taxi\"\n",
    "    (\"z\", \"z\"):\"s\",  # voiced alveolar fricative\n",
    "    (\"p\", \"p\"):\"p\",  # voiceless bilabial plosive\n",
    "    (\"h\", \"h\"):\"h\",  # voiceless glottal fricative\n",
    "    (\"w\", \"w\"):\"w\",  # voiced labial-velar approximant\n",
    "    (\"y\", \"j\"):\"y\",  # voiced palatal approximant\n",
    "    (\"j\", \"dʒ\"):\"g\", # voiced palato-alveolar affricate, often \"j\" sound\n",
    "    (\"j\", \"ʤ\"):\"g\", # voiced palato-alveolar affricate, often \"j\" sound\n",
    "    (\"j\", \"ʒ\"):\"g\", # voiced palato-alveolar affricate, often \"j\" sound\n",
    "    (\"ng\", \"ŋ\"):\"ng\", # voiced velar nasal\n",
    "    (\"n\", \"ŋ\"):\"n\", # voiced velar nasal\n",
    "    (\"ch\", \"ʧ\"):\"C\", # voiceless palato-alveolar affricate\n",
    "    (\"ch\", \"tʃ\"):\"C\", # voiceless palato-alveolar fricative\n",
    "    (\"sh\", \"tʃ\"):\"S\", # voiceless palato-alveolar fricative\n",
    "    (\"sh\", \"ʧ\"):\"S\", # voiceless palato-alveolar affricate\n",
    "    (\"sh\", \"ʃ\"):\"S\", # voiceless palato-alveolar fricative\n",
    "    (\"ch\", \"ʃ\"):\"C\", # voiceless palato-alveolar fricative\n",
    "    (\"th\", \"θ\"):\"T\", # voiceless dental fricative\n",
    "    (\"th\", \"ð\"):\"T\", # voiced dental fricative, as in \"this\"\n",
    "    (\"t\", \"tʃ\"):\"t\", # voiceless palato-alveolar fricative\n",
    "    (\"g\", \"ʒ\"):\"g\",  # voiced palato-alveolar fricative\n",
    "    (\"s\", \"z\"):\"s\",  # voiced alveolar fricative, usually in \"dogs\"\n",
    "    (\"s\", \"ʒ\"):\"s\",  # voiced palato-alveolar fricative, as in \"measure\"\n",
    "    (\"z\", \"ʒ\"):\"s\",  # voiced palato-alveolar fricative, as in \"measure\"\n",
    "    (\"z\", \"ʃ\"):\"s\",  # voiceless palato-alveolar fricative, as in \"champagne\"\n",
    "    (\"s\", \"ʃ\"):\"s\",  # voiceless palato-alveolar fricative, as in \"champagne\"\n",
    "    (\"c\", \"ʃ\"):\"c\",  # voiceless palato-alveolar fricative, as in \"champagne\"\n",
    "    (\"ck\", \"k\"):\"k\", # voiceless velar plosive, as in \"back\"\n",
    "    (\"ps\", \"s\"):\"s\", # \"ps\" matching \"s\"\n",
    "    (\"q\", \"kw\"):\"q\", # voiceless velar and labial-velar approximant, usually in \"queen\"\n",
    "    (\"b\", \"\"):\"\",   # silent b\n",
    "    (\"h\", \"\"):\"\",   # silent h\n",
    "    (\"w\", \"\"):\"\",   # silent w\n",
    "    (\"y\", \"\"):\"\",   # sometimes silent y\n",
    "    (\"f\", \"v\"):\"f\",  # voiced labiodental fricative\n",
    "    (\"ph\", \"f\"):\"f\", # voiceless labiodental fricative, usually in \"phone\"\n",
    "    (\"gh\", \"f\"):\"f\", # voiceless labiodental fricative, as in \"laugh\"\n",
    "    (\"gh\", \"\"):\"\",  # silent \"gh\", as in \"though\"\n",
    "    (\"kn\", \"n\"):\"n\", # silent \"k\", as in \"knee\"\n",
    "    (\"gn\", \"n\"):\"n\", # silent \"g\", as in \"gnome\"\n",
    "    (\"mn\", \"m\"):\"m\", # \"mn\" matching \"m\"\n",
    "    (\"mn\", \"n\"):\"n\", # \"mn\" matching \"n\"\n",
    "    (\"wr\", \"r\"):\"r\", # silent \"w\", as in \"write\" with matching \"r\"\n",
    "    (\"wh\", \"w\"):\"w\", # voiced labial-velar approximant, as in \"what\"\n",
    "    (\"sc\", \"s\"):\"s\", # \"sc\" matching \"s\"\n",
    "    (\"c\", \"s\"):\"s\",  # voiceless alveolar fricative, usually in \"cent\"\n",
    "    (\"ld\", \"d\"):\"d\", # \"dd\" matching \"d\"\n",
    "    (\"dd\", \"d\"):\"d\", # \"dd\" matching \"d\"\n",
    "    (\"ll\", \"l\"):\"l\", # \"ll\" matching \"l\"\n",
    "    (\"ll\", \"ɫ\"):\"l\",\n",
    "    (\"mm\", \"m\"):\"m\", # \"mm\" matching \"m\"\n",
    "    (\"nn\", \"n\"):\"n\", # \"nn\" matching \"n\"\n",
    "    (\"rr\", \"r\"):\"r\", # \"nn\" matching \"n\"\n",
    "    (\"pp\", \"p\"):\"p\", # \"pp\" matching \"p\"\n",
    "    (\"ff\", \"f\"):\"f\", # \"pp\" matching \"p\"\n",
    "    (\"tt\", \"t\"):\"t\", # \"tt\" matching \"t\"\n",
    "    (\"ss\", \"s\"):\"s\", # \"ss\" matching \"s\"\n",
    "    (\"ss\", \"z\"):\"s\", # \"ss\" matching \"z\"\n",
    "    (\"ss\", \"ʒ\"):\"s\", # \"ss\" matching \"ʒ\"\n",
    "    (\"ss\", \"ʃ\"):\"s\", # \"ss\" matching \"ʃ\"\n",
    "    (\"t\", \"ʒ\"):\"t\",  # \"t\" matching \"ʒ\"\n",
    "    (\"t\", \"ʃ\"):\"t\",  # \"t\" matching \"ʃ\"\n",
    "    (\"x\", \"gz\"):\"x\", # voiced velar-alveolar fricative, \"exam\"\n",
    "    (\"x\", \"ɡz\"):\"x\", # voiced velar-alveolar fricative, \"exam\"\n",
    "    (\"x\", \"z\"):\"x\",  # voiced alveolar fricative, \"xylophone\"\n",
    "    (\"zz\", \"ts\"):\"z\",# \"zz\" matching \"ts\"\n",
    "    (\"z\", \"ts\"):\"z\", # \"z\" matching \"ts\"\n",
    "    (\"g\", \"dʒ\"):\"g\", # voiced palato-alveolar affricate\n",
    "    (\"g\", \"ʤ\"):\"g\", # voiced palato-alveolar affricate\n",
    "    (\"gg\", \"g\"):\"g\", # \"gg\" matching voiced velar plosive\n",
    "    (\"gg\", \"ɡ\"):\"g\",\n",
    "    (\"gg\", \"ʒ\"):\"g\", # \"gg\" matching voiced palato-alveolar fricative\n",
    "    (\"gg\", \"dʒ\"):\"g\",# \"gg\" matching voiced palato-alveolar affricate\n",
    "    (\"gg\", \"ʤ\"):\"g\",# \"gg\" matching voiced palato-alveolar affricate\n",
    "    (\"ch\", \"k\"):\"C\", # voiceless velar plosive, as in \"school\"\n",
    "    (\"cc\", \"k\"):\"k\",  # voiceless velar plosive, usually in \"cat\"\n",
    "    (\"\", \"j\"):\"\",   # silent matching \"j\"\n",
    "    (\"p\",\"\"):\"\",    # silent \"p\" in words like \"pnumonia\" or \"coup\"\n",
    "    (\"t\",\"\"):\"\",    # silent \"p\" in words like \"pnumonia\" or \"coup\"\n",
    "    (\"sl\",\"l\"):\"l\",  # silent \"s\" like in \"island\"\n",
    "    (\"ch\",\"\"):\"\",   # to handle \"yacht\"\n",
    "    (\"q\",\"k\"):\"k\",   # to handle \"queue\"\n",
    "    (\"l\",\"r\"):\"r\",   # to handle \"colonel\"\n",
    "    (\"s\",\"\"):\"\",    # silent \"s\" in words like \"debris\"\n",
    "    (\"ct\",\"t\"):\"t\",  # to handle \"indict\"\n",
    "    (\"l\",\"\"):\"\",    # silent \"l\" in words like \"salmon\"\n",
    "    (\"dn\",\"n\"):\"n\",  # to handle \"wednesday\"\n",
    "    (\"z\",\"\"):\"\",    # to handle \"rendezvous\"\n",
    "    (\"r\",\"\"):\"\",    # to handle \"dossier\"\n",
    "    (\"ç\",\"s\"):\"s\",   # to handle \"façade\"\n",
    "    (\"gm\",\"m\"):\"m\",  # to handle \"rendezvous\"\n",
    "    (\"\",\"w\"):\"\",    # to handle \"bourgeois\"\n",
    "    (\"sc\",\"ʃ\"):\"s\",  # \"concience\"\n",
    "    (\"d\",\"\"):\"\",    # \"handkerchief\"\n",
    "    (\"cq\",\"kw\"):\"k\",  # \"aquire\"\n",
    "    (\"ng\",\"n\"):\"ng\",\n",
    "    (\"kd\",\"t\"):\"kt\",\n",
    "    (\"nm\",\"m\"):\"m\", #government\n",
    "    (\"d\",\"dʒ\"):\"d\", #education\n",
    "    (\"x\",\"kʃ\"):\"x\",\n",
    "    (\"x\",\"k\"):\"x\", #excellent\n",
    "    (\"x\",\"ʃ\"):\"x\", #anxious\n",
    "    (\"l\",\"ɝ\"):\"l\", #colonel\n",
    "    (\"x\",\"ɡʒ\"):\"x\", #luxury\n",
    "    (\"s\",\"tʃ\"):\"s\" #tensions\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 338,
   "id": "4d50ce6d-ce7a-4d06-a22d-cc6f066772af",
   "metadata": {},
   "outputs": [],
   "source": [
    "def taylor_raw(word):\n",
    "    consonants = \"\".join([taylor_convert[x] for x in process_word(word,ipa_dict, consonant_pairs)])\n",
    "    vowels = \"aeiouAEIOUɑæɪɔʊəɜɛːɒʌ\"\n",
    "    ipa = ipa_dict[word]\n",
    "    punctuation_and_spaces = \"ˈˌ. ,-'\\\";:!?()[]{}<>/@#%^&*~`\"\n",
    "    ipa =  ''.join([char for char in ipa if char not in punctuation_and_spaces])\n",
    "    initial, final = False, False\n",
    "    if ipa[0] in vowels:\n",
    "        initial = True\n",
    "    if len(consonants) > 0 and ipa[-1] in vowels:\n",
    "        final = True\n",
    "    if initial:\n",
    "        consonants = \"'\" + consonants\n",
    "    if final:\n",
    "        consonants = consonants + \"'\"\n",
    "    return consonants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 339,
   "id": "01af9b78-8b6f-4cc0-8f44-25a49ba6873d",
   "metadata": {},
   "outputs": [],
   "source": [
    "taylor_endings = {\n",
    "    'ing':(',',2),\n",
    "    'ings':(\"_\",3),\n",
    "    'ble':(\"-b\",2),\n",
    "    'ful':(\"-f\",2),\n",
    "    'ly':(\".\",2),\n",
    "    'ment':(\"-m\",3),\n",
    "    'ments':(\"-m\",4),\n",
    "    'ness':(\"-n\",2),\n",
    "    'rary':(\"-r\",3),\n",
    "    'self':(\"-s\",3),\n",
    "    'selvs':(\"-s\",4),\n",
    "    'ward':(\"-w\",3),\n",
    "    'ship':(\"-S\",2),\n",
    "    'ious':(\"I\",1),\n",
    "    'eous':(\"I\",1),\n",
    "    'uous':(\"I\",1),\n",
    "    'tion':(\"*\",2),\n",
    "    'sion':(\"*\",2),\n",
    "    'tions':(\"^\",3),\n",
    "    'sions':(\"^\",3),\n",
    "}\n",
    "\n",
    "taylor_endings = {\n",
    "    'ing':(',',2),\n",
    "    'ings':(\"_\",3),\n",
    "    'ble':(\"b\",2),\n",
    "    'ful':(\"f\",2),\n",
    "    'ly':(\".\",2),\n",
    "    'ment':(\"m\",3),\n",
    "    'ments':(\"m\",4),\n",
    "    'ness':(\"n\",2),\n",
    "    'rary':(\"r\",3),\n",
    "    'self':(\"s\",3),\n",
    "    'selvs':(\"s\",4),\n",
    "    'ward':(\"w\",3),\n",
    "    'ship':(\"S\",2),\n",
    "    'ious':(\"I\",1),\n",
    "    'eous':(\"I\",1),\n",
    "    'uous':(\"I\",1),\n",
    "    'tion':(\"*\",2),\n",
    "    'sion':(\"*\",2),\n",
    "    'tions':(\"^\",3),\n",
    "    'sions':(\"^\",3),\n",
    "}\n",
    "\n",
    "taylor_briefs = {\n",
    "    \"be\":\"b\",\n",
    "    \"by\":\"b\",\n",
    "    \"been\":\"b\",\n",
    "    \"do\":\"d\",\n",
    "    \"did\":\"d\",\n",
    "    \"of\":\"f\",\n",
    "    \"off\":\"f\",\n",
    "    \"if\":\"f\",\n",
    "    \"god\":\"g\",\n",
    "    \"give\":\"g\",\n",
    "    \"go\":\"g\",\n",
    "    \"have\":\"h\",\n",
    "    \"he\":\"h\",\n",
    "    \"know\":\"k\",\n",
    "    \"known\":\"k\",\n",
    "    \"no\":\"k\",\n",
    "    \"lord\":\"l\",\n",
    "    \"all\":\"l\",\n",
    "    \"me\":\"m\",\n",
    "    \"my\":\"m\",\n",
    "    \"many\":\"m\",\n",
    "    \"hand\":\"n\",\n",
    "    \"and\":\"n\",\n",
    "    \"an\":\"n\",\n",
    "    \"in\":\"n\",\n",
    "    \"peace\":\"p\",\n",
    "    \"person\":\"p\",\n",
    "    \"are\":\"r\",\n",
    "    \"air\":\"r\",\n",
    "    \"our\":\"r\",\n",
    "    \"or\":\"r\",\n",
    "    \"his\":\"s\",\n",
    "    \"is\":\"s\",\n",
    "    \"as\":\"s\",\n",
    "    \"us\":\"s\",\n",
    "    \"that\":\"t\",\n",
    "    \"time\":\"t\",\n",
    "    \"with\":\"w\",\n",
    "    \"which\":\"w\",\n",
    "    \"who\":\"w\",\n",
    "    \"example\":\"x\",\n",
    "    \"except\":\"x\",\n",
    "    \"you\":\"y\",\n",
    "    \"your\":\"y\",\n",
    "    \"year\":\"y\",\n",
    "    \"such\":\"C\",\n",
    "    \"chance\":\"C\",\n",
    "    \"shalt\":\"S\",\n",
    "    \"shall\":\"S\",\n",
    "    \"the\":\"T\",\n",
    "    \"thee\":\"T\",\n",
    "    \"they\":\"T\",\n",
    "    \"conscious\":\"I\",\n",
    "    \"judicious\":\"I\"\n",
    "}\n",
    "\n",
    "def taylor_full(word):\n",
    "    if word in taylor_briefs:\n",
    "        return taylor_briefs[word]\n",
    "    for ending in taylor_endings:\n",
    "        if len(word) >= len(ending) and word[-len(ending):] == ending:\n",
    "            symb, trim = taylor_endings[ending]\n",
    "            return taylor_raw(word)[:-trim][:4] + symb\n",
    "    return taylor_raw(word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 340,
   "id": "f2781f13-7f34-425c-8fe6-78852ea6da98",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2283, 'out of', 125926, 'failed.')"
      ]
     },
     "execution_count": 340,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "taylor = {}\n",
    "failures = 0\n",
    "\n",
    "for word in ipa_dict:\n",
    "    try:\n",
    "        taylor[word] = taylor_full(word)\n",
    "    except:\n",
    "        failures+=1\n",
    "\n",
    "failures, \"out of\", len(ipa_dict), \"failed.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 341,
   "id": "5bbc1e86-e0a4-4c7b-b220-d57bb4bc9dba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mean_binary_length': 12.003069219198045,\n",
       " 'mean_binary_excess': 5.358130249730938,\n",
       " 'mean_entropy_shift': 2.2383583238376072,\n",
       " 'mean_transmission_cost': 9.922202289347016,\n",
       " 'mean_transmission_excess': 3.2772633198799097,\n",
       " 'mean_transmission_shift': 0.15749139398657874,\n",
       " 'reconstruction_entropy': 0.7987322326991305,\n",
       " 'reconstruction_error': 0.2218609124591735,\n",
       " 'alphabet': \"s t r ' n f l d T k m p w b g h , * . S C y x ^ q c _ I z\"}"
      ]
     },
     "execution_count": 341,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "code_statistics([x for x in one_grams_dict if x in taylor], taylor)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8cecc239-ffdb-45ef-9247-7b2329a290bf",
   "metadata": {},
   "source": [
    "# Odell/Times Variant Approximation\n",
    "\n",
    "This attempts to construct an allignment of the spelling and of the IPA sounds then construct the Taylor representation of that word. This is not a true implementation of Odell/Times Variant, but instead just allows for the the names of initial and final vowels to be kept."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 342,
   "id": "c2e1a6e8-c566-4d1a-931b-93b89d887fb3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# just kinda madeup lol\n",
    "vowel_map = {\n",
    "    \"a\":\"a\",\n",
    "    \"e\":\"e\",\n",
    "    \"i\":\"i\",\n",
    "    \"o\":\"o\",\n",
    "    \"u\":\"u\",\n",
    "    \"ɑ\":\"a\",\n",
    "    \"æ\":\"a\",\n",
    "    \"ɪ\":\"i\",\n",
    "    \"ɔ\":\"o\",\n",
    "    \"ʊ\":\"u\",\n",
    "    \"ə\":\"e\",\n",
    "    \"ɜ\":\"e\",\n",
    "    \"ɛ\":\"e\",\n",
    "    \"ɒ\":\"a\",\n",
    "    \"ʌ\":\"a\",\n",
    "}\n",
    "\n",
    "def taylor_plus_raw(word):\n",
    "    consonants = \"\".join([taylor_convert[x] for x in process_word(word,ipa_dict, consonant_pairs)])\n",
    "    vowels = \"aeiouAEIOUɑæɪɔʊəɜɛːɒʌ\"\n",
    "    ipa = ipa_dict[word]\n",
    "    punctuation_and_spaces = \"ˈˌ. ,-'\\\";:!?()[]{}<>/@#%^&*~`\"\n",
    "    ipa =  ''.join([char for char in ipa if char not in punctuation_and_spaces])\n",
    "    initial, final = False, False\n",
    "    if ipa[0] in vowels:\n",
    "        initial = vowel_map[ipa[0]]\n",
    "        if word[0] in vowels:\n",
    "            initial = word[0]\n",
    "    if len(consonants) > 0 and ipa[-1] in vowels:\n",
    "        final = vowel_map[ipa[-1]]\n",
    "        if word[-1] in vowels:\n",
    "            final = word[-1]\n",
    "    if initial:\n",
    "        consonants = initial + consonants\n",
    "    if final:\n",
    "        consonants = consonants + final\n",
    "    return consonants\n",
    "\n",
    "def taylor_plus_full(word):\n",
    "    if word in taylor_briefs:\n",
    "        return taylor_briefs[word]\n",
    "    for ending in taylor_endings:\n",
    "        if len(word) >= len(ending) and word[-len(ending):] == ending:\n",
    "            symb, trim = taylor_endings[ending]\n",
    "            return taylor_plus_raw(word)[:-trim][:4] + symb\n",
    "    return taylor_plus_raw(word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 343,
   "id": "93c37c40-4b9d-411f-a6b1-432e7d0c866f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2283, 'out of', 125926, 'failed.')"
      ]
     },
     "execution_count": 343,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "taylor_plus = {}\n",
    "failures = 0\n",
    "\n",
    "for word in ipa_dict:\n",
    "    try:\n",
    "        taylor_plus[word] = taylor_plus_full(word)\n",
    "    except:\n",
    "        failures+=1\n",
    "\n",
    "failures, \"out of\", len(ipa_dict), \"failed.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 344,
   "id": "3cda90d7-f849-4f74-a047-cb223dba5e7d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mean_binary_length': 12.463657606792484,\n",
       " 'mean_binary_excess': 5.818718637325378,\n",
       " 'mean_entropy_shift': 2.698946711432047,\n",
       " 'mean_transmission_cost': 10.487605401556818,\n",
       " 'mean_transmission_excess': 3.842666432089712,\n",
       " 'mean_transmission_shift': 0.722894506196381,\n",
       " 'reconstruction_entropy': 0.6857898653998239,\n",
       " 'reconstruction_error': 0.1900756428174497,\n",
       " 'alphabet': 's t r n f l d T k m p i a o w b g h e , * u . S C y x ^ q c _ I z'}"
      ]
     },
     "execution_count": 344,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "code_statistics([x for x in one_grams_dict if x in taylor_plus], taylor_plus)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a1980117-ddd4-41c5-a5ff-64663bdac288",
   "metadata": {},
   "source": [
    "# Characterie\n",
    "\n",
    "This uses a dictionary I made for [my webpage](https://characterie.neocities.org/).  This requires a custom alphabet and length measure that takes into account the strange way that characterie represents words."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 345,
   "id": "eec30f49-be85-425a-a9e6-d291be3bdc85",
   "metadata": {},
   "outputs": [],
   "source": [
    "words = {\n",
    "    'a':['abound','about','accept','accuse','advance','air','again','age','all','almost','also','although','alter','am','amend','anger','anoint','apparel','appertain','appoint','arm','art','as','at','awe','away'], #added awe and away from the table of english words\n",
    "    'b':['banish','bargain','bear','beast','beat','before','beg','begin','belly','bend','benefit','bestow','between','beware','by','because','bird','bishop','bite','blaze','blows','blood','blue','bless','bone','book','borrow','both','bottom','bread','break','breed','breast','bright','brittle','brother','bruise','burn','busy','but'], #blussy to blush by table guess\n",
    "    'c':['call','can','captious','care','case','cave','cause','certain','challenge','change','christian','church','choose','kill','kind','circumstance','city','cloth','know','coin','color','command','comfort','common','compare','company','compel','continue','conceive','condition','contained','consider','confess','conscience','constant','convey','content','come','corner','corrupt','cover','council','count','cry','question','quit','compass','cut'], # caii to call and first cause to case by consulting Table.\n",
    "    'd':['day','danger','deceive','declare','dedicate','dear','defend','delight','deprive','deputy','descend','desire','despise','destitute','destroy','diet','differ','dig','diligence','dissemble','distress','dizzy','do','doubt','draw','dream','dry','drink','drive','drop','due','double'], #dont to doubt by brachyography\n",
    "    'e':['earth','edge','even','element','eloquence','enough','enter','enterprise','erect','err','escape','ever','example','except','exercise','expect','expert'], #every changed to ever due to overlap with all, second except changed to expect based on Bales\n",
    "    'f':['face','faith','fair','fall','fare','far','fast','fat','fear','feast','feed','field','fetter','fight','fickle','fill','filthy','fine','find','fire','fish','flatter','flesh','fly','fling','flourish','follow','ford','force','forsake','fortune','foundation','fountain','free','friend','from','frown','fruit','furnish'], #third fair homophone changed to far.  Matches back table and Bales.\n",
    "    'g':['gape','guard','garment','gather','gentle','guest','get','guide','given','go','god','good','gospel','glass','glory','grace','grass','grain','gravel','grave','great','grief','grove','grow'], #grassy to gravel by back table\n",
    "    'h':['half','hand','hang','hard','heart','harvest','haste','have','halt','haunt','heal','he','head','help','herb','here','heat','hitherto','heaven','high','hill','history','hit','holy','hollow','honest','hope','how','hold','house','husband','hurt'], #hie to high and missing h to holy based on table\n",
    "    'i':['yet','if','inheritance','enjoy','innocent','inquire','instrument','entertain','invent','you','join','young','judge','jewel'],\n",
    "    'l':['labor','last','late','laugh','lean','learn','let','leather','leave','lie','liberality','life','light','like','limit','line','load','loose','love'], #ly to lie, lie to light by consulting back, removed louge for lack of matching backmatter word (all represented after other change)\n",
    "    'm':['mad','make','man','manner','many','merchant','mark','marry','marvel','mass','master','matter','mean','measure','meet','mercy','merit','message','metal','mind','mine','mirth','mixed','mock','modesty','more','move','mouth','much','murmur'], #mast to master due to use with lord and maste going to rod in table (no master listed), many for money due to back and Bales\n",
    "    'n':['nail','nature','necessary','neighbor','neither','net','nevertheless','nip','no','noble','nothing','nonetheless','now'],\n",
    "    'o':['obey','office','offend','offer','oft','oh','omit','one','open','oppose','oppress','or','order','oath','other','over','overmuch','overtake','ought','own','our','out','outward'], \n",
    "    'p':['patient','parent','part','pass','peace','people','perfect','persuade','physique','place','plague','play','plain','plead','pledge','point','possible','power','pray','praise','preach','prejudice','prepare','present','pretend','prevail','prevent','prick','prince','promise','prophesy','proportion','prosper','prove','pulpit','punish','purge','purpose'],\n",
    "    'r':['race','reign','rebuke','reach','recover','read','ready','region','rejoice','religion','remember','reap','repent','reason','resolve','rest','restore','reward','revenge','revile','rich','right','ripe','rob','rod','root','rough','rub','rule','rush'],\n",
    "    's':['salute','save','scarce','school','slander','see','seed','seem','since','shine','ship','shoot','side','sink','sing','sit','skill','slip','smatter','smoke','sudden','soever','some','sore','sound','space','spare','spark','speak','spice','spit','spring','stay','start','step','steward','stone','strain','strong','study','stuff','stumble','substance','such','sweet','swell','surfeit','sun'], #shore to shoot to match back table, string to spring to also increase alignment\n",
    "    't':['tame','taste','tear','temper','tempest','thank','that','then','thence','there','thither','thine','thing','think','this','thrive','tidings','till','time','together','tongue','touch','trade','treason','tree','tribute','triumph','trouble','true','turn'], # second tree changed to true (matches English Words)\n",
    "    'w':['way','vain','wait','wear','warn','watch','water','weapon','weary','venture','very','virtue','vessel','weather','what','where','wherefore','which','whore','will','vine','wind','winter','violence','wife','visit','witness','wood','word','world','worship','worthy','up','uproar','wrinkle','write','use'] #wayne to way, vnrove to uproar based on back\n",
    "}\n",
    "\n",
    "particles = ['the', 'we', 'i', 'well', 'etc', 'be', 'fie', 'hence', 'they', 'myself', 'ourselves', 'so', 'so_as', 'and', 'in', 'of', 'to', 'a', 'for', 'with', 'it', 'it_is', 'it_self', 'as_it_were', 'that_is_to_say', 'that', 'least_that', 'thou', 'ye', 'self', 'ward', 'amen']\n",
    "\n",
    "encode = {\n",
    "    \"0\":\"aa\",\n",
    "    \"1\":\"ba\",\n",
    "    \"2\":\"ca\",\n",
    "    \"3\":\"da\",\n",
    "    \"4\":\"ea\",\n",
    "    \"5\":\"fa\",\n",
    "    \"6\":\"ga\",\n",
    "    \"7\":\"ha\",\n",
    "    \"8\":\"ia\",\n",
    "    \"9\":\"la\"\n",
    "}\n",
    "\n",
    "num_encode = \"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ@#$%^&*()[]{}\"\n",
    "\n",
    "for letter in words:\n",
    "    for i in range(len(words[letter])):\n",
    "        encode[words[letter][i]] = num_encode[i] + letter\n",
    "\n",
    "for i in range(len(particles)):\n",
    "    encode[\".\" + particles[i]] = num_encode[i] + \".\" "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 346,
   "id": "6bb1d2b1-a499-4080-b953-6d105e69237b",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('manual_curate.json') as f:\n",
    "    characterie_raw = json.load(f)\n",
    "\n",
    "def characterie_length(in_string):\n",
    "    count = 0\n",
    "    words = in_string.split(\" \")\n",
    "    for word in words:\n",
    "        parts = word.split(\"-\")\n",
    "        count += len(parts)-1\n",
    "        segments = parts[0].split(\">\")\n",
    "        for segment in segments:\n",
    "            if \".\" in segment:\n",
    "                count += 1\n",
    "            elif segment.isupper():\n",
    "                count += len(segment)\n",
    "            else:\n",
    "                count += 2\n",
    "    return count\n",
    "\n",
    "parts_map = {\n",
    "    \"past\":\"-\", #left dot\n",
    "    \"future\":\"+\", #right dot\n",
    "    \"mark\":\"+\",\n",
    "    \"er\":\":\",\n",
    "    \"s\":\"+\",\n",
    "    \"ing\":\"_\",\n",
    "    \"not\":\"!\",\n",
    "    \"ship\":\" As\",\n",
    "    \"hood\":\" As\",\n",
    "}\n",
    "\n",
    "def characterie_encode(in_word):\n",
    "    in_string = characterie_raw[in_word]\n",
    "    output = []\n",
    "    words = in_string.split(\" \")\n",
    "    for word in words:\n",
    "        parts = word.split(\"-\")\n",
    "        segments = parts[0].split(\">\")\n",
    "        out_seg = []\n",
    "        for segment in segments:\n",
    "            if segment.isupper():\n",
    "                out_seg.append(segment.lower())\n",
    "            else:\n",
    "                out_seg.append(encode[segment])\n",
    "        output_word = \"\".join(out_seg)\n",
    "        for i in range(1,len(parts)):\n",
    "            output_word += parts_map[parts[i]]\n",
    "        output.append(output_word)\n",
    "    return \" \".join(output)\n",
    "    \n",
    "characterie = {w:characterie_encode(w) for w in characterie_raw}\n",
    "    \n",
    "# # the letters, left dot, right dot, right double dot, under double dot, negation\n",
    "# # the 48 foot/orientation combinations possible\n",
    "# # the 32 particles (some also letters, and thus somewhat overcounted)\n",
    "# characterie_alphabet = set()\n",
    "# characterie_alphabet = characterie_alphabet | set(\"abcdefghilmnoprsztu1.:_!\")\n",
    "# characterie_alphabet = characterie_alphabet | set(f\"o{i}\" for i in range(48))\n",
    "# characterie_alphabet = characterie_alphabet | set(f\"p{i}\" for i in range(32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 347,
   "id": "02bf50d1-7343-43e5-b996-f411825e8d7b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mean_binary_length': 15.229091785638134,\n",
       " 'mean_binary_excess': 9.756127863469656,\n",
       " 'mean_entropy_shift': 6.8783592319593065,\n",
       " 'mean_transmission_cost': 13.066362734274271,\n",
       " 'mean_transmission_excess': 7.593398812105793,\n",
       " 'mean_transmission_shift': 4.7156301805954435,\n",
       " 'reconstruction_entropy': 0.21794984827213726,\n",
       " 'reconstruction_error': 0.0662984787750377,\n",
       " 'alphabet': '. 0 s h t a E c w o F B D m b G p f 5 H n i 8 r l I 1 d 7 e 2 - 6 K g J M 9 C S + N A 3 P L Z   T % 4 _ V O Q R W : @ * X $ # Y U ( ] ) & { ^ ['}"
      ]
     },
     "execution_count": 347,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "code_statistics([x for x in one_grams_dict if x in characterie], characterie)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b562b903-8c30-4bf5-a583-6162e89428a0",
   "metadata": {},
   "source": [
    "# Teeline\n",
    "\n",
    "Using the Anki deck from [here](https://ankiweb.net/shared/info/310534731)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 348,
   "id": "960c9c6a-e574-4c12-8c31-1eab57c7bedf",
   "metadata": {},
   "outputs": [],
   "source": [
    "from ankipandas import Collection\n",
    "\n",
    "col = Collection(\"collection.anki21\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 349,
   "id": "ebe04e56-03b7-4a10-a1e3-ff0cb6ef9dc6",
   "metadata": {},
   "outputs": [],
   "source": [
    "teeline = {}\n",
    "\n",
    "stuff_map = {'+':'',\n",
    " '+C':' C',\n",
    " '+CH':' CH',\n",
    " '+CHmn':' CHmn',\n",
    " '+CHr':' CHr',\n",
    " '+CHs':' CHs',\n",
    " '+Cd':' Cd',\n",
    " '+Ci':' Ci',\n",
    " '+Cs':' Cs',\n",
    " '+ING':' i',\n",
    " '+N':' N',\n",
    " '+ONG':' o',\n",
    " '+S':' S',\n",
    " '+Si':' Si',\n",
    " '+W':' W',\n",
    " '+and':' and',\n",
    " '+ang':' a',\n",
    " '+ange':' ae',\n",
    " '+c':' c',\n",
    " '+ch':' ch',\n",
    " '+ci':' ci',\n",
    " '+cl':' cl',\n",
    " '+cr':' cr',\n",
    " '+eng':' e',\n",
    " '+iN':' iN',\n",
    " '+ibble':' I',\n",
    " '+ing':' i',\n",
    " '+ingC':' iC',\n",
    " '+ingc':' ic',\n",
    " '+inge':' ie',\n",
    " '+ingle':' ile',\n",
    " '+ingr':' ir',\n",
    " '+l':' l',\n",
    " '+li':' li',\n",
    " '+ly':' ly',\n",
    " '+nch':' nch',\n",
    " '+ong':' o',\n",
    " '+ung':' u'}\n",
    "\n",
    "for _,row in col.notes.iterrows():\n",
    "    info = row['nflds']\n",
    "    head = info[1].replace(\"&nbsp;\",\"\")\n",
    "    tail = info[2].replace(\"&nbsp;\",\"\")\n",
    "    \n",
    "    if \"<\" in info[1]:\n",
    "        continue\n",
    "\n",
    "    output = tail.split(\" \")\n",
    "    for i in range(len(output)):\n",
    "        part = output[i]\n",
    "        if len(part) > 0 and part[0] == '+':\n",
    "            output[i] = stuff_map[part]\n",
    "        else:\n",
    "            output[i] = \" \"+output[i]\n",
    "    final = \"\".join(output)[1:]\n",
    "\n",
    "    if len(final) == 0:\n",
    "        continue\n",
    "        \n",
    "    for word in head.split(\" \"):\n",
    "        teeline[word] = final.lower()\n",
    "\n",
    "teeline['things'] = 'th is'\n",
    "teeline['sung'] = 's u'\n",
    "teeline['compelling'] = 'cmpl i' # teeline['compelling'] = 'CMPL i'\n",
    "teeline['mangle'] = 'm al'\n",
    "teeline['tongue'] = 't o'\n",
    "teeline['tongs'] = 't os'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 350,
   "id": "137692ae-e445-444d-8c35-995cd0342b6d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mean_binary_length': 13.596477070553554,\n",
       " 'mean_binary_excess': 8.630874974793834,\n",
       " 'mean_entropy_shift': 5.8566119086553075,\n",
       " 'mean_transmission_cost': 11.296584494109258,\n",
       " 'mean_transmission_excess': 6.330982398349538,\n",
       " 'mean_transmission_shift': 3.556719332211011,\n",
       " 'reconstruction_entropy': 0.21836014216752608,\n",
       " 'reconstruction_error': 0.054781492737337456,\n",
       " 'alphabet': 'n t s i p r v h a w l d c   m o f b e u g y k x j q z / 8 -'}"
      ]
     },
     "execution_count": 350,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "code_statistics([x for x in one_grams_dict if x in teeline], teeline)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bab20cdf-7cfb-4721-95df-95f07ffea681",
   "metadata": {},
   "source": [
    "# Dearborn Speedwriting\n",
    "\n",
    "A dictionary of the original speedwriting from [here](https://pastebin.com/Ca0aRexr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 351,
   "id": "2a57787e-4342-46b7-a8ba-397f5b7530be",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"dearborn.txt\", \"r\") as file:\n",
    "    speedwriting_raw = file.readlines()\n",
    "\n",
    "speedwriting = {}\n",
    "\n",
    "for line in speedwriting_raw:\n",
    "    splits = line.replace(\"\\n\",\"\").split(\" = \")\n",
    "    if len(splits) != 2:\n",
    "        continue\n",
    "\n",
    "    word = splits[0]\n",
    "    rep = splits[1]\n",
    "    speedwriting[word.lower()] = rep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 352,
   "id": "1457cb4c-1970-4e0a-958c-f3d677ccbb35",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mean_binary_length': 9.891620202359249,\n",
       " 'mean_binary_excess': 4.734476718881333,\n",
       " 'mean_entropy_shift': 1.9143101331526058,\n",
       " 'mean_transmission_cost': 7.7219945382249655,\n",
       " 'mean_transmission_excess': 2.5648510547470496,\n",
       " 'mean_transmission_shift': -0.2553155309816777,\n",
       " 'reconstruction_entropy': 0.6123703821986793,\n",
       " 'reconstruction_error': 0.18216280866243562,\n",
       " 'alphabet': \"t a s o n v l m w b r k f d i h p e u g , j c - z / y Z x S O ; K ' q N M A V W L U T E B F R P C G D H 2 J 3 X Q Y I\"}"
      ]
     },
     "execution_count": 352,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "code_statistics([x for x in one_grams_dict if x in speedwriting], speedwriting)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3caee0db-33a8-4ccd-b0e4-a0e0672ccf5d",
   "metadata": {},
   "source": [
    "# Keyscript\n",
    "\n",
    "Using a Keyscript dictionary extracted from the original PDF."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 353,
   "id": "e0a9835c-4b5d-47d4-8d68-130a359e0465",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('keyscript.json') as f:\n",
    "    keyscript_raw = json.load(f)\n",
    "\n",
    "keyscript = {}\n",
    "for word in keyscript_raw:\n",
    "    keyscript[word.lower()] = keyscript_raw[word]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 354,
   "id": "d1e5d4aa-a204-4b2c-aac0-ab0b18e8ad64",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mean_binary_length': 6.332050766621434,\n",
       " 'mean_binary_excess': 2.061107371693759,\n",
       " 'mean_entropy_shift': -0.5181941605179468,\n",
       " 'mean_transmission_cost': 5.808437712965305,\n",
       " 'mean_transmission_excess': 1.5374943180376306,\n",
       " 'mean_transmission_shift': -1.0418072141740753,\n",
       " 'reconstruction_entropy': 1.0871969584544927,\n",
       " 'reconstruction_error': 0.27275805499274164,\n",
       " 'alphabet': 'h a s v t i r c n l w e o x y u z m q d k p f j b g 8 9'}"
      ]
     },
     "execution_count": 354,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "code_statistics([x for x in one_grams_dict if x in keyscript], keyscript)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "55815d97",
   "metadata": {},
   "source": [
    "# Superwrite\n",
    "\n",
    "A superwrite dictionary created and contributed to this project by [Keith Rowe](https://github.com/keithrowe)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 355,
   "id": "03ac6c64",
   "metadata": {},
   "outputs": [],
   "source": [
    "superwrite_raw = pd.read_csv(\"superwrite.csv\",names=['form','word'],header = None)\n",
    "\n",
    "superwrite = {}\n",
    "\n",
    "for _,line in superwrite_raw.iterrows():\n",
    "    superwrite[str(line['word']).lower()] = str(line['form'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 356,
   "id": "80aa2462",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mean_binary_length': 9.580136164172522,\n",
       " 'mean_binary_excess': 5.153402737882618,\n",
       " 'mean_entropy_shift': 2.561370833984843,\n",
       " 'mean_transmission_cost': 8.089306894505098,\n",
       " 'mean_transmission_excess': 3.662573468215194,\n",
       " 'mean_transmission_shift': 1.070541564317419,\n",
       " 'reconstruction_entropy': 0.3157422119141371,\n",
       " 'reconstruction_error': 0.12144321942914205,\n",
       " 'alphabet': 'n | t o s r a l w m d f e b c u h p i y g S v C k - 1 j x   q 2 O U N z T'}"
      ]
     },
     "execution_count": 356,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "code_statistics([x for x in one_grams_dict if x in superwrite], superwrite)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "859f7acf-0297-43a0-95f9-13fcdf41a445",
   "metadata": {},
   "source": [
    "# QC-Line Base"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 357,
   "id": "f557c73b-9a8c-4ec6-b376-afd01093c601",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Pattern to remove vowels that are not the first or last letter, and double consonants\n",
    "vowel_pattern = r\"(?<!^)[aeiou](?!$)\"\n",
    "double_consonant_pattern = r\"(?<=(.))\\1\"\n",
    "\n",
    "def replace_qc(s): #this is approximate, but should be good enough to plot it\n",
    "    # Replace 'c' followed by 'e', 'i', or 'y' with 's'\n",
    "    s = re.sub(r\"c(?=[eiy])\", \"s\", s)\n",
    "    # Replace remaining 'c's with 'k' and 'q's with 'k'\n",
    "    s = re.sub(r\"[cq]\", \"k\", s)\n",
    "    return s\n",
    "\n",
    "# Apply the vowel removal pattern first, then the double consonant pattern\n",
    "qc_line = {word:re.sub(vowel_pattern, '', re.sub(double_consonant_pattern, '', replace_qc(word))) for word in one_grams_dict}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 358,
   "id": "8f219be6-e2c7-4da9-807b-7628033ef81a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mean_binary_length': 15.39536827849569,\n",
       " 'mean_binary_excess': 8.748415458722716,\n",
       " 'mean_entropy_shift': 5.628221908026719,\n",
       " 'mean_transmission_cost': 13.513070834782372,\n",
       " 'mean_transmission_excess': 6.866118015009398,\n",
       " 'mean_transmission_shift': 3.7459244643134006,\n",
       " 'reconstruction_entropy': 0.2327331929045948,\n",
       " 'reconstruction_error': 0.0647610299624436,\n",
       " 'alphabet': \"t n s r h e d l k a o m f w p g y i b v u x j z ' é\"}"
      ]
     },
     "execution_count": 358,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "code_statistics([x for x in one_grams_dict if x in qc_line], qc_line)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e3fc5bb9-2e05-4ee7-b3ef-e4c1cd2168a4",
   "metadata": {},
   "source": [
    "# Yash\n",
    "\n",
    "A rough approximation to Yash."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 359,
   "id": "9dfb459d-dbcf-4673-885e-d86f73092818",
   "metadata": {},
   "outputs": [],
   "source": [
    "# THIS IS THE ONE THAT WORKS MOSTLY\n",
    "\n",
    "# Remove punctuation and stress markers\n",
    "def clean_word(word):\n",
    "    punctuation_and_spaces = \"ˈˌ. ,-'\\\";:!?()[]{}<>/@#%^&*~`\"\n",
    "    return ''.join([char for char in word if char not in punctuation_and_spaces])\n",
    "\n",
    "# Recursive alignment function with debug statements\n",
    "def find_allowable_pairs_recursive(str1, str2, allowable_pairs, debug=False):\n",
    "    def helper(s1, s2, pairs, path):\n",
    "        if not s1 and not s2:\n",
    "            return path\n",
    "        for pair in pairs:\n",
    "            l1, l2 = len(pair[0]), len(pair[1])\n",
    "            if s1[:l1] == pair[0] and s2[:l2] == pair[1]:\n",
    "                if debug:\n",
    "                    print(f\"Matching: {pair} with {s1[:l1]} and {s2[:l2]}\")\n",
    "                result = helper(s1[l1:], s2[l2:], pairs, path + [pair])\n",
    "                if result is not None:\n",
    "                    return result\n",
    "        if debug:\n",
    "            print(f\"Failed to match: {s1} with {s2}\")\n",
    "        return None\n",
    "\n",
    "    return helper(str1, str2, allowable_pairs, [])\n",
    "\n",
    "# Function to process the word and find alignment with debug statements\n",
    "def process_word(word, ipa_representation, allowable_pairs, debug=False):\n",
    "    cleaned_word = clean_word(word)\n",
    "    cleaned_ipa = clean_word(ipa_representation)\n",
    "    \n",
    "    if debug:\n",
    "        print(f\"Processing '{word}' -> '{cleaned_word}'\")\n",
    "        print(f\"IPA '{ipa_representation}' -> '{cleaned_ipa}'\")\n",
    "    \n",
    "    alignment = find_allowable_pairs_recursive(cleaned_word, cleaned_ipa, allowable_pairs, debug=debug)\n",
    "    \n",
    "    return alignment\n",
    "\n",
    "# Combined allowable pairs for both consonants and vowels\n",
    "allowable_pairs = [\n",
    "    # Consonant Combinations\n",
    "    (\"ch\", \"ʧ\"), (\"ch\", \"tʃ\"), (\"sh\", \"tʃ\"), (\"sh\", \"ʧ\"), (\"sh\", \"ʃ\"),\n",
    "    (\"ch\", \"ʃ\"), (\"th\", \"θ\"), (\"th\", \"ð\"), (\"t\", \"tʃ\"), (\"g\", \"ʒ\"),\n",
    "    (\"s\", \"z\"), (\"s\", \"ʒ\"), (\"s\", \"ʃ\"), (\"z\", \"ʒ\"), (\"z\", \"ʃ\"), (\"c\", \"ʃ\"),\n",
    "    (\"ck\", \"k\"), (\"ps\", \"s\"), (\"q\", \"kw\"), (\"gh\", \"f\"), (\"ph\", \"f\"),\n",
    "    (\"kn\", \"n\"), (\"gn\", \"n\"), (\"wr\", \"r\"), (\"wh\", \"w\"), (\"sc\", \"s\"),\n",
    "    (\"c\", \"s\"), (\"ld\", \"d\"), (\"dd\", \"d\"), (\"ll\", \"l\"), (\"ll\", \"ɫ\"),\n",
    "    (\"mm\", \"m\"), (\"nn\", \"n\"), (\"mn\",\"m\"), (\"rr\", \"r\"), (\"pp\", \"p\"), (\"ff\", \"f\"),\n",
    "    (\"tt\", \"t\"), (\"ss\", \"s\"), (\"ss\", \"z\"), (\"ss\", \"ʒ\"), (\"ss\", \"ʃ\"),\n",
    "    (\"t\", \"ʒ\"), (\"t\", \"ʃ\"), (\"x\", \"gz\"), (\"x\", \"z\"), (\"zz\", \"ts\"),\n",
    "    (\"z\", \"ts\"), (\"g\", \"dʒ\"), (\"g\", \"ʤ\"), (\"gg\", \"g\"), (\"gg\", \"ʒ\"),\n",
    "    (\"gg\", \"dʒ\"), (\"gg\", \"ʤ\"), (\"ch\", \"k\"), (\"cc\", \"k\"), \n",
    "    (\"sl\", \"l\"), (\"q\", \"k\"), (\"l\", \"r\"), \n",
    "    (\"ct\", \"t\"), (\"dn\", \"n\"),(\"ç\", \"s\"),\n",
    "    (\"gm\", \"m\"),(\"sc\", \"ʃ\"), (\"x\", \"ɡz\"), (\"gg\", \"ɡ\"),\n",
    "    (\"cq\", \"kw\"), (\"ng\", \"n\"), (\"kd\", \"t\"), (\"nm\", \"m\"), (\"d\", \"dʒ\"),\n",
    "    (\"x\", \"kʃ\"), (\"x\", \"k\"), (\"x\", \"ʃ\"), (\"l\", \"ɝ\"), (\"x\", \"ɡʒ\"), (\"s\", \"tʃ\"),\n",
    "    (\"f\", \"v\"), (\"fth\",\"θ\"), (\"cz\",\"tʃ\"), (\"k\",\"keɪ\"),\n",
    "    (\"kn\", \"n\"), (\"gn\", \"n\"), (\"wr\", \"r\"), (\"mb\", \"m\"),\n",
    "    (\"gu\", \"g\"), (\"que\", \"k\"),(\"wh\",\"hw\"), (\"g\", \"k\"), (\"c\",\"tʃ\"),(\"an\",\"æm\"),(\"th\",\"tθ\"),\n",
    "\n",
    "    # Consonants\n",
    "    (\"c\", \"k\"), (\"t\", \"t\"), (\"d\", \"d\"), (\"d\", \"t\"), (\"t\", \"d\"), (\"n\", \"n\"),\n",
    "    (\"s\", \"s\"), (\"l\", \"l\"), (\"l\", \"ɫ\"), (\"r\", \"r\"), (\"r\", \"ɹ\"), (\"r\", \"ɝ\"),\n",
    "    (\"m\", \"m\"), (\"b\", \"b\"), (\"k\", \"k\"), (\"f\", \"f\"), (\"g\", \"g\"), (\"g\", \"ɡ\"),\n",
    "    (\"v\", \"v\"), (\"x\", \"ks\"), (\"z\", \"z\"), (\"p\", \"p\"), (\"h\", \"h\"), (\"w\", \"w\"), (\"w\", \"hw\"),\n",
    "    (\"y\", \"j\"), (\"j\", \"dʒ\"), (\"j\", \"ʤ\"), (\"j\", \"ʒ\"), (\"ng\", \"ŋ\"), (\"n\", \"ŋ\"),\n",
    "\n",
    "    # Vowel combinations\n",
    "    (\"ee\",\"ɪ\"),(\"ee\",\"ə\"),\n",
    "    (\"oo\", \"u\"), (\"oo\", \"ʊ\"), (\"ea\", \"i\"), (\"ee\", \"i\"), \n",
    "    (\"ai\", \"eɪ\"), (\"ay\", \"eɪ\"), (\"ei\", \"eɪ\"), (\"ey\", \"eɪ\"), (\"ai\",\"ɛ\"), (\"ai\",\"iɪ\"),\n",
    "    (\"ie\", \"aɪ\"), (\"igh\", \"aɪ\"),\n",
    "    (\"oa\", \"oʊ\"), (\"ow\", \"oʊ\"), (\"ow\",\"aʊ\"),\n",
    "    (\"ew\", \"ju\"), (\"ue\", \"ju\"),\n",
    "    (\"au\", \"ɔ\"), (\"aw\", \"ɔ\"), (\"au\",\"aʊ\"), ('au', 'ə'), (\"au\",\"ɑ\"),\n",
    "    (\"ou\", \"aʊ\"), (\"ou\", \"u\"), (\"ou\", \"ʊ\"), (\"ou\",\"ɔ\"),\n",
    "    (\"oi\", \"ɔɪ\"), (\"oy\", \"ɔɪ\"), (\"oo\",\"ɪ\"), (\"oo\",\"wɑ\"), (\"oo\",\"wɔ\"),\n",
    "    (\"ea\",\"ɪ\"),(\"ea\",\"iə\"),\n",
    "    (\"ay\", \"i\"),(\"ay\",\"ɛ\"),\n",
    "    (\"eau\",\"oʊ\"),\n",
    "    (\"ui\",\"wɪ\"),\n",
    "    (\"ua\",\"wə\"),\n",
    "    (\"ua\",\"weɪ\"),(\"ua\",\"wɪ\"),\n",
    "    (\"oi\",\"wɑ\"), (\"ui\",\"wi\"),\n",
    "    \n",
    "    # Vowels\n",
    "    (\"a\", \"æ\"), (\"a\", \"ə\"), (\"a\", \"ɑ\"), (\"a\", \"ɔ\"), (\"a\", \"eɪ\"), (\"a\", \"ɒ\"), (\"a\",\"ɛ\"), (\"a\",\"ɪ\"), (\"a\",\"aɪ\"), (\"a\", \"ə\"), \n",
    "    (\"e\", \"ɛ\"), (\"e\", \"i\"), (\"e\", \"ə\"), (\"e\", \"ɪ\"), (\"e\",\"æ\"), (\"e\", \"ɑ\"), (\"e\",\"eɪ\"),\n",
    "    (\"i\", \"ɪ\"), (\"i\", \"aɪ\"), (\"i\", \"i\"), (\"i\", \"ɜ\"), (\"i\", \"ə\"), (\"i\",\"j\"), (\"i\",\"jɪ\"),\n",
    "    (\"o\", \"ɔ\"), (\"o\", \"ə\"), (\"o\", \"oʊ\"), (\"o\", \"ɒ\"), (\"o\", \"ɑ\"), (\"o\", \"u\"), (\"o\",\"ʊ\"), (\"o\",\"ɪ\"), (\"o\", \"wə\"),\n",
    "    (\"u\", \"ʊ\"), (\"u\", \"ə\"), (\"u\", \"ju\"), (\"u\", \"ʌ\"), (\"u\", \"ɜ\"), (\"u\", \"u\"), (\"u\", \"ɪ\"), (\"u\",\"j\"), (\"u\",\"jʊ\"), (\"u\",\"ɛ\"), (\"u\",\"jəw\"), (\"u\", \"juw\"), (\"u\",\"ɑ\"), (\"u\",\"uw\"),(\"u\",\"ɔ\"),\n",
    "    (\"y\", \"ɪ\"), (\"y\", \"aɪ\"), (\"y\", \"i\"), (\"y\", \"ə\"),\n",
    "\n",
    "    # stupid r/w stuff\n",
    "    (\"ar\", \"ɝ\"),  (\"er\", \"ɝ\"),  (\"or\", \"ɝ\"),  (\"yr\", \"ɝ\"),  (\"re\", \"ɝ\"),  (\"ir\", \"ɝ\"),  (\"ir\", \"aɪɝ\"),\n",
    "    (\"ar\", \"ɹɝ\"), (\"er\", \"ɹɝ\"), (\"or\", \"ɹɝ\"), (\"yr\", \"ɹɝ\"), (\"re\", \"ɹɝ\"), (\"ir\", \"ɹɝ\"), (\"ir\", \"aɪɹɝ\"),\n",
    "    (\"ar\", \"ɝɹ\"), (\"er\", \"ɝɹ\"), (\"or\", \"ɝɹ\"), (\"yr\", \"ɝɹ\"), (\"re\", \"ɝɹ\"), (\"ir\", \"ɝɹ\"), (\"ir\", \"aɪɝɹ\"),\n",
    "    (\"u\", \"ɝ\"), (\"ew\", \"u\"),\n",
    "    \n",
    "    # Silent Fallbacks and magic schwas\n",
    "    (\"t\", \"\"), (\"l\", \"\"), (\"ch\", \"\"), (\"s\", \"\"), (\"z\", \"\"), (\"r\", \"\"), (\"d\", \"\"), (\"n\",\"\"), (\"k\",\"\"), (\"p\", \"\"),\n",
    "    (\"a\", \"\"), (\"e\", \"\"), (\"i\", \"\"), (\"o\", \"\"), (\"u\", \"\"), (\"gh\", \"\"), (\"b\", \"\"), (\"h\", \"\"), (\"w\", \"\"), \n",
    "    (\"\", \"ə\"), \n",
    "\n",
    "    # Magic appearing \"z\" ??\n",
    "    (\"\",\"z\"),\n",
    "\n",
    "    # Speaking the letters\n",
    "    (\"s\",\"ɛs\"), (\"m\",\"ɛm\"), (\"d\",\"di\"), (\"c\", \"si\")\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 360,
   "id": "2d8a9187-e1b0-494c-aed9-5a47eae8844d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def aligned(word):\n",
    "    if word not in ipa_dict:\n",
    "        return None\n",
    "    return process_word(word, ipa_dict[word], allowable_pairs, debug=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 361,
   "id": "746b95e2-a72b-49f8-99f5-da0b0a01ef9a",
   "metadata": {},
   "outputs": [],
   "source": [
    "yash_pair_mapping = {\n",
    "    # Consonant Combinations\n",
    "    (\"ch\", \"ʧ\"):'j', (\"ch\", \"tʃ\"):'j', (\"sh\", \"tʃ\"):'j', (\"sh\", \"ʧ\"):'j', (\"sh\", \"ʃ\"):'c',\n",
    "    (\"ch\", \"ʃ\"):'c', (\"th\", \"θ\"):'y', (\"th\", \"ð\"):'y', (\"t\", \"tʃ\"):'t', (\"g\", \"ʒ\"):'g',\n",
    "    (\"s\", \"z\"):'s', (\"s\", \"ʒ\"):'s', (\"s\", \"ʃ\"):'s', (\"z\", \"ʒ\"):'ts', (\"z\", \"ʃ\"):'ts', (\"c\", \"ʃ\"):'s',\n",
    "    (\"ck\", \"k\"):'k', (\"ps\", \"s\"):'ps', (\"q\", \"kw\"):'kv', (\"gh\", \"f\"):'f', (\"ph\", \"f\"):'f',\n",
    "    (\"kn\", \"n\"):'n', (\"gn\", \"n\"):'n', (\"wr\", \"r\"):'r', (\"wh\", \"w\"):'v', (\"sc\", \"s\"):'sc',\n",
    "    (\"c\", \"s\"):'s', (\"ld\", \"d\"):'ld', (\"dd\", \"d\"):'d', (\"ll\", \"l\"):'l', (\"ll\", \"ɫ\"):'l',\n",
    "    (\"mm\", \"m\"):'m', (\"nn\", \"n\"):'n', (\"mn\",\"m\"):'mn', (\"rr\", \"r\"):'r', (\"pp\", \"p\"):'p', (\"ff\", \"f\"):'f',\n",
    "    (\"tt\", \"t\"):'t', (\"ss\", \"s\"):'s', (\"ss\", \"z\"):'s', (\"ss\", \"ʒ\"):'s', (\"ss\", \"ʃ\"):'s',\n",
    "    (\"t\", \"ʒ\"):'c', (\"t\", \"ʃ\"):'c', (\"x\", \"gz\"):'ks', (\"x\", \"z\"):'ks', (\"zz\", \"ts\"):'ts',\n",
    "    (\"z\", \"ts\"):'ts', (\"g\", \"dʒ\"):'g', (\"g\", \"ʤ\"):'g', (\"gg\", \"g\"):'g', (\"gg\", \"ʒ\"):'g',\n",
    "    (\"gg\", \"dʒ\"):'g', (\"gg\", \"ʤ\"):'g', (\"ch\", \"k\"):'k', (\"cc\", \"k\"):'k', \n",
    "    (\"sl\", \"l\"):'sl', (\"q\", \"k\"):'k', (\"l\", \"r\"):'l', \n",
    "    (\"ct\", \"t\"):'ct', (\"dn\", \"n\"):'dn',(\"ç\", \"s\"):'s',\n",
    "    (\"gm\", \"m\"):'gm',(\"sc\", \"ʃ\"):'sc', (\"x\", \"ɡz\"):'ks', (\"gg\", \"ɡ\"):'g',\n",
    "    (\"cq\", \"kw\"):'kv', (\"ng\", \"n\"):'ng', (\"kd\", \"t\"):'kd', (\"nm\", \"m\"):'nm', (\"d\", \"dʒ\"):'d',\n",
    "    (\"x\", \"kʃ\"):'ks', (\"x\", \"k\"):'ks', (\"x\", \"ʃ\"):'ks', (\"l\", \"ɝ\"):'l', (\"x\", \"ɡʒ\"):'ks', (\"s\", \"tʃ\"):'s',\n",
    "    (\"f\", \"v\"):'f', (\"fth\",\"θ\"):'fy', (\"cz\",\"tʃ\"):'cts', \n",
    "    (\"kn\", \"n\"):'kn', (\"gn\", \"n\"):'gn', (\"wr\", \"r\"):'r', (\"mb\", \"m\"):'mb',\n",
    "    (\"gu\", \"g\"):'g', (\"que\", \"k\"):'k',(\"wh\",\"hw\"):'v', (\"g\", \"k\"):'g', (\"c\",\"tʃ\"):'c',(\"an\",\"æm\"):'n',(\"th\",\"tθ\"):'y',\n",
    "\n",
    "    # Consonants\n",
    "    (\"c\", \"k\"):'k', (\"t\", \"t\"):'t', (\"d\", \"d\"):'d', (\"d\", \"t\"):'d', (\"t\", \"d\"):'t', (\"n\", \"n\"):'n',\n",
    "    (\"s\", \"s\"):'s', (\"l\", \"l\"):'l', (\"l\", \"ɫ\"):'l', (\"r\", \"r\"):'r', (\"r\", \"ɹ\"):'r', (\"r\", \"ɝ\"):'r',\n",
    "    (\"m\", \"m\"):'m', (\"b\", \"b\"):'b', (\"k\", \"k\"):'k', (\"f\", \"f\"):'f', (\"g\", \"g\"):'g', (\"g\", \"ɡ\"):'g',\n",
    "    (\"v\", \"v\"):'v', (\"x\", \"ks\"):'ks', (\"z\", \"z\"):'ts', (\"p\", \"p\"):'p', (\"h\", \"h\"):'h', (\"w\", \"w\"):'v', (\"w\", \"hw\"):'v',\n",
    "    (\"y\", \"j\"):'i', (\"j\", \"dʒ\"):'j', (\"j\", \"ʤ\"):'j', (\"j\", \"ʒ\"):'i', (\"ng\", \"ŋ\"):'q', (\"n\", \"ŋ\"):'n',\n",
    "\n",
    "    # Vowel combinations\n",
    "    (\"ee\",\"ɪ\"):'i', (\"ee\",\"ə\"):'',\n",
    "    (\"oo\", \"u\"):'o', (\"oo\", \"ʊ\"):'o', (\"ea\", \"i\"):'i', (\"ee\", \"i\"):'i', \n",
    "    (\"ai\", \"eɪ\"):'a', (\"ay\", \"eɪ\"):'a', (\"ei\", \"eɪ\"):'e', (\"ey\", \"eɪ\"):'e', (\"ai\",\"ɛ\"):'a', (\"ai\",\"iɪ\"):'i',\n",
    "    (\"ie\", \"aɪ\"):'i', (\"igh\", \"aɪ\"):'a',\n",
    "    (\"oa\", \"oʊ\"):'o', (\"ow\", \"oʊ\"):'v', (\"ow\",\"aʊ\"):'v',\n",
    "    (\"ew\", \"ju\"):'v', (\"ue\", \"ju\"):'u',\n",
    "    (\"au\", \"ɔ\"):'o', (\"aw\", \"ɔ\"):'v', (\"au\",\"aʊ\"):'a', ('au', 'ə'):'a', (\"au\",\"ɑ\"):'a',\n",
    "    (\"ou\", \"aʊ\"):'o', (\"ou\", \"u\"):'u', (\"ou\", \"ʊ\"):'o', (\"ou\",\"ɔ\"):'o',\n",
    "    (\"oi\", \"ɔɪ\"):'i', (\"oy\", \"ɔɪ\"):'i', (\"oo\",\"ɪ\"):'i', (\"oo\",\"wɑ\"):'o', (\"oo\",\"wɔ\"):'o',\n",
    "    (\"ea\",\"ɪ\"):'i', (\"ea\",\"iə\"):'e',\n",
    "    (\"ay\", \"i\"):'i', (\"ay\",\"ɛ\"):'e',\n",
    "    (\"eau\",\"oʊ\"):'o',\n",
    "    (\"ui\",\"wɪ\"):'i',\n",
    "    (\"ua\",\"wə\"):'e',\n",
    "    (\"ua\",\"weɪ\"):'a',(\"ua\",\"wɪ\"):'i',\n",
    "    (\"oi\",\"wɑ\"):'o', (\"ui\",\"wi\"):'u',\n",
    "    \n",
    "    # Vowels\n",
    "    (\"a\", \"æ\"):'.', (\"a\", \"ə\"):'', (\"a\", \"ɑ\"):'.', (\"a\", \"ɔ\"):'.', (\"a\", \"eɪ\"):'.', (\"a\", \"ɒ\"):'.', (\"a\",\"ɛ\"):'', (\"a\",\"ɪ\"):'', (\"a\",\"aɪ\"):'.', (\"a\", \"ə\"):'', \n",
    "    (\"e\", \"ɛ\"):'', (\"e\", \"i\"):'.', (\"e\", \"ə\"):'', (\"e\", \"ɪ\"):'', (\"e\",\"æ\"):'.', (\"e\", \"ɑ\"):'.', (\"e\",\"eɪ\"):'..',\n",
    "    (\"i\", \"ɪ\"):'.', (\"i\", \"aɪ\"):'.', (\"i\", \"i\"):'.', (\"i\", \"ɜ\"):'', (\"i\", \"ə\"):'', (\"i\",\"j\"):'.', (\"i\",\"jɪ\"):'.',\n",
    "    (\"o\", \"ɔ\"):'.', (\"o\", \"ə\"):'', (\"o\", \"oʊ\"):'.', (\"o\", \"ɒ\"):'.', (\"o\", \"ɑ\"):'.', (\"o\", \"u\"):'.', (\"o\",\"ʊ\"):'.', (\"o\",\"ɪ\"):'', (\"o\", \"wə\"):'.',\n",
    "    (\"u\", \"ʊ\"):'.', (\"u\", \"ə\"):'', (\"u\", \"ju\"):'.', (\"u\", \"ʌ\"):'', (\"u\", \"ɜ\"):'', (\"u\", \"u\"):'.', (\"u\", \"ɪ\"):'', (\"u\",\"j\"):'.', (\"u\",\"jʊ\"):'.', (\"u\",\"ɛ\"):'', (\"u\",\"jəw\"):'.', (\"u\", \"juw\"):'.', (\"u\",\"ɑ\"):'.', (\"u\",\"uw\"):'.',(\"u\",\"ɔ\"):'.',\n",
    "    (\"y\", \"ɪ\"):'.', (\"y\", \"aɪ\"):'.', (\"y\", \"i\"):'i', (\"y\", \"ə\"):'',\n",
    "\n",
    "    # stupid r/w stuff\n",
    "    (\"ar\", \"ɝ\"):'r',  (\"er\", \"ɝ\"):'r',  (\"or\", \"ɝ\"):'r',  (\"yr\", \"ɝ\"):'r',  (\"re\", \"ɝ\"):'r',  (\"ir\", \"ɝ\"):'r',  (\"ir\", \"aɪɝ\"):'r',\n",
    "    (\"ar\", \"ɹɝ\"):'r', (\"er\", \"ɹɝ\"):'r', (\"or\", \"ɹɝ\"):'r', (\"yr\", \"ɹɝ\"):'r', (\"re\", \"ɹɝ\"):'r', (\"ir\", \"ɹɝ\"):'r', (\"ir\", \"aɪɹɝ\"):'r',\n",
    "    (\"ar\", \"ɝɹ\"):'r', (\"er\", \"ɝɹ\"):'r', (\"or\", \"ɝɹ\"):'r', (\"yr\", \"ɝɹ\"):'r', (\"re\", \"ɝɹ\"):'r', (\"ir\", \"ɝɹ\"):'r', (\"ir\", \"aɪɝɹ\"):'r',\n",
    "    (\"u\", \"ɝ\"):'', (\"ew\", \"u\"):'v',\n",
    "    \n",
    "    # Silent Fallbacks and magic schwas\n",
    "    (\"t\", \"\"):'', (\"l\", \"\"):'', (\"ch\", \"\"):'', (\"s\", \"\"):'', (\"z\", \"\"):'', (\"r\", \"\"):'', (\"d\", \"\"):'', (\"n\",\"\"):'', (\"k\",\"\"):'', (\"p\", \"\"):'',\n",
    "    (\"a\", \"\"):'', (\"e\", \"\"):'', (\"i\", \"\"):'', (\"o\", \"\"):'', (\"u\", \"\"):'', (\"gh\", \"\"):'', (\"b\", \"\"):'', (\"h\", \"\"):'', (\"w\", \"\"):'', \n",
    "    (\"\", \"ə\"):'', \n",
    "\n",
    "    # Magic appearing \"z\" ??\n",
    "    (\"\",\"z\"):'',\n",
    "\n",
    "    # Speaking the letters\n",
    "    (\"s\",\"ɛs\"):'s', (\"m\",\"ɛm\"):'m', (\"d\",\"di\"):'d', (\"c\", \"si\"):'s', (\"k\",\"keɪ\"):'k'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 362,
   "id": "fa3f957d-115e-4cbf-81f5-2ebcc7e633b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def yash_aligned(word, verbose = False):\n",
    "    if word == \"a\":\n",
    "        return \"a\"\n",
    "    if word == \"i\":\n",
    "        return \"i\"\n",
    "    if word == \"not\":\n",
    "        return \"x\"\n",
    "    \n",
    "    base = aligned(word)\n",
    "    if base == None:\n",
    "        return None\n",
    "\n",
    "    if verbose: \n",
    "        print(base)\n",
    "\n",
    "    mapped = \"\".join([yash_pair_mapping[p] for p in base])\n",
    "\n",
    "    if verbose: \n",
    "        print(mapped)\n",
    "\n",
    "    mapped = mapped.replace(\"nd\",\"x\").replace(\"nt\",\"x\") # X rule\n",
    "    mapped = mapped.replace(\"st\",\"z\") # Z rule\n",
    "    mapped = mapped.replace(\"rld\",\"w\").replace(\"rt\",\"w\").replace(\"tr\",\"w\").replace(\"rd\",\"w\").replace(\"ld\",\"w\").replace(\"dl\",\"w\").replace(\"lt\",\"w\").replace(\"td\",\"w\") # W rules\n",
    "\n",
    "    if verbose:\n",
    "        print(mapped)\n",
    "\n",
    "    # not a listed rule, but he seems to almost always leave off the \n",
    "    if mapped[-2:] == \"cn\":\n",
    "        mapped = mapped[:-1]\n",
    "\n",
    "    if mapped[-3:] == \"cns\":\n",
    "        mapped = mapped[:-2] + \"s\"\n",
    "\n",
    "    mapped = mapped.replace(\".\",\"\") # get rid of significant vowel markes\n",
    "\n",
    "    return mapped"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 363,
   "id": "e6793e27-fa59-473b-aa98-256a5e837781",
   "metadata": {},
   "outputs": [],
   "source": [
    "yash = {x:yash_aligned(x) for x in one_grams_dict if yash_aligned(x) != None}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 364,
   "id": "aa9f47c8-7f83-417f-a626-478c356dd356",
   "metadata": {},
   "outputs": [],
   "source": [
    "for w in list(yash):\n",
    "    if yash[w] == '':\n",
    "        del yash[w]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 365,
   "id": "bf5a9978-0345-4375-91fc-2ff6fdb8acf1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mean_binary_length': 11.047253230410739,\n",
       " 'mean_binary_excess': 4.403937749275396,\n",
       " 'mean_entropy_shift': 1.2842536453698834,\n",
       " 'mean_transmission_cost': 10.24483395861936,\n",
       " 'mean_transmission_excess': 3.601518477484017,\n",
       " 'mean_transmission_shift': 0.4818343735785042,\n",
       " 'reconstruction_entropy': 0.6985334759219873,\n",
       " 'reconstruction_error': 0.18388474099533814,\n",
       " 'alphabet': 's r n t y l v k m f x i d p w b h a g z c o q j u e'}"
      ]
     },
     "execution_count": 365,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "code_statistics([x for x in one_grams_dict if x in yash], yash)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "18a36c18-a211-41d2-969f-040af3a093bf",
   "metadata": {},
   "source": [
    "# Jeake Philosophical Transaction No. 487\n",
    "\n",
    "[This](https://royalsocietypublishing.org/doi/epdf/10.1098/rstl.1748.0041) system is very simple, and related closely to an experimental system I had code for already."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 366,
   "id": "1a3e8c58-95b4-4ae9-9d55-a63af3905842",
   "metadata": {},
   "outputs": [],
   "source": [
    "def translate_jotter(string, rules, default = \".\"):\n",
    "    temp = \"\".join(map(lambda x: rules.get(x,\"\"), string))\n",
    "    if len(temp) > 0:\n",
    "        return temp\n",
    "    else:\n",
    "        return default\n",
    "\n",
    "def compile_rules_jotter(merge_list):\n",
    "    rule_dict = {}\n",
    "    for equiv_class in merge_list:\n",
    "        head = equiv_class[0]\n",
    "        for letter in equiv_class:\n",
    "            rule_dict[letter] = head\n",
    "    return rule_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 367,
   "id": "2257745f-3b0c-4e3c-ada9-8828e4317a2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "jeake_rule = ['dt','lr','mn','uvw','sxz','bfp','qkg','y','c'] # Jeake Philosophical Transaction No. 487\n",
    "compiled_jeake = compile_rules_jotter(jeake_rule)\n",
    "jeake = {w:translate_jotter(w,compiled_jeake) for w in one_grams_dict}\n",
    "\n",
    "def replace_c(s): #this is approximate, but should be good enough to plot it\n",
    "    # Replace 'c' followed by 'e', 'i', or 'y' with 's'\n",
    "    s = re.sub(r\"c(?=[eiy])\", \"s\", s)\n",
    "    # Replace remaining 'c's with 'k'\n",
    "    s = re.sub(r\"c\", \"g\", s)\n",
    "    return s\n",
    "\n",
    "def combine_letters(s):\n",
    "    # Use regex to find consecutive identical letters and replace them with a single capital letter\n",
    "    result = re.sub(r\"(.)\\1+\", lambda m: m.group(1).upper(), s)\n",
    "    return result\n",
    "\n",
    "jeake = {w:combine_letters(replace_c(jeake[w])) for w in jeake}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 368,
   "id": "9bb6ba38-d7e6-4e75-b628-94820d89b02e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mean_binary_length': 10.531745445747445,\n",
       " 'mean_binary_excess': 3.8847926259744714,\n",
       " 'mean_entropy_shift': 0.7645990752784737,\n",
       " 'mean_transmission_cost': 8.438665250814323,\n",
       " 'mean_transmission_excess': 1.7917124310413488,\n",
       " 'mean_transmission_shift': -1.3284811196546489,\n",
       " 'reconstruction_entropy': 1.4819684205694112,\n",
       " 'reconstruction_error': 0.35139657535538305,\n",
       " 'alphabet': 'd m l s u b g q y . D L M S B U G Q'}"
      ]
     },
     "execution_count": 368,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "code_statistics([x for x in one_grams_dict if x in jeake], jeake)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3d323922-ecbc-46a0-b360-cc7968e9bb93",
   "metadata": {},
   "source": [
    "# A Readable Polyphonic Cipher\n",
    "Not shorthand per se, but related, [here](https://digitalcommons.butler.edu/wordways/vol8/iss1/16/)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 369,
   "id": "52975dd4-0fc9-4f96-a6e6-4a759d914991",
   "metadata": {},
   "outputs": [],
   "source": [
    "polyphonic_rule = ['e','txz','acq','ilb','ogj','npkv','ryw','sfm','hdu']\n",
    "compiled_polyphonic = compile_rules_jotter(polyphonic_rule)\n",
    "polyphonic = {w:translate_jotter(w,compiled_polyphonic) for w in one_grams_dict}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 370,
   "id": "52d0fe42-0b64-41cf-b917-67d94b1465b1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mean_binary_length': 14.335438221763479,\n",
       " 'mean_binary_excess': 7.688485401990505,\n",
       " 'mean_entropy_shift': 4.568291851294507,\n",
       " 'mean_transmission_cost': 14.302604190212872,\n",
       " 'mean_transmission_excess': 7.655651370439898,\n",
       " 'mean_transmission_shift': 4.5354578197439,\n",
       " 'reconstruction_entropy': 0.08207522153529219,\n",
       " 'reconstruction_error': 0.023091815341297117,\n",
       " 'alphabet': 'i e h a s n o t r'}"
      ]
     },
     "execution_count": 370,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "code_statistics([x for x in one_grams_dict if x in polyphonic], polyphonic)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "be9542bb",
   "metadata": {},
   "source": [
    "# Pitman 2000\n",
    "\n",
    "This uses the output from [this online translator](https://steno.tu-clausthal.de/Pitman.php) with `proof:7` which returns the underlying ascii information that it uses to feed into metafont to draw the proper Pitman form.  This online translator uses a machine-generated outline, and is known to not be particularly accurate, so this dot location should taken with a grain of salt.  This is also essentially unabbreviated Pitman (aside from some brief forms), so it is not fully representative of true practical performance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 371,
   "id": "9ba8f24b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "999 999\n"
     ]
    }
   ],
   "source": [
    "pitman2k_raw = pd.read_csv(\"Pitman.txt\",sep=r\"\\|\\|\",engine='python',names=['word','outline'],header=None)\n",
    "\n",
    "pitman2k = {}\n",
    "pitman2k_novowel = {}\n",
    "\n",
    "pitman_translation_dictionary = {\n",
    "    # initial positional vowels\n",
    "    \"{1}\":[\"1\"],\n",
    "    \"{2}\":[\"2\"],\n",
    "    \"{3}\":[\"3\"],\n",
    "    # connecters (ignoring concatenation)\n",
    "    \"&\":[],\n",
    "    \"/\":[\"/\"],\n",
    "    \" \":[\" \"],\n",
    "    \"=\":[\"=\"],\n",
    "    # basic consonants\n",
    "    \"(b)\":[\"b\"],\n",
    "    \"(p)\":[\"p\"],\n",
    "    \"(d)\":[\"d\"],\n",
    "    \"(t)\":[\"t\"],\n",
    "    \"(v)\":[\"v\"],\n",
    "    \"(f)\":[\"f\"],\n",
    "    \"(dh)\":[\"dh\"],\n",
    "    \"(th)\":[\"th\"],\n",
    "    \"(zh)\":[\"zh\"],\n",
    "    \"(sh)\":[\"sh\"],\n",
    "    \"(ng)\":[\"ng\"],\n",
    "    \"(n)\":[\"n\"],\n",
    "    \"(m)\":[\"m\"],\n",
    "    \"(l)\":[\"l\"],\n",
    "    \"(r)\":[\"r\"],\n",
    "    \"(w)\":[\"w\"],\n",
    "    \"(hw)\":[\"hw\"],\n",
    "    \"(y)\":[\"y\"],\n",
    "    \"(h)\":[\"h\"],\n",
    "    \"(s)\":[\"s\"],\n",
    "    \"(z)\":[\"z\"],\n",
    "    \"(ch)\":[\"ch\"],\n",
    "    \"(jh)\":[\"jh\"],\n",
    "    \"(k)\":[\"k\"],\n",
    "    \"(g)\":[\"g\"],\n",
    "    \"(_r)\":[\"_r\"],\n",
    "    # combining consonants with l\n",
    "    \"(b,l)\":[\"bl\"],\n",
    "    \"(p,l)\":[\"pl\"],\n",
    "    \"(d,l)\":[\"dl\"],\n",
    "    \"(t,l)\":[\"tl\"],\n",
    "    \"(v,l)\":[\"vl\"],\n",
    "    \"(f,l)\":[\"fl\"],\n",
    "    \"(dh,l)\":[\"dhl\"],\n",
    "    \"(th,l)\":[\"thl\"],\n",
    "    \"(zh,l)\":[\"zhl\"],\n",
    "    \"(sh,l)\":[\"shl\"],\n",
    "    \"(ng,l)\":[\"ngl\"],\n",
    "    \"(n,l)\":[\"nl\"],\n",
    "    \"(m,l)\":[\"ml\"],\n",
    "    \"(l,l)\":[\"ll\"],\n",
    "    \"(r,l)\":[\"rl\"],\n",
    "    \"(w,l)\":[\"wl\"],\n",
    "    \"(hw,l)\":[\"hwl\"],\n",
    "    \"(y,l)\":[\"yl\"],\n",
    "    \"(h,l)\":[\"hl\"],\n",
    "    \"(s,l)\":[\"sl\"],\n",
    "    \"(z,l)\":[\"zl\"],\n",
    "    \"(ch,l)\":[\"chl\"],\n",
    "    \"(jh,l)\":[\"jhl\"],\n",
    "    \"(k,l)\":[\"kl\"],\n",
    "    \"(g,l)\":[\"gl\"],\n",
    "    \"(_r,l)\":[\"_rl\"],\n",
    "    # combining consonants with r\n",
    "    \"(b,r)\":[\"br\"],\n",
    "    \"(p,r)\":[\"pr\"],\n",
    "    \"(d,r)\":[\"dr\"],\n",
    "    \"(t,r)\":[\"tr\"],\n",
    "    \"(v,r)\":[\"vr\"],\n",
    "    \"(f,r)\":[\"fr\"],\n",
    "    \"(dh,r)\":[\"dhr\"],\n",
    "    \"(th,r)\":[\"thr\"],\n",
    "    \"(zh,r)\":[\"zhr\"],\n",
    "    \"(sh,r)\":[\"shr\"],\n",
    "    \"(ng,r)\":[\"ngr\"],\n",
    "    \"(n,r)\":[\"nr\"],\n",
    "    \"(m,r)\":[\"mr\"],\n",
    "    \"(l,r)\":[\"lr\"],\n",
    "    \"(r,r)\":[\"rr\"],\n",
    "    \"(w,r)\":[\"wr\"],\n",
    "    \"(hw,r)\":[\"hwr\"],\n",
    "    \"(y,r)\":[\"yr\"],\n",
    "    \"(h,r)\":[\"hr\"],\n",
    "    \"(s,r)\":[\"sr\"],\n",
    "    \"(z,r)\":[\"zr\"],\n",
    "    \"(ch,r)\":[\"chr\"],\n",
    "    \"(jh,r)\":[\"jhr\"],\n",
    "    \"(k,r)\":[\"kr\"],\n",
    "    \"(g,r)\":[\"gr\"],\n",
    "    \"(_r,r)\":[\"_rr\"],\n",
    "    # basic vowels\n",
    "    \"[a]\":[\"a\"],\n",
    "    \"[aa]\":[\"aa\"],\n",
    "    \"[o]\":[\"o\"],\n",
    "    \"[oo]\":[\"oo\"],\n",
    "    \"[e]\":[\"e\"],\n",
    "    \"[ei]\":[\"ei\"],\n",
    "    \"[uh]\":[\"uh\"],\n",
    "    \"[ou]\":[\"ou\"],\n",
    "    \"[i]\":[\"i\"],\n",
    "    \"[ii]\":[\"ii\"],\n",
    "    \"[u]\":[\"u\"],\n",
    "    \"[uu]\":[\"uu\"],\n",
    "    \"[ai]\":[\"ai\"],\n",
    "    \"[oi]\":[\"oi\"],\n",
    "    \"[ow]\":[\"ow\"],\n",
    "    \"[yuu]\":[\"yuu\"],\n",
    "    # punctuation\n",
    "    \"(_period_)\":[\"punc:.\"],\n",
    "    # briefs\n",
    "    \"(_i_)\":[\"brief:i\"],\n",
    "    \"(_a_)\":[\"brief:a\"],\n",
    "    \"(_all_)\":[\"brief:all\"],\n",
    "    \"(_and_)\":[\"brief:and\"],\n",
    "    \"(_as_)\":[\"brief:as\"],\n",
    "    \"(_but_)\":[\"brief:but\"],\n",
    "    \"(_you_)\":[\"brief:_u_\"],\n",
    "    \"(_first_)\":[\"brief:first\"],\n",
    "    \"(_would_)\":[\"brief:would\"],\n",
    "    \"(_with_)\":[\"brief:with\"],\n",
    "    \"(_is_)\":[\"brief:is\"],\n",
    "    \"(_how_)\":[\"brief:how\"],\n",
    "    \"(_in_)\":[\"brief:in\"],\n",
    "    \"(_who_)\":[\"brief:who\"],\n",
    "    \"(_you_)\":[\"brief:you\"],\n",
    "    \"(_to_)\":[\"brief:to\"],\n",
    "    \"(_too_)\":[\"brief:too\"],\n",
    "    \"(_of_)\":[\"brief:of\"],\n",
    "    \"(_oh_)\":[\"brief:oh\"],\n",
    "    \"(_on_)\":[\"brief:on\"],\n",
    "    \"(_should_)\":[\"brief:should\"],\n",
    "    \"(_the_)\":[\"brief:the\"],\n",
    "    \"(_u_)\":[\"brief:_u_\"],\n",
    "    # numbers\n",
    "    \"(_zero_)\":[\"#0\"],\n",
    "    \"(_one_)\":[\"#1\"],\n",
    "    \"(_two_)\":[\"#2\"],\n",
    "    \"(_three_)\":[\"#3\"],\n",
    "    \"(_four_)\":[\"#4\"],\n",
    "    \"(_five_)\":[\"#5\"],\n",
    "    \"(_six_)\":[\"#6\"],\n",
    "    \"(_seven_)\":[\"#7\"],\n",
    "    \"(_eight_)\":[\"#8\"],\n",
    "    \"(_nine_)\":[\"#9\"],\n",
    "    # decorations (need to fix to attach to previous consonant?)\n",
    "    \",_str\":[\"dec:str\"],\n",
    "    \",st\":[\"dec:st\"],\n",
    "    \",ses\":[\"dec:ses\"],\n",
    "    \",sis\":[\"dec:sis\"],\n",
    "    \",sais\":[\"dec:sais\"],\n",
    "    \",s\":[\"dec:s\"],\n",
    "    \",S\":[\"dec:s\"],\n",
    "    \";n\":[\"dec:n\"],\n",
    "    \";sishn\":[\"dec:sishn\"],\n",
    "    \";shn\":[\"dec:shn\"],\n",
    "    \";Shn\":[\"dec:shn\"],\n",
    "    \";f\":[\"dec:f\"],\n",
    "    \";v\":[\"dec:v\"],\n",
    "    # prefix\n",
    "    \"^con\":[\"prefix:con\"],\n",
    "    \"^h\":[\"prefix:h\"],\n",
    "    # suffix\n",
    "    \"~ing\":[\"suffix:ing\"],\n",
    "    # sizes\n",
    "    \":tr\":[\"size:2\"],\n",
    "    \":t\":[\"size:0.5\"],\n",
    "    \":dr\":[\"size:2\"],\n",
    "    \":dhr\":[\"size:2\"],\n",
    "    \":d\":[\"size:0.5\"],\n",
    "    # unknown bits, might be translated wrong\n",
    "    \"[_ow]\":[\"ow\"],\n",
    "    \"[ow_]\":[\"ow\"],\n",
    "    \"[_yuu]\":[\"yuu\"],\n",
    "    \"[i_]\":[\"i\"],\n",
    "    \"[ii_]\":[\"ii\"],\n",
    "    \"[ai_]\":[\"ai\"],\n",
    "    \"[_ai]\":[\"ai\"],\n",
    "    \"(_v,r)\":[\"vr\"],\n",
    "    \"(_v,l)\":[\"vl\"],\n",
    "    \"(k,w)\":[\"kw\"],\n",
    "    \"(g,w)\":[\"gw\"],\n",
    "    \"[ai{(0u,-13u)}]\":[\"ai\"],\n",
    "    \"[ou{(5u,-5u)}]\":[\"ou\"],\n",
    "    \"[e{(10u,5u)}]\":[\"e\"],\n",
    "    \"[i{(2u,10u)}]\":[\"i\"],\n",
    "    \"[ai{(-2u,-25u)}]\":[\"ai\"],\n",
    "    \"[e{(5u,0u)}]\":[\"e\"],\n",
    "    \"[e{(-7u,7u)}]\":[\"e\"],\n",
    "    \"(_dh,r)\":[\"dhr\"],\n",
    "    \"(_l_)\":[\"l\"],\n",
    "    \"(_sh)\":[\"sh\"],\n",
    "    \"(_l)\":[\"l\"],\n",
    "    \"(l_)\":[\"l\"],\n",
    "    \"{_three_}\":[\"3\"],\n",
    "    \"(m_)\":[\"m\"],\n",
    "    \"(n_)\":[\"n\"],\n",
    "    \"(_m)\":[\"m\"],\n",
    "    \"(_n)\":[\"n\"],\n",
    "    \"[ou_]\":[\"ou\"],\n",
    "    \"(_zh)\":[\"zh\"],\n",
    "    \"[uu_]\":[\"uu\"]\n",
    "}\n",
    "\n",
    "pitman_vowels = {\"a\",\"aa\",\"o\",\"oo\",\"e\",\"ei\",\"uh\",\"ou\",\"i\",\"ii\",\"u\",\"uu\",\"ai\",\"oi\",\"ow\",\"yuu\"}\n",
    "\n",
    "def pitman_translate(outline, verbose = False, vowel_drop_percent = 0.0):\n",
    "    pos = 0\n",
    "    out = []\n",
    "    \n",
    "    while pos < len(outline):\n",
    "        matched = False\n",
    "        for head in pitman_translation_dictionary:\n",
    "            if head == outline[pos:pos+len(head)]:\n",
    "                out+=pitman_translation_dictionary[head]\n",
    "                pos += len(head)\n",
    "                matched = True\n",
    "                break\n",
    "        if not matched:\n",
    "            if verbose:\n",
    "                print(f\"Failed in {outline} at {outline[pos:]}\")\n",
    "            return None\n",
    "    vowel_dropped = []\n",
    "    for x in out:\n",
    "        if x not in pitman_vowels or random.random() > vowel_drop_percent:\n",
    "            vowel_dropped.append(x)\n",
    "    #print(out,vowel_dropped)\n",
    "    return vowel_dropped\n",
    "\n",
    "for _,line in pitman2k_raw.iterrows():\n",
    "    translated = pitman_translate(line['outline'],verbose=True,vowel_drop_percent=0.0)\n",
    "    translated_novowel = pitman_translate(line['outline'],verbose=True,vowel_drop_percent=1.0)\n",
    "    if translated == None:\n",
    "        #print(f\"{line['word']:<10}{line['outline']:<20}{translated}\")\n",
    "        continue\n",
    "    else:\n",
    "        pitman2k[line['word']] = tuple(translated)\n",
    "        pitman2k_novowel[line['word']] = tuple(translated_novowel)\n",
    "\n",
    "print(len(pitman2k),len(pitman2k_raw))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 372,
   "id": "b451ca08",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('1', 'brief:all', 'l', 's'),\n",
       " ('1', 'brief:all', 'w', 'dec:s'),\n",
       " ('1', 'n'),\n",
       " ('3', 'd', 'dec:f'),\n",
       " ('3', 'dr', 'e', 'k', 'size:0.5'),\n",
       " ('1', 'd'),\n",
       " ('1', 'f'),\n",
       " ('1', 'd'),\n",
       " ('3', 'h', 'w', 'dec:s'),\n",
       " ('3', 'r'),\n",
       " ('3', 'f'),\n",
       " ('3', 'dec:s', 't'),\n",
       " ('1', 'jh'),\n",
       " ('1', 'mr'),\n",
       " ('3', 'r'),\n",
       " ('1', 'pr', 'size:0.5'),\n",
       " ('2', 'pr', 'h', 'a', 'p', 'dec:s'),\n",
       " ('3', 'p'),\n",
       " ('1', 'shr', 'size:0.5'),\n",
       " ('1', 'th'),\n",
       " ('1', 'dh', 'size:0.5'),\n",
       " ('3', 'l'),\n",
       " ('3', '_r')]"
      ]
     },
     "execution_count": 372,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[pitman2k[x] for x in pitman2k if pitman2k[x][0] in ['1','2','3']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 373,
   "id": "fc940f1a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mean_binary_length': 13.634455336659894,\n",
       " 'mean_binary_excess': 9.008772205004034,\n",
       " 'mean_entropy_shift': 6.377524380264968,\n",
       " 'mean_transmission_cost': 11.62421978023691,\n",
       " 'mean_transmission_excess': 6.99853664858105,\n",
       " 'mean_transmission_shift': 4.367288823841984,\n",
       " 'reconstruction_entropy': 0.06492057146871243,\n",
       " 'reconstruction_error': 0.0188530058822729,\n",
       " 'alphabet': 'brief:the e size:0.5 dec:s m dec:n brief:of l brief:and 1 i brief:to dh _r ii d brief:a k f n a uh b ai ei t r ou brief:in o w p prefix:h dec:st brief:as brief:i v ch s aa ow hw brief:with g 3 brief:on sh brief:_u_ uu _rr brief:is zr brief:all size:2 jh yuu h brief:but pr th ng vr oo u tr pl dhr gr brief:would mr bl y brief:too brief:who thr br prefix:con dec:v brief:how dr kl oi brief:first tl fr kw brief:should dec:shn suffix:ing dec:ses nr dec:f kr vl shl gl fl jhr   #0 nl gw shr 2 dec:sishn dec:str brief:oh dl #1 #2 zhr dec:sais ml z zh'}"
      ]
     },
     "execution_count": 373,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "code_statistics([x for x in one_grams_dict if x in pitman2k], pitman2k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 374,
   "id": "5e297d53",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mean_binary_length': 10.107191714684433,\n",
       " 'mean_binary_excess': 5.481508583028573,\n",
       " 'mean_entropy_shift': 2.850260758289507,\n",
       " 'mean_transmission_cost': 8.37665637336233,\n",
       " 'mean_transmission_excess': 3.75097324170647,\n",
       " 'mean_transmission_shift': 1.1197254169674045,\n",
       " 'reconstruction_entropy': 0.41180440117056166,\n",
       " 'reconstruction_error': 0.12174817795441739,\n",
       " 'alphabet': 'brief:the size:0.5 dec:s m dec:n brief:of l brief:and 1 brief:to dh _r d brief:a k f n b t r brief:in w p prefix:h dec:st brief:as brief:i v ch s hw brief:with g 3 brief:on sh brief:_u_ _rr brief:is zr brief:all size:2 jh h brief:but pr th ng vr tr pl dhr gr brief:would mr bl y brief:too brief:who thr br prefix:con dec:v brief:how dr kl brief:first tl fr kw brief:should dec:shn suffix:ing dec:ses nr dec:f kr vl shl gl fl jhr   #0 nl gw shr 2 dec:sishn dec:str brief:oh dl #1 #2 zhr dec:sais ml z zh'}"
      ]
     },
     "execution_count": 374,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "code_statistics([x for x in one_grams_dict if x in pitman2k_novowel], pitman2k_novowel)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 375,
   "id": "37ffa4b3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['p', 'p', 'l', 'size:0.5'],\n",
       " ['p', 'o', 'p', 'l', 'size:0.5'],\n",
       " ['p', 'p', 'yuu', 'l', 'size:0.5'],\n",
       " ['p', 'p', 'l', 'ei', 'size:0.5'],\n",
       " ['p', 'o', 'p', 'yuu', 'l', 'size:0.5'],\n",
       " ['p', 'o', 'p', 'l', 'ei', 'size:0.5'],\n",
       " ['p', 'p', 'yuu', 'l', 'ei', 'size:0.5'],\n",
       " ['p', 'o', 'p', 'yuu', 'l', 'ei', 'size:0.5']]"
      ]
     },
     "execution_count": 375,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from itertools import combinations\n",
    "\n",
    "# Define the iterator class\n",
    "class VowelFilterIterator:\n",
    "    def __init__(self, outline, vowel_list):\n",
    "        self.outline = outline\n",
    "        self.vowel_list = set(vowel_list)\n",
    "        self.vowel_indices = [i for i, ch in enumerate(outline) if ch in self.vowel_list]\n",
    "        self.current_count = 0\n",
    "        self.to_keep_combinations = []\n",
    "\n",
    "    def __iter__(self):\n",
    "        return self\n",
    "\n",
    "    def __next__(self):\n",
    "        # If all combinations for the current count are exhausted, increment count\n",
    "        if not self.to_keep_combinations:\n",
    "            if self.current_count > len(self.vowel_indices):\n",
    "                raise StopIteration\n",
    "            # Generate new combinations for the current count\n",
    "            self.to_keep_combinations = list(combinations(self.vowel_indices, self.current_count))\n",
    "            self.current_count += 1\n",
    "\n",
    "        # Pick the next combination of vowel indices to keep\n",
    "        keep_indices = set(self.to_keep_combinations.pop(0))\n",
    "        sub_list = [ch for i, ch in enumerate(self.outline) if i not in self.vowel_indices or i in keep_indices]\n",
    "\n",
    "        return sub_list\n",
    "\n",
    "# Test case as per user's example\n",
    "outline = list(pitman2k[\"populate\"])\n",
    "vowel_list = pitman_vowels\n",
    "\n",
    "# Run the iterator and collect results for analysis\n",
    "iterator = VowelFilterIterator(outline, vowel_list)\n",
    "results = list(iterator)\n",
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 376,
   "id": "9e27e8b1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "24"
      ]
     },
     "execution_count": 376,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pitman_vowel_optimized = {}\n",
    "used = set()\n",
    "\n",
    "failures = 0\n",
    "for word in one_grams_dict:\n",
    "    if word not in pitman2k:\n",
    "        continue\n",
    "    \n",
    "    for candidate in VowelFilterIterator(list(pitman2k[word]), pitman_vowels):\n",
    "        test = tuple(candidate)\n",
    "        if test not in used:\n",
    "            pitman_vowel_optimized[word] = test\n",
    "            used.add(test)\n",
    "            break\n",
    "    \n",
    "    if word not in pitman_vowel_optimized:\n",
    "        failures += 1\n",
    "        pitman_vowel_optimized[word] = pitman2k[word]\n",
    "\n",
    "failures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 377,
   "id": "bd7b09a6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mean_binary_length': 11.136021518086377,\n",
       " 'mean_binary_excess': 6.510338386430517,\n",
       " 'mean_entropy_shift': 3.8790905616914513,\n",
       " 'mean_transmission_cost': 9.291760572933075,\n",
       " 'mean_transmission_excess': 4.666077441277215,\n",
       " 'mean_transmission_shift': 2.0348296165381496,\n",
       " 'reconstruction_entropy': 0.07683250655124771,\n",
       " 'reconstruction_error': 0.022772203395222124,\n",
       " 'alphabet': 'brief:the size:0.5 dec:s m dec:n brief:of l brief:and 1 brief:to dh _r d brief:a k f n b t r brief:in w p prefix:h ai dec:st ii brief:as brief:i v ch s ei hw brief:with g 3 brief:on sh brief:_u_ ou _rr brief:is zr e brief:all size:2 jh h brief:but pr th a ng vr tr pl uh o dhr yuu gr brief:would mr bl ow u y brief:too brief:who i thr br prefix:con dec:v oo brief:how dr kl brief:first tl fr aa kw brief:should dec:shn uu suffix:ing dec:ses nr dec:f kr vl shl gl fl oi jhr   #0 nl gw shr 2 dec:sishn dec:str brief:oh dl #1 #2 zhr dec:sais ml z zh'}"
      ]
     },
     "execution_count": 377,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "code_statistics([x for x in one_grams_dict if x in pitman_vowel_optimized], pitman_vowel_optimized)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "af077fa5",
   "metadata": {},
   "source": [
    "# Swiftograph - Abbott 15 \n",
    "\n",
    "This is a simple orthographic system which can be efficiently implemented."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 378,
   "id": "d26507c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "swiftograph_briefs = abbreviations = {\n",
    "    \"a\": [\"a\"],\n",
    "    \"an\": [\"a\"],\n",
    "    \"about\": [\"a\", \"b\", \"t\"],\n",
    "    \"always\": [\"a\", \"l\", \"s\"],\n",
    "    \"am\": [\"m\"],\n",
    "    \"amount\": [\"a\", \"m\", \"t\"],\n",
    "    \"and\": [\"n\", \"d\"],\n",
    "    \"are\": [\"r\"],\n",
    "    \"be, but\": [\"b\"],\n",
    "    \"been\": [\"b\", \"n\"],\n",
    "    \"because\": [\"b\", \"c\"],\n",
    "    \"between\": [\"b\", \"t\"],\n",
    "    \"can\": [\"c\", \"n\"],\n",
    "    \"could\": [\"c\", \"d\"],\n",
    "    \"do\": [\"d\"],\n",
    "    \"ever\": [\"e\", \"v\"],\n",
    "    \"for\": [\"f\"],\n",
    "    \"from\": [\"f\", \"r\"],\n",
    "    \"friend\": [\"f\", \"d\"],\n",
    "    \"good, go\": [\"g\"],\n",
    "    \"great\": [\"g\", \"r\"],\n",
    "    \"government\": [\"g\", \"v\"],\n",
    "    \"had\": [\"a\", \"d\"],\n",
    "    \"have\": [\"v\"],\n",
    "    \"he\": [\"h\"],\n",
    "    \"in\": [\"n\"],\n",
    "    \"no\": [\"n\"],\n",
    "    \"made\": [\"m\", \"d\"],\n",
    "    \"man\": [\"m\", \"n\"],\n",
    "    \"any\": [\"a\", \"y\"],\n",
    "    \"more\": [\"m\", \"o\"],\n",
    "    \"must\": [\"m\", \"s\", \"t\"],\n",
    "    \"most\": [\"m\", \"s\", \"t\"],\n",
    "    \"name\": [\"n\", \"a\"],\n",
    "    \"not\": [\"n\", \"t\"],\n",
    "    \"of\": [\"o\"],\n",
    "    \"only\": [\"n\", \"l\", \"y\"],\n",
    "    \"parliament\": [\"p\", \"a\", \"r\"],\n",
    "    \"people\": [\"p\"],\n",
    "    \"question\": [\"q\", \"u\"],\n",
    "    \"said\": [\"s\", \"d\"],\n",
    "    \"shall\": [\"s\", \"h\"],\n",
    "    \"should\": [\"s\", \"u\"],\n",
    "    \"to\": [\"t\"],\n",
    "    \"the\": [\"t\", \"h\"],\n",
    "    \"than\": [\"t\", \"n\"],\n",
    "    \"they\": [\"t\", \"i\"],\n",
    "    \"that\": [\"t\", \"a\"],\n",
    "    \"this\": [\"t\", \"s\"],\n",
    "    \"their\": [\"t\", \"r\"],\n",
    "    \"there\": [\"t\", \"r\"],\n",
    "    \"through\": [\"t\", \"u\"],\n",
    "    \"under\": [\"u\"],\n",
    "    \"very\": [\"v\", \"r\"],\n",
    "    \"was\": [\"w\"],\n",
    "    \"what\": [\"w\", \"t\"],\n",
    "    \"whatsoever\": [\"w\", \"t\", \"s\", \"r\"],\n",
    "    \"where\": [\"w\", \"r\"],\n",
    "    \"were\": [\"w\", \"r\"],\n",
    "    \"when\": [\"w\", \"e\", \"n\"],\n",
    "    \"who\": [\"w\", \"o\"],\n",
    "    \"whole\": [\"o\", \"l\"],\n",
    "    \"which\": [\"c\", \"h\"],\n",
    "    \"will\": [\"l\", \"o\"],\n",
    "    \"with\": [\"w\", \"t\", \"h\"],\n",
    "    \"world\": [\"w\", \"l\", \"d\"],\n",
    "    \"would\": [\"w\", \"d\"],\n",
    "    \"you\": [\"y\"]\n",
    "}\n",
    "\n",
    "\n",
    "swiftograph_blends = {\n",
    "    \"ai\":\"ai\", \"ea\":\"ea\", \"ao\":\"ao\", \"ei\":\"ei\", \"ie\":\"ie\", \"eu\":\"eu\", \"sg\":\"sg\",\n",
    "    \"em\":\"m\", \"im\":\"m\",\n",
    "    \"en\":\"n\", \"in\":\"n\",\n",
    "    \"te\":\"t\", \"ti\":\"t\",\n",
    "    \"de\":\"d\", \"di\":\"d\",\n",
    "    \"bd\":\"bd\", \"bt\":\"bt\",\n",
    "    \"ch\":\"ch\", \"th\":\"th\",\n",
    "    \"au\":\"au\", \"oi\":\"oi\", \"ou\":\"ou\"\n",
    "}\n",
    "\n",
    "def make_swiftographic(word):\n",
    "    # handle briefs\n",
    "    if word in swiftograph_briefs:\n",
    "        sequence = swiftograph_briefs[word]\n",
    "        pos = 0\n",
    "        out = []\n",
    "        while pos < len(sequence):\n",
    "            if pos+1 < len(sequence) and sequence[pos] + sequence[pos+1] in swiftograph_blends:\n",
    "                out.append(swiftograph_blends[sequence[pos] + sequence[pos+1]])\n",
    "                pos += 2\n",
    "            else:\n",
    "                out.append(sequence[pos])\n",
    "                pos += 1\n",
    "        return tuple(out), tuple(out)\n",
    "    \n",
    "    # process word endings\n",
    "    patterns = [\n",
    "        (r'ed\\b', 'd'),   # Replace \"ed\" at the end with \"d\"\n",
    "        (r'ing\\b', 'n'),  # Replace \"ing\" at the end with \"n\"\n",
    "        (r'tion\\b', 'un'), # Replace \"tion\" at the end with \"un\"\n",
    "        (r'ight\\b', ' t') # Replace \"ight\" at the end with \" t\"\n",
    "    ]\n",
    "    \n",
    "    previous_word = None\n",
    "    while word != previous_word:  # Keep applying replacements until no more changes\n",
    "        previous_word = word\n",
    "        for pattern, replacement in patterns:\n",
    "            word = re.sub(pattern, replacement, word)\n",
    "\n",
    "    # additional abbreviation principles\n",
    "    word = word.replace(\"ee\",\"e\") # first trim double e's to a single \"e\"\n",
    "    word = word.replace(\"qu\",\"q\") # u always assumed after q\n",
    "    word = re.sub(r'(\\w)\\1', r'\\1_', word) # replace all other double letters with the letter followed by an underscore\n",
    "    word = re.sub(r'(?<!\\b)[ao](?=[mn])', '', word) # remove a and o before m or n except when it is initial\n",
    "\n",
    "    # implement blends\n",
    "    sequence = list(word)\n",
    "\n",
    "    pos = 0\n",
    "    out = []\n",
    "    while pos < len(sequence):\n",
    "        if pos+1 < len(sequence) and sequence[pos] + sequence[pos+1] in swiftograph_blends:\n",
    "            out.append(swiftograph_blends[sequence[pos] + sequence[pos+1]])\n",
    "            pos += 2\n",
    "        else:\n",
    "            out.append(sequence[pos])\n",
    "            pos += 1\n",
    "\n",
    "    # don't write i with a dot or underscores\n",
    "    out2 = []\n",
    "    for i in out:\n",
    "        if i == \"i\":\n",
    "            out2.append(\"e\")\n",
    "        elif i == \"_\":\n",
    "            continue\n",
    "        else:\n",
    "            out2.append(i)\n",
    "\n",
    "    return tuple(out), tuple(out2[:5]) #return both the \"by the book\" version and the \"by the examples\" version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 379,
   "id": "e297c7ff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'the': ('th',),\n",
       " 'of': ('o',),\n",
       " 'and': ('n', 'd'),\n",
       " 'to': ('t',),\n",
       " 'a': ('a',),\n",
       " 'in': ('n',),\n",
       " 'that': ('t', 'a'),\n",
       " 'is': ('e', 's'),\n",
       " 'for': ('f',),\n",
       " 'i': ('e',),\n",
       " 'it': ('e', 't'),\n",
       " 'was': ('w',),\n",
       " 'as': ('a', 's'),\n",
       " 'not': ('n', 't'),\n",
       " 'with': ('w', 'th'),\n",
       " 'he': ('h',),\n",
       " 'on': ('o', 'n'),\n",
       " 'you': ('y',),\n",
       " 'be': ('b', 'e'),\n",
       " 'his': ('h', 'e', 's'),\n",
       " 'this': ('t', 's'),\n",
       " 'by': ('b', 'y'),\n",
       " 'or': ('o', 'r'),\n",
       " 'are': ('r',),\n",
       " 'at': ('a', 't'),\n",
       " 'her': ('h', 'e', 'r'),\n",
       " 'from': ('f', 'r'),\n",
       " 'she': ('s', 'h', 'e'),\n",
       " 'had': ('a', 'd'),\n",
       " 'but': ('b', 'u', 't'),\n",
       " 'have': ('v',),\n",
       " 'an': ('a',),\n",
       " 'they': ('t',),\n",
       " 'we': ('w', 'e'),\n",
       " 'were': ('w', 'r'),\n",
       " 'one': ('o', 'n', 'e'),\n",
       " 'all': ('a', 'l'),\n",
       " 'can': ('c', 'n'),\n",
       " 'which': ('ch',),\n",
       " 'their': ('t', 'r'),\n",
       " 'my': ('m', 'y'),\n",
       " 'if': ('e', 'f'),\n",
       " 'what': ('w', 't'),\n",
       " 'do': ('d',),\n",
       " 'there': ('t', 'r'),\n",
       " 'when': ('w', 'n'),\n",
       " 'would': ('w', 'd'),\n",
       " 'him': ('h', 'm'),\n",
       " 'so': ('s', 'o'),\n",
       " 'more': ('m', 'o'),\n",
       " 'me': ('m', 'e'),\n",
       " 'will': ('l', 'o'),\n",
       " 'been': ('b', 'n'),\n",
       " 'out': ('ou', 't'),\n",
       " 'up': ('u', 'p'),\n",
       " 'about': ('a', 'bt'),\n",
       " 'who': ('w', 'o'),\n",
       " 'has': ('h', 'a', 's'),\n",
       " 'no': ('n',),\n",
       " 'into': ('n', 't', 'o'),\n",
       " 'time': ('t', 'm', 'e'),\n",
       " 'other': ('o', 'th', 'e', 'r'),\n",
       " 'them': ('th', 'm'),\n",
       " 'your': ('y', 'ou', 'r'),\n",
       " 'said': ('s', 'd'),\n",
       " 'could': ('c', 'd'),\n",
       " 'did': ('d', 'd'),\n",
       " 'then': ('th', 'n'),\n",
       " 'some': ('s', 'm', 'e'),\n",
       " 'these': ('th', 'e', 's', 'e'),\n",
       " 'also': ('a', 'l', 's', 'o'),\n",
       " 'like': ('l', 'e', 'k', 'e'),\n",
       " 'than': ('t', 'n'),\n",
       " 'its': ('e', 't', 's'),\n",
       " 'may': ('m', 'a', 'y'),\n",
       " 'only': ('n', 'l', 'y'),\n",
       " 'see': ('s', 'e'),\n",
       " 'new': ('n', 'e', 'w'),\n",
       " 'how': ('h', 'o', 'w'),\n",
       " 'two': ('t', 'w', 'o'),\n",
       " 'such': ('s', 'u', 'ch'),\n",
       " 'over': ('o', 'v', 'e', 'r'),\n",
       " 'our': ('ou', 'r'),\n",
       " 'first': ('f', 'e', 'r', 's', 't'),\n",
       " 'any': ('a', 'y'),\n",
       " 'just': ('j', 'u', 's', 't'),\n",
       " 'after': ('a', 'f', 't', 'r'),\n",
       " 'back': ('b', 'a', 'c', 'k'),\n",
       " 'now': ('n', 'o', 'w'),\n",
       " 'through': ('t', 'u'),\n",
       " 'even': ('e', 'v', 'n'),\n",
       " 'people': ('p',),\n",
       " 'where': ('w', 'r'),\n",
       " 'well': ('w', 'e', 'l'),\n",
       " 'most': ('m', 's', 't'),\n",
       " 'between': ('bt',),\n",
       " 'way': ('w', 'a', 'y'),\n",
       " 'because': ('b', 'c'),\n",
       " 'know': ('k', 'n', 'o', 'w'),\n",
       " 'should': ('s', 'u'),\n",
       " 'before': ('b', 'e', 'f', 'o', 'r'),\n",
       " 'many': ('m', 'n', 'y'),\n",
       " 'down': ('d', 'o', 'w', 'n'),\n",
       " 'very': ('v', 'r'),\n",
       " 'made': ('m', 'd'),\n",
       " 'life': ('l', 'e', 'f', 'e'),\n",
       " 'work': ('w', 'o', 'r', 'k'),\n",
       " 'those': ('th', 'o', 's', 'e'),\n",
       " 'us': ('u', 's'),\n",
       " 'here': ('h', 'e', 'r', 'e'),\n",
       " 'get': ('g', 'e', 't'),\n",
       " 'use': ('u', 's', 'e'),\n",
       " 'make': ('m', 'a', 'k', 'e'),\n",
       " 'being': ('b', 'n'),\n",
       " 'good': ('g', 'o', 'd'),\n",
       " 'much': ('m', 'u', 'ch'),\n",
       " 'each': ('ea', 'ch'),\n",
       " 'both': ('b', 'o', 'th'),\n",
       " 'man': ('m', 'n'),\n",
       " 'right': ('r', ' ', 't'),\n",
       " 'while': ('w', 'h', 'e', 'l', 'e'),\n",
       " 'used': ('u', 's', 'd'),\n",
       " 'world': ('w', 'l', 'd'),\n",
       " 'same': ('s', 'm', 'e'),\n",
       " 'must': ('m', 's', 't'),\n",
       " 'long': ('l', 'n', 'g'),\n",
       " 'go': ('g', 'o'),\n",
       " 'years': ('y', 'ea', 'r', 's'),\n",
       " 'still': ('s', 't', 'l'),\n",
       " 'day': ('d', 'a', 'y'),\n",
       " 'own': ('o', 'w', 'n'),\n",
       " 'does': ('d', 'o', 'e', 's'),\n",
       " 'too': ('t', 'o'),\n",
       " 'under': ('u',),\n",
       " 'take': ('t', 'a', 'k', 'e'),\n",
       " 'another': ('a', 'n', 'o', 'th', 'e'),\n",
       " 'part': ('p', 'a', 'r', 't'),\n",
       " 'state': ('s', 't', 'a', 't'),\n",
       " 'little': ('l', 'e', 't', 'l', 'e'),\n",
       " 'however': ('h', 'o', 'w', 'e', 'v'),\n",
       " 'off': ('o', 'f'),\n",
       " 'three': ('th', 'r', 'e'),\n",
       " 'around': ('a', 'r', 'ou', 'n', 'd'),\n",
       " 'think': ('th', 'n', 'k'),\n",
       " 'need': ('n', 'd'),\n",
       " 'never': ('n', 'e', 'v', 'e', 'r'),\n",
       " 'come': ('c', 'm', 'e'),\n",
       " 'might': ('m', ' ', 't'),\n",
       " 'different': ('d', 'f', 'e', 'r', 'n'),\n",
       " 'again': ('a', 'g', 'ai', 'n'),\n",
       " 'without': ('w', 'e', 'th', 'ou', 't'),\n",
       " 'god': ('g', 'o', 'd'),\n",
       " 'system': ('s', 'y', 's', 't', 'm'),\n",
       " 'going': ('g', 'n'),\n",
       " 'want': ('w', 'n', 't'),\n",
       " 'hand': ('h', 'n', 'd'),\n",
       " 'social': ('s', 'o', 'c', 'e', 'a'),\n",
       " 'during': ('d', 'u', 'r', 'n'),\n",
       " 'place': ('p', 'l', 'a', 'c', 'e'),\n",
       " 'thought': ('th', 'ou', 'g', 'h', 't'),\n",
       " 'against': ('a', 'g', 'ai', 'n', 's'),\n",
       " 'something': ('s', 'm', 'e', 'th', 'n'),\n",
       " 'found': ('f', 'ou', 'n', 'd'),\n",
       " 'high': ('h', 'e', 'g', 'h'),\n",
       " 'case': ('c', 'a', 's', 'e'),\n",
       " 'say': ('s', 'a', 'y'),\n",
       " 'why': ('w', 'h', 'y'),\n",
       " 'away': ('a', 'w', 'a', 'y'),\n",
       " 'information': ('n', 'f', 'o', 'r', 'm'),\n",
       " 'head': ('h', 'ea', 'd'),\n",
       " 'within': ('w', 'e', 'th', 'n'),\n",
       " 'let': ('l', 'e', 't'),\n",
       " 'eyes': ('e', 'y', 'e', 's'),\n",
       " 'number': ('n', 'u', 'm', 'b', 'e'),\n",
       " 'left': ('l', 'e', 'f', 't'),\n",
       " 'old': ('o', 'l', 'd'),\n",
       " 'every': ('e', 'v', 'e', 'r', 'y'),\n",
       " 'great': ('g', 'r'),\n",
       " 'example': ('e', 'x', 'm', 'p', 'l'),\n",
       " 'since': ('s', 'n', 'c', 'e'),\n",
       " 'data': ('d', 'a', 't', 'a'),\n",
       " 'came': ('c', 'm', 'e'),\n",
       " 'men': ('m', 'n'),\n",
       " 'set': ('s', 'e', 't'),\n",
       " 'look': ('l', 'o', 'k'),\n",
       " 'children': ('ch', 'e', 'l', 'd', 'r'),\n",
       " 'water': ('w', 'a', 't', 'r'),\n",
       " 'university': ('u', 'n', 'e', 'v', 'e'),\n",
       " 'few': ('f', 'e', 'w'),\n",
       " 'face': ('f', 'a', 'c', 'e'),\n",
       " 'research': ('r', 'e', 's', 'ea', 'r'),\n",
       " 'chapter': ('ch', 'a', 'p', 't', 'r'),\n",
       " 'small': ('s', 'm', 'a', 'l'),\n",
       " 'things': ('th', 'n', 'g', 's'),\n",
       " 'year': ('y', 'ea', 'r'),\n",
       " 'based': ('b', 'a', 's', 'd'),\n",
       " 'find': ('f', 'n', 'd'),\n",
       " 'end': ('n', 'd'),\n",
       " 'human': ('h', 'u', 'm', 'n'),\n",
       " 'last': ('l', 'a', 's', 't'),\n",
       " 'home': ('h', 'm', 'e'),\n",
       " 'always': ('a', 'l', 's'),\n",
       " 'power': ('p', 'o', 'w', 'e', 'r'),\n",
       " 'asked': ('a', 's', 'k', 'd'),\n",
       " 'figure': ('f', 'e', 'g', 'u', 'r'),\n",
       " 'often': ('o', 'f', 't', 'n'),\n",
       " 'order': ('o', 'r', 'd', 'r'),\n",
       " 'family': ('f', 'm', 'e', 'l', 'y'),\n",
       " 'important': ('m', 'p', 'o', 'r', 't'),\n",
       " 'using': ('u', 's', 'n'),\n",
       " 'women': ('w', 'm', 'n'),\n",
       " 'process': ('p', 'r', 'o', 'c', 'e'),\n",
       " 'house': ('h', 'ou', 's', 'e'),\n",
       " 'point': ('p', 'oi', 'n', 't'),\n",
       " 'though': ('th', 'ou', 'g', 'h'),\n",
       " 'law': ('l', 'a', 'w'),\n",
       " 'took': ('t', 'o', 'k'),\n",
       " 'states': ('s', 't', 'a', 't', 's'),\n",
       " 'called': ('c', 'a', 'l', 'd'),\n",
       " 'looked': ('l', 'o', 'k', 'd'),\n",
       " 'second': ('s', 'e', 'c', 'n', 'd'),\n",
       " 'public': ('p', 'u', 'b', 'l', 'e'),\n",
       " 'given': ('g', 'e', 'v', 'n'),\n",
       " 'got': ('g', 'o', 't'),\n",
       " 'help': ('h', 'e', 'l', 'p'),\n",
       " 'once': ('o', 'n', 'c', 'e'),\n",
       " 'body': ('b', 'o', 'd', 'y'),\n",
       " 'until': ('u', 'n', 't', 'l'),\n",
       " 'change': ('ch', 'n', 'g', 'e'),\n",
       " 'put': ('p', 'u', 't'),\n",
       " 'development': ('d', 'v', 'e', 'l', 'o'),\n",
       " 'went': ('w', 'n', 't'),\n",
       " 'room': ('r', 'o', 'm'),\n",
       " 'group': ('g', 'r', 'ou', 'p'),\n",
       " 'next': ('n', 'e', 'x', 't'),\n",
       " 'general': ('g', 'n', 'e', 'r', 'a'),\n",
       " 'love': ('l', 'o', 'v', 'e'),\n",
       " 'am': ('m',),\n",
       " 'form': ('f', 'o', 'r', 'm'),\n",
       " 'side': ('s', 'e', 'd'),\n",
       " 'following': ('f', 'o', 'l', 'o', 'w'),\n",
       " 'give': ('g', 'e', 'v', 'e'),\n",
       " 'large': ('l', 'a', 'r', 'g', 'e'),\n",
       " 'less': ('l', 'e', 's'),\n",
       " 'told': ('t', 'o', 'l', 'd'),\n",
       " 'better': ('b', 'e', 't', 'e', 'r'),\n",
       " 'knew': ('k', 'n', 'e', 'w'),\n",
       " 'yet': ('y', 'e', 't'),\n",
       " 'table': ('t', 'a', 'b', 'l', 'e'),\n",
       " 'control': ('c', 'n', 't', 'r', 'o'),\n",
       " 'upon': ('u', 'p', 'n'),\n",
       " 'later': ('l', 'a', 't', 'r'),\n",
       " 'study': ('s', 't', 'u', 'd', 'y'),\n",
       " 'others': ('o', 'th', 'e', 'r', 's'),\n",
       " 'himself': ('h', 'm', 's', 'e', 'l'),\n",
       " 'school': ('s', 'ch', 'o', 'l'),\n",
       " 'press': ('p', 'r', 'e', 's'),\n",
       " 'war': ('w', 'a', 'r'),\n",
       " 'rather': ('r', 'a', 'th', 'e', 'r'),\n",
       " 'best': ('b', 'e', 's', 't'),\n",
       " 'government': ('g', 'v'),\n",
       " 'book': ('b', 'o', 'k'),\n",
       " 'american': ('a', 'm', 'e', 'r', 'e'),\n",
       " 'person': ('p', 'e', 'r', 's', 'n'),\n",
       " 'business': ('b', 'u', 's', 'n', 'e'),\n",
       " 'nothing': ('n', 'o', 'th', 'n'),\n",
       " 'tell': ('t', 'l'),\n",
       " 'fact': ('f', 'a', 'c', 't'),\n",
       " 'mind': ('m', 'n', 'd'),\n",
       " 'although': ('a', 'l', 'th', 'ou', 'g'),\n",
       " 'really': ('r', 'ea', 'l', 'y'),\n",
       " 'level': ('l', 'e', 'v', 'e', 'l'),\n",
       " 'early': ('ea', 'r', 'l', 'y'),\n",
       " 'father': ('f', 'a', 'th', 'e', 'r'),\n",
       " 'national': ('n', 'a', 't', 'n', 'a'),\n",
       " 'political': ('p', 'o', 'l', 'e', 't'),\n",
       " 'name': ('n', 'a'),\n",
       " 'shall': ('s', 'h'),\n",
       " 'thus': ('th', 'u', 's'),\n",
       " 'felt': ('f', 'e', 'l', 't'),\n",
       " 'words': ('w', 'o', 'r', 'd', 's'),\n",
       " 'enough': ('n', 'ou', 'g', 'h'),\n",
       " 'among': ('a', 'm', 'n', 'g'),\n",
       " 'seen': ('s', 'n'),\n",
       " 'far': ('f', 'a', 'r'),\n",
       " 'course': ('c', 'ou', 'r', 's', 'e'),\n",
       " 'night': ('n', ' ', 't'),\n",
       " 'health': ('h', 'ea', 'l', 'th'),\n",
       " 'history': ('h', 'e', 's', 't', 'o'),\n",
       " 'making': ('m', 'a', 'k', 'n'),\n",
       " 'together': ('t', 'o', 'g', 'e', 'th'),\n",
       " 'open': ('o', 'p', 'n'),\n",
       " 'times': ('t', 'm', 'e', 's'),\n",
       " 'possible': ('p', 'o', 's', 'e', 'b'),\n",
       " 'door': ('d', 'o', 'r'),\n",
       " 'value': ('v', 'a', 'l', 'u', 'e'),\n",
       " 'mother': ('m', 'o', 'th', 'e', 'r'),\n",
       " 'young': ('y', 'ou', 'n', 'g'),\n",
       " 'several': ('s', 'e', 'v', 'e', 'r'),\n",
       " 'model': ('m', 'o', 'd', 'l'),\n",
       " 'self': ('s', 'e', 'l', 'f'),\n",
       " 'light': ('l', ' ', 't'),\n",
       " 'four': ('f', 'ou', 'r'),\n",
       " 'days': ('d', 'a', 'y', 's'),\n",
       " 'care': ('c', 'a', 'r', 'e'),\n",
       " 'become': ('b', 'e', 'c', 'm', 'e'),\n",
       " 'thing': ('th', 'n'),\n",
       " 'having': ('h', 'a', 'v', 'n'),\n",
       " 'saw': ('s', 'a', 'w'),\n",
       " 'ever': ('e', 'v'),\n",
       " 'least': ('l', 'ea', 's', 't'),\n",
       " 'whether': ('w', 'h', 'e', 'th', 'e'),\n",
       " 'international': ('n', 't', 'r', 'n', 'a'),\n",
       " 'white': ('w', 'h', 'e', 't'),\n",
       " 'woman': ('w', 'm', 'n'),\n",
       " 'section': ('s', 'e', 'c', 'u', 'n'),\n",
       " 'sure': ('s', 'u', 'r', 'e'),\n",
       " 'wanted': ('w', 'n', 't', 'd'),\n",
       " 'means': ('m', 'n', 's'),\n",
       " 'turned': ('t', 'u', 'r', 'n', 'd'),\n",
       " 'including': ('n', 'c', 'l', 'u', 'd'),\n",
       " 'already': ('a', 'l', 'r', 'ea', 'd'),\n",
       " 'done': ('d', 'n', 'e'),\n",
       " 'along': ('a', 'l', 'n', 'g'),\n",
       " 'whole': ('o', 'l'),\n",
       " 'across': ('a', 'c', 'r', 'o', 's'),\n",
       " 'question': ('q', 'u'),\n",
       " 'known': ('k', 'n', 'o', 'w', 'n'),\n",
       " 'black': ('b', 'l', 'a', 'c', 'k'),\n",
       " 'feel': ('f', 'e', 'l'),\n",
       " 'rights': ('r', 'e', 'g', 'h', 't'),\n",
       " 'analysis': ('a', 'n', 'a', 'l', 'y'),\n",
       " 'city': ('c', 'e', 't', 'y'),\n",
       " 'heart': ('h', 'ea', 'r', 't'),\n",
       " 'support': ('s', 'u', 'p', 'o', 'r'),\n",
       " 'child': ('ch', 'e', 'l', 'd'),\n",
       " 'present': ('p', 'r', 'e', 's', 'n'),\n",
       " 'able': ('a', 'b', 'l', 'e'),\n",
       " 'almost': ('a', 'l', 'm', 'o', 's'),\n",
       " 'line': ('l', 'n', 'e'),\n",
       " 'keep': ('k', 'e', 'p'),\n",
       " 'act': ('a', 'c', 't'),\n",
       " 'hands': ('h', 'n', 'd', 's'),\n",
       " 'above': ('a', 'b', 'o', 'v', 'e'),\n",
       " 'problem': ('p', 'r', 'o', 'b', 'l'),\n",
       " 'anything': ('a', 'n', 'y', 'th', 'n'),\n",
       " 'further': ('f', 'u', 'r', 'th', 'e'),\n",
       " 'either': ('ei', 'th', 'e', 'r'),\n",
       " 'area': ('a', 'r', 'ea'),\n",
       " 'sense': ('s', 'n', 's', 'e'),\n",
       " 'common': ('c', 'm', 'n'),\n",
       " 'real': ('r', 'ea', 'l'),\n",
       " 'local': ('l', 'o', 'c', 'a', 'l'),\n",
       " 'low': ('l', 'o', 'w'),\n",
       " 'taken': ('t', 'a', 'k', 'n'),\n",
       " 'experience': ('e', 'x', 'p', 'e', 'r'),\n",
       " 'behind': ('b', 'e', 'h', 'n', 'd'),\n",
       " 'moment': ('m', 'm', 'n', 't'),\n",
       " 'full': ('f', 'u', 'l'),\n",
       " 'free': ('f', 'r', 'e'),\n",
       " 'education': ('e', 'd', 'u', 'c', 'au'),\n",
       " 'management': ('m', 'n', 'a', 'g', 'm'),\n",
       " 'looking': ('l', 'o', 'k', 'n'),\n",
       " 'yes': ('y', 'e', 's'),\n",
       " 'show': ('s', 'h', 'o', 'w'),\n",
       " 'service': ('s', 'e', 'r', 'v', 'e'),\n",
       " 'therefore': ('th', 'e', 'r', 'e', 'f'),\n",
       " 'students': ('s', 't', 'u', 'd', 'n'),\n",
       " 'nature': ('n', 'a', 't', 'u', 'r'),\n",
       " 'death': ('d', 'a', 'th'),\n",
       " 'studies': ('s', 't', 'u', 'd', 'e'),\n",
       " 'five': ('f', 'e', 'v', 'e'),\n",
       " 'learning': ('l', 'ea', 'r', 'n'),\n",
       " 'knowledge': ('k', 'n', 'o', 'w', 'l'),\n",
       " 'word': ('w', 'o', 'r', 'd'),\n",
       " 'company': ('c', 'm', 'p', 'n', 'y'),\n",
       " 'voice': ('v', 'oi', 'c', 'e'),\n",
       " 'economic': ('e', 'c', 'n', 'm', 'e'),\n",
       " 'energy': ('n', 'e', 'r', 'g', 'y'),\n",
       " 'country': ('c', 'ou', 'n', 't', 'r'),\n",
       " 'past': ('p', 'a', 's', 't'),\n",
       " 'mean': ('m', 'n'),\n",
       " 'air': ('ai', 'r'),\n",
       " 'kind': ('k', 'n', 'd'),\n",
       " 'began': ('b', 'e', 'g', 'n'),\n",
       " 'hard': ('h', 'a', 'r', 'd'),\n",
       " 'type': ('t', 'y', 'p', 'e'),\n",
       " 'society': ('s', 'o', 'c', 'ie', 't'),\n",
       " 'view': ('v', 'ie', 'w'),\n",
       " 'results': ('r', 'e', 's', 'u', 'l'),\n",
       " 'particular': ('p', 'a', 'r', 't', 'c'),\n",
       " 'language': ('l', 'n', 'g', 'u', 'a'),\n",
       " 'half': ('h', 'a', 'l', 'f'),\n",
       " 'working': ('w', 'o', 'r', 'k', 'n'),\n",
       " 'role': ('r', 'o', 'l', 'e'),\n",
       " 'certain': ('c', 'e', 'r', 't', 'ai'),\n",
       " 'food': ('f', 'o', 'd'),\n",
       " 'due': ('d', 'u', 'e'),\n",
       " 'effect': ('e', 'f', 'e', 'c', 't'),\n",
       " 'court': ('c', 'ou', 'r', 't'),\n",
       " 'call': ('c', 'a', 'l'),\n",
       " 'policy': ('p', 'o', 'l', 'e', 'c'),\n",
       " 'true': ('t', 'r', 'u', 'e'),\n",
       " 'individual': ('n', 'd', 'v', 'e', 'd'),\n",
       " 'party': ('p', 'a', 'r', 't', 'y'),\n",
       " 'money': ('m', 'n', 'e', 'y'),\n",
       " 'result': ('r', 'e', 's', 'u', 'l'),\n",
       " 'risk': ('r', 'e', 's', 'k'),\n",
       " 'age': ('a', 'g', 'e'),\n",
       " 'gave': ('g', 'a', 'v', 'e'),\n",
       " 'approach': ('a', 'p', 'r', 'o', 'a'),\n",
       " 'doing': ('d', 'n'),\n",
       " 'per': ('p', 'e', 'r'),\n",
       " 'systems': ('s', 'y', 's', 't', 'm'),\n",
       " 'community': ('c', 'm', 'u', 'n', 'e'),\n",
       " 'action': ('a', 'c', 'u', 'n'),\n",
       " 'seemed': ('s', 'm', 'd'),\n",
       " 'front': ('f', 'r', 'n', 't'),\n",
       " 'available': ('a', 'v', 'ai', 'l', 'a'),\n",
       " 'matter': ('m', 'a', 't', 'e', 'r'),\n",
       " 'close': ('c', 'l', 'o', 's', 'e'),\n",
       " 'provide': ('p', 'r', 'o', 'v', 'e'),\n",
       " 'specific': ('s', 'p', 'e', 'c', 'e'),\n",
       " 'terms': ('t', 'r', 'm', 's'),\n",
       " 'held': ('h', 'e', 'l', 'd'),\n",
       " 'idea': ('e', 'd', 'a'),\n",
       " 'english': ('n', 'g', 'l', 'e', 's'),\n",
       " 'toward': ('t', 'o', 'w', 'a', 'r'),\n",
       " 'period': ('p', 'e', 'r', 'e', 'o'),\n",
       " 'theory': ('th', 'e', 'o', 'r', 'y'),\n",
       " 'everything': ('e', 'v', 'e', 'r', 'y'),\n",
       " 'heard': ('h', 'ea', 'r', 'd'),\n",
       " 'interest': ('n', 't', 'r', 'e', 's'),\n",
       " 'term': ('t', 'r', 'm'),\n",
       " 'non': ('n', 'n'),\n",
       " 'short': ('s', 'h', 'o', 'r', 't'),\n",
       " 'perhaps': ('p', 'e', 'r', 'h', 'a'),\n",
       " 'class': ('c', 'l', 'a', 's'),\n",
       " 'needed': ('n', 'e', 'd'),\n",
       " 'became': ('b', 'e', 'c', 'm', 'e'),\n",
       " 'space': ('s', 'p', 'a', 'c', 'e'),\n",
       " 'function': ('f', 'u', 'n', 'c', 'u'),\n",
       " 'century': ('c', 'n', 't', 'u', 'r'),\n",
       " 'future': ('f', 'u', 't', 'u', 'r'),\n",
       " 'journal': ('j', 'ou', 'r', 'n', 'a'),\n",
       " 'evidence': ('e', 'v', 'e', 'd', 'n'),\n",
       " 'land': ('l', 'n', 'd'),\n",
       " 'rate': ('r', 'a', 't'),\n",
       " 'practice': ('p', 'r', 'a', 'c', 't'),\n",
       " 'believe': ('b', 'e', 'l', 'ie', 'v'),\n",
       " 'itself': ('e', 't', 's', 'e', 'l'),\n",
       " 'someone': ('s', 'm', 'n', 'e'),\n",
       " 'position': ('p', 'o', 's', 'e', 'u'),\n",
       " 'field': ('f', 'ie', 'l', 'd'),\n",
       " 'third': ('th', 'e', 'r', 'd'),\n",
       " 'single': ('s', 'n', 'g', 'l', 'e'),\n",
       " 'according': ('a', 'c', 'o', 'r', 'd'),\n",
       " 'patients': ('p', 'a', 't', 'n', 't'),\n",
       " 'clear': ('c', 'l', 'ea', 'r'),\n",
       " 'especially': ('e', 's', 'p', 'e', 'c'),\n",
       " 'blood': ('b', 'l', 'o', 'd'),\n",
       " 'understand': ('u', 'n', 'd', 'r', 's'),\n",
       " 'turn': ('t', 'u', 'r', 'n'),\n",
       " 'big': ('b', 'e', 'g'),\n",
       " 'themselves': ('th', 'm', 's', 'e', 'l'),\n",
       " 'quite': ('q', 'e', 't'),\n",
       " 'report': ('r', 'e', 'p', 'o', 'r'),\n",
       " 'says': ('s', 'a', 'y', 's'),\n",
       " 'treatment': ('t', 'r', 'ea', 't', 'm'),\n",
       " 'reason': ('r', 'ea', 's', 'n'),\n",
       " 'cases': ('c', 'a', 's', 'e', 's'),\n",
       " 'market': ('m', 'a', 'r', 'k', 'e'),\n",
       " 'services': ('s', 'e', 'r', 'v', 'e'),\n",
       " 'office': ('o', 'f', 'e', 'c', 'e'),\n",
       " 'taking': ('t', 'a', 'k', 'n'),\n",
       " 'members': ('m', 'm', 'b', 'e', 'r'),\n",
       " 'related': ('r', 'e', 'l', 'a', 't'),\n",
       " 'story': ('s', 't', 'o', 'r', 'y'),\n",
       " 'read': ('r', 'ea', 'd'),\n",
       " 'soon': ('s', 'o', 'n'),\n",
       " 'king': ('k', 'n'),\n",
       " 'problems': ('p', 'r', 'o', 'b', 'l'),\n",
       " 'similar': ('s', 'm', 'e', 'l', 'a'),\n",
       " 'major': ('m', 'a', 'j', 'o', 'r'),\n",
       " 'include': ('n', 'c', 'l', 'u', 'd'),\n",
       " 'higher': ('h', 'e', 'g', 'h', 'e'),\n",
       " 'changes': ('ch', 'n', 'g', 'e', 's'),\n",
       " 'finally': ('f', 'n', 'a', 'l', 'y'),\n",
       " 'outside': ('ou', 't', 's', 'e', 'd'),\n",
       " 'groups': ('g', 'r', 'ou', 'p', 's'),\n",
       " 'church': ('ch', 'u', 'r', 'ch'),\n",
       " 'various': ('v', 'a', 'r', 'e', 'ou'),\n",
       " 'instead': ('n', 's', 't', 'a', 'd'),\n",
       " 'living': ('l', 'e', 'v', 'n'),\n",
       " 'leave': ('l', 'ea', 'v', 'e'),\n",
       " 'inside': ('n', 's', 'e', 'd'),\n",
       " 'top': ('t', 'o', 'p'),\n",
       " 'south': ('s', 'ou', 'th'),\n",
       " 'relationship': ('r', 'e', 'l', 'a', 't'),\n",
       " 'conditions': ('c', 'n', 'd', 't', 'n'),\n",
       " 'started': ('s', 't', 'a', 'r', 't'),\n",
       " 'sometimes': ('s', 'm', 'e', 't', 'm'),\n",
       " 'required': ('r', 'e', 'q', 'e', 'r'),\n",
       " 'today': ('t', 'o', 'd', 'a', 'y'),\n",
       " 'science': ('s', 'c', 'ie', 'n', 'c'),\n",
       " 'questions': ('q', 'e', 's', 't', 'n'),\n",
       " 'total': ('t', 'o', 't', 'a', 'l'),\n",
       " 'shown': ('s', 'h', 'o', 'w', 'n'),\n",
       " 'effects': ('e', 'f', 'e', 'c', 't'),\n",
       " 'current': ('c', 'u', 'r', 'n', 't'),\n",
       " 'subject': ('s', 'u', 'b', 'j', 'e'),\n",
       " 'else': ('e', 'l', 's', 'e'),\n",
       " 'project': ('p', 'r', 'o', 'j', 'e'),\n",
       " 'force': ('f', 'o', 'r', 'c', 'e'),\n",
       " 'structure': ('s', 't', 'r', 'u', 'c'),\n",
       " 'play': ('p', 'l', 'a', 'y'),\n",
       " 'ways': ('w', 'a', 'y', 's'),\n",
       " 'usually': ('u', 's', 'u', 'a', 'l'),\n",
       " 'method': ('m', 'e', 'th', 'o', 'd'),\n",
       " 'natural': ('n', 'a', 't', 'u', 'r'),\n",
       " 'performance': ('p', 'e', 'r', 'f', 'o'),\n",
       " 'son': ('s', 'n'),\n",
       " 'design': ('d', 's', 'e', 'g', 'n'),\n",
       " 'brought': ('b', 'r', 'ou', 'g', 'h'),\n",
       " 'considered': ('c', 'n', 's', 'e', 'd'),\n",
       " 'main': ('m', 'ai', 'n'),\n",
       " 'lord': ('l', 'o', 'r', 'd'),\n",
       " 'likely': ('l', 'e', 'k', 'e', 'l'),\n",
       " 'red': ('r', 'd'),\n",
       " 'review': ('r', 'e', 'v', 'ie', 'w'),\n",
       " 'ask': ('a', 's', 'k'),\n",
       " 'needs': ('n', 'e', 'd', 's'),\n",
       " 'test': ('t', 's', 't'),\n",
       " 'probably': ('p', 'r', 'o', 'b', 'a'),\n",
       " 'thinking': ('th', 'n', 'k', 'n'),\n",
       " 'production': ('p', 'r', 'o', 'd', 'u'),\n",
       " 'response': ('r', 'e', 's', 'p', 'n'),\n",
       " 'key': ('k', 'e', 'y'),\n",
       " 'live': ('l', 'e', 'v', 'e'),\n",
       " 'building': ('b', 'u', 'e', 'l', 'd'),\n",
       " 'necessary': ('n', 'e', 'c', 'e', 's'),\n",
       " 'answer': ('a', 'n', 's', 'w', 'e'),\n",
       " 'plan': ('p', 'l', 'n'),\n",
       " 'personal': ('p', 'e', 'r', 's', 'n'),\n",
       " 'culture': ('c', 'u', 'l', 't', 'u'),\n",
       " 'ground': ('g', 'r', 'ou', 'n', 'd'),\n",
       " 'program': ('p', 'r', 'o', 'g', 'r'),\n",
       " 'morning': ('m', 'o', 'r', 'n'),\n",
       " 'values': ('v', 'a', 'l', 'u', 'e'),\n",
       " 'late': ('l', 'a', 't'),\n",
       " 'attention': ('a', 't', 'n', 'u', 'n'),\n",
       " 'herself': ('h', 'e', 'r', 's', 'e'),\n",
       " 'oh': ('o', 'h'),\n",
       " 'strong': ('s', 't', 'r', 'n', 'g'),\n",
       " 'move': ('m', 'o', 'v', 'e'),\n",
       " 'talk': ('t', 'a', 'l', 'k'),\n",
       " 'security': ('s', 'e', 'c', 'u', 'r'),\n",
       " 'cell': ('c', 'e', 'l'),\n",
       " 'trying': ('t', 'r', 'y', 'n'),\n",
       " 'makes': ('m', 'a', 'k', 'e', 's'),\n",
       " 'return': ('r', 'e', 't', 'u', 'r'),\n",
       " 'note': ('n', 'o', 't'),\n",
       " 'longer': ('l', 'n', 'g', 'e', 'r'),\n",
       " 'understanding': ('u', 'n', 'd', 'r', 's'),\n",
       " 'start': ('s', 't', 'a', 'r', 't'),\n",
       " 'run': ('r', 'u', 'n'),\n",
       " 'associated': ('a', 's', 'o', 'c', 'e'),\n",
       " 'special': ('s', 'p', 'e', 'c', 'e'),\n",
       " 'job': ('j', 'o', 'b'),\n",
       " 'physical': ('p', 'h', 'y', 's', 'e'),\n",
       " 'north': ('n', 'o', 'r', 'th'),\n",
       " 'lot': ('l', 'o', 't'),\n",
       " 'coming': ('c', 'm', 'n'),\n",
       " 'maybe': ('m', 'a', 'y', 'b', 'e'),\n",
       " 'feet': ('f', 'e', 't'),\n",
       " 'countries': ('c', 'ou', 'n', 't', 'r'),\n",
       " 'account': ('a', 'c', 'ou', 'n', 't'),\n",
       " 'cost': ('c', 'o', 's', 't'),\n",
       " 'whose': ('w', 'h', 'o', 's', 'e'),\n",
       " 'series': ('s', 'e', 'r', 'ie', 's'),\n",
       " 'growth': ('g', 'r', 'o', 'w', 'th'),\n",
       " 'activity': ('a', 'c', 't', 'v', 'e'),\n",
       " 'rest': ('r', 'e', 's', 't'),\n",
       " 'car': ('c', 'a', 'r'),\n",
       " 'lost': ('l', 'o', 's', 't'),\n",
       " 'art': ('a', 'r', 't'),\n",
       " 'alone': ('a', 'l', 'n', 'e'),\n",
       " 'potential': ('p', 'o', 't', 'n', 't'),\n",
       " 'cause': ('c', 'au', 's', 'e'),\n",
       " 'getting': ('g', 'e', 't', 'n'),\n",
       " 'decision': ('d', 'c', 'e', 's', 'n'),\n",
       " 'issues': ('e', 's', 'u', 'e', 's'),\n",
       " 'friends': ('f', 'r', 'ie', 'n', 'd'),\n",
       " 'increase': ('n', 'c', 'r', 'ea', 's'),\n",
       " 'myself': ('m', 'y', 's', 'e', 'l'),\n",
       " 'minutes': ('m', 'n', 'u', 't', 's'),\n",
       " 'near': ('n', 'ea', 'r'),\n",
       " 'hours': ('h', 'ou', 'r', 's'),\n",
       " 'provided': ('p', 'r', 'o', 'v', 'e'),\n",
       " 'stood': ('s', 't', 'o', 'd'),\n",
       " 'material': ('m', 'a', 't', 'r', 'e'),\n",
       " 'lower': ('l', 'o', 'w', 'e', 'r'),\n",
       " 'actually': ('a', 'c', 't', 'u', 'a'),\n",
       " 'patient': ('p', 'a', 't', 'n', 't'),\n",
       " 'quality': ('q', 'a', 'l', 'e', 't'),\n",
       " 'simply': ('s', 'm', 'p', 'l', 'y'),\n",
       " 'content': ('c', 'n', 't', 'n', 't'),\n",
       " 'source': ('s', 'ou', 'r', 'c', 'e'),\n",
       " 'paper': ('p', 'a', 'p', 'e', 'r'),\n",
       " 'step': ('s', 't', 'p'),\n",
       " 'significant': ('s', 'e', 'g', 'n', 'e'),\n",
       " 'surface': ('s', 'u', 'r', 'f', 'a'),\n",
       " 'hair': ('h', 'ai', 'r'),\n",
       " 'six': ('s', 'e', 'x'),\n",
       " 'areas': ('a', 'r', 'ea', 's'),\n",
       " 'deep': ('d', 'p'),\n",
       " 'fire': ('f', 'e', 'r', 'e'),\n",
       " 'bed': ('bd',),\n",
       " 'hear': ('h', 'ea', 'r'),\n",
       " 'central': ('c', 'n', 't', 'r', 'a'),\n",
       " 'friend': ('f', 'd'),\n",
       " 'towards': ('t', 'o', 'w', 'a', 'r'),\n",
       " 'sound': ('s', 'ou', 'n', 'd'),\n",
       " 'try': ('t', 'r', 'y'),\n",
       " 'financial': ('f', 'n', 'n', 'c', 'e'),\n",
       " 'tried': ('t', 'r', 'e', 'd'),\n",
       " 'beyond': ('b', 'e', 'y', 'n', 'd'),\n",
       " 'dark': ('d', 'a', 'r', 'k'),\n",
       " 'cultural': ('c', 'u', 'l', 't', 'u'),\n",
       " 'stop': ('s', 't', 'o', 'p'),\n",
       " 'range': ('r', 'n', 'g', 'e'),\n",
       " 'size': ('s', 'e', 'z', 'e'),\n",
       " 'percent': ('p', 'e', 'r', 'c', 'n'),\n",
       " 'girl': ('g', 'e', 'r', 'l'),\n",
       " 'remember': ('r', 'm', 'm', 'b', 'e'),\n",
       " 'disease': ('d', 's', 'ea', 's', 'e'),\n",
       " 'situation': ('s', 'e', 't', 'u', 'au'),\n",
       " 'continued': ('c', 'n', 't', 'n', 'u'),\n",
       " 'seems': ('s', 'm', 's'),\n",
       " 'middle': ('m', 'e', 'd', 'l', 'e'),\n",
       " 'indeed': ('n', 'd'),\n",
       " 'private': ('p', 'r', 'e', 'v', 'a'),\n",
       " 'gone': ('g', 'n', 'e'),\n",
       " 'population': ('p', 'o', 'p', 'u', 'l'),\n",
       " 'months': ('m', 'n', 'th', 's'),\n",
       " 'cells': ('c', 'e', 'l', 's'),\n",
       " 'activities': ('a', 'c', 't', 'v', 'e'),\n",
       " 'difficult': ('d', 'f', 'e', 'c', 'u'),\n",
       " 'followed': ('f', 'o', 'l', 'o', 'w'),\n",
       " 'jesus': ('j', 'e', 's', 'u', 's'),\n",
       " 'parents': ('p', 'a', 'r', 'n', 't'),\n",
       " 'pain': ('p', 'ai', 'n'),\n",
       " 'sat': ('s', 'a', 't'),\n",
       " 'described': ('d', 's', 'c', 'r', 'e'),\n",
       " 'british': ('b', 'r', 'e', 't', 's'),\n",
       " 'bring': ('b', 'r', 'n'),\n",
       " 'wife': ('w', 'e', 'f', 'e'),\n",
       " 'earth': ('ea', 'r', 'th'),\n",
       " 'everyone': ('e', 'v', 'e', 'r', 'y'),\n",
       " 'property': ('p', 'r', 'o', 'p', 'e'),\n",
       " 'image': ('m', 'a', 'g', 'e'),\n",
       " 'consider': ('c', 'n', 's', 'e', 'd'),\n",
       " 'levels': ('l', 'e', 'v', 'e', 'l'),\n",
       " 'bad': ('b', 'a', 'd'),\n",
       " 'hope': ('h', 'o', 'p', 'e'),\n",
       " 'amount': ('a', 'm', 't'),\n",
       " 'below': ('b', 'e', 'l', 'o', 'w'),\n",
       " 'center': ('c', 'n', 't', 'r'),\n",
       " 'writing': ('w', 'r', 'e', 't', 'n'),\n",
       " 'written': ('w', 'r', 'e', 't', 'n'),\n",
       " 'modern': ('m', 'o', 'd', 'r', 'n'),\n",
       " 'dead': ('d', 'a', 'd'),\n",
       " 'addition': ('a', 'd', 'e', 'u', 'n'),\n",
       " 'arms': ('a', 'r', 'm', 's'),\n",
       " 'greater': ('g', 'r', 'ea', 't', 'r'),\n",
       " 'technology': ('t', 'ch', 'n', 'o', 'l'),\n",
       " 'truth': ('t', 'r', 'u', 'th'),\n",
       " 'access': ('a', 'c', 'e', 's'),\n",
       " 'poor': ('p', 'o', 'r'),\n",
       " 'movement': ('m', 'o', 'v', 'm', 'n'),\n",
       " 'shows': ('s', 'h', 'o', 'w', 's'),\n",
       " 'hold': ('h', 'o', 'l', 'd'),\n",
       " 'feeling': ('f', 'e', 'l', 'n'),\n",
       " 'focus': ('f', 'o', 'c', 'u', 's'),\n",
       " 'european': ('eu', 'r', 'o', 'p', 'n'),\n",
       " 'music': ('m', 'u', 's', 'e', 'c'),\n",
       " 'road': ('r', 'o', 'a', 'd'),\n",
       " 'increased': ('n', 'c', 'r', 'ea', 's'),\n",
       " 'simple': ('s', 'm', 'p', 'l', 'e'),\n",
       " 'street': ('s', 't', 'r', 'e', 't'),\n",
       " 'resources': ('r', 'e', 's', 'ou', 'r'),\n",
       " 'nor': ('n', 'o', 'r'),\n",
       " 'global': ('g', 'l', 'o', 'b', 'a'),\n",
       " 'text': ('t', 'x', 't'),\n",
       " 'lead': ('l', 'ea', 'd'),\n",
       " 'kept': ('k', 'e', 'p', 't'),\n",
       " 'factors': ('f', 'a', 'c', 't', 'o'),\n",
       " 'training': ('t', 'r', 'ai', 'n'),\n",
       " 'trade': ('t', 'r', 'a', 'd'),\n",
       " 'department': ('d', 'p', 'a', 'r', 't'),\n",
       " 'environment': ('n', 'v', 'e', 'r', 'n'),\n",
       " 'town': ('t', 'o', 'w', 'n'),\n",
       " 'methods': ('m', 'e', 'th', 'o', 'd'),\n",
       " 'developed': ('d', 'v', 'e', 'l', 'o'),\n",
       " 'news': ('n', 'e', 'w', 's'),\n",
       " 'led': ('l', 'd'),\n",
       " 'involved': ('n', 'v', 'o', 'l', 'v'),\n",
       " 'particularly': ('p', 'a', 'r', 't', 'c'),\n",
       " 'quickly': ('q', 'e', 'c', 'k', 'l'),\n",
       " 'moved': ('m', 'o', 'v', 'd'),\n",
       " 'books': ('b', 'o', 'k', 's'),\n",
       " 'positive': ('p', 'o', 's', 'e', 't'),\n",
       " 'standard': ('s', 't', 'n', 'd', 'a'),\n",
       " 'reading': ('r', 'ea', 'd', 'n'),\n",
       " 'west': ('w', 'e', 's', 't'),\n",
       " 'forward': ('f', 'o', 'r', 'w', 'a'),\n",
       " 'pressure': ('p', 'r', 'e', 's', 'u'),\n",
       " 'mouth': ('m', 'ou', 'th'),\n",
       " 'military': ('m', 'e', 'l', 'e', 't'),\n",
       " 'create': ('c', 'r', 'ea', 't'),\n",
       " 'medical': ('m', 'e', 'd', 'c', 'a'),\n",
       " 'legal': ('l', 'e', 'g', 'a', 'l'),\n",
       " 'application': ('a', 'p', 'l', 'e', 'c'),\n",
       " 'organization': ('o', 'r', 'g', 'n', 'e'),\n",
       " 'bit': ('b', 'e', 't'),\n",
       " 'loss': ('l', 'o', 's'),\n",
       " 'context': ('c', 'n', 't', 'x', 't'),\n",
       " 'behavior': ('b', 'e', 'h', 'a', 'v'),\n",
       " 'team': ('t', 'm'),\n",
       " 'foreign': ('f', 'o', 'r', 'ei', 'g'),\n",
       " 'wall': ('w', 'a', 'l'),\n",
       " 'forms': ('f', 'o', 'r', 'm', 's'),\n",
       " 'works': ('w', 'o', 'r', 'k', 's'),\n",
       " 'meet': ('m', 'e', 't'),\n",
       " 'rule': ('r', 'u', 'l', 'e'),\n",
       " 'happened': ('h', 'a', 'p', 'n', 'd'),\n",
       " 'effective': ('e', 'f', 'e', 'c', 't'),\n",
       " 'events': ('e', 'v', 'n', 't', 's'),\n",
       " 'whom': ('w', 'h', 'm'),\n",
       " 'complex': ('c', 'm', 'p', 'l', 'e'),\n",
       " 'met': ('m', 'e', 't'),\n",
       " 'product': ('p', 'r', 'o', 'd', 'u'),\n",
       " 'issue': ('e', 's', 'u', 'e'),\n",
       " 'reached': ('r', 'ea', 'ch', 'd'),\n",
       " 'walked': ('w', 'a', 'l', 'k', 'd'),\n",
       " 'points': ('p', 'oi', 'n', 't', 's'),\n",
       " 'impact': ('m', 'p', 'a', 'c', 't'),\n",
       " 'added': ('a', 'd', 'd'),\n",
       " 'follow': ('f', 'o', 'l', 'o', 'w'),\n",
       " 'sent': ('s', 'n', 't'),\n",
       " 'rules': ('r', 'u', 'l', 'e', 's'),\n",
       " 'lives': ('l', 'e', 'v', 'e', 's'),\n",
       " 'influence': ('n', 'f', 'l', 'u', 'n'),\n",
       " 'cut': ('c', 'u', 't'),\n",
       " 'green': ('g', 'r', 'n'),\n",
       " 'expression': ('e', 'x', 'p', 'r', 'e'),\n",
       " 'earlier': ('ea', 'r', 'l', 'ie', 'r'),\n",
       " 'media': ('m', 'e', 'd', 'a'),\n",
       " 'generally': ('g', 'n', 'e', 'r', 'a'),\n",
       " 'floor': ('f', 'l', 'o', 'r'),\n",
       " 'individuals': ('n', 'd', 'v', 'e', 'd'),\n",
       " 'purpose': ('p', 'u', 'r', 'p', 'o'),\n",
       " 'pay': ('p', 'a', 'y'),\n",
       " 'ten': ('t', 'n'),\n",
       " 'america': ('a', 'm', 'e', 'r', 'e'),\n",
       " 'stay': ('s', 't', 'a', 'y'),\n",
       " 'final': ('f', 'n', 'a', 'l'),\n",
       " 'comes': ('c', 'm', 'e', 's'),\n",
       " 'types': ('t', 'y', 'p', 'e', 's'),\n",
       " 'wrong': ('w', 'r', 'n', 'g'),\n",
       " 'complete': ('c', 'm', 'p', 'l', 'e'),\n",
       " 'ready': ('r', 'ea', 'd', 'y'),\n",
       " 'normal': ('n', 'o', 'r', 'm', 'a'),\n",
       " 'basis': ('b', 'a', 's', 'e', 's'),\n",
       " 'list': ('l', 'e', 's', 't'),\n",
       " 'presence': ('p', 'r', 'e', 's', 'n'),\n",
       " 'stage': ('s', 't', 'a', 'g', 'e'),\n",
       " 'critical': ('c', 'r', 'e', 't', 'c'),\n",
       " 'twenty': ('t', 'w', 'n', 't', 'y'),\n",
       " 'seem': ('s', 'm'),\n",
       " 'french': ('f', 'r', 'n', 'ch'),\n",
       " 'difference': ('d', 'f', 'e', 'r', 'n'),\n",
       " 'meaning': ('m', 'n'),\n",
       " 'anyone': ('a', 'n', 'y', 'n', 'e'),\n",
       " 'sea': ('s', 'ea'),\n",
       " 'saying': ('s', 'a', 'y', 'n'),\n",
       " 'cold': ('c', 'o', 'l', 'd'),\n",
       " 'basic': ('b', 'a', 's', 'e', 'c'),\n",
       " 'direct': ('d', 'r', 'e', 'c', 't'),\n",
       " 'beginning': ('b', 'e', 'g', 'n', 'n'),\n",
       " 'received': ('r', 'e', 'c', 'ei', 'v'),\n",
       " 'week': ('w', 'e', 'k'),\n",
       " 'fear': ('f', 'ea', 'r'),\n",
       " 'expected': ('e', 'x', 'p', 'e', 'c'),\n",
       " 'price': ('p', 'r', 'e', 'c', 'e'),\n",
       " 'cross': ('c', 'r', 'o', 's'),\n",
       " 'capital': ('c', 'a', 'p', 'e', 't'),\n",
       " 'police': ('p', 'o', 'l', 'e', 'c'),\n",
       " 'china': ('ch', 'n', 'a'),\n",
       " 'respect': ('r', 'e', 's', 'p', 'e'),\n",
       " 'spirit': ('s', 'p', 'e', 'r', 'e'),\n",
       " 'clearly': ('c', 'l', 'ea', 'r', 'l'),\n",
       " 'fine': ('f', 'n', 'e'),\n",
       " 'president': ('p', 'r', 'e', 's', 'e'),\n",
       " 'opened': ('o', 'p', 'n', 'd'),\n",
       " 'religious': ('r', 'e', 'l', 'e', 'g'),\n",
       " 'east': ('ea', 's', 't'),\n",
       " 'limited': ('l', 'm', 'e', 't', 'd'),\n",
       " 'additional': ('a', 'd', 'e', 't', 'n'),\n",
       " 'post': ('p', 'o', 's', 't'),\n",
       " 'boy': ('b', 'o', 'y'),\n",
       " 'page': ('p', 'a', 'g', 'e'),\n",
       " 'ability': ('a', 'b', 'e', 'l', 'e'),\n",
       " 'speak': ('s', 'p', 'ea', 'k'),\n",
       " 'brother': ('b', 'r', 'o', 'th', 'e'),\n",
       " 'income': ('n', 'c', 'm', 'e'),\n",
       " 'defined': ('d', 'f', 'n', 'd'),\n",
       " 'letter': ('l', 'e', 't', 'e', 'r'),\n",
       " 'parts': ('p', 'a', 'r', 't', 's'),\n",
       " 'share': ('s', 'h', 'a', 'r', 'e'),\n",
       " 'easy': ('ea', 's', 'y'),\n",
       " 'eye': ('e', 'y', 'e'),\n",
       " 'products': ('p', 'r', 'o', 'd', 'u'),\n",
       " 'processes': ('p', 'r', 'o', 'c', 'e'),\n",
       " 'river': ('r', 'e', 'v', 'e', 'r'),\n",
       " 'choice': ('ch', 'oi', 'c', 'e'),\n",
       " 'recent': ('r', 'e', 'c', 'n', 't'),\n",
       " 'pulled': ('p', 'u', 'l', 'd'),\n",
       " 'published': ('p', 'u', 'b', 'l', 'e'),\n",
       " 'date': ('d', 'a', 't'),\n",
       " 'network': ('n', 'e', 't', 'w', 'o'),\n",
       " 'created': ('c', 'r', 'ea', 't', 'd'),\n",
       " 'deal': ('d', 'a', 'l'),\n",
       " 'wide': ('w', 'e', 'd'),\n",
       " 'object': ('o', 'b', 'j', 'e', 'c'),\n",
       " 'game': ('g', 'm', 'e'),\n",
       " 'federal': ('f', 'e', 'd', 'r', 'a'),\n",
       " 'meeting': ('m', 'e', 't', 'n'),\n",
       " 'fall': ('f', 'a', 'l'),\n",
       " 'bank': ('b', 'n', 'k'),\n",
       " 'blue': ('b', 'l', 'u', 'e'),\n",
       " 'smile': ('s', 'm', 'e', 'l', 'e'),\n",
       " 'directly': ('d', 'r', 'e', 'c', 't'),\n",
       " 'passed': ('p', 'a', 's', 'd'),\n",
       " 'region': ('r', 'e', 'g', 'n'),\n",
       " 'site': ('s', 'e', 't'),\n",
       " 'skin': ('s', 'k', 'n'),\n",
       " 'allow': ('a', 'l', 'o', 'w'),\n",
       " 'reported': ('r', 'e', 'p', 'o', 'r'),\n",
       " 'solution': ('s', 'o', 'l', 'u', 'n'),\n",
       " 'established': ('e', 's', 't', 'a', 'b'),\n",
       " 'sir': ('s', 'e', 'r'),\n",
       " 'authority': ('au', 'th', 'o', 'r', 'e'),\n",
       " 'provides': ('p', 'r', 'o', 'v', 'e'),\n",
       " 'phone': ('p', 'h', 'n', 'e'),\n",
       " 'appropriate': ('a', 'p', 'r', 'o', 'p'),\n",
       " 'primary': ('p', 'r', 'm', 'a', 'r'),\n",
       " 'despite': ('d', 's', 'p', 'e', 't'),\n",
       " 'brain': ('b', 'r', 'ai', 'n'),\n",
       " 'memory': ('m', 'm', 'o', 'r', 'y'),\n",
       " 'mark': ('m', 'a', 'r', 'k'),\n",
       " 'original': ('o', 'r', 'e', 'g', 'n'),\n",
       " 'direction': ('d', 'r', 'e', 'c', 'u'),\n",
       " 'hour': ('h', 'ou', 'r'),\n",
       " 'oil': ('oi', 'l'),\n",
       " 'box': ('b', 'o', 'x'),\n",
       " 'distance': ('d', 's', 't', 'n', 'c'),\n",
       " 'learn': ('l', 'ea', 'r', 'n'),\n",
       " 'arm': ('a', 'r', 'm'),\n",
       " 'except': ('e', 'x', 'c', 'e', 'p'),\n",
       " 'status': ('s', 't', 'a', 't', 'u'),\n",
       " 'communication': ('c', 'm', 'u', 'n', 'e'),\n",
       " 'flow': ('f', 'l', 'o', 'w'),\n",
       " 'takes': ('t', 'a', 'k', 'e', 's'),\n",
       " 'member': ('m', 'm', 'b', 'e', 'r'),\n",
       " 'closed': ('c', 'l', 'o', 's', 'd'),\n",
       " 'ago': ('a', 'g', 'o'),\n",
       " 'die': ('d', 'e'),\n",
       " 'elements': ('e', 'l', 'm', 'n', 't'),\n",
       " 'worked': ('w', 'o', 'r', 'k', 'd'),\n",
       " 'please': ('p', 'l', 'ea', 's', 'e'),\n",
       " 'window': ('w', 'n', 'd', 'o', 'w'),\n",
       " 'contact': ('c', 'n', 't', 'a', 'c'),\n",
       " 'models': ('m', 'o', 'd', 'l', 's'),\n",
       " 'board': ('b', 'o', 'a', 'r', 'd'),\n",
       " 'lines': ('l', 'n', 'e', 's'),\n",
       " 'ideas': ('e', 'd', 'a', 's'),\n",
       " 'average': ('a', 'v', 'e', 'r', 'a'),\n",
       " 'yourself': ('y', 'ou', 'r', 's', 'e'),\n",
       " 'species': ('s', 'p', 'e', 'c', 'ie'),\n",
       " 'college': ('c', 'o', 'l', 'e', 'g'),\n",
       " 'standing': ('s', 't', 'n', 'd', 'n'),\n",
       " 'distribution': ('d', 's', 't', 'r', 'e'),\n",
       " 'trust': ('t', 'r', 'u', 's', 't'),\n",
       " 'heat': ('h', 'ea', 't'),\n",
       " 'immediately': ('m', 'e', 'd', 'a', 't'),\n",
       " 'justice': ('j', 'u', 's', 't', 'c'),\n",
       " 'included': ('n', 'c', 'l', 'u', 'd'),\n",
       " 'throughout': ('th', 'r', 'ou', 'g', 'h'),\n",
       " 'sun': ('s', 'u', 'n'),\n",
       " 'entire': ('n', 't', 'r', 'e'),\n",
       " 'army': ('a', 'r', 'm', 'y'),\n",
       " 'condition': ('c', 'n', 'd', 'u', 'n'),\n",
       " 'industry': ('n', 'd', 'u', 's', 't'),\n",
       " 'tax': ('t', 'a', 'x'),\n",
       " 'applied': ('a', 'p', 'l', 'e', 'd'),\n",
       " 'volume': ('v', 'o', 'l', 'u', 'm'),\n",
       " 'round': ('r', 'ou', 'n', 'd'),\n",
       " 'decided': ('d', 'c', 'e', 'd'),\n",
       " 'add': ('a', 'd'),\n",
       " 'meant': ('m', 'n', 't'),\n",
       " 'happy': ('h', 'a', 'p', 'y'),\n",
       " 'whatever': ('w', 'h', 'a', 't', 'v'),\n",
       " 'stand': ('s', 't', 'n', 'd'),\n",
       " 'temperature': ('t', 'm', 'p', 'e', 'r'),\n",
       " 'mass': ('m', 'a', 's'),\n",
       " 'discussion': ('d', 's', 'c', 'u', 's'),\n",
       " 'weight': ('w', 'e', ' ', 't'),\n",
       " 'concept': ('c', 'n', 'c', 'e', 'p'),\n",
       " 'costs': ('c', 'o', 's', 't', 's'),\n",
       " 'running': ('r', 'u', 'n', 'n'),\n",
       " 'talking': ('t', 'a', 'l', 'k', 'n'),\n",
       " 'features': ('f', 'ea', 't', 'u', 'r'),\n",
       " 'million': ('m', 'e', 'l', 'n'),\n",
       " 'environmental': ('n', 'v', 'e', 'r', 'n'),\n",
       " 'moving': ('m', 'o', 'v', 'n'),\n",
       " 'german': ('g', 'e', 'r', 'm', 'n'),\n",
       " 'giving': ('g', 'e', 'v', 'n'),\n",
       " 'association': ('a', 's', 'o', 'c', 'e'),\n",
       " 'placed': ('p', 'l', 'a', 'c', 'd'),\n",
       " 'stopped': ('s', 't', 'o', 'p', 'd'),\n",
       " 'union': ('u', 'n', 'n'),\n",
       " 're': ('r', 'e'),\n",
       " 'europe': ('eu', 'r', 'o', 'p', 'e'),\n",
       " 'degree': ('d', 'g', 'r', 'e'),\n",
       " 'identity': ('e', 'd', 'n', 't', 't'),\n",
       " 'historical': ('h', 'e', 's', 't', 'o'),\n",
       " 'negative': ('n', 'e', 'g', 'a', 't'),\n",
       " 'student': ('s', 't', 'u', 'd', 'n'),\n",
       " 'success': ('s', 'u', 'c', 'e', 's'),\n",
       " 'suddenly': ('s', 'u', 'd', 'n', 'l'),\n",
       " 'offer': ('o', 'f', 'e', 'r'),\n",
       " 'code': ('c', 'o', 'd'),\n",
       " 'sleep': ('s', 'l', 'e', 'p'),\n",
       " 'scale': ('s', 'c', 'a', 'l', 'e'),\n",
       " 'character': ('ch', 'a', 'r', 'a', 'c'),\n",
       " 'clinical': ('c', 'l', 'n', 'e', 'c'),\n",
       " 'staff': ('s', 't', 'a', 'f'),\n",
       " 'christ': ('ch', 'r', 'e', 's', 't'),\n",
       " 'event': ('e', 'v', 'n', 't'),\n",
       " 'relations': ('r', 'e', 'l', 'a', 't'),\n",
       " 'western': ('w', 'e', 's', 't', 'r'),\n",
       " 'changed': ('ch', 'n', 'g', 'd'),\n",
       " 'walk': ('w', 'a', 'l', 'k'),\n",
       " 'wrote': ('w', 'r', 'o', 't'),\n",
       " 'workers': ('w', 'o', 'r', 'k', 'e'),\n",
       " 'returned': ('r', 'e', 't', 'u', 'r'),\n",
       " 'politics': ('p', 'o', 'l', 'e', 't'),\n",
       " 'lack': ('l', 'a', 'c', 'k'),\n",
       " 'multiple': ('m', 'u', 'l', 't', 'p'),\n",
       " 'certainly': ('c', 'e', 'r', 't', 'ai'),\n",
       " 'faith': ('f', 'ai', 'th'),\n",
       " 'length': ('l', 'n', 'g', 'th'),\n",
       " 'materials': ('m', 'a', 't', 'r', 'e'),\n",
       " 'march': ('m', 'a', 'r', 'ch'),\n",
       " 'previous': ('p', 'r', 'e', 'v', 'e'),\n",
       " 'write': ('w', 'r', 'e', 't'),\n",
       " 'presented': ('p', 'r', 'e', 's', 'n'),\n",
       " 'phase': ('p', 'h', 'a', 's', 'e'),\n",
       " 'article': ('a', 'r', 't', 'c', 'l'),\n",
       " 'author': ('au', 'th', 'o', 'r'),\n",
       " 'appear': ('a', 'p', 'ea', 'r'),\n",
       " 'literature': ('l', 'e', 't', 'r', 'a'),\n",
       " 'assessment': ('a', 's', 'e', 's', 'm'),\n",
       " 'peace': ('p', 'ea', 'c', 'e'),\n",
       " 'husband': ('h', 'u', 's', 'b', 'n'),\n",
       " 'strength': ('s', 't', 'r', 'n', 'g'),\n",
       " 'forces': ('f', 'o', 'r', 'c', 'e'),\n",
       " 'chance': ('ch', 'n', 'c', 'e'),\n",
       " 'onto': ('o', 'n', 't', 'o'),\n",
       " 'allowed': ('a', 'l', 'o', 'w', 'd'),\n",
       " 'continue': ('c', 'n', 't', 'n', 'u'),\n",
       " 'sort': ('s', 'o', 'r', 't'),\n",
       " 'weeks': ('w', 'e', 'k', 's'),\n",
       " 'england': ('n', 'g', 'l', 'n', 'd'),\n",
       " 'steps': ('s', 't', 'p', 's'),\n",
       " 'task': ('t', 'a', 's', 'k'),\n",
       " 'miss': ('m', 'e', 's'),\n",
       " 'christian': ('ch', 'r', 'e', 's', 't'),\n",
       " 'carried': ('c', 'a', 'r', 'e', 'd'),\n",
       " 'civil': ('c', 'e', 'v', 'e', 'l'),\n",
       " 'produced': ('p', 'r', 'o', 'd', 'u'),\n",
       " 'address': ('a', 'd', 'r', 'e', 's'),\n",
       " 'numbers': ('n', 'u', 'm', 'b', 'e'),\n",
       " 'differences': ('d', 'f', 'e', 'r', 'n'),\n",
       " 'hundred': ('h', 'u', 'n', 'd', 'r'),\n",
       " 'bible': ('b', 'e', 'b', 'l', 'e'),\n",
       " 'active': ('a', 'c', 't', 'v', 'e'),\n",
       " 'observed': ('o', 'b', 's', 'e', 'r'),\n",
       " 'nearly': ('n', 'ea', 'r', 'l', 'y'),\n",
       " ...}"
      ]
     },
     "execution_count": 379,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "swiftograph_trim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 380,
   "id": "74699da2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(('c', 'n', 's', 't', 'n', 't'), ('c', 'n', 's', 't', 'n'))"
      ]
     },
     "execution_count": 380,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "make_swiftographic(\"constant\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 381,
   "id": "a489ad75",
   "metadata": {},
   "outputs": [],
   "source": [
    "swiftograph = {w:make_swiftographic(w)[0] for w in one_grams_dict}\n",
    "swiftograph_trim = {w:make_swiftographic(w)[1] for w in one_grams_dict}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 382,
   "id": "6b0b5beb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mean_binary_length': 18.635976768037462,\n",
       " 'mean_binary_excess': 11.989023948264489,\n",
       " 'mean_entropy_shift': 8.868830397568491,\n",
       " 'mean_transmission_cost': 15.410821790316401,\n",
       " 'mean_transmission_excess': 8.763868970543427,\n",
       " 'mean_transmission_shift': 5.64367541984743,\n",
       " 'reconstruction_entropy': 0.09461940533612552,\n",
       " 'reconstruction_error': 0.02466666732051115,\n",
       " 'alphabet': \"n t e s r a o d l i c th m p u w _ y h f b g v k ou ea ch ai au x ie j q bt   z oi ei eu bd ao ' sg é\"}"
      ]
     },
     "execution_count": 382,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "code_statistics([x for x in one_grams_dict if x in swiftograph], swiftograph)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 383,
   "id": "0fe354f6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mean_binary_length': 15.903743546645053,\n",
       " 'mean_binary_excess': 9.25679072687208,\n",
       " 'mean_entropy_shift': 6.136597176176082,\n",
       " 'mean_transmission_cost': 12.893786392223394,\n",
       " 'mean_transmission_excess': 6.24683357245042,\n",
       " 'mean_transmission_shift': 3.1266400217544223,\n",
       " 'reconstruction_entropy': 0.39707130141218433,\n",
       " 'reconstruction_error': 0.10909772718267308,\n",
       " 'alphabet': \"e n t r s a o d l th c m p w u f h b y g v k ea ou ch ai x ie j au q bt   oi ei z eu bd ao ' sg é\"}"
      ]
     },
     "execution_count": 383,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "code_statistics([x for x in one_grams_dict if x in swiftograph_trim], swiftograph_trim)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "091ba4af-1b96-4f82-8a7b-f3a7bd2e08cd",
   "metadata": {},
   "source": [
    "# Plotting\n",
    "\n",
    "Now we plot!  I will also include an allowable region based upon using Fano's inequality to this problem. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 384,
   "id": "a2036085-84d5-4bc3-b592-091e4310bb1c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def word_list(dict):\n",
    "    return [x for x in one_grams_dict if x in dict]\n",
    "\n",
    "scores = []\n",
    "scores.append((\"Spelling\",code_statistics(word_list(spelling), spelling),'top center'))\n",
    "scores.append((\"IPA\",code_statistics(word_list(reduced_ipa), reduced_ipa),'top center'))\n",
    "scores.append((\"Carter</br></br>Briefhand\",code_statistics(word_list(briefhand), briefhand),'top center'))\n",
    "scores.append((\"Dutton</br></br>Speedwords\",code_statistics(word_list(speedwords), speedwords),'bottom center'))\n",
    "scores.append((\"Gregg</br></br>Anniversary\",code_statistics(word_list(gregg_anniversary), gregg_anniversary, length_function = gregg_length, letter_making = gregg_letters),'top center'))\n",
    "scores.append((\"Gregg</br></br>Notehand\",code_statistics(word_list(gregg_notehand), gregg_notehand, length_function = gregg_length, letter_making = gregg_letters),'bottom left'))\n",
    "scores.append((\"Gregg</br></br>Simplified\",code_statistics(word_list(gregg_simplified), gregg_simplified),'bottom center'))\n",
    "scores.append((\"bref\",code_statistics(word_list(bref), bref),'top center'))\n",
    "scores.append((\"Yublin\",code_statistics(word_list(yublin), yublin),'top center'))\n",
    "scores.append((\"Cut Spelng\",code_statistics(word_list(cut_spelng), cut_spelng),'top center'))\n",
    "scores.append((\"Taylor\",code_statistics(word_list(taylor), taylor),'top center'))\n",
    "scores.append((\"Taylor+\",code_statistics(word_list(taylor_plus), taylor_plus),'top center'))\n",
    "scores.append((\"Characterie\",code_statistics(word_list(characterie), characterie),'top center'))\n",
    "scores.append((\"Teeline\",code_statistics(word_list(teeline), teeline),'bottom center'))\n",
    "scores.append((\"Keyscript\",code_statistics(word_list(keyscript), keyscript),'top right'))\n",
    "scores.append((\"Dearborn</br></br>Speedwriting\",code_statistics(word_list(speedwriting), speedwriting),'top center'))\n",
    "scores.append((\"QC-line\",code_statistics(word_list(qc_line), qc_line),'middle right'))\n",
    "scores.append((\"Yash\",code_statistics(word_list(yash), yash),'bottom center'))\n",
    "scores.append((\"Jeake</br></br>Ph. Tr. 487\",code_statistics(word_list(jeake), jeake),'top right'))\n",
    "scores.append((\"Polyphonic</br></br>Cipher\",code_statistics(word_list(polyphonic), polyphonic),'top right'))\n",
    "scores.append((\"Pitman</br></br>2000\",code_statistics(word_list(pitman2k), pitman2k),'bottom left'))\n",
    "scores.append((\"Pitman 2000</br></br>(No Vowel)\",code_statistics(word_list(pitman2k_novowel), pitman2k_novowel),'top right'))\n",
    "#scores.append((\"Pitman 2000</br></br>(Optimal Vowel)\",code_statistics(word_list(pitman_vowel_optimized), pitman_vowel_optimized),'bottom left'))\n",
    "scores.append((\"Superwrite\",code_statistics(word_list(superwrite), superwrite),'top left'))\n",
    "scores.append((\"Swiftograph</br></br>Curtailed\",code_statistics(word_list(swiftograph_trim), swiftograph_trim),'top center'))\n",
    "scores.append((\"Swiftograph</br></br>Full\",code_statistics(word_list(swiftograph), swiftograph),'middle right'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 385,
   "id": "8ffd00cb-8c5b-4981-a155-00b593eff352",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(f\"system_scores_clean.json\", \"w\") as f:\n",
    "            f.write(json.dumps(scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 389,
   "id": "24dfb5a1-517a-4c80-bdf9-bf3a70c77acd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "config": {
        "plotlyServerURL": "https://plot.ly"
       },
       "data": [
        {
         "fill": "tozeroy",
         "fillcolor": "#EF553B",
         "mode": "none",
         "name": "Excluded Region",
         "type": "scatter",
         "x": {
          "bdata": "AAAAAAAA+L8AAAAAAADwvwAAAAAAAOC/AAAAAAAAAAAAAAAAAADgPwAAAAAAAPA/AAAAAAAA+D8AAAAAAAAAQAAAAAAAAARAAAAAAAAACEAAAAAAAAAMQAAAAAAAABBAAAAAAAAAEkAAAAAAAAAUQAAAAAAAABZAAAAAAAAAGEAAAAAAAAAaQAAAAAAAABxAAAAAAAAAHkAAAAAAAAAgQAAAAAAAACFAAAAAAAAAIkAAAAAAAAAjQAAAAAAAACRA",
          "dtype": "f8"
         },
         "y": [
          0.375,
          0.25,
          0.125,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
         ]
        },
        {
         "fill": "tozeroy",
         "fillcolor": "#EF553B",
         "mode": "none",
         "showlegend": false,
         "type": "scatter",
         "x": {
          "bdata": "AAAAAAAA+L8AAAAAAADwvwAAAAAAAOC/AAAAAAAAAAAAAAAAAADgPwAAAAAAAPA/AAAAAAAA+D8AAAAAAAAAQAAAAAAAAARAAAAAAAAACEAAAAAAAAAMQAAAAAAAABBAAAAAAAAAEkAAAAAAAAAUQAAAAAAAABZAAAAAAAAAGEAAAAAAAAAaQAAAAAAAABxAAAAAAAAAHkAAAAAAAAAgQAAAAAAAACFAAAAAAAAAIkAAAAAAAAAjQAAAAAAAACRA",
          "dtype": "f8"
         },
         "y": [
          -1,
          -1,
          -1,
          -1,
          -1,
          -1,
          -1,
          -1,
          -1,
          -1,
          -1,
          -1,
          -1,
          -1,
          -1,
          -1,
          -1,
          -1,
          -1,
          -1,
          -1,
          -1,
          -1,
          -1
         ]
        },
        {
         "fill": "none",
         "line": {
          "color": "#EF553B"
         },
         "mode": "lines",
         "showlegend": false,
         "type": "scatter",
         "x": {
          "bdata": "AAAAAAAA+L8AAAAAAADwvwAAAAAAAOC/AAAAAAAAAAAAAAAAAADgPwAAAAAAAPA/AAAAAAAA+D8AAAAAAAAAQAAAAAAAAARAAAAAAAAACEAAAAAAAAAMQAAAAAAAABBAAAAAAAAAEkAAAAAAAAAUQAAAAAAAABZAAAAAAAAAGEAAAAAAAAAaQAAAAAAAABxAAAAAAAAAHkAAAAAAAAAgQAAAAAAAACFAAAAAAAAAIkAAAAAAAAAjQAAAAAAAACRA",
          "dtype": "f8"
         },
         "y": [
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0,
          0
         ]
        },
        {
         "line": {
          "color": "#636EFA"
         },
         "mode": "markers+text",
         "name": "Shorthand Systems",
         "text": [
          "Spelling",
          "IPA",
          "Carter</br></br>Briefhand",
          "Dutton</br></br>Speedwords",
          "Gregg</br></br>Anniversary",
          "Gregg</br></br>Notehand",
          "Gregg</br></br>Simplified",
          "bref",
          "Yublin",
          "Cut Spelng",
          "Taylor",
          "Taylor+",
          "Characterie",
          "Teeline",
          "Keyscript",
          "Dearborn</br></br>Speedwriting",
          "QC-line",
          "Yash",
          "Jeake</br></br>Ph. Tr. 487",
          "Polyphonic</br></br>Cipher",
          "Pitman</br></br>2000",
          "Pitman 2000</br></br>(No Vowel)",
          "Superwrite",
          "Swiftograph</br></br>Curtailed",
          "Swiftograph</br></br>Full"
         ],
         "textposition": [
          "top center",
          "top center",
          "top center",
          "bottom center",
          "top center",
          "bottom left",
          "bottom center",
          "top center",
          "top center",
          "top center",
          "top center",
          "top center",
          "top center",
          "bottom center",
          "top right",
          "top center",
          "middle right",
          "bottom center",
          "top right",
          "top right",
          "bottom left",
          "top right",
          "top left",
          "top center",
          "middle right"
         ],
         "type": "scatter",
         "x": [
          9.032685167501281,
          8.957894660731132,
          -0.3404847284724646,
          0.5043864856355338,
          2.0157196431571123,
          3.3728810587563522,
          2.5752095048045787,
          1.7994265664138087,
          4.948696388815664,
          7.57562594835035,
          0.15749139398657874,
          0.722894506196381,
          4.7156301805954435,
          3.556719332211011,
          -1.0418072141740753,
          -0.2553155309816777,
          3.7459244643134006,
          0.4818343735785042,
          -1.3284811196546489,
          4.5354578197439,
          4.367288823841984,
          1.1197254169674045,
          1.070541564317419,
          3.1266400217544223,
          5.64367541984743
         ],
         "y": [
          0,
          0.014930232236564533,
          0.3048400643277738,
          0.11017558339765876,
          0.07505138574335446,
          0.033954582356027774,
          0.06600137338137413,
          0,
          0,
          0.005325956756667138,
          0.2218609124591735,
          0.1900756428174497,
          0.0662984787750377,
          0.054781492737337456,
          0.27275805499274164,
          0.18216280866243562,
          0.0647610299624436,
          0.18388474099533814,
          0.35139657535538305,
          0.023091815341297117,
          0.0188530058822729,
          0.12174817795441739,
          0.12144321942914205,
          0.10909772718267308,
          0.02466666732051115
         ]
        }
       ],
       "layout": {
        "height": 1024,
        "template": {
         "data": {
          "bar": [
           {
            "error_x": {
             "color": "#2a3f5f"
            },
            "error_y": {
             "color": "#2a3f5f"
            },
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "bar"
           }
          ],
          "barpolar": [
           {
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             },
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "barpolar"
           }
          ],
          "carpet": [
           {
            "aaxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "baxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "type": "carpet"
           }
          ],
          "choropleth": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "choropleth"
           }
          ],
          "contour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "contour"
           }
          ],
          "contourcarpet": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "contourcarpet"
           }
          ],
          "heatmap": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmap"
           }
          ],
          "histogram": [
           {
            "marker": {
             "pattern": {
              "fillmode": "overlay",
              "size": 10,
              "solidity": 0.2
             }
            },
            "type": "histogram"
           }
          ],
          "histogram2d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2d"
           }
          ],
          "histogram2dcontour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2dcontour"
           }
          ],
          "mesh3d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "mesh3d"
           }
          ],
          "parcoords": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "parcoords"
           }
          ],
          "pie": [
           {
            "automargin": true,
            "type": "pie"
           }
          ],
          "scatter": [
           {
            "fillpattern": {
             "fillmode": "overlay",
             "size": 10,
             "solidity": 0.2
            },
            "type": "scatter"
           }
          ],
          "scatter3d": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter3d"
           }
          ],
          "scattercarpet": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattercarpet"
           }
          ],
          "scattergeo": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergeo"
           }
          ],
          "scattergl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergl"
           }
          ],
          "scattermap": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattermap"
           }
          ],
          "scattermapbox": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattermapbox"
           }
          ],
          "scatterpolar": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolar"
           }
          ],
          "scatterpolargl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolargl"
           }
          ],
          "scatterternary": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterternary"
           }
          ],
          "surface": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "surface"
           }
          ],
          "table": [
           {
            "cells": {
             "fill": {
              "color": "#EBF0F8"
             },
             "line": {
              "color": "white"
             }
            },
            "header": {
             "fill": {
              "color": "#C8D4E3"
             },
             "line": {
              "color": "white"
             }
            },
            "type": "table"
           }
          ]
         },
         "layout": {
          "annotationdefaults": {
           "arrowcolor": "#2a3f5f",
           "arrowhead": 0,
           "arrowwidth": 1
          },
          "autotypenumbers": "strict",
          "coloraxis": {
           "colorbar": {
            "outlinewidth": 0,
            "ticks": ""
           }
          },
          "colorscale": {
           "diverging": [
            [
             0,
             "#8e0152"
            ],
            [
             0.1,
             "#c51b7d"
            ],
            [
             0.2,
             "#de77ae"
            ],
            [
             0.3,
             "#f1b6da"
            ],
            [
             0.4,
             "#fde0ef"
            ],
            [
             0.5,
             "#f7f7f7"
            ],
            [
             0.6,
             "#e6f5d0"
            ],
            [
             0.7,
             "#b8e186"
            ],
            [
             0.8,
             "#7fbc41"
            ],
            [
             0.9,
             "#4d9221"
            ],
            [
             1,
             "#276419"
            ]
           ],
           "sequential": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ],
           "sequentialminus": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ]
          },
          "colorway": [
           "#636efa",
           "#EF553B",
           "#00cc96",
           "#ab63fa",
           "#FFA15A",
           "#19d3f3",
           "#FF6692",
           "#B6E880",
           "#FF97FF",
           "#FECB52"
          ],
          "font": {
           "color": "#2a3f5f"
          },
          "geo": {
           "bgcolor": "white",
           "lakecolor": "white",
           "landcolor": "#E5ECF6",
           "showlakes": true,
           "showland": true,
           "subunitcolor": "white"
          },
          "hoverlabel": {
           "align": "left"
          },
          "hovermode": "closest",
          "mapbox": {
           "style": "light"
          },
          "paper_bgcolor": "white",
          "plot_bgcolor": "#E5ECF6",
          "polar": {
           "angularaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "radialaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "scene": {
           "xaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "yaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "zaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           }
          },
          "shapedefaults": {
           "line": {
            "color": "#2a3f5f"
           }
          },
          "ternary": {
           "aaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "baxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "caxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "title": {
           "x": 0.05
          },
          "xaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          },
          "yaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          }
         }
        },
        "width": 1280,
        "xaxis": {
         "range": [
          -1.5,
          10
         ],
         "title": {
          "text": "Average Outline Complexity Overhead (bits)"
         }
        },
        "yaxis": {
         "range": [
          -0.01,
          0.4
         ],
         "title": {
          "text": "Reconstruction Error (probability)"
         }
        }
       }
      }
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = go.Figure()\n",
    "\n",
    "fig.add_trace(go.Scatter(x=np.linspace(-1.5,10,24), y=[max(0,-x/4) for x in np.linspace(-1.5,10,24)], mode='none', fill='tozeroy', name = \"Excluded Region\", fillcolor='#EF553B'))\n",
    "fig.add_trace(go.Scatter(x=np.linspace(-1.5,10,24), y=[-1 for x in np.linspace(-1.5,10,24)], mode='none', fill='tozeroy', showlegend=False, fillcolor='#EF553B'))\n",
    "fig.add_trace(go.Scatter(x=np.linspace(-1.5,10,24), y=[0 for x in np.linspace(-1.5,10,24)], mode='lines', fill='none', showlegend=False,line=dict(color='#EF553B')))\n",
    "\n",
    "fig.add_trace(go.Scatter(x=[x[1]['mean_transmission_shift'] for x in scores],\n",
    "                         y=[x[1]['reconstruction_error'] for x in scores],\n",
    "                         text=[x[0] for x in scores], textposition=[x[2] for x in scores],\n",
    "                         mode='markers+text', name=\"Shorthand Systems\", line=dict(color='#636EFA')))\n",
    "fig.update_layout(width=1280, height=1024, xaxis_range=[-1.5, 10], yaxis_range=[-0.01, 0.4],\n",
    "                  xaxis_title=\"Average Outline Complexity Overhead (bits)\",\n",
    "                  yaxis_title=\"Reconstruction Error (probability)\")\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 388,
   "id": "35e92eff-4937-4dce-995f-6e747c93dbe6",
   "metadata": {},
   "outputs": [],
   "source": [
    "fig.write_image(\"comparison_graph.svg\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "388c148a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}