homework-01.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "source": [
    "## Практическая работа №1. Азат Сибагатулин"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    " !pip install scipy matplotlib statsmodels pandas seaborn tqdm scikit-learn numba tweedie"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import scipy\n",
    "import matplotlib.pyplot as plt\n",
    "import scipy.stats as stats\n",
    "from tweedie.tweedie_dist import tweedie_gen\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "import os\n",
    "from tqdm import tqdm\n",
    "from sklearn.utils import shuffle\n",
    "import hashlib\n",
    "from typing import Callable\n",
    "from base64 import b64encode\n",
    "import collections\n",
    "import datetime\n",
    "\n",
    "from numba import njit, float32\n",
    "from joblib import Parallel, delayed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "pd.set_option('display.max_colwidth', None)\n",
    "pd.set_option('display.max_columns', 1000)\n",
    "pd.set_option('display.max_rows', 20000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "## Загрузка KION датасета"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "\n",
    "# download dataset by chunks\n",
    "url = \"https://github.com/sharthZ23/your-second-recsys/raw/master/data_kion.zip\"\n",
    "\n",
    "req = requests.get(url, stream=True)\n",
    "\n",
    "with open('data_kion.zip', \"wb\") as fd:\n",
    "    total_size_in_bytes = int(req.headers.get('Content-Length', 0))\n",
    "    progress_bar = tqdm(desc='kion dataset download', total=total_size_in_bytes, unit='iB', unit_scale=True)\n",
    "    for chunk in req.iter_content(chunk_size=2 ** 20):\n",
    "        progress_bar.update(len(chunk))\n",
    "        fd.write(chunk)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "# Распаковываем архив из 3 файлов\n",
    "import zipfile\n",
    "\n",
    "with zipfile.ZipFile('data_kion.zip', 'r') as zip_ref:\n",
    "    zip_ref.extractall()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "jp-MarkdownHeadingCollapsed": true,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "source": [
    "## 1. Извлекаем просмотры \n",
    "\n",
    "- за две последних недели ```week2```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "interactions = pd.read_csv('data_kion/interactions_df.csv')\n",
    "\n",
    "interactions['last_watch_dt'] = pd.to_datetime(interactions['last_watch_dt']).map(lambda x: x.date())\n",
    "interactions['user_id'] = interactions['user_id'].astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "max_date = interactions['last_watch_dt'].max()\n",
    "\n",
    "week2 = interactions[interactions['last_watch_dt'] >= max_date - datetime.timedelta(days=14)]\n",
    "week2 = week2.groupby('user_id', as_index=False)[\"total_dur\"].sum()\n",
    "week2.total_dur = week2.total_dur.astype(np.int32)\n",
    "\n",
    "del interactions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Уникальных пользователей за две недели: 279129\n"
     ]
    }
   ],
   "source": [
    "print(\"Уникальных пользователей за две недели:\", week2.user_id.nunique())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Удаление выбросов ```total_dur```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Week2(total_dur): mean=21896 std=74516\n"
     ]
    }
   ],
   "source": [
    "print(f\"Week2(total_dur): mean={week2.total_dur.mean():.0f} std={week2.total_dur.std():.0f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgkAAAHFCAYAAAB4oGqqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAj2ElEQVR4nO3deVTU9f7H8dcwbC6AErmgoElhi7Tp0Swp7VYUtli3yPIaVmYdbc/Kbr+udTVt0+p2zbJSy+u5XuuWdZKyMshK21zKFE1T01AwywQXEJjP749+zI+Rt8ugDiDPxzmcI9/5znfe89GapzPfL3qcc04AAAB7CKvrAQAAQP1EJAAAABORAAAATEQCAAAwEQkAAMBEJAAAABORAAAATEQCAAAwEQkAAMBEJAD1xNSpU+XxePTNN98c0uNu375drVq10vTp0w/pcQ9WXl6ePB6P8vLyDvpYVWu3bt26gz5WdS+88IKSk5NVVlZ2SI8LNBREAtDAzJ07VzfccINSU1PVtGlTderUSYMHD9amTZvM/Z999lnFxMSof//+/m05OTl6+OGHD2qOMWPGaNasWQd1jPpu0KBB2r17t1588cW6HgWoE0QC0MDcf//9ysvL0+WXX65//OMf6t+/v2bOnKnTTjtNhYWFAfuWl5fr2Wef1eDBg+X1ev3bc3Jy9MgjjxzUHI0hEqKjo5Wdna3x48eLf+YGjRGRADQw48eP1+rVq/X4449r8ODBGjNmjN59910VFRXpn//8Z8C+7777rn755RdlZWXV0bQN044dO/y/zsrK0k8//aTc3Nw6nAioG0QCGr3vvvtOHo9H77zzjn/bwoUL5fF4dPrppwfse9FFF6lHjx7+79977z2lp6erWbNmiomJUd++fbVs2bIaj7FixQpdeeWVio+PV3R0tLp16xbweHuzdetWde/eXe3bt9fKlSslSWeffbbCwgL/0z377LMVHx+v/Pz8gO2zZs1Sx44dlZKS4t82aNAgTZgwQZLk8Xj8X1V27Nihe+65R0lJSYqKilLnzp311FNPBfxN2uPxaMeOHXr11Vf99x80aJAk6aefftLQoUPVuXNnNWnSREcddZSuuuqqQ3a+wLJly3TuueeqSZMmat++vUaPHi2fz1djP4/HY36k0rFjR/+s0v+fz/DJJ59o6NChatWqldq3b++/vWvXroqPj9fbb799SOYHGpLwuh4AqGtdunRRixYtNG/ePF166aWSpE8//VRhYWH69ttvVVxcrNjYWPl8Ps2fP19DhgyRJE2bNk3Z2dnKyMjQ448/rp07d2rixInq1auXFi9erI4dO0r640XtrLPOUrt27TRixAg1a9ZMM2fOVL9+/fTf//5Xl19+uTnXli1bdP755+u3337TJ598EvBCv6ft27dr+/btSkhICNg+f/78GqFz8803a+PGjfrwww81bdq0gNucc7r00kuVm5urG2+8UaeeeqrmzJmje++9VwUFBXr66af9z33w4MHq3r27fz2q5vv66681f/589e/fX+3bt9e6des0ceJE9e7dW8uXL1fTpk0P5LfFVFhYqD59+qiiosK/lpMmTVKTJk1qfcwqQ4cO1dFHH62//e1vAe8kSNLpp5+uzz///KAfA2hwHADXt29f1717d//3V1xxhbviiiuc1+t17733nnPOuUWLFjlJ7u2333YlJSWuRYsW7qabbgo4TmFhoYuLiwvY/qc//cmlpaW50tJS/zafz+fOPPNMd9xxx/m3TZkyxUlyX3/9tdu0aZM76aSTXKdOndy6dev2O/+oUaOcJDd37lz/tvLycufxeNw999xTY/9hw4Y56z//WbNmOUlu9OjRAduvvPJK5/F43OrVq/3bmjVr5rKzs2scY+fOnTW2LViwwElyr732mn9bbm6uk+Ryc3P3+/yq3HnnnU6S+/LLL/3bNm/e7OLi4pwkt3btWv92SW7kyJE1jtGhQ4eAuavWvVevXq6iosJ83CFDhrgmTZoc8JzAkYKPGwBJ6enpWrRokf9vkJ999pkyMzN16qmn6tNPP5X0x7sLHo9HvXr10ocffqjff/9d11xzjbZs2eL/8nq96tGjh//z699++00ff/yxsrKyVFJS4t/v119/VUZGhlatWqWCgoKAWX7++Wedc845Ki8v17x589ShQ4d9zj5v3jw98sgjysrK0rnnnuvf/ttvv8k5p5YtWx7wOuTk5Mjr9er2228P2H7PPffIOaf33ntvv8eo/rf68vJy/frrrzr22GPVokULLVq06IBn2dt8Z5xxhrp37+7fdvTRR2vAgAEHdVxJuummmwJO7qyuZcuW2rVrl3bu3HnQjwM0JHzcAOiPSKioqNCCBQuUlJSkzZs3Kz09XcuWLQuIhBNPPFHx8fFatWqVJAW8KFcXGxsrSVq9erWcc3rooYf00EMPmftu3rxZ7dq1838/cOBAhYeHKz8/X23atNnn3CtWrNDll1+uLl266OWXXzb3cUGclf/TTz8pMTFRMTExAdtPOOEE/+37s2vXLo0dO1ZTpkxRQUFBwONv27btgGfZ23zVzwmp0rlz54M6riQdc8wxe72t6jlUP3cDaAyIBEBSt27dFB0drXnz5ik5OVmtWrVSamqq0tPT9fzzz6usrEyffvqp//yBqhPlpk2bZr6Qh4eHB+w3fPhwZWRkmI997LHHBnx/xRVX6LXXXtOzzz6rsWPH7nXmDRs26IILLlBcXJxycnJqvLDHx8fL4/Fo69atB7gKh8Ztt92mKVOm6M4771TPnj0VFxcnj8ej/v37mycYhlplZaW5fV/nNWzdulVNmzY9JOc+AA0JkQBIioyMVPfu3fXpp58qOTlZ6enpkv54h6GsrEzTp09XUVGRzj77bEn/f5Jeq1atdN555+31uJ06dZIkRURE7HO/6m677TYde+yx+tvf/qa4uDiNGDGixj6//vqrLrjgApWVlWnu3Llq27ZtjX3Cw8OVkpKitWvX1rhtb38j7tChgz766COVlJQERMeKFSv8t+/vGG+88Yays7M1btw4/7bS0lL9/vvv9hMOQocOHfzv4lRXdeVHdS1btqzxmLt3797rD53al7Vr1/rfTQEaE85JAP5Penq6vvzyS+Xm5vojISEhQSeccIIef/xx/z6SlJGRodjYWI0ZM0bl5eU1jvXLL79I+iMievfurRdffNF8carab08PPfSQhg8frgceeEATJ04MuG3Hjh3KzMxUQUGBcnJydNxxx+31OfXs2dP8Mc/NmjWTpBovopmZmaqsrKzx8xaefvppeTweXXTRRQHHsF74vV5vjY84nnvuub3+DT4YmZmZ+uKLL/TVV1/5t/3yyy/mj5xOSUnRvHnzArZNmjSpVnMsWrRIZ555ZvADAw0c7yQA/yc9PV2PPvqoNmzY4I8B6Y+fQfDiiy+qY8eO/uvnY2NjNXHiRA0cOFCnn366+vfvr6OPPlrr16/X7NmzddZZZ/lfaCdMmKBevXopLS1NN910kzp16qSioiItWLBAP//8s7799ltznieffFLbtm3TsGHDFBMTo7/85S+SpAEDBuirr77SDTfcoPz8/ICfjdC8eXP169fP//1ll12madOm6YcfflBqaqp/e9euXSVJt99+uzIyMuT1etW/f39dcskl6tOnjx588EGtW7dOp5xyij744AO9/fbbuvPOOwMuw+zatas++ugjjR8/XomJiTrmmGPUo0cPXXzxxZo2bZri4uJ04oknasGCBfroo4901FFHHeTvkHTfffdp2rRpuvDCC3XHHXf4L4Hs0KGDvvvuu4B9Bw8erFtuuUV//vOfdf755+vbb7/VnDlzalwmuj8LFy7Ub7/9pssuu+yg5wcanDq8sgKoV4qLi53X63UxMTEBl8L961//cpLcwIEDa9wnNzfXZWRkuLi4OBcdHe1SUlLcoEGD3DfffBOw348//uiuu+4616ZNGxcREeHatWvnLr74YvfGG2/496l+CWSVyspKd80117jw8HA3a9Ys59wfl/BJMr86dOgQ8LhlZWUuISHBjRo1KmB7RUWFu+2229zRRx/tPB5PwOWQJSUl7q677nKJiYkuIiLCHXfcce7JJ590Pp8v4BgrVqxwZ599tmvSpImT5L+scOvWre766693CQkJrnnz5i4jI8OtWLGixqWHtbkE0jnnvvvuO3fOOee46Oho165dOzdq1Cj3yiuv1LgEsrKy0t1///0uISHBNW3a1GVkZLjVq1fv9RLI6ute3f333++Sk5NrPH+gMfA4xw8kB45ko0aN0pQpU7Rq1aq9XuIHW1lZmTp27KgRI0bojjvuqOtxgJDjnATgCHfXXXdp+/btmjFjRl2P0uBMmTJFERERuuWWW+p6FKBO8E4CgDq3a9eu/f4Mhfj4eEVGRoZoIgASJy4CqAf+85//6Prrr9/nPrm5uerdu3doBgIgiXcSANQDmzZtMv/1zOq6du0a1I+YBnDwiAQAAGDixEUAAGCq9TkJPp9PGzduVExMDP/oCQAADYRzTiUlJUpMTFRY2L7fK6h1JGzcuFFJSUm1vTsAAKhDGzZs8P8U2b2pdSRU/eMvGzZs8P+zuAAAoH4rLi5WUlJSjX851lLrSKj6iCE2NpZIAACggTmQUwU4cREAAJiIBAAAYCISAACAiUgAAAAmIgEAAJiIBAAAYCISAACAiUgAAAAmIgEAAJiIBAAAYCISAACAiUgAAAAmIgEAAJiIBAAAYCISAACAiUgAAAAmIgEAAJiIBAAAYCISAACAiUgAAAAmIgEAAJiIBAAAYCISAACAiUgAAAAmIgEAAJiIBAAAYCISAACAiUgAAAAmIgEAAJiIBAAAYCISAACAiUgAAACmehkJRUVFKioqqusxAABo1OpdJBQVFWnAXwbqmmuu1YoVK+p6HAAAGq16Fwnbtm1TRflu+XyV2rBhQ12PAwBAo1XvIgEAANQPRAIAADARCQAAwEQkAAAAE5EAAABMRAIAADARCQAAwEQkAAAAE5EAAABMRAIAADARCQAAwEQkAAAAE5EAAABMRAIAADARCQAAwEQkAAAAE5EAAABMRAIAADARCQAAwEQkAAAAE5EAAABMRAIAADARCQAAwEQkAAAAE5EAAABMRAIAADARCQAAwEQkAAAAE5EAAABMRAIAADARCQAAwEQkAAAAE5EAAABMRAIAADARCQAAwEQkAAAAE5EAAABMRAIAADARCQAAwEQkAAAAE5EAAABMRAIAADARCQAAwEQkAAAAE5EAAABMRAIAADARCQAAwEQkAAAAE5EAAABMRAIAADARCQAAwEQkAAAAE5EAAABMRAIAADARCQAAwEQkAAAAE5EAAABMRAIAADARCQAAwEQkAAAAE5EAAABMRAIAADARCQAAwEQkAAAAE5EAAABMRAIAADARCQAAwEQkAAAAE5EAAABMRAIAADARCQAAwFTvIqGsrMz/6/Lycv3www8qLS2tw4kAAGic6l0kFBYW+n+9ZcsWDRkyROvXr6/DiQAAaJzqXSQAAID6gUgAAAAmIgEAAJiIBAAAYCISAACAiUgAAAAmIgEAAJiIBAAAYCISAACAiUgAAAAmIgEAAJiIBAAAYCISAACAiUgAAAAmIgEAAJiIBAAAYCISAACAiUgAAAAmIgEAAJiIBAAAYCISAACAiUgAAAAmIgEAAJiIBAAAYCISAACAiUgAAAAmIgEAAJiIBAAAYCISAACAiUgAAAAmIgEAAJiIBAAAYCISAACAiUgAAAAmIgEAAJiIBAAAYCISAACAiUgAAAAmIgEAAJiIBAAAYCISAACAiUgAAAAmIgEAAJiIBAAAYCISAACAiUgAAAAmIgEAAJiIBAAAYCISAACAiUgAAAAmIgEAAJiIBAAAYCISAACAiUgAAAAmIgEAAJiIBAAAYCISAACAiUgAAAAmIgEAAJiIBAAAYCISAACAiUgAAAAmIgEAAJiIBAAAYCISAACAiUgAAAAmIgEAAJiIBAAAYCISAACAiUgAAACm8LoeYF8mT54sSRoyZEgdTxK8sLAwRURESJIqKytVUVEhSYqKitLu3bvlnPPv6/V61aRJE8XFxen3339XaWmpIiIi1Lx5czVv3lyRkZEKDw+X1+tVeXm5kpKS1LZtWx1//PEqLS3Vxo0b9fPPP8vj8SguLk7FxcVyzmn79u364Ycf5PP51KJFC8XGxio8PFxdunRRt27dVFpaqsLCQs2fP19er1cpKSmKjY1VTEyMunTpolatWkmSNm/erIKCArVr106SVFBQoF27dmnFihWKj49X586dVVpaql27dmnhwoXauXOnjj/+ePXs2VOStGzZMjnnAo5ZXfXj73n7vm7bnz3n/v777+XxeHTSSScFHKs2j5Gfn6+lS5cqLS1NJ5xwQlBzBTv7oVyTUGtIs9ZHjXX9Guvzrq6+rIHHVX+1CkJxcbHi4uK0bds2xcbGHrKBPvzwQz366KOH7HioHY/Ho+HDh0uSxo0bJ5/PJ4/HI0mq5R8Z/zH79u3r3zZ79mz/8cPCwnTPPff4b9/XbftT/b4ejydg5upz1OYxxo4dqzlz5vi/z8jI0AMPPBDMUhzw7IdyTUKtIc1aHzXW9Wusz7u6w70Gwbx+H/DHDWVlZSouLg74wpHLOadx48bpqaeeUmZmpp555hn/9v2JjIwM+L5nz57q3bu3JGn8+PHavHmzpD9Kedy4ccrMzNRrr72mzMxM/+37um1/qt/3mWee8c98xhln+OcYN26c8vPzg36M/Px8zZkzR8nJyXr88ceVnJysOXPmKD8/f79zHYjDtSah1pBmrY8a6/o11uddXX1bgwOOhLFjxyouLs7/lZSUdDjnQj3g8/nknFNWVpaccwf8DkK/fv0Cvu/du7duuOEGOedUWVmpgoICSX98bOHz+ZSVlaXk5GRlZWX5b9/XbftT/b7VZ+7Tp49/Dp/Pp6VLlwb9GEuXLpUkDRs2TD169NCwYcMk/fFRxqFwuNYk1BrSrPVRY12/xvq8q6tva3DAkfDAAw9o27Zt/q8NGzYczrlQD4SFhcnj8WjmzJnyeDz+jxv2Z9asWQHf5+XlafLkyfJ4PPJ6vf5zBNq1a6ewsDDNnDlT69ev18yZM/237+u2/al+3+oz5+bm+ucICwtTWlpa0I+RlpYmSZowYYK+/PJLTZgwQZLUpUuXA1qbYGY/lGsSag1p1vqosa5fY33e1dW3NeCcBJiqPgeT/viIoLKy0h8KPp/vgI9T/XwA67O12bNn+4/v9Xp19913B3z+vrfb9qf6fcPCwgLeCdnznIRgHyMU5yQcjjUJtYY0a33UWNevsT7v6g73GgTz+k0kHCaH+uqGiIgIhYWFqaKiQu3bt1diYqI6d+6ssrIyFRQUaOPGjZLk/z2RpO3bt2vlypX+qxvi4uLk9XqVlpamrl27+q9uWLBggf/qhpiYGMXGxgZcAbC3qxtWrlyp+Ph4paam+q9uWLRokXbu3KnOnTsHXN0gqcZVBVVCdXXD3uao7dUN33//vbp06cLVDfvQkGatjxrr+jXW513d4VyDIyYSbrjhBk2ePFmTJk1SamrqIXsMAAAaq8NydQMAAGhciAQAAGAiEgAAgIlIAAAAJiIBAACYiAQAAGAiEgAAgIlIAAAAJiIBAACYiAQAAGAiEgAAgIlIAAAAJiIBAACYiAQAAGAiEgAAgIlIAAAAJiIBAACYiAQAAGAiEgAAgIlIAAAAJiIBAACYiAQAAGAiEgAAgIlIAAAAJiIBAACYiAQAAGAiEgAAgIlIAAAAJiIBAACYiAQAAGAiEgAAgIlIAAAAJiIBAACYiAQAAGAiEgAAgIlIAAAAJiIBAACYiAQAAGAiEgAAgIlIAAAAJiIBAACYiAQAAGAiEgAAgIlIAAAAJiIBAACYiAQAAGAiEgAAgIlIAAAAJiIBAACYiAQAAGAiEgAAgIlIAAAAJiIBAACYiAQAAGAiEgAAgIlIAAAAJiIBAACYiAQAAGAiEgAAgIlIAAAAJiIBAACYiAQAAGAiEgAAgIlIAAAAJiIBAACYiAQAAGAiEgAAgIlIAAAAJiIBAACYiAQAAGCqd5HQpk0b/68TEhI0adIkJScn1+FEAAA0TuF1PcCeoqKi/L+OiIhQampqHU4DAEDjVe/eSQAAAPUDkQAAAExEAgAAMBEJAADARCQAAAATkQAAAExEAgAAMBEJAADARCQAAAATkQAAAExEAgAAMBEJAADARCQAAAATkQAAAExEAgAAMBEJAADARCQAAAATkQAAAExEAgAAMBEJAADARCQAAAATkQAAAExEAgAAMBEJAADARCQAAAATkQAAAExEAgAAMBEJAADARCQAAAATkQAAAExEAgAAMBEJAADARCQAAAATkQAAAExEAgAAMBEJAADARCQAAAATkQAAAExEAgAAMBEJAADARCQAAAATkQAAAExEAgAAMBEJAADARCQAAAATkQAAAExEAgAAMBEJAADARCQAAAATkQAAAExEAgAAMBEJAADARCQAAAATkQAAAExEAgAAMBEJAADARCQAAAATkQAAAExEAgAAMBEJAADARCQAAAATkQAAAExEAgAAMBEJAADARCQAAAATkQAAAExEAgAAMBEJAADARCQAAAATkQAAAExEAgAAMNW7SIiLi1N4RKTCwrxKSkqq63EAAGi0wut6gD21bt1a0/81zf9rAABQN+pdJEjEAQAA9UG9+7gBAADUD0QCAAAwEQkAAMBEJAAAABORAAAATEQCAAAwEQkAAMBEJAAAABORAAAATEQCAAAwEQkAAMBEJAAAABORAAAATEQCAAAwEQkAAMBEJAAAABORAAAATEQCAAAwEQkAAMBEJAAAABORAAAATEQCAAAwEQkAAMBEJAAAABORAAAATEQCAAAwEQkAAMBEJAAAABORAAAATEQCAAAwEQkAAMBEJAAAABORAAAATOG1vaNzTpJUXFx8yIYBAACHV9XrdtXr+L7UOhJKSkokSUlJSbU9BAAAqCMlJSWKi4vb5z4edyApYfD5fNq4caNiYmLk8XhqNaCluLhYSUlJ2rBhg2JjYw/ZcbF3rHnoseahxXqHHmseege65s45lZSUKDExUWFh+z7roNbvJISFhal9+/a1vft+xcbG8gcrxFjz0GPNQ4v1Dj3WPPQOZM339w5CFU5cBAAAJiIBAACY6l0kREVFaeTIkYqKiqrrURoN1jz0WPPQYr1DjzUPvcOx5rU+cREAABzZ6t07CQAAoH4gEgAAgIlIAAAAJiIBAACYiAQAAGCqk0iYMGGCOnbsqOjoaPXo0UNfffXVPvd//fXXdfzxxys6OlppaWnKyckJ0aRHjmDW/KWXXlJ6erpatmypli1b6rzzztvv7xFqCvbPeZUZM2bI4/GoX79+h3fAI0yw6/37779r2LBhatu2raKiopSamsr/W4IU7Jo/88wz6ty5s5o0aaKkpCTdddddKi0tDdG0Ddu8efN0ySWXKDExUR6PR7NmzdrvffLy8nT66acrKipKxx57rKZOnRr8A7sQmzFjhouMjHSTJ092y5YtczfddJNr0aKFKyoqMvf//PPPndfrdU888YRbvny5+5//+R8XERHhli5dGuLJG65g1/zaa691EyZMcIsXL3b5+flu0KBBLi4uzv38888hnrzhCnbNq6xdu9a1a9fOpaenu8suuyw0wx4Bgl3vsrIy161bN5eZmek+++wzt3btWpeXl+eWLFkS4skbrmDXfPr06S4qKspNnz7drV271s2ZM8e1bdvW3XXXXSGevGHKyclxDz74oHvzzTedJPfWW2/tc/81a9a4pk2burvvvtstX77cPffcc87r9br3338/qMcNeSR0797dDRs2zP99ZWWlS0xMdGPHjjX3z8rKcn379g3Y1qNHD3fzzTcf1jmPJMGu+Z4qKipcTEyMe/XVVw/XiEec2qx5RUWFO/PMM93LL7/ssrOziYQgBLveEydOdJ06dXK7d+8O1YhHnGDXfNiwYe7cc88N2Hb33Xe7s84667DOeSQ6kEi477773EknnRSw7eqrr3YZGRlBPVZIP27YvXu3Fi5cqPPOO8+/LSwsTOedd54WLFhg3mfBggUB+0tSRkbGXvdHoNqs+Z527typ8vJyxcfHH64xjyi1XfO///3vatWqlW688cZQjHnEqM16v/POO+rZs6eGDRum1q1bq0uXLhozZowqKytDNXaDVps1P/PMM7Vw4UL/RxJr1qxRTk6OMjMzQzJzY3OoXjtr/a9A1saWLVtUWVmp1q1bB2xv3bq1VqxYYd6nsLDQ3L+wsPCwzXkkqc2a7+n+++9XYmJijT9wsNVmzT/77DO98sorWrJkSQgmPLLUZr3XrFmjjz/+WAMGDFBOTo5Wr16toUOHqry8XCNHjgzF2A1abdb82muv1ZYtW9SrVy8551RRUaFbbrlFf/3rX0MxcqOzt9fO4uJi7dq1S02aNDmg43B1A/bpscce04wZM/TWW28pOjq6rsc5IpWUlGjgwIF66aWXlJCQUNfjNAo+n0+tWrXSpEmT1LVrV1199dV68MEH9cILL9T1aEesvLw8jRkzRs8//7wWLVqkN998U7Nnz9aoUaPqejTsQ0jfSUhISJDX61VRUVHA9qKiIrVp08a8T5s2bYLaH4Fqs+ZVnnrqKT322GP66KOPdPLJJx/OMY8owa75jz/+qHXr1umSSy7xb/P5fJKk8PBwrVy5UikpKYd36AasNn/G27Ztq4iICHm9Xv+2E044QYWFhdq9e7ciIyMP68wNXW3W/KGHHtLAgQM1ePBgSVJaWpp27NihIUOG6MEHH1RYGH9nPZT29toZGxt7wO8iSCF+JyEyMlJdu3bV3Llz/dt8Pp/mzp2rnj17mvfp2bNnwP6S9OGHH+51fwSqzZpL0hNPPKFRo0bp/fffV7du3UIx6hEj2DU//vjjtXTpUi1ZssT/demll6pPnz5asmSJkpKSQjl+g1ObP+NnnXWWVq9e7Y8xSfrhhx/Utm1bAuEA1GbNd+7cWSMEqiLN8e8MHnKH7LUzuHMqD96MGTNcVFSUmzp1qlu+fLkbMmSIa9GihSssLHTOOTdw4EA3YsQI//6ff/65Cw8Pd0899ZTLz893I0eO5BLIIAW75o899piLjIx0b7zxhtu0aZP/q6SkpK6eQoMT7JrviasbghPseq9fv97FxMS4W2+91a1cudK9++67rlWrVm706NF19RQanGDXfOTIkS4mJsb9+9//dmvWrHEffPCBS0lJcVlZWXX1FBqUkpISt3jxYrd48WInyY0fP94tXrzY/fTTT84550aMGOEGDhzo37/qEsh7773X5efnuwkTJjSMSyCdc+65555zycnJLjIy0nXv3t198cUX/tvOOeccl52dHbD/zJkzXWpqqouMjHQnnXSSmz17dognbviCWfMOHTo4STW+Ro4cGfrBG7Bg/5xXRyQEL9j1nj9/vuvRo4eLiopynTp1co8++qirqKgI8dQNWzBrXl5e7h5++GGXkpLioqOjXVJSkhs6dKjbunVr6AdvgHJzc83/L1etcXZ2tjvnnHNq3OfUU091kZGRrlOnTm7KlClBP67HOd7nAQAANXGmCAAAMBEJAADARCQAAAATkQAAAExEAgAAMBEJAADARCQAAAATkQAAQD0yb948XXLJJUpMTJTH49GsWbOCuv/DDz8sj8dT46tZs2ZBz0IkAABQj+zYsUOnnHKKJkyYUKv7Dx8+XJs2bQr4OvHEE3XVVVcFfSwiAQCAeuSiiy7S6NGjdfnll5u3l5WVafjw4WrXrp2aNWumHj16KC8vz3978+bN1aZNG/9XUVGRli9frhtvvDHoWYgEAAAakFtvvVULFizQjBkz9N133+mqq67ShRdeqFWrVpn7v/zyy0pNTVV6enrQj0UkAADQQKxfv15TpkzR66+/rvT0dKWkpGj48OHq1auXpkyZUmP/0tJSTZ8+vVbvIkhS+MEODAAAQmPp0qWqrKxUampqwPaysjIdddRRNfZ/6623VFJSouzs7Fo9HpEAAEADsX37dnm9Xi1cuFBerzfgtubNm9fY/+WXX9bFF1+s1q1b1+rxiAQAABqI0047TZWVldq8efN+zzFYu3atcnNz9c4779T68YgEAADqke3bt2v16tX+79euXaslS5YoPj5eqampGjBggK677jqNGzdOp512mn755RfNnTtXJ598svr27eu/3+TJk9W2bVtddNFFtZ7F45xzB/VsAADAIZOXl6c+ffrU2J6dna2pU6eqvLxco0eP1muvvaaCggIlJCTojDPO0COPPKK0tDRJks/nU4cOHXTdddfp0UcfrfUsRAIAADBxCSQAADARCQAAwEQkAAAAE5EAAABMRAIAADARCQAAwEQkAAAAE5EAAABMRAIAADARCQAAwEQkAAAA0/8CdaRcqMKC5eUAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.boxplot(week2['total_dur'].values, fliersize=3, orient='h').set_title('week2(total_dur)');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(279129, 2)\n",
      "(275616, 2)\n"
     ]
    }
   ],
   "source": [
    "# Удалим выбросы в 3-х стандартных отклонениях\n",
    "\n",
    "print(week2.shape)\n",
    "week2 = week2[(week2['total_dur'] < week2['total_dur'].std() * 3)]\n",
    "print(week2.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkIAAAGzCAYAAADDgXghAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA2AklEQVR4nO3deXRU9f3/8VcWsrBMwpaEQIAgmywFAQmRpVpSBolVlFaglCICVg2UTQWOAlqtbG6oCNr2C9a6Qb8Vlc0fhlWIAQKyk0ILgugEJGQmIISQfH5/eHK/DAmbBELyeT7OmXOYz+c9977vXJ15nZt77wQYY4wAAAAsFFjWDQAAAJQVghAAALAWQQgAAFiLIAQAAKxFEAIAANYiCAEAAGsRhAAAgLUIQgAAwFoEIQAAYC2CEIDrZt68eQoICNCmTZtKdbknTpxQVFSU3n333VJd7tVatWqVAgICtGrVqqteVtF7d+DAgate1rnmzJmj+vXrKy8vr1SXC5QXBCEAN5zU1FQ9+OCDatq0qSpXrqxGjRpp6NCh+u6770qsnzlzpqpVq6Z+/fo5Y0uWLNHTTz99VX08//zzWrhw4VUt40b3wAMP6MyZM3rzzTfLuhWgTBCEANxwxo0bp1WrVunee+/Vq6++qn79+mn+/Pm65ZZb5PF4/Grz8/M1c+ZMDR06VEFBQc74kiVL9Mwzz1xVHzYEobCwMA0aNEgvvfSS+OlJ2IggBOCG89JLL2nfvn2aNm2ahg4dqueff16LFi1SVlaWXn/9db/aRYsW6ejRo7r//vvLqNvy6eTJk86/77//fn399ddauXJlGXYElA2CEGCBbdu2KSAgQJ988okzlpGRoYCAALVr186v9s4771RCQoLzfOnSperatauqVKmiatWqKTk5WTt37iy2jj179ujXv/61atSoobCwMHXo0MFvfRdy/PhxdezYUfXq1VNmZqYkqVu3bgoM9P946tatm2rUqKHdu3f7jS9cuFANGzbUTTfd5Iw98MADmjVrliQpICDAeRQ5efKkxo4dq7i4OIWGhqpZs2Z64YUX/I6IBAQE6OTJk3r77bed1z/wwAOSpK+//lqPPvqomjVrpvDwcNWsWVO/+c1vSu38nZ07d+oXv/iFwsPDVa9ePT333HMqLCwsVhcQEFDin/8aNmzo9Cr93/lFq1ev1qOPPqqoqCjVq1fPmW/fvr1q1Kihjz/+uFT6B8qT4LJuAMC116pVK0VGRmrNmjW6++67JUlr165VYGCgtm7dKp/PJ5fLpcLCQq1fv14PPfSQJOmdd97RoEGD5Ha7NW3aNP3www+aPXu2unTpoi1btqhhw4aSfvzi7ty5s+rWravx48erSpUqmj9/vnr37q3//d//1b333ltiX99//71++ctfKjs7W6tXr/YLM+c7ceKETpw4oVq1avmNr1+/vliY+8Mf/qBvv/1Wy5cv1zvvvOM3Z4zR3XffrZUrV2rIkCFq27atPvvsMz3++OM6fPiwXn75ZWfbhw4dqo4dOzrvR1F/Gzdu1Pr169WvXz/Vq1dPBw4c0OzZs3X77bdr165dqly58uXslhJ5PB7dcccdOnv2rPNevvXWWwoPD//Jyyzy6KOPqnbt2po0aZLfESFJateundatW3fV6wDKHQPACsnJyaZjx47O8/vuu8/cd999JigoyCxdutQYY8zmzZuNJPPxxx+b3NxcExkZaYYNG+a3HI/HYyIiIvzGu3fvblq3bm1Onz7tjBUWFprbbrvNNGnSxBmbO3eukWQ2btxovvvuO9OyZUvTqFEjc+DAgUv2/+yzzxpJJjU11RnLz883AQEBZuzYscXqU1JSTEkfcQsXLjSSzHPPPec3/utf/9oEBASYffv2OWNVqlQxgwYNKraMH374odhYWlqakWT+/ve/O2MrV640kszKlSsvuX1FRo0aZSSZ9PR0Z+zIkSMmIiLCSDL79+93xiWZyZMnF1tGgwYN/Pouet+7dOlizp49W+J6H3roIRMeHn7ZfQIVBX8aAyzRtWtXbd682TkS8MUXX6hXr15q27at1q5dK+nHo0QBAQHq0qWLli9frpycHPXv31/ff/+98wgKClJCQoJzPkl2drZWrFih+++/X7m5uU7dsWPH5Ha7tXfvXh0+fNivl2+++UY///nPlZ+frzVr1qhBgwYX7X3NmjV65plndP/99+sXv/iFM56dnS1jjKpXr37Z78OSJUsUFBSkP/7xj37jY8eOlTFGS5cuveQyzj06k5+fr2PHjqlx48aKjIzU5s2bL7uXC/XXqVMndezY0RmrXbu2BgwYcFXLlaRhw4b5nVB+rurVq+vUqVP64Ycfrno9QHnCn8YAS3Tt2lVnz55VWlqa4uLidOTIEXXt2lU7d+70C0ItWrRQjRo1tHfvXknyCx7ncrlckqR9+/bJGKOJEydq4sSJJdYeOXJEdevWdZ4PHDhQwcHB2r17t2JiYi7a9549e3TvvfeqVatW+utf/1pijbmCq52+/vprxcbGqlq1an7jN998szN/KadOndKUKVM0d+5cHT582G/9Xq/3snu5UH/nnqNVpFmzZle1XEmKj4+/4FzRNpx7LhVgA4IQYIkOHTooLCxMa9asUf369RUVFaWmTZuqa9eueuONN5SXl6e1a9c65/MUnZz7zjvvlBhWgoOD/eoee+wxud3uEtfduHFjv+f33Xef/v73v2vmzJmaMmXKBXs+dOiQevTooYiICC1ZsqRYeKlRo4YCAgJ0/Pjxy3wXSseIESM0d+5cjRo1SomJiYqIiFBAQID69etX4knN11tBQUGJ4xc7z+j48eOqXLlyqZyLBJQnBCHAEiEhIerYsaPWrl2r+vXrq2vXrpJ+PFKUl5end999V1lZWerWrZuk/zsxOCoqSklJSRdcbqNGjSRJlSpVumjduUaMGKHGjRtr0qRJioiI0Pjx44vVHDt2TD169FBeXp5SU1NVp06dYjXBwcG66aabtH///mJzFzqy0aBBA33++efKzc31C1Z79uxx5i+1jH/+858aNGiQXnzxRWfs9OnTysnJKXmDr0CDBg2co3HnKrqi7lzVq1cvts4zZ85c8MaTF7N//37nqBhgE84RAizStWtXpaena+XKlU4QqlWrlm6++WZNmzbNqZEkt9stl8ul559/Xvn5+cWWdfToUUk/BqXbb79db775ZolfwEV155s4caIee+wxTZgwQbNnz/abO3nypHr16qXDhw9ryZIlatKkyQW3KTExscSf7KhSpYokFQsKvXr1UkFBQbH7Eb388ssKCAjQnXfe6beMksJNUFBQsT/Hvfbaaxc8EnMlevXqpS+//FIbNmxwxo4ePVriz4fcdNNNWrNmjd/YW2+99ZP62Lx5s2677bYrbxgo5zgiBFika9eu+vOf/6xDhw45gUf68R49b775pho2bOjcX8blcmn27NkaOHCg2rVrp379+ql27do6ePCgFi9erM6dOzthYtasWerSpYtat26tYcOGqVGjRsrKylJaWpq++eYbbd26tcR+ZsyYIa/Xq5SUFFWrVk2/+93vJEkDBgzQhg0b9OCDD2r37t1+9w6qWrWqevfu7Ty/55579M477+jf//63mjZt6oy3b99ekvTHP/5RbrdbQUFB6tevn371q1/pjjvu0JNPPqkDBw6oTZs2+n//7//p448/1qhRo/wu4W/fvr0+//xzvfTSS4qNjVV8fLwSEhJ011136Z133lFERIRatGihtLQ0ff7556pZs+ZV7iHpiSee0DvvvKOePXtq5MiRzuXzDRo00LZt2/xqhw4dqocfflh9+vTRL3/5S23dulWfffZZsVsMXEpGRoays7N1zz33XHX/QLlThlesAbjOfD6fCQoKMtWqVfO7jPof//iHkWQGDhxY7DUrV640brfbREREmLCwMHPTTTeZBx54wGzatMmv7j//+Y/5/e9/b2JiYkylSpVM3bp1zV133WX++c9/OjXnXj5fpKCgwPTv398EBwebhQsXGmN+vPxbUomPBg0a+K03Ly/P1KpVyzz77LN+42fPnjUjRowwtWvXNgEBAX6X0ufm5prRo0eb2NhYU6lSJdOkSRMzY8YMU1hY6LeMPXv2mG7dupnw8HAjybkk/fjx42bw4MGmVq1apmrVqsbtdps9e/YUu2z9p1w+b4wx27ZtMz//+c9NWFiYqVu3rnn22WfN3/72t2KXzxcUFJhx48aZWrVqmcqVKxu322327dt3wcvnz33fzzVu3DhTv379YtsP2CDAGH5cBkD59uyzz2ru3Lnau3fvBS8PR8ny8vLUsGFDjR8/XiNHjizrdoDrjnOEAJR7o0eP1okTJ/TBBx+UdSvlzty5c1WpUiU9/PDDZd0KUCY4IgQA18GpU6cueY+hGjVqKCQk5Dp1BEDiZGkAuC4+/PBDDR48+KI1K1eu1O233359GgIgiSNCAHBdfPfdd9q5c+dFa9q3b39FPxcC4OoRhAAAgLU4WRoAAFiLc4QuorCwUN9++62qVavGDxECAFBOGGOUm5ur2NhYBQZe/JgPQegivv32W8XFxZV1GwAA4Cc4dOiQc7f8C7niILRmzRrNmDFDGRkZ+u677/TRRx/53e7eGKPJkyfrL3/5i3JyctS5c2fNnj3b77eCsrOzNWLECH366acKDAxUnz59NHPmTFWtWtWp2bZtm1JSUrRx40bVrl1bI0aM0BNPPOHXy4IFCzRx4kQdOHBATZo00bRp09SrV68r6uViin6Q8dChQ3K5XFf6VgEAgDLg8/kUFxfn98PKF3Slt6JesmSJefLJJ82//vUvI8l89NFHfvNTp041ERERZuHChWbr1q3m7rvvNvHx8ebUqVNOTc+ePU2bNm3Ml19+adauXWsaN25s+vfv78x7vV4THR1tBgwYYHbs2GHef/99Ex4ebt58802nZt26dSYoKMhMnz7d7Nq1yzz11FOmUqVKZvv27VfUy8V4vV4jyXi93it9mwAAQBm5ku/vq/qtsfODUGFhoYmJiTEzZsxwxnJyckxoaKh5//33jTHG7Nq1q9hv3ixdutQEBASYw4cPG2OMeeONN0z16tVNXl6eUzNu3DjTrFkz5/n9999vkpOT/fpJSEgwf/jDHy67l0shCAEAUP5cyfd3qV41tn//fnk8HiUlJTljERERSkhIUFpamiQpLS1NkZGR6tChg1OTlJSkwMBApaenOzXdunXzu8Oq2+1WZmamjh8/7tScu56imqL1XE4v58vLy5PP5/N7AACAiqtUg5DH45EkRUdH+41HR0c7cx6PR1FRUX7zwcHBqlGjhl9NScs4dx0Xqjl3/lK9nG/KlCmKiIhwHpwoDQBAxcZ9hM4xYcIEeb1e53Ho0KGybgkAAFxDpRqEYmJiJElZWVl+41lZWc5cTEyMjhw54jd/9uxZZWdn+9WUtIxz13GhmnPnL9XL+UJDQ+VyufweAACg4irVIBQfH6+YmBilpqY6Yz6fT+np6UpMTJQkJSYmKicnRxkZGU7NihUrVFhYqISEBKdmzZo1ys/Pd2qWL1+uZs2aOb/Dk5iY6Leeopqi9VxOLwAAwHJXeiZ2bm6u2bJli9myZYuRZF566SWzZcsW8/XXXxtjfrxkPTIy0nz88cdm27Zt5p577inx8vlbbrnFpKenmy+++MI0adLE7/L5nJwcEx0dbQYOHGh27NhhPvjgA1O5cuVil88HBwebF154wezevdtMnjy5xMvnL9XLxXDVGAAA5c81vXx+5cqVRlKxx6BBg4wxP162PnHiRBMdHW1CQ0NN9+7dTWZmpt8yjh07Zvr372+qVq1qXC6XGTx4sMnNzfWr2bp1q+nSpYsJDQ01devWNVOnTi3Wy/z5803Tpk1NSEiIadmypVm8eLHf/OX0cjEEIQAAyp8r+f7m1+cvwufzKSIiQl6vl/OFAAAoJ67k+5urxgAAgLUIQgAAwFoEIQAAYC2CEAAAsBZBCAAAWCu4rBuwWcPxi/2eH5iaXEadAABgJ44IAQAAaxGEAACAtQhCAADAWgQhAABgLYIQAACwFkEIAABYiyAEAACsRRACAADWIggBAABrEYQAAIC1CEIAAMBaBCEAAGAtghAAALAWQQgAAFiLIAQAAKxFEAIAANYiCAEAAGsRhAAAgLUIQgAAwFoEIQAAYC2CEAAAsBZBCAAAWIsgBAAArEUQAgAA1iIIAQAAaxGEAACAtQhCAADAWgQhAABgLYIQAACwFkEIAABYiyAEAACsRRACAADWIggBAABrEYQAAIC1CEIAAMBaBCEAAGAtghAAALAWQQgAAFiLIAQAAKxFEAIAANYiCAEAAGsRhAAAgLUIQgAAwFoEIQAAYC2CEAAAsBZBCAAAWIsgBAAArEUQAgAA1iIIAQAAaxGEAACAtQhCAADAWgQhAABgLYIQAACwFkEIAABYiyAEAACsRRACAADWKvUgVFBQoIkTJyo+Pl7h4eG66aab9Oyzz8oY49QYYzRp0iTVqVNH4eHhSkpK0t69e/2Wk52drQEDBsjlcikyMlJDhgzRiRMn/Gq2bdumrl27KiwsTHFxcZo+fXqxfhYsWKDmzZsrLCxMrVu31pIlS0p7kwEAQDlV6kFo2rRpmj17tl5//XXt3r1b06ZN0/Tp0/Xaa685NdOnT9err76qOXPmKD09XVWqVJHb7dbp06edmgEDBmjnzp1avny5Fi1apDVr1uihhx5y5n0+n3r06KEGDRooIyNDM2bM0NNPP6233nrLqVm/fr369++vIUOGaMuWLerdu7d69+6tHTt2lPZmAwCAcijAnHuophTcddddio6O1t/+9jdnrE+fPgoPD9c//vEPGWMUGxursWPH6rHHHpMkeb1eRUdHa968eerXr592796tFi1aaOPGjerQoYMkadmyZerVq5e++eYbxcbGavbs2XryySfl8XgUEhIiSRo/frwWLlyoPXv2SJL69u2rkydPatGiRU4vnTp1Utu2bTVnzpxLbovP51NERIS8Xq9cLlepvUdFGo5f7Pf8wNTkUl8HAAC2uZLv71I/InTbbbcpNTVV//73vyVJW7du1RdffKE777xTkrR//355PB4lJSU5r4mIiFBCQoLS0tIkSWlpaYqMjHRCkCQlJSUpMDBQ6enpTk23bt2cECRJbrdbmZmZOn78uFNz7nqKaorWc768vDz5fD6/BwAAqLiCS3uB48ePl8/nU/PmzRUUFKSCggL9+c9/1oABAyRJHo9HkhQdHe33uujoaGfO4/EoKirKv9HgYNWoUcOvJj4+vtgyiuaqV68uj8dz0fWcb8qUKXrmmWd+ymYDAIByqNSPCM2fP1/vvvuu3nvvPW3evFlvv/22XnjhBb399tulvapSN2HCBHm9Xudx6NChsm4JAABcQ6V+ROjxxx/X+PHj1a9fP0lS69at9fXXX2vKlCkaNGiQYmJiJElZWVmqU6eO87qsrCy1bdtWkhQTE6MjR474Lffs2bPKzs52Xh8TE6OsrCy/mqLnl6opmj9faGioQkNDf8pmAwCAcqjUjwj98MMPCgz0X2xQUJAKCwslSfHx8YqJiVFqaqoz7/P5lJ6ersTERElSYmKicnJylJGR4dSsWLFChYWFSkhIcGrWrFmj/Px8p2b58uVq1qyZqlev7tScu56imqL1AAAAu5V6EPrVr36lP//5z1q8eLEOHDigjz76SC+99JLuvfdeSVJAQIBGjRql5557Tp988om2b9+u3//+94qNjVXv3r0lSTfffLN69uypYcOGacOGDVq3bp2GDx+ufv36KTY2VpL029/+ViEhIRoyZIh27typDz/8UDNnztSYMWOcXkaOHKlly5bpxRdf1J49e/T0009r06ZNGj58eGlvNgAAKI9MKfP5fGbkyJGmfv36JiwszDRq1Mg8+eSTJi8vz6kpLCw0EydONNHR0SY0NNR0797dZGZm+i3n2LFjpn///qZq1arG5XKZwYMHm9zcXL+arVu3mi5dupjQ0FBTt25dM3Xq1GL9zJ8/3zRt2tSEhISYli1bmsWLF1/2tni9XiPJeL3eK3wXLk+DcYv8HgAA4Opdyfd3qd9HqCLhPkIAAJQ/ZXofIQAAgPKCIAQAAKxFEAIAANYiCAEAAGsRhAAAgLUIQgAAwFoEIQAAYC2CEAAAsBZBCAAAWIsgBAAArEUQAgAA1iIIAQAAaxGEAACAtQhCAADAWgQhAABgLYIQAACwFkEIAABYiyAEAACsRRACAADWIggBAABrEYQAAIC1CEIAAMBaBCEAAGAtghAAALAWQQgAAFiLIAQAAKxFEAIAANYiCAEAAGsRhAAAgLUIQgAAwFoEIQAAYC2CEAAAsBZBCAAAWIsgBAAArEUQAgAA1iIIAQAAaxGEAACAtQhCAADAWgQhAABgLYIQAACwFkEIAABYiyAEAACsRRACAADWIggBAABrEYQAAIC1CEIAAMBaBCEAAGAtghAAALAWQQgAAFiLIAQAAKxFEAIAANYiCAEAAGsRhAAAgLUIQgAAwFoEIQAAYC2CEAAAsBZBCAAAWIsgBAAArEUQAgAA1iIIAQAAaxGEAACAtQhCAADAWtckCB0+fFi/+93vVLNmTYWHh6t169batGmTM2+M0aRJk1SnTh2Fh4crKSlJe/fu9VtGdna2BgwYIJfLpcjISA0ZMkQnTpzwq9m2bZu6du2qsLAwxcXFafr06cV6WbBggZo3b66wsDC1bt1aS5YsuRabDAAAyqFSD0LHjx9X586dValSJS1dulS7du3Siy++qOrVqzs106dP16uvvqo5c+YoPT1dVapUkdvt1unTp52aAQMGaOfOnVq+fLkWLVqkNWvW6KGHHnLmfT6fevTooQYNGigjI0MzZszQ008/rbfeesupWb9+vfr3768hQ4Zoy5Yt6t27t3r37q0dO3aU9mYDAIByKMAYY0pzgePHj9e6deu0du3aEueNMYqNjdXYsWP12GOPSZK8Xq+io6M1b9489evXT7t371aLFi20ceNGdejQQZK0bNky9erVS998841iY2M1e/ZsPfnkk/J4PAoJCXHWvXDhQu3Zs0eS1LdvX508eVKLFi1y1t+pUye1bdtWc+bMueS2+Hw+RUREyOv1yuVyXdX7UpKG4xf7PT8wNbnU1wEAgG2u5Pu71I8IffLJJ+rQoYN+85vfKCoqSrfccov+8pe/OPP79++Xx+NRUlKSMxYREaGEhASlpaVJktLS0hQZGemEIElKSkpSYGCg0tPTnZpu3bo5IUiS3G63MjMzdfz4cafm3PUU1RSt53x5eXny+Xx+DwAAUHGVehD673//q9mzZ6tJkyb67LPP9Mgjj+iPf/yj3n77bUmSx+ORJEVHR/u9Ljo62pnzeDyKiorymw8ODlaNGjX8akpaxrnruFBN0fz5pkyZooiICOcRFxd3xdsPAADKj1IPQoWFhWrXrp2ef/553XLLLXrooYc0bNiwy/pTVFmbMGGCvF6v8zh06FBZtwQAAK6hUg9CderUUYsWLfzGbr75Zh08eFCSFBMTI0nKysryq8nKynLmYmJidOTIEb/5s2fPKjs726+mpGWcu44L1RTNny80NFQul8vvAQAAKq5SD0KdO3dWZmam39i///1vNWjQQJIUHx+vmJgYpaamOvM+n0/p6elKTEyUJCUmJionJ0cZGRlOzYoVK1RYWKiEhASnZs2aNcrPz3dqli9frmbNmjlXqCUmJvqtp6imaD0AAMBupR6ERo8erS+//FLPP/+89u3bp/fee09vvfWWUlJSJEkBAQEaNWqUnnvuOX3yySfavn27fv/73ys2Nla9e/eW9OMRpJ49e2rYsGHasGGD1q1bp+HDh6tfv36KjY2VJP32t79VSEiIhgwZop07d+rDDz/UzJkzNWbMGKeXkSNHatmyZXrxxRe1Z88ePf3009q0aZOGDx9e2psNAADKI3MNfPrpp6ZVq1YmNDTUNG/e3Lz11lt+84WFhWbixIkmOjrahIaGmu7du5vMzEy/mmPHjpn+/fubqlWrGpfLZQYPHmxyc3P9arZu3Wq6dOliQkNDTd26dc3UqVOL9TJ//nzTtGlTExISYlq2bGkWL1582dvh9XqNJOP1eq9g6y9fg3GL/B4AAODqXcn3d6nfR6gi4T5CAACUP2V6HyEAAIDygiAEAACsRRACAADWIggBAABrEYQAAIC1CEIAAMBaBCEAAGAtghAAALAWQQgAAFiLIAQAAKxFEAIAANYiCAEAAGsRhAAAgLUIQgAAwFoEIQAAYC2CEAAAsBZBCAAAWIsgBAAArEUQAgAA1iIIAQAAaxGEAACAtQhCAADAWgQhAABgLYIQAACwFkEIAABYiyAEAACsRRACAADWIggBAABrEYQAAIC1CEIAAMBaBCEAAGAtghAAALAWQQgAAFiLIAQAAKxFEAIAANYiCAEAAGsRhAAAgLUIQgAAwFoEIQAAYC2CEAAAsBZBCAAAWIsgBAAArEUQAgAA1iIIAQAAaxGEAACAtQhCAADAWgQhAABgLYIQAACwFkEIAABYiyAEAACsRRACAADWIggBAABrEYQAAIC1CEIAAMBaBCEAAGAtghAAALAWQQgAAFiLIAQAAKxFEAIAANYiCAEAAGsRhAAAgLUIQgAAwFrXPAhNnTpVAQEBGjVqlDN2+vRppaSkqGbNmqpatar69OmjrKwsv9cdPHhQycnJqly5sqKiovT444/r7NmzfjWrVq1Su3btFBoaqsaNG2vevHnF1j9r1iw1bNhQYWFhSkhI0IYNG67FZgIAgHLomgahjRs36s0339TPfvYzv/HRo0fr008/1YIFC7R69Wp9++23uu+++5z5goICJScn68yZM1q/fr3efvttzZs3T5MmTXJq9u/fr+TkZN1xxx366quvNGrUKA0dOlSfffaZU/Phhx9qzJgxmjx5sjZv3qw2bdrI7XbryJEj13KzAQBAeWGukdzcXNOkSROzfPly8/Of/9yMHDnSGGNMTk6OqVSpklmwYIFTu3v3biPJpKWlGWOMWbJkiQkMDDQej8epmT17tnG5XCYvL88YY8wTTzxhWrZs6bfOvn37Grfb7Tzv2LGjSUlJcZ4XFBSY2NhYM2XKlMvaBq/XayQZr9d7ZRt/mRqMW+T3AAAAV+9Kvr+v2RGhlJQUJScnKykpyW88IyND+fn5fuPNmzdX/fr1lZaWJklKS0tT69atFR0d7dS43W75fD7t3LnTqTl/2W6321nGmTNnlJGR4VcTGBiopKQkp+Z8eXl58vl8fg8AAFBxBV+LhX7wwQfavHmzNm7cWGzO4/EoJCREkZGRfuPR0dHyeDxOzbkhqGi+aO5iNT6fT6dOndLx48dVUFBQYs2ePXtK7HvKlCl65plnLn9DAQBAuVbqR4QOHTqkkSNH6t1331VYWFhpL/6amjBhgrxer/M4dOhQWbcEAACuoVIPQhkZGTpy5IjatWun4OBgBQcHa/Xq1Xr11VcVHBys6OhonTlzRjk5OX6vy8rKUkxMjCQpJiam2FVkRc8vVeNyuRQeHq5atWopKCioxJqiZZwvNDRULpfL7wEAACquUg9C3bt31/bt2/XVV185jw4dOmjAgAHOvytVqqTU1FTnNZmZmTp48KASExMlSYmJidq+fbvf1V3Lly+Xy+VSixYtnJpzl1FUU7SMkJAQtW/f3q+msLBQqampTg0AALBbqZ8jVK1aNbVq1cpvrEqVKqpZs6YzPmTIEI0ZM0Y1atSQy+XSiBEjlJiYqE6dOkmSevTooRYtWmjgwIGaPn26PB6PnnrqKaWkpCg0NFSS9PDDD+v111/XE088oQcffFArVqzQ/PnztXjxYme9Y8aM0aBBg9ShQwd17NhRr7zyik6ePKnBgweX9mYDAIBy6JqcLH0pL7/8sgIDA9WnTx/l5eXJ7XbrjTfecOaDgoK0aNEiPfLII0pMTFSVKlU0aNAg/elPf3Jq4uPjtXjxYo0ePVozZ85UvXr19Ne//lVut9up6du3r44ePapJkybJ4/Gobdu2WrZsWbETqAEAgJ0CjDGmrJu4Ufl8PkVERMjr9V6T84Uajl/s9/zA1ORSXwcAALa5ku9vfmsMAABYiyAEAACsRRACAADWIggBAABrEYQAAIC1CEIAAMBaBCEAAGAtghAAALAWQQgAAFiLIAQAAKxFEAIAANYiCAEAAGsRhAAAgLUIQgAAwFoEIQAAYC2CEAAAsBZBCAAAWIsgBAAArEUQAgAA1iIIAQAAaxGEAACAtQhCAADAWgQhAABgLYIQAACwFkEIAABYiyAEAACsRRACAADWIggBAABrEYQAAIC1CEIAAMBaBCEAAGAtghAAALAWQQgAAFiLIAQAAKxFEAIAANYiCAEAAGsRhAAAgLUIQgAAwFoEIQAAYC2CEAAAsFZwWTeA/9Nw/OJiYwemJpdBJwAA2IEjQgAAwFoEIQAAYC2CEAAAsBZBCAAAWIuTpW9w559AzcnTAACUHo4IAQAAaxGEAACAtQhCAADAWgQhAABgLYIQAACwFkEIAABYiyAEAACsRRACAADWIggBAABrEYQAAIC1CEIAAMBaBCEAAGAtghAAALAWQQgAAFiLIAQAAKxFEAIAANYq9SA0ZcoU3XrrrapWrZqioqLUu3dvZWZm+tWcPn1aKSkpqlmzpqpWrao+ffooKyvLr+bgwYNKTk5W5cqVFRUVpccff1xnz571q1m1apXatWun0NBQNW7cWPPmzSvWz6xZs9SwYUOFhYUpISFBGzZsKO1NBgAA5VSpB6HVq1crJSVFX375pZYvX678/Hz16NFDJ0+edGpGjx6tTz/9VAsWLNDq1av17bff6r777nPmCwoKlJycrDNnzmj9+vV6++23NW/ePE2aNMmp2b9/v5KTk3XHHXfoq6++0qhRozR06FB99tlnTs2HH36oMWPGaPLkydq8ebPatGkjt9utI0eOlPZmAwCAcijAGGOu5QqOHj2qqKgorV69Wt26dZPX61Xt2rX13nvv6de//rUkac+ePbr55puVlpamTp06aenSpbrrrrv07bffKjo6WpI0Z84cjRs3TkePHlVISIjGjRunxYsXa8eOHc66+vXrp5ycHC1btkySlJCQoFtvvVWvv/66JKmwsFBxcXEaMWKExo8fX6zXvLw85eXlOc99Pp/i4uLk9XrlcrlK/b1pOH7xFb/mwNTkUu8DAICKxOfzKSIi4rK+v6/5OUJer1eSVKNGDUlSRkaG8vPzlZSU5NQ0b95c9evXV1pamiQpLS1NrVu3dkKQJLndbvl8Pu3cudOpOXcZRTVFyzhz5owyMjL8agIDA5WUlOTUnG/KlCmKiIhwHnFxcVe7+QAA4AZ2TYNQYWGhRo0apc6dO6tVq1aSJI/Ho5CQEEVGRvrVRkdHy+PxODXnhqCi+aK5i9X4fD6dOnVK33//vQoKCkqsKVrG+SZMmCCv1+s8Dh069NM2HAAAlAvB13LhKSkp2rFjh7744otruZpSExoaqtDQ0LJuAwAAXCfX7IjQ8OHDtWjRIq1cuVL16tVzxmNiYnTmzBnl5OT41WdlZSkmJsapOf8qsqLnl6pxuVwKDw9XrVq1FBQUVGJN0TIAAIDdSj0IGWM0fPhwffTRR1qxYoXi4+P95tu3b69KlSopNTXVGcvMzNTBgweVmJgoSUpMTNT27dv9ru5avny5XC6XWrRo4dScu4yimqJlhISEqH379n41hYWFSk1NdWoAAIDdSv1PYykpKXrvvff08ccfq1q1as75OBEREQoPD1dERISGDBmiMWPGqEaNGnK5XBoxYoQSExPVqVMnSVKPHj3UokULDRw4UNOnT5fH49FTTz2llJQU509XDz/8sF5//XU98cQTevDBB7VixQrNnz9fixf/35VYY8aM0aBBg9ShQwd17NhRr7zyik6ePKnBgweX9mYDAIByqNSD0OzZsyVJt99+u9/43Llz9cADD0iSXn75ZQUGBqpPnz7Ky8uT2+3WG2+84dQGBQVp0aJFeuSRR5SYmKgqVapo0KBB+tOf/uTUxMfHa/HixRo9erRmzpypevXq6a9//avcbrdT07dvXx09elSTJk2Sx+NR27ZttWzZsmInUAMAADtd8/sIlWdXch+Cn+Kn3EeoJNxbCACA/3ND3UcIAADgRkUQAgAA1iIIAQAAaxGEAACAtQhCAADAWgQhAABgLYIQAACwFkEIAABYiyAEAACsRRACAADWIggBAABrEYQAAIC1CEIAAMBaBCEAAGAtghAAALAWQQgAAFiLIAQAAKxFEAIAANYiCAEAAGsRhAAAgLWCy7oBXL2G4xf7PT8wNbmMOgEAoHzhiBAAALAWQQgAAFiLIAQAAKxFEAIAANYiCAEAAGsRhAAAgLW4fL4COv9yeolL6gEAKAlHhAAAgLUIQgAAwFoEIQAAYC2CEAAAsBZBCAAAWIsgBAAArMXl85bgF+oBACiOI0IAAMBaBCEAAGAtghAAALAWQQgAAFiLk6Utxe+RAQDAESEAAGAxghAAALAWfxqDg3sNAQBswxEhAABgLYIQAACwFkEIAABYi3OEcEFcYg8AqOg4IgQAAKzFESFcEa4sAwBUJBwRAgAA1iIIAQAAa/GnMVwVTqgGAJRnBCGUOsIRAKC8IAjhuuAkawDAjYgghDLBUSMAwI2AIIQbBkeNAADXG0EINyyOGgEArjWCEMoVjhoBAEoTQQjlGkeNAABXgyCECqekcHQphCcAsBNBCBBHlgDAVlYEoVmzZmnGjBnyeDxq06aNXnvtNXXs2LGs28INjiNLAFDxVfgg9OGHH2rMmDGaM2eOEhIS9Morr8jtdiszM1NRUVFl3R4qmJ8SnkpCoAKA6yPAGGPKuolrKSEhQbfeeqtef/11SVJhYaHi4uI0YsQIjR8//qKv9fl8ioiIkNfrlcvlKvXeSutLEyhtBDEA5dmVfH9X6CNCZ86cUUZGhiZMmOCMBQYGKikpSWlpacXq8/LylJeX5zz3er2SfnxDr4XCvB+uyXKBq1V/9IKybuGGseMZ9yVrWk3+rFSWA6B0FH1vX86xngodhL7//nsVFBQoOjrabzw6Olp79uwpVj9lyhQ988wzxcbj4uKuWY8AbmwRr9xYywFw+XJzcxUREXHRmgodhK7UhAkTNGbMGOd5YWGhsrOzVbNmTQUEBJTqunw+n+Li4nTo0KFr8mc3XDn2yY2HfXLjYZ/cWNgfJTPGKDc3V7GxsZesrdBBqFatWgoKClJWVpbfeFZWlmJiYorVh4aGKjQ01G8sMjLyWrYol8vFf7w3GPbJjYd9cuNhn9xY2B/FXepIUJHAa9xHmQoJCVH79u2VmprqjBUWFio1NVWJiYll2BkAALgRVOgjQpI0ZswYDRo0SB06dFDHjh31yiuv6OTJkxo8eHBZtwYAAMpYhQ9Cffv21dGjRzVp0iR5PB61bdtWy5YtK3YC9fUWGhqqyZMnF/tTHMoO++TGwz658bBPbizsj6tX4e8jBAAAcCEV+hwhAACAiyEIAQAAaxGEAACAtQhCAADAWgQhAABgLYJQGZg1a5YaNmyosLAwJSQkaMOGDWXdUrn09NNPKyAgwO/RvHlzZ/706dNKSUlRzZo1VbVqVfXp06fYXcYPHjyo5ORkVa5cWVFRUXr88cd19uxZv5pVq1apXbt2Cg0NVePGjTVv3rxivdi6T9esWaNf/epXio2NVUBAgBYuXOg3b4zRpEmTVKdOHYWHhyspKUl79+71q8nOztaAAQPkcrkUGRmpIUOG6MSJE34127ZtU9euXRUWFqa4uDhNnz69WC8LFixQ8+bNFRYWptatW2vJkiVX3EtFcKl98sADDxT7/6Znz55+NeyT0jNlyhTdeuutqlatmqKiotS7d29lZmb61dxIn1WX00uFY3BdffDBByYkJMT8z//8j9m5c6cZNmyYiYyMNFlZWWXdWrkzefJk07JlS/Pdd985j6NHjzrzDz/8sImLizOpqalm06ZNplOnTua2225z5s+ePWtatWplkpKSzJYtW8ySJUtMrVq1zIQJE5ya//73v6Zy5cpmzJgxZteuXea1114zQUFBZtmyZU6Nzft0yZIl5sknnzT/+te/jCTz0Ucf+c1PnTrVREREmIULF5qtW7eau+++28THx5tTp045NT179jRt2rQxX375pVm7dq1p3Lix6d+/vzPv9XpNdHS0GTBggNmxY4d5//33TXh4uHnzzTedmnXr1pmgoCAzffp0s2vXLvPUU0+ZSpUqme3bt19RLxXBpfbJoEGDTM+ePf3+v8nOzvarYZ+UHrfbbebOnWt27NhhvvrqK9OrVy9Tv359c+LECafmRvqsulQvFRFB6Drr2LGjSUlJcZ4XFBSY2NhYM2XKlDLsqnyaPHmyadOmTYlzOTk5plKlSmbBggXO2O7du40kk5aWZoz58QsjMDDQeDwep2b27NnG5XKZvLw8Y4wxTzzxhGnZsqXfsvv27WvcbrfznH36o/O/dAsLC01MTIyZMWOGM5aTk2NCQ0PN+++/b4wxZteuXUaS2bhxo1OzdOlSExAQYA4fPmyMMeaNN94w1atXd/aJMcaMGzfONGvWzHl+//33m+TkZL9+EhISzB/+8IfL7qUiulAQuueeey74GvbJtXXkyBEjyaxevdoYc2N9Vl1OLxURfxq7js6cOaOMjAwlJSU5Y4GBgUpKSlJaWloZdlZ+7d27V7GxsWrUqJEGDBiggwcPSpIyMjKUn5/v9143b95c9evXd97rtLQ0tW7d2u8u4263Wz6fTzt37nRqzl1GUU3RMtinF7Z//355PB6/9yYiIkIJCQl++yAyMlIdOnRwapKSkhQYGKj09HSnplu3bgoJCXFq3G63MjMzdfz4cafmYvvpcnqxyapVqxQVFaVmzZrpkUce0bFjx5w59sm15fV6JUk1atSQdGN9Vl1OLxURQeg6+v7771VQUFDs5z2io6Pl8XjKqKvyKyEhQfPmzdOyZcs0e/Zs7d+/X127dlVubq48Ho9CQkIUGRnp95pz32uPx1Piviiau1iNz+fTqVOn2KcXUbT9F3tvPB6PoqKi/OaDg4NVo0aNUtlP585fqhdb9OzZU3//+9+VmpqqadOmafXq1brzzjtVUFAgiX1yLRUWFmrUqFHq3LmzWrVqJUk31GfV5fRSEVX43xpDxXXnnXc6//7Zz36mhIQENWjQQPPnz1d4eHgZdgbcuPr16+f8u3Xr1vrZz36mm266SatWrVL37t3LsLOKLyUlRTt27NAXX3xR1q3gHBwRuo5q1aqloKCgYmfgZ2VlKSYmpoy6qjgiIyPVtGlT7du3TzExMTpz5oxycnL8as59r2NiYkrcF0VzF6txuVwKDw9nn15E0fZf7L2JiYnRkSNH/ObPnj2r7OzsUtlP585fqhdbNWrUSLVq1dK+ffsksU+uleHDh2vRokVauXKl6tWr54zfSJ9Vl9NLRUQQuo5CQkLUvn17paamOmOFhYVKTU1VYmJiGXZWMZw4cUL/+c9/VKdOHbVv316VKlXye68zMzN18OBB571OTEzU9u3b/T70ly9fLpfLpRYtWjg15y6jqKZoGezTC4uPj1dMTIzfe+Pz+ZSenu63D3JycpSRkeHUrFixQoWFhUpISHBq1qxZo/z8fKdm+fLlatasmapXr+7UXGw/XU4vtvrmm2907Ngx1alTRxL7pLQZYzR8+HB99NFHWrFiheLj4/3mb6TPqsvppUIq67O1bfPBBx+Y0NBQM2/ePLNr1y7z0EMPmcjISL+rAXB5xo4da1atWmX2799v1q1bZ5KSkkytWrXMkSNHjDE/XgZav359s2LFCrNp0yaTmJhoEhMTndcXXZLao0cP89VXX5lly5aZ2rVrl3hJ6uOPP252795tZs2aVeIlqbbu09zcXLNlyxazZcsWI8m89NJLZsuWLebrr782xvx4eXRkZKT5+OOPzbZt28w999xT4uXzt9xyi0lPTzdffPGFadKkid+l2jk5OSY6OtoMHDjQ7Nixw3zwwQemcuXKxS7VDg4ONi+88ILZvXu3mTx5comXal+ql4rgYvskNzfXPPbYYyYtLc3s37/ffP7556Zdu3amSZMm5vTp084y2Cel55FHHjERERFm1apVfrcs+OGHH5yaG+mz6lK9VEQEoTLw2muvmfr165uQkBDTsWNH8+WXX5Z1S+VS3759TZ06dUxISIipW7eu6du3r9m3b58zf+rUKfPoo4+a6tWrm8qVK5t7773XfPfdd37LOHDggLnzzjtNeHi4qVWrlhk7dqzJz8/3q1m5cqVp27atCQkJMY0aNTJz584t1out+3TlypVGUrHHoEGDjDE/XiI9ceJEEx0dbUJDQ0337t1NZmam3zKOHTtm+vfvb6pWrWpcLpcZPHiwyc3N9avZunWr6dKliwkNDTV169Y1U6dOLdbL/PnzTdOmTU1ISIhp2bKlWbx4sd/85fRSEVxsn/zwww+mR48epnbt2qZSpUqmQYMGZtiwYcVCO/uk9JS0LyT5fY7cSJ9Vl9NLRRNgjDHX+ygUAADAjYBzhAAAgLUIQgAAwFoEIQAAYC2CEAAAsBZBCAAAWIsgBAAArEUQAgAA1iIIAQAAaxGEAACAtQhCAADAWgQhAABgrf8Ppt3o5hsdqIkAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "week2['total_dur'].hist(bins=100, grid=False).set_title(\"week2(total_dur)\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Week2(total_dur): mean=16222 std=28341\n"
     ]
    }
   ],
   "source": [
    "print(f\"Week2(total_dur): mean={week2.total_dur.mean():.0f} std={week2.total_dur.std():.0f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "jp-MarkdownHeadingCollapsed": true,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "source": [
    "## 2. Подбор пары: метрика + статистический тест"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def salt_generator() -> str:\n",
    "    salt = os.urandom(8)\n",
    "    return b64encode(salt).decode('ascii')\n",
    "\n",
    "\n",
    "def groups_splitter(df: pd.DataFrame, \n",
    "                    columns: list[str], \n",
    "                    user_salt: str | None = None) -> pd.DataFrame:\n",
    "    salt = user_salt if user_salt else salt_generator()\n",
    "    \n",
    "    df = df.copy()\n",
    "    df['hash'] = ((df['user_id'].astype(str)) + '#' + salt).apply(lambda x: hashlib.sha256(x.encode('utf-8')).hexdigest())\n",
    "    df['group'] = ((df['hash'].str.slice(start=-6).apply(int, base=16) % 2).map(lambda x: 'A' if x == 0 else 'B'))\n",
    "    return df[columns]\n",
    "\n",
    "\n",
    "def calc_pvalue(week: pd.DataFrame, \n",
    "                col: str = 'total_dur', \n",
    "                criterion: Callable = stats.mannwhitneyu) -> float:\n",
    "    week_groups = groups_splitter(week, columns=['user_id', col, 'group'])\n",
    "    _, p = criterion(\n",
    "        week_groups[week_groups['group'] == 'A'][col].values,\n",
    "        week_groups[week_groups['group'] == 'B'][col].values\n",
    "    )\n",
    "    return p"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "Валидация:\n",
    "\n",
    "**Распределение ```p_value```** при корректном подборе пары метрика + критерий - равномерное распределение\n",
    "\n",
    "**Корректность** - вероятность того, что найдет различие, когда его нет\n",
    "\n",
    "$$significance = P(\\text{reject }H_0 | H_0 \\text{ is true})$$\n",
    "$$ \\alpha = significance $$\n",
    "\n",
    "**Мощность** - вероятность того, что критерий найдет различие в случае, когда оно есть\n",
    "\n",
    "$$power = P(\\text{reject }H_0 | H_1 \\text{ is true})$$\n",
    "$$ \\beta = 1 - power = P(\\text{accept }H_0 | H_1 \\text{ is true}) $$\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "metric_column = \"total_dur\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- **Манна-Уитни**:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 1min 58s\n",
      "Wall time: 4min 4s\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh8AAAGdCAYAAACyzRGfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAeNElEQVR4nO3df3TV9X348Vf4kRsqJAiWhMxg0daCP2grVozo2rJsObaH6TFn2tY56mGyrtFNss6aWk1treF4OnX2BJiOwnaOjJWd4mprcV06cNqAGuUcVyzVwgYdJs6tJEhHQPL5/tGvd42gckPyhhsfj3PuOb2f+7mf++JNJM9+cj+5JVmWZQEAkMio4z0AAPDOIj4AgKTEBwCQlPgAAJISHwBAUuIDAEhKfAAASYkPACCpMcd7gDfq7++P3bt3x4QJE6KkpOR4jwMAHIUsy2Lv3r1RXV0do0a99bmNEy4+du/eHTU1Ncd7DABgEHbt2hWnnnrqW+5zwsXHhAkTIuJXw5eXlx/naQCAo9Hb2xs1NTX57+Nv5YSLj9d/1FJeXi4+AKDIHM1bJrzhFABISnwAAEmJDwAgKfEBACQlPgCApMQHAJCU+AAAkhIfAEBS4gMASEp8AABJFRwf//mf/xm///u/H5MnT45x48bFueeeG08//XT+8SzL4rbbboupU6fGuHHjoq6uLl544YUhHRoAKF4FxccvfvGLmDt3bowdOza+//3vx9atW+Mv/uIv4uSTT87vc9ddd8V9990Xy5cvj82bN8dJJ50U9fX1sX///iEfHgAoPiVZlmVHu/PNN98cTzzxRPzrv/7rER/Psiyqq6vjz/7sz+Lzn/98RET09PREZWVlrFq1Kj75yU++7Wv09vZGRUVF9PT0+GA5ACgShXz/LujMx3e+8504//zz4/d+7/diypQp8aEPfSgeeOCB/OM7duyIrq6uqKury2+rqKiIOXPmREdHxxGP2dfXF729vQNuAMDINaaQnbdv3x7Lli2Lpqam+OIXvxhPPfVU/Mmf/EmUlpbGggULoqurKyIiKisrBzyvsrIy/9gbtba2xu233z7I8QGK2+0lw/fvX0vWMmzHhmNR0JmP/v7+OO+88+LOO++MD33oQ7Fo0aK47rrrYvny5YMeoLm5OXp6evK3Xbt2DfpYAMCJr6D4mDp1apx11lkDts2cOTN27twZERFVVVUREdHd3T1gn+7u7vxjb5TL5aK8vHzADQAYuQqKj7lz58a2bdsGbPvpT38ap512WkRETJ8+PaqqqqK9vT3/eG9vb2zevDlqa2uHYFwAoNgV9J6PxYsXx0UXXRR33nlnXHnllfHkk0/G/fffH/fff39ERJSUlMSNN94Yd9xxR7zvfe+L6dOnx6233hrV1dVx+eWXD8f8AECRKSg+PvzhD8e6deuiubk5vvKVr8T06dPj3nvvjauvvjq/z0033RT79u2LRYsWxZ49e+Liiy+O9evXR1lZ2ZAPDwAUn4J+z0cKfs8H8E7iahdGimH7PR8AAMdKfAAASYkPACAp8QEAJCU+AICkxAcAkJT4AACSEh8AQFLiAwBISnwAAEmJDwAgqYI+WA4AhtNwfdaNz7k5sTjzAQAkJT4AgKTEBwCQlPgAAJISHwBAUq52ARihXDnCicqZDwAgKfEBACQlPgCApMQHAJCU+AAAknK1C0XHO/iL23D9/UX4O4Ri4cwHAJCU+AAAkhIfAEBS4gMASEp8AABJiQ8AICnxAQAkJT4AgKTEBwCQlPgAAJLy69WLgF9HDcfXcP43SPErxq+P4/1vvzMfAEBS4gMASEp8AABJiQ8AICnxAQAk9Y672mW43pV8vN85DJBKMV7dwYnFmQ8AICnxAQAkJT4AgKTEBwCQlPgAAJJ6x13twkCu/mEkcRUGFAdnPgCApAqKjy9/+ctRUlIy4DZjxoz84/v374/GxsaYPHlyjB8/PhoaGqK7u3vIhwYAilfBZz7OPvvseOmll/K3xx9/PP/Y4sWL4+GHH461a9fGxo0bY/fu3XHFFVcM6cAAQHEr+D0fY8aMiaqqqsO29/T0xIoVK2L16tUxb968iIhYuXJlzJw5MzZt2hQXXnjhsU8LABS9gs98vPDCC1FdXR2nn356XH311bFz586IiOjs7IyDBw9GXV1dft8ZM2bEtGnToqOj402P19fXF729vQNuAMDIVVB8zJkzJ1atWhXr16+PZcuWxY4dO+KSSy6JvXv3RldXV5SWlsbEiRMHPKeysjK6urre9Jitra1RUVGRv9XU1AzqDwIAFIeCfuxy6aWX5v/3rFmzYs6cOXHaaafFt771rRg3btygBmhubo6mpqb8/d7eXgECACPYMV1qO3HixDjzzDPjxRdfjKqqqjhw4EDs2bNnwD7d3d1HfI/I63K5XJSXlw+4AQAj1zHFx6uvvho/+9nPYurUqTF79uwYO3ZstLe35x/ftm1b7Ny5M2pra495UABgZCjoxy6f//znY/78+XHaaafF7t27o6WlJUaPHh2f+tSnoqKiIhYuXBhNTU0xadKkKC8vjxtuuCFqa2td6QIA5BUUHz//+c/jU5/6VPz3f/93vPvd746LL744Nm3aFO9+97sjIuKee+6JUaNGRUNDQ/T19UV9fX0sXbp0WAYHAIpTQfGxZs2at3y8rKws2traoq2t7ZiGAo7OcH6Wic/nAYaLz3YBAJISHwBAUuIDAEhKfAAASYkPACCpgj/VFo7GcF6FAUBxc+YDAEhKfAAASYkPACAp8QEAJCU+AICkXO0C/5/PSQFIw5kPACAp8QEAJCU+AICkxAcAkJT4AACSEh8AQFLiAwBISnwAAEmJDwAgKfEBACQlPgCApMQHAJCU+AAAkhIfAEBS4gMASEp8AABJiQ8AICnxAQAkJT4AgKTEBwCQlPgAAJISHwBAUuIDAEhKfAAASYkPACAp8QEAJCU+AICkxAcAkJT4AACSEh8AQFLiAwBISnwAAEmJDwAgKfEBACQ15ngPAADD7faS24/3CPyaYzrzsWTJkigpKYkbb7wxv23//v3R2NgYkydPjvHjx0dDQ0N0d3cf65wAwAgx6Ph46qmn4q/+6q9i1qxZA7YvXrw4Hn744Vi7dm1s3Lgxdu/eHVdcccUxDwoAjAyDio9XX301rr766njggQfi5JNPzm/v6emJFStWxN133x3z5s2L2bNnx8qVK+NHP/pRbNq0aciGBgCK16Dio7GxMT7xiU9EXV3dgO2dnZ1x8ODBAdtnzJgR06ZNi46OjmObFAAYEQp+w+maNWvimWeeiaeeeuqwx7q6uqK0tDQmTpw4YHtlZWV0dXUd8Xh9fX3R19eXv9/b21voSABAESnozMeuXbviT//0T+PBBx+MsrKyIRmgtbU1Kioq8reampohOS4AcGIqKD46Ozvj5ZdfjvPOOy/GjBkTY8aMiY0bN8Z9990XY8aMicrKyjhw4EDs2bNnwPO6u7ujqqrqiMdsbm6Onp6e/G3Xrl2D/sMAACe+gn7s8lu/9Vvx3HPPDdh27bXXxowZM+ILX/hC1NTUxNixY6O9vT0aGhoiImLbtm2xc+fOqK2tPeIxc7lc5HK5QY4PABSbguJjwoQJcc455wzYdtJJJ8XkyZPz2xcuXBhNTU0xadKkKC8vjxtuuCFqa2vjwgsvHLqpAYCiNeS/4fSee+6JUaNGRUNDQ/T19UV9fX0sXbp0qF8GAChSxxwfGzZsGHC/rKws2traoq2t7VgPDRxHfh01MFx8sBwAkJT4AACSEh8AQFLiAwBISnwAAEkN+aW2wOFcOQLwf5z5AACSEh8AQFLiAwBISnwAAEmJDwAgKVe7DBFXMwDA0XHmAwBISnwAAEmJDwAgKfEBACQlPgCApMQHAJCU+AAAkhIfAEBS4gMASEp8AABJiQ8AICnxAQAkJT4AgKTEBwCQlPgAAJISHwBAUuIDAEhKfAAASYkPACAp8QEAJCU+AICkxAcAkJT4AACSEh8AQFLiAwBISnwAAEmJDwAgKfEBACQlPgCApMQHAJCU+AAAkhIfAEBS4gMASEp8AABJiQ8AICnxAQAkVVB8LFu2LGbNmhXl5eVRXl4etbW18f3vfz//+P79+6OxsTEmT54c48ePj4aGhuju7h7yoQGA4lVQfJx66qmxZMmS6OzsjKeffjrmzZsXl112Wfz4xz+OiIjFixfHww8/HGvXro2NGzfG7t2744orrhiWwQGA4jSmkJ3nz58/4P7Xvva1WLZsWWzatClOPfXUWLFiRaxevTrmzZsXERErV66MmTNnxqZNm+LCCy8cuqkBgKI16Pd8HDp0KNasWRP79u2L2tra6OzsjIMHD0ZdXV1+nxkzZsS0adOio6PjTY/T19cXvb29A24AwMhVcHw899xzMX78+MjlcvHZz3421q1bF2eddVZ0dXVFaWlpTJw4ccD+lZWV0dXV9abHa21tjYqKivytpqam4D8EAFA8Co6P97///bFly5bYvHlz/PEf/3EsWLAgtm7dOugBmpubo6enJ3/btWvXoI8FAJz4CnrPR0REaWlpvPe9742IiNmzZ8dTTz0Vf/mXfxlXXXVVHDhwIPbs2TPg7Ed3d3dUVVW96fFyuVzkcrnCJwcAitIx/56P/v7+6Ovri9mzZ8fYsWOjvb09/9i2bdti586dUVtbe6wvAwCMEAWd+Whubo5LL700pk2bFnv37o3Vq1fHhg0b4tFHH42KiopYuHBhNDU1xaRJk6K8vDxuuOGGqK2tdaULAJBXUHy8/PLL8Qd/8Afx0ksvRUVFRcyaNSseffTR+O3f/u2IiLjnnnti1KhR0dDQEH19fVFfXx9Lly4dlsEBgOJUkmVZdryH+HW9vb1RUVERPT09UV5ePuTHv73k9iE/JgAUk5asZciPWcj3b5/tAgAkJT4AgKTEBwCQlPgAAJISHwBAUuIDAEhKfAAASYkPACAp8QEAJCU+AICkxAcAkJT4AACSEh8AQFLiAwBISnwAAEmJDwAgKfEBACQlPgCApMQHAJCU+AAAkhIfAEBS4gMASEp8AABJiQ8AICnxAQAkJT4AgKTEBwCQlPgAAJISHwBAUuIDAEhKfAAASYkPACAp8QEAJCU+AICkxAcAkJT4AACSEh8AQFLiAwBISnwAAEmJDwAgKfEBACQlPgCApMQHAJCU+AAAkhIfAEBS4gMASEp8AABJiQ8AIKmC4qO1tTU+/OEPx4QJE2LKlClx+eWXx7Zt2wbss3///mhsbIzJkyfH+PHjo6GhIbq7u4d0aACgeBUUHxs3bozGxsbYtGlT/OAHP4iDBw/G7/zO78S+ffvy+yxevDgefvjhWLt2bWzcuDF2794dV1xxxZAPDgAUpzGF7Lx+/foB91etWhVTpkyJzs7O+M3f/M3o6emJFStWxOrVq2PevHkREbFy5cqYOXNmbNq0KS688MKhmxwAKErH9J6Pnp6eiIiYNGlSRER0dnbGwYMHo66uLr/PjBkzYtq0adHR0XHEY/T19UVvb++AGwAwcg06Pvr7++PGG2+MuXPnxjnnnBMREV1dXVFaWhoTJ04csG9lZWV0dXUd8Titra1RUVGRv9XU1Ax2JACgCAw6PhobG+Pf/u3fYs2aNcc0QHNzc/T09ORvu3btOqbjAQAntoLe8/G666+/Pr773e/GY489Fqeeemp+e1VVVRw4cCD27Nkz4OxHd3d3VFVVHfFYuVwucrncYMYAAIpQQWc+siyL66+/PtatWxc//OEPY/r06QMenz17dowdOzba29vz27Zt2xY7d+6M2traoZkYAChqBZ35aGxsjNWrV8c//uM/xoQJE/Lv46ioqIhx48ZFRUVFLFy4MJqammLSpElRXl4eN9xwQ9TW1rrSBQCIiALjY9myZRER8dGPfnTA9pUrV8ZnPvOZiIi45557YtSoUdHQ0BB9fX1RX18fS5cuHZJhAYDiV1B8ZFn2tvuUlZVFW1tbtLW1DXooAGDk8tkuAEBS4gMASEp8AABJiQ8AICnxAQAkJT4AgKTEBwCQlPgAAJISHwBAUuIDAEhKfAAASYkPACAp8QEAJCU+AICkxAcAkJT4AACSEh8AQFLiAwBISnwAAEmJDwAgKfEBACQlPgCApMQHAJCU+AAAkhIfAEBS4gMASEp8AABJiQ8AICnxAQAkJT4AgKTEBwCQlPgAAJISHwBAUuIDAEhKfAAASYkPACAp8QEAJCU+AICkxAcAkJT4AACSEh8AQFLiAwBISnwAAEmJDwAgKfEBACQlPgCApMQHAJBUwfHx2GOPxfz586O6ujpKSkrioYceGvB4lmVx2223xdSpU2PcuHFRV1cXL7zwwlDNCwAUuYLjY9++ffGBD3wg2trajvj4XXfdFffdd18sX748Nm/eHCeddFLU19fH/v37j3lYAKD4jSn0CZdeemlceumlR3wsy7K4995740tf+lJcdtllERHxt3/7t1FZWRkPPfRQfPKTnzy2aQGAojek7/nYsWNHdHV1RV1dXX5bRUVFzJkzJzo6OobypQCAIlXwmY+30tXVFRERlZWVA7ZXVlbmH3ujvr6+6Ovry9/v7e0dypEAgBPMcb/apbW1NSoqKvK3mpqa4z0SADCMhjQ+qqqqIiKiu7t7wPbu7u78Y2/U3NwcPT09+duuXbuGciQA4AQzpPExffr0qKqqivb29vy23t7e2Lx5c9TW1h7xOblcLsrLywfcAICRq+D3fLz66qvx4osv5u/v2LEjtmzZEpMmTYpp06bFjTfeGHfccUe8733vi+nTp8ett94a1dXVcfnllw/l3ABAkSo4Pp5++un42Mc+lr/f1NQUERELFiyIVatWxU033RT79u2LRYsWxZ49e+Liiy+O9evXR1lZ2dBNDQAUrZIsy7LjPcSv6+3tjYqKiujp6RmWH8HcXnL7kB8TAIpJS9Yy5Mcs5Pv3cb/aBQB4ZxEfAEBS4gMASEp8AABJiQ8AICnxAQAkJT4AgKTEBwCQlPgAAJISHwBAUuIDAEhKfAAASYkPACAp8QEAJCU+AICkxAcAkJT4AACSEh8AQFLiAwBISnwAAEmJDwAgKfEBACQlPgCApMQHAJCU+AAAkhIfAEBS4gMASEp8AABJiQ8AICnxAQAkJT4AgKTEBwCQlPgAAJISHwBAUuIDAEhKfAAASYkPACAp8QEAJCU+AICkxAcAkJT4AACSEh8AQFLiAwBISnwAAEmJDwAgKfEBACQlPgCApMQHAJDUsMVHW1tbvOc974mysrKYM2dOPPnkk8P1UgBAERmW+Pj7v//7aGpqipaWlnjmmWfiAx/4QNTX18fLL788HC8HABSRYYmPu+++O6677rq49tpr46yzzorly5fHu971rvjmN785HC8HABSRMUN9wAMHDkRnZ2c0Nzfnt40aNSrq6uqio6PjsP37+vqir68vf7+npyciInp7e4d6tIiI2B/7h+W4AFAshuN77OvHzLLsbfcd8vh45ZVX4tChQ1FZWTlge2VlZfzkJz85bP/W1ta4/fbbD9teU1Mz1KMBABGxpGLJsB177969UVFR8Zb7DHl8FKq5uTmampry9/v7++N//ud/YvLkyVFSUjJkr9Pb2xs1NTWxa9euKC8vH7LjcjhrnYZ1TsM6p2Ot0xiudc6yLPbu3RvV1dVvu++Qx8cpp5wSo0ePju7u7gHbu7u7o6qq6rD9c7lc5HK5AdsmTpw41GPllZeX+6JOxFqnYZ3TsM7pWOs0hmOd3+6Mx+uG/A2npaWlMXv27Ghvb89v6+/vj/b29qitrR3qlwMAisyw/NilqakpFixYEOeff35ccMEFce+998a+ffvi2muvHY6XAwCKyLDEx1VXXRX/9V//Fbfddlt0dXXFBz/4wVi/fv1hb0JNKZfLRUtLy2E/4mHoWes0rHMa1jkda53GibDOJdnRXBMDADBEfLYLAJCU+AAAkhIfAEBS4gMASGpExUdbW1u85z3vibKyspgzZ048+eSTb7n/2rVrY8aMGVFWVhbnnntuPPLII4kmLX6FrPUDDzwQl1xySZx88slx8sknR11d3dv+3fArhX5Nv27NmjVRUlISl19++fAOOEIUus579uyJxsbGmDp1auRyuTjzzDP9+3GUCl3re++9N97//vfHuHHjoqamJhYvXhz79/uMrjfz2GOPxfz586O6ujpKSkrioYceetvnbNiwIc4777zI5XLx3ve+N1atWjXsc0Y2QqxZsyYrLS3NvvnNb2Y//vGPs+uuuy6bOHFi1t3dfcT9n3jiiWz06NHZXXfdlW3dujX70pe+lI0dOzZ77rnnEk9efApd609/+tNZW1tb9uyzz2bPP/989pnPfCarqKjIfv7znyeevLgUus6v27FjR/Ybv/Eb2SWXXJJddtllaYYtYoWuc19fX3b++ednH//4x7PHH38827FjR7Zhw4Zsy5YtiScvPoWu9YMPPpjlcrnswQcfzHbs2JE9+uij2dSpU7PFixcnnrx4PPLII9ktt9ySffvb384iIlu3bt1b7r99+/bsXe96V9bU1JRt3bo1+8Y3vpGNHj06W79+/bDOOWLi44ILLsgaGxvz9w8dOpRVV1dnra2tR9z/yiuvzD7xiU8M2DZnzpzsj/7oj4Z1zpGg0LV+o9deey2bMGFC9jd/8zfDNeKIMJh1fu2117KLLroo++u//utswYIF4uMoFLrOy5Yty04//fTswIEDqUYcMQpd68bGxmzevHkDtjU1NWVz584d1jlHiqOJj5tuuik7++yzB2y76qqrsvr6+mGcLMtGxI9dDhw4EJ2dnVFXV5ffNmrUqKirq4uOjo4jPqejo2PA/hER9fX1b7o/vzKYtX6jX/7yl3Hw4MGYNGnScI1Z9Aa7zl/5yldiypQpsXDhwhRjFr3BrPN3vvOdqK2tjcbGxqisrIxzzjkn7rzzzjh06FCqsYvSYNb6oosuis7OzvyPZrZv3x6PPPJIfPzjH08y8zvB8fpeeNw/1XYovPLKK3Ho0KHDfoNqZWVl/OQnPznic7q6uo64f1dX17DNORIMZq3f6Atf+EJUV1cf9gXP/xnMOj/++OOxYsWK2LJlS4IJR4bBrPP27dvjhz/8YVx99dXxyCOPxIsvvhif+9zn4uDBg9HS0pJi7KI0mLX+9Kc/Ha+88kpcfPHFkWVZvPbaa/HZz342vvjFL6YY+R3hzb4X9vb2xv/+7//GuHHjhuV1R8SZD4rHkiVLYs2aNbFu3booKys73uOMGHv37o1rrrkmHnjggTjllFOO9zgjWn9/f0yZMiXuv//+mD17dlx11VVxyy23xPLly4/3aCPOhg0b4s4774ylS5fGM888E9/+9rfje9/7Xnz1q1893qNxjEbEmY9TTjklRo8eHd3d3QO2d3d3R1VV1RGfU1VVVdD+/Mpg1vp1X//612PJkiXxz//8zzFr1qzhHLPoFbrOP/vZz+Lf//3fY/78+flt/f39ERExZsyY2LZtW5xxxhnDO3QRGszX89SpU2Ps2LExevTo/LaZM2dGV1dXHDhwIEpLS4d15mI1mLW+9dZb45prrok//MM/jIiIc889N/bt2xeLFi2KW265JUaN8v+fj9WbfS8sLy8ftrMeESPkzEdpaWnMnj072tvb89v6+/ujvb09amtrj/ic2traAftHRPzgBz940/35lcGsdUTEXXfdFV/96ldj/fr1cf7556cYtagVus4zZsyI5557LrZs2ZK//e7v/m587GMfiy1btkRNTU3K8YvGYL6e586dGy+++GI+7iIifvrTn8bUqVOFx1sYzFr/8pe/PCwwXo++zMeSDYnj9r1wWN/OmtCaNWuyXC6XrVq1Ktu6dWu2aNGibOLEiVlXV1eWZVl2zTXXZDfffHN+/yeeeCIbM2ZM9vWvfz17/vnns5aWFpfaHqVC13rJkiVZaWlp9g//8A/ZSy+9lL/t3bv3eP0RikKh6/xGrnY5OoWu886dO7MJEyZk119/fbZt27bsu9/9bjZlypTsjjvuOF5/hKJR6Fq3tLRkEyZMyP7u7/4u2759e/ZP//RP2RlnnJFdeeWVx+uPcMLbu3dv9uyzz2bPPvtsFhHZ3XffnT377LPZf/zHf2RZlmU333xzds011+T3f/1S2z//8z/Pnn/++aytrc2ltoX6xje+kU2bNi0rLS3NLrjggmzTpk35xz7ykY9kCxYsGLD/t771rezMM8/MSktLs7PPPjv73ve+l3ji4lXIWp922mlZRBx2a2lpST94kSn0a/rXiY+jV+g6/+hHP8rmzJmT5XK57PTTT8++9rWvZa+99lriqYtTIWt98ODB7Mtf/nJ2xhlnZGVlZVlNTU32uc99LvvFL36RfvAi8S//8i9H/Pf29XVdsGBB9pGPfOSw53zwgx/MSktLs9NPPz1buXLlsM9ZkmXOXQEA6YyI93wAAMVDfAAASYkPACAp8QEAJCU+AICkxAcAkJT4AACSEh8AQFLiAwBISnwAAEmJDwAgKfEBACT1/wAKiPtxcTAQQwAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%time\n",
    "criterion=stats.mannwhitneyu\n",
    "\n",
    "pvalues_week2 = Parallel(n_jobs=-1)(delayed(calc_pvalue)(week2, metric_column, criterion) for _ in range(1000))\n",
    "plt.hist(pvalues_week2, bins=20, color='purple');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "p_value = 0.23430875387568695 - p_values равномерное распределение\n",
      "I type error:  0.051\n"
     ]
    }
   ],
   "source": [
    "stat, p_value = scipy.stats.kstest(pvalues_week2, scipy.stats.uniform(loc=0.0, scale=1.0).cdf)\n",
    "if p_value < 0.05:\n",
    "    print(f'p_value = {p_value} - p_values неравномерное распределение')\n",
    "else:\n",
    "    print(f'p_value = {p_value} - p_values равномерное распределение')\n",
    "\n",
    "x = np.array(pvalues_week2)\n",
    "print(\"I type error: \", x[x < 0.05].shape[0] / x.shape[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- **t-test**:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 1min 41s\n",
      "Wall time: 2min 46s\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh8AAAGdCAYAAACyzRGfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAeJklEQVR4nO3df3DX9X3A8Vf4kYQKCYIlITNYtCr4g7ZixYiuLcuWsz2mZ27a1jnqMVnb6Ca5zTbzR0prDfM6ZfYCTEehvZOxsiuuthbXpQPPNqBGubPFUi1s0GHi3EqCdAQkn/2x87tG8cc3fPOGb/p43H3uzOf7+X7yytucefrJ95NvSZZlWQAAJDLqeA8AAPxmER8AQFLiAwBISnwAAEmJDwAgKfEBACQlPgCApMQHAJDUmOM9wOsNDAzE3r17Y8KECVFSUnK8xwEA3oEsy2L//v1RU1MTo0a99bWNEy4+9u7dG7W1tcd7DABgCPbs2ROnnnrqWx5zwsXHhAkTIuL/hq+oqDjO0wAA70RfX1/U1tbmfo6/lRMuPl77VUtFRYX4AIAi805eMuEFpwBAUuIDAEhKfAAASYkPACAp8QEAJCU+AICkxAcAkJT4AACSEh8AQFLiAwBISnwAAEmJDwAgKfEBACQlPgCApMYc7wEYmZaULBm2c7dmrcN2bgCGnysfAEBSecfHf/zHf8Qf/uEfxuTJk2PcuHFx/vnnx1NPPZV7PMuyuOOOO2Lq1Kkxbty4qK+vj+eff76gQwMAxSuv+PjlL38Zc+fOjbFjx8b3vve92L59e/z1X/91nHzyyblj7r777rjvvvti5cqVsXXr1jjppJOioaEhDh48WPDhAYDik9drPv7qr/4qamtrY/Xq1bl906dPz/1zlmWxbNmyuO222+KKK66IiIhvfOMbUVVVFQ899FB8/OMfL9DYAECxyuvKx7e//e248MIL4w/+4A9iypQp8YEPfCAeeOCB3OO7du2K7u7uqK+vz+2rrKyMOXPmRGdn51HP2d/fH319fYM2AGDkyis+du7cGStWrIgzzzwzHn300fjMZz4Tf/qnfxpf//rXIyKiu7s7IiKqqqoGPa+qqir32Ou1tbVFZWVlbqutrR3K1wEAFIm84mNgYCAuuOCCuOuuu+IDH/hALFq0KG644YZYuXLlkAdoaWmJ3t7e3LZnz54hnwsAOPHlFR9Tp06Nc845Z9C+mTNnxu7duyMiorq6OiIienp6Bh3T09OTe+z1ysrKoqKiYtAGAIxcecXH3LlzY8eOHYP2/exnP4vTTjstIv7vxafV1dXR0dGRe7yvry+2bt0adXV1BRgXACh2ed3tsnjx4rjkkkvirrvuiquvvjqeeOKJuP/+++P++++PiIiSkpK4+eab484774wzzzwzpk+fHrfffnvU1NTElVdeORzzAwBFJq/4+OAHPxgbNmyIlpaW+OIXvxjTp0+PZcuWxbXXXps75pZbbokDBw7EokWLYt++fXHppZfGxo0bo7y8vODDAwDFpyTLsux4D/Hr+vr6orKyMnp7e73+I4HhfA+W4eK9XQBOPPn8/PbeLgBAUuIDAEhKfAAASYkPACAp8QEAJCU+AICkxAcAkJT4AACSEh8AQFLiAwBIKq/3dgFOLMP55/H9GXtguLjyAQAkJT4AgKTEBwCQlPgAAJISHwBAUu52AWDEc2fYicWVDwAgKfEBACQlPgCApMQHAJCU+AAAknK3C5CUuw4AVz4AgKTEBwCQlPgAAJISHwBAUuIDAEjK3S4Ab8MdOukM51pz4nDlAwBISnwAAEmJDwAgKfEBACQlPgCApMQHAJCU+AAAkhIfAEBS4gMASEp8AABJ+fPqFJ3h+vPL/sw1QBqufAAASYkPACAp8QEAJCU+AICkxAcAkJS7XQDgGLgDL3+ufAAASeUVH1/4wheipKRk0DZjxozc4wcPHoympqaYPHlyjB8/PhobG6Onp6fgQwMAxSvvKx/nnntuvPjii7nt8ccfzz22ePHiePjhh2P9+vWxefPm2Lt3b1x11VUFHRgAKG55v+ZjzJgxUV1d/Yb9vb29sWrVqli7dm3MmzcvIiJWr14dM2fOjC1btsTFF1987NMCAEUv7ysfzz//fNTU1MTpp58e1157bezevTsiIrq6uuLw4cNRX1+fO3bGjBkxbdq06OzsfNPz9ff3R19f36ANABi58rryMWfOnFizZk2cffbZ8eKLL8aSJUvisssuix//+MfR3d0dpaWlMXHixEHPqaqqiu7u7jc9Z1tbWyxZMjyvFB4phuuV1PBWfN8BwyWv+Lj88stz/zxr1qyYM2dOnHbaafHNb34zxo0bN6QBWlpaorm5OfdxX19f1NbWDulcAMCJ75hutZ04cWKcddZZ8cILL0R1dXUcOnQo9u3bN+iYnp6eo75G5DVlZWVRUVExaAMARq5jio9XXnklfv7zn8fUqVNj9uzZMXbs2Ojo6Mg9vmPHjti9e3fU1dUd86AAwMiQ169d/vzP/zzmz58fp512WuzduzdaW1tj9OjR8YlPfCIqKytj4cKF0dzcHJMmTYqKioq46aaboq6uzp0uAEBOXvHxi1/8Ij7xiU/Ef/3Xf8W73/3uuPTSS2PLli3x7ne/OyIi7r333hg1alQ0NjZGf39/NDQ0xPLly4dlcACgOOUVH+vWrXvLx8vLy6O9vT3a29uPaSgATlzuhOJYeW8XACAp8QEAJCU+AICkxAcAkFTebywHQOEM54s3W7PWYTs3HAtXPgCApMQHAJCU+AAAkhIfAEBS4gMASEp8AABJiQ8AICnxAQAkJT4AgKTEBwCQlPgAAJISHwBAUuIDAEhKfAAASYkPACAp8QEAJCU+AICkxAcAkJT4AACSEh8AQFLiAwBISnwAAEmNOd4DpLakZMmwnLc1ax2W8zIyDNf3HYNZ58GsR3Ebzn9/x/tnlisfAEBS4gMASEp8AABJiQ8AICnxAQAk9Rt3twu8GXcGAKThygcAkJT4AACSEh8AQFLiAwBISnwAAEmJDwAgKfEBACQlPgCApMQHAJCU+AAAkhIfAEBS3tulQLwvCAC8M8d05WPp0qVRUlISN998c27fwYMHo6mpKSZPnhzjx4+PxsbG6OnpOdY5AYARYsjx8eSTT8bf/u3fxqxZswbtX7x4cTz88MOxfv362Lx5c+zduzeuuuqqYx4UABgZhhQfr7zySlx77bXxwAMPxMknn5zb39vbG6tWrYp77rkn5s2bF7Nnz47Vq1fHj370o9iyZUvBhgYAiteQ4qOpqSk+9rGPRX19/aD9XV1dcfjw4UH7Z8yYEdOmTYvOzs6jnqu/vz/6+voGbQDAyJX3C07XrVsXTz/9dDz55JNveKy7uztKS0tj4sSJg/ZXVVVFd3f3Uc/X1tYWS5Z4sSYA/KbI68rHnj174s/+7M/iwQcfjPLy8oIM0NLSEr29vbltz549BTkvAHBiyis+urq64qWXXooLLrggxowZE2PGjInNmzfHfffdF2PGjImqqqo4dOhQ7Nu3b9Dzenp6orq6+qjnLCsri4qKikEbADBy5fVrl9/5nd+JZ599dtC+66+/PmbMmBGf+9znora2NsaOHRsdHR3R2NgYERE7duyI3bt3R11dXeGmBgCKVl7xMWHChDjvvPMG7TvppJNi8uTJuf0LFy6M5ubmmDRpUlRUVMRNN90UdXV1cfHFFxduagCgaBX8L5zee++9MWrUqGhsbIz+/v5oaGiI5cuXF/rTAABF6pjjY9OmTYM+Li8vj/b29mhvbz/WUwMAI5A3lgMAkhIfAEBS4gMASEp8AABJiQ8AICnxAQAkJT4AgKTEBwCQlPgAAJISHwBAUuIDAEhKfAAASYkPACAp8QEAJCU+AICkxAcAkJT4AACSEh8AQFLiAwBISnwAAEmJDwAgKfEBACQlPgCApMQHAJCU+AAAkhIfAEBS4gMASEp8AABJiQ8AICnxAQAkJT4AgKTEBwCQlPgAAJISHwBAUuIDAEhKfAAASYkPACAp8QEAJCU+AICkxAcAkJT4AACSEh8AQFLiAwBISnwAAEmJDwAgKfEBACQlPgCApPKKjxUrVsSsWbOioqIiKioqoq6uLr73ve/lHj948GA0NTXF5MmTY/z48dHY2Bg9PT0FHxoAKF55xcepp54aS5cuja6urnjqqadi3rx5ccUVV8RPfvKTiIhYvHhxPPzww7F+/frYvHlz7N27N6666qphGRwAKE5j8jl4/vz5gz7+8pe/HCtWrIgtW7bEqaeeGqtWrYq1a9fGvHnzIiJi9erVMXPmzNiyZUtcfPHFhZsaAChaQ37Nx5EjR2LdunVx4MCBqKuri66urjh8+HDU19fnjpkxY0ZMmzYtOjs73/Q8/f390dfXN2gDAEauvOPj2WefjfHjx0dZWVl8+tOfjg0bNsQ555wT3d3dUVpaGhMnThx0fFVVVXR3d7/p+dra2qKysjK31dbW5v1FAADFI+/4OPvss2Pbtm2xdevW+MxnPhMLFiyI7du3D3mAlpaW6O3tzW179uwZ8rkAgBNfXq/5iIgoLS2N9773vRERMXv27HjyySfjb/7mb+Kaa66JQ4cOxb59+wZd/ejp6Ynq6uo3PV9ZWVmUlZXlPzkAUJSO+e98DAwMRH9/f8yePTvGjh0bHR0ducd27NgRu3fvjrq6umP9NADACJHXlY+Wlpa4/PLLY9q0abF///5Yu3ZtbNq0KR599NGorKyMhQsXRnNzc0yaNCkqKiripptuirq6One6AAA5ecXHSy+9FH/0R38UL774YlRWVsasWbPi0Ucfjd/93d+NiIh77703Ro0aFY2NjdHf3x8NDQ2xfPnyYRkcAChOJVmWZcd7iF/X19cXlZWV0dvbGxUVFQU//5KSJQU/JwAUk9asteDnzOfnt/d2AQCSEh8AQFLiAwBISnwAAEmJDwAgKfEBACQlPgCApMQHAJCU+AAAkhIfAEBS4gMASEp8AABJiQ8AICnxAQAkJT4AgKTEBwCQlPgAAJISHwBAUuIDAEhKfAAASYkPACAp8QEAJCU+AICkxAcAkJT4AACSEh8AQFLiAwBISnwAAEmJDwAgKfEBACQlPgCApMQHAJCU+AAAkhIfAEBS4gMASEp8AABJiQ8AICnxAQAkJT4AgKTEBwCQlPgAAJISHwBAUuIDAEhKfAAASYkPACAp8QEAJCU+AICk8oqPtra2+OAHPxgTJkyIKVOmxJVXXhk7duwYdMzBgwejqakpJk+eHOPHj4/Gxsbo6ekp6NAAQPHKKz42b94cTU1NsWXLlvj+978fhw8fjt/7vd+LAwcO5I5ZvHhxPPzww7F+/frYvHlz7N27N6666qqCDw4AFKcx+Ry8cePGQR+vWbMmpkyZEl1dXfHbv/3b0dvbG6tWrYq1a9fGvHnzIiJi9erVMXPmzNiyZUtcfPHFhZscAChKx/Saj97e3oiImDRpUkREdHV1xeHDh6O+vj53zIwZM2LatGnR2dl51HP09/dHX1/foA0AGLmGHB8DAwNx8803x9y5c+O8886LiIju7u4oLS2NiRMnDjq2qqoquru7j3qetra2qKyszG21tbVDHQkAKAJDjo+mpqb48Y9/HOvWrTumAVpaWqK3tze37dmz55jOBwCc2PJ6zcdrbrzxxvjOd74Tjz32WJx66qm5/dXV1XHo0KHYt2/foKsfPT09UV1dfdRzlZWVRVlZ2VDGAACKUF5XPrIsixtvvDE2bNgQP/jBD2L69OmDHp89e3aMHTs2Ojo6cvt27NgRu3fvjrq6usJMDAAUtbyufDQ1NcXatWvjn/7pn2LChAm513FUVlbGuHHjorKyMhYuXBjNzc0xadKkqKioiJtuuinq6urc6QIARESe8bFixYqIiPjwhz88aP/q1avjU5/6VERE3HvvvTFq1KhobGyM/v7+aGhoiOXLlxdkWACg+OUVH1mWve0x5eXl0d7eHu3t7UMeCgAYuby3CwCQlPgAAJISHwBAUuIDAEhKfAAASYkPACAp8QEAJCU+AICkxAcAkJT4AACSEh8AQFLiAwBISnwAAEmJDwAgKfEBACQlPgCApMQHAJCU+AAAkhIfAEBS4gMASEp8AABJiQ8AICnxAQAkJT4AgKTEBwCQlPgAAJISHwBAUuIDAEhKfAAASYkPACAp8QEAJCU+AICkxAcAkJT4AACSEh8AQFLiAwBISnwAAEmJDwAgKfEBACQlPgCApMQHAJCU+AAAkhIfAEBS4gMASEp8AABJiQ8AIKm84+Oxxx6L+fPnR01NTZSUlMRDDz006PEsy+KOO+6IqVOnxrhx46K+vj6ef/75Qs0LABS5vOPjwIED8b73vS/a29uP+vjdd98d9913X6xcuTK2bt0aJ510UjQ0NMTBgwePeVgAoPiNyfcJl19+eVx++eVHfSzLsli2bFncdtttccUVV0RExDe+8Y2oqqqKhx56KD7+8Y8f27QAQNEr6Gs+du3aFd3d3VFfX5/bV1lZGXPmzInOzs5CfioAoEjlfeXjrXR3d0dERFVV1aD9VVVVucder7+/P/r7+3Mf9/X1FXIkAOAEc9zvdmlra4vKysrcVltbe7xHAgCGUUHjo7q6OiIienp6Bu3v6enJPfZ6LS0t0dvbm9v27NlTyJEAgBNMQeNj+vTpUV1dHR0dHbl9fX19sXXr1qirqzvqc8rKyqKiomLQBgCMXHm/5uOVV16JF154Iffxrl27Ytu2bTFp0qSYNm1a3HzzzXHnnXfGmWeeGdOnT4/bb789ampq4sorryzk3ABAkco7Pp566qn4yEc+kvu4ubk5IiIWLFgQa9asiVtuuSUOHDgQixYtin379sWll14aGzdujPLy8sJNDQAUrZIsy7LjPcSv6+vri8rKyujt7R2WX8EsKVlS8HMCQDFpzVoLfs58fn4f97tdAIDfLOIDAEhKfAAASYkPACAp8QEAJCU+AICkxAcAkJT4AACSEh8AQFLiAwBISnwAAEmJDwAgKfEBACQlPgCApMQHAJCU+AAAkhIfAEBS4gMASEp8AABJiQ8AICnxAQAkJT4AgKTEBwCQlPgAAJISHwBAUuIDAEhKfAAASYkPACAp8QEAJCU+AICkxAcAkJT4AACSEh8AQFLiAwBISnwAAEmJDwAgKfEBACQlPgCApMQHAJCU+AAAkhIfAEBS4gMASEp8AABJiQ8AICnxAQAkJT4AgKTEBwCQ1LDFR3t7e7znPe+J8vLymDNnTjzxxBPD9akAgCIyLPHxD//wD9Hc3Bytra3x9NNPx/ve975oaGiIl156aTg+HQBQRIYlPu6555644YYb4vrrr49zzjknVq5cGe9617via1/72nB8OgCgiIwp9AkPHToUXV1d0dLSkts3atSoqK+vj87Ozjcc39/fH/39/bmPe3t7IyKir6+v0KNFRMTBODgs5wWAYjEcP2NfO2eWZW97bMHj4+WXX44jR45EVVXVoP1VVVXx05/+9A3Ht7W1xZIlS96wv7a2ttCjAQARsbRy6bCde//+/VFZWfmWxxQ8PvLV0tISzc3NuY8HBgbiv//7v2Py5MlRUlJS0M/V19cXtbW1sWfPnqioqCjoufl/1jkN65yOtU7DOqcxXOucZVns378/ampq3vbYgsfHKaecEqNHj46enp5B+3t6eqK6uvoNx5eVlUVZWdmgfRMnTiz0WINUVFT4xk7AOqdhndOx1mlY5zSGY53f7orHawr+gtPS0tKYPXt2dHR05PYNDAxER0dH1NXVFfrTAQBFZlh+7dLc3BwLFiyICy+8MC666KJYtmxZHDhwIK6//vrh+HQAQBEZlvi45ppr4j//8z/jjjvuiO7u7nj/+98fGzdufMOLUFMrKyuL1tbWN/yah8KyzmlY53SsdRrWOY0TYZ1LsndyTwwAQIF4bxcAICnxAQAkJT4AgKTEBwCQ1IiLj/b29njPe94T5eXlMWfOnHjiiSfe8vj169fHjBkzory8PM4///x45JFHEk1a3PJZ5wceeCAuu+yyOPnkk+Pkk0+O+vr6t/33wv/J9/v5NevWrYuSkpK48sorh3fAESLfdd63b180NTXF1KlTo6ysLM466yz/7XiH8l3rZcuWxdlnnx3jxo2L2traWLx4cRw86D263spjjz0W8+fPj5qamigpKYmHHnrobZ+zadOmuOCCC6KsrCze+973xpo1a4Z3yGwEWbduXVZaWpp97Wtfy37yk59kN9xwQzZx4sSsp6fnqMf/8Ic/zEaPHp3dfffd2fbt27PbbrstGzt2bPbss88mnry45LvOn/zkJ7P29vbsmWeeyZ577rnsU5/6VFZZWZn94he/SDx5ccl3nV+za9eu7Ld+67eyyy67LLviiivSDFvE8l3n/v7+7MILL8w++tGPZo8//ni2a9eubNOmTdm2bdsST1588l3rBx98MCsrK8sefPDBbNeuXdmjjz6aTZ06NVu8eHHiyYvLI488kt16663Zt771rSwisg0bNrzl8Tt37sze9a53Zc3Nzdn27duzr371q9no0aOzjRs3DtuMIyo+LrrooqypqSn38ZEjR7Kampqsra3tqMdfffXV2cc+9rFB++bMmZP9yZ/8ybDOWezyXefXe/XVV7MJEyZkX//614drxBFhKOv86quvZpdcckn2d3/3d9mCBQvExzuQ7zqvWLEiO/3007NDhw6lGnHEyHetm5qasnnz5g3a19zcnM2dO3dY5xxJ3kl83HLLLdm55547aN8111yTNTQ0DNtcI+bXLocOHYqurq6or6/P7Rs1alTU19dHZ2fnUZ/T2dk56PiIiIaGhjc9nqGt8+v96le/isOHD8ekSZOGa8yiN9R1/uIXvxhTpkyJhQsXphiz6A1lnb/97W9HXV1dNDU1RVVVVZx33nlx1113xZEjR1KNXZSGstaXXHJJdHV15X41s3PnznjkkUfiox/9aJKZf1Mcj5+Fx/1dbQvl5ZdfjiNHjrzhr6hWVVXFT3/606M+p7u7+6jHd3d3D9ucxW4o6/x6n/vc56KmpuYN3+z8v6Gs8+OPPx6rVq2Kbdu2JZhwZBjKOu/cuTN+8IMfxLXXXhuPPPJIvPDCC/HZz342Dh8+HK2trSnGLkpDWetPfvKT8fLLL8ell14aWZbFq6++Gp/+9KfjL//yL1OM/BvjzX4W9vX1xf/8z//EuHHjCv45R8yVD4rD0qVLY926dbFhw4YoLy8/3uOMGPv374/rrrsuHnjggTjllFOO9zgj2sDAQEyZMiXuv//+mD17dlxzzTVx6623xsqVK4/3aCPOpk2b4q677orly5fH008/Hd/61rfiu9/9bnzpS1863qNxjEbMlY9TTjklRo8eHT09PYP29/T0RHV19VGfU11dndfxDG2dX/OVr3wlli5dGv/yL/8Ss2bNGs4xi16+6/zzn/88/u3f/i3mz5+f2zcwMBAREWPGjIkdO3bEGWecMbxDF6GhfD9PnTo1xo4dG6NHj87tmzlzZnR3d8ehQ4eitLR0WGcuVkNZ69tvvz2uu+66+OM//uOIiDj//PPjwIEDsWjRorj11ltj1Cj//1wIb/azsKKiYliuekSMoCsfpaWlMXv27Ojo6MjtGxgYiI6Ojqirqzvqc+rq6gYdHxHx/e9//02PZ2jrHBFx9913x5e+9KXYuHFjXHjhhSlGLWr5rvOMGTPi2WefjW3btuW23//934+PfOQjsW3btqitrU05ftEYyvfz3Llz44UXXsjFXUTEz372s5g6darweAtDWetf/epXbwiM16Iv87ZkBXNcfhYO20tZj4N169ZlZWVl2Zo1a7Lt27dnixYtyiZOnJh1d3dnWZZl1113Xfb5z38+d/wPf/jDbMyYMdlXvvKV7LnnnstaW1vdavsO5LvOS5cuzUpLS7N//Md/zF588cXctn///uP1JRSFfNf59dzt8s7ku867d+/OJkyYkN14443Zjh07su985zvZlClTsjvvvPN4fQlFI9+1bm1tzSZMmJD9/d//fbZz587sn//5n7Mzzjgju/rqq4/Xl1AU9u/fnz3zzDPZM888k0VEds8992TPPPNM9u///u9ZlmXZ5z//+ey6667LHf/arbZ/8Rd/kT333HNZe3u7W23z9dWvfjWbNm1aVlpaml100UXZli1bco996EMfyhYsWDDo+G9+85vZWWedlZWWlmbnnntu9t3vfjfxxMUpn3U+7bTTsoh4w9ba2pp+8CKT7/fzrxMf71y+6/yjH/0omzNnTlZWVpadfvrp2Ze//OXs1VdfTTx1ccpnrQ8fPpx94QtfyM4444ysvLw8q62tzT772c9mv/zlL9MPXkT+9V//9aj/zX1tbRcsWJB96EMfesNz3v/+92elpaXZ6aefnq1evXpYZyzJMteuAIB0RsxrPgCA4iA+AICkxAcAkJT4AACSEh8AQFLiAwBISnwAAEmJDwAgKfEBACQlPgCApMQHAJCU+AAAkvpf3F3r91RJVPUAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%time\n",
    "criterion=stats.ttest_ind\n",
    "\n",
    "pvalues_week2_ttest = Parallel(n_jobs=-1)(delayed(calc_pvalue)(week2, metric_column, criterion) for _ in range(1000))\n",
    "plt.hist(pvalues_week2_ttest, bins=20, color='purple');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "p_value = 0.8484785028403148 - p_values равномерное распределение\n",
      "I type error:  0.044\n"
     ]
    }
   ],
   "source": [
    "stat, p_value = scipy.stats.kstest(pvalues_week2_ttest, scipy.stats.uniform(loc=0.0, scale=1.0).cdf)\n",
    "if p_value < 0.05:\n",
    "    print(f'p_value = {p_value} - p_values неравномерное распределение')\n",
    "else:\n",
    "    print(f'p_value = {p_value} - p_values равномерное распределение')\n",
    "\n",
    "x = np.array(pvalues_week2_ttest)\n",
    "print(\"I type error: \", x[x < 0.05].shape[0] / x.shape[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Вывод**:\n",
    "\n",
    "- все распределения ```p_values``` равномерны\n",
    "- ошибка первого рода варьируется около 0.05 для t-test и Манна-Уитни в зависимости от разбиения"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "source": [
    "## 3. Дизайн A/B на Бутстрапе"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "N = 1000"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "source": [
    "### 3.1. Корректность"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "@njit(parallel=True)\n",
    "def calc_bootstrap_mean(arr: np.array, size = 1000) -> np.array:\n",
    "    samplings = np.empty(size, dtype=float32)\n",
    "    for i in range(size):\n",
    "        sampling = np.random.choice(arr, size=arr.size, replace=True)\n",
    "        mean = np.mean(sampling)\n",
    "        samplings[i] = mean\n",
    "    return samplings\n",
    "\n",
    "\n",
    "def get_p_value(dist):\n",
    "    dist = np.array(dist)\n",
    "    x = (dist > 0).mean()\n",
    "    pvalue = min(x, 1 - x) * 2\n",
    "    return pvalue\n",
    "\n",
    "\n",
    "def bootstrap_correctness(week: pd.DataFrame, metric_column: str, alpha=0.05) -> (np.array, bool):\n",
    "    new_df = groups_splitter(week, columns=['user_id', metric_column, 'group'])\n",
    "\n",
    "    vec_a = new_df[(new_df['group'] == 'A')][metric_column]\n",
    "    vec_b = new_df[(new_df['group'] == 'B')][metric_column]\n",
    "\n",
    "    sample_a_mean = calc_bootstrap_mean(vec_a.to_numpy(), size=1000)\n",
    "    sample_b_mean = calc_bootstrap_mean(vec_b.to_numpy(), size=1000)\n",
    "    diff_mean = sample_a_mean - sample_b_mean\n",
    "\n",
    "    # Compute the confidence interval (e.g., 95% CI)\n",
    "    left_side, right_side = np.percentile(diff_mean, [100 * alpha / 2., 100 * (1 - alpha / 2.)])\n",
    "    # Reject equality of means when it's true\n",
    "    is_this_first_type_error = not left_side <= 0 <= right_side\n",
    "    \n",
    "    return get_p_value(diff_mean), is_this_first_type_error\n",
    "\n",
    "\n",
    "def test_correctness(week, metric_column):\n",
    "    result = Parallel(n_jobs=-1)(delayed(bootstrap_correctness)(week, metric_column) for _ in range(N))\n",
    "\n",
    "    p_values = [x[0] for x in result]\n",
    "\n",
    "    first_type_errors = [x[1] for x in result]\n",
    "    alpha = sum(first_type_errors) / len(first_type_errors)\n",
    "\n",
    "    print(f\"I type error: {alpha:.2f}\")\n",
    "\n",
    "    stat, p_value = scipy.stats.kstest(p_values, scipy.stats.uniform(loc=0.0, scale=1.0).cdf)\n",
    "    if p_value < 0.05:\n",
    "        print(f'p_value = {p_value} - p_values неравномерное распределение')\n",
    "    else:\n",
    "        print(f'p_value = {p_value} - p_values равномерное распределение')\n",
    "    return p_values, alpha"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "metric_column=\"total_dur\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I type error: 0.05\n",
      "p_value = 0.023344526390774223 - p_values неравномерное распределение\n",
      "CPU times: total: 2min 31s\n",
      "Wall time: 14min 16s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "p_values, alpha = test_correctness(week2, metric_column)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAEpCAYAAACz/8hbAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAApt0lEQVR4nO3de1xUZf4H8M9wm0GEUVwZwEVA1MXM1sREvEQqRUauJJs3UjQT1qBW+W0mKZJXvEcqXmu9lK6KqZupuOa1DDVRe1leU1TKQE1hUJbrPL8/fHnWgVEZGMBn/Lxfr3m9nOc85znfZ+bw4cyZw1ElhBAgIiJp2NR3AUREZB4GNxGRZBjcRESSYXATEUmGwU1EJBkGNxGRZBjcRESSYXATEUmGwU1EJBkGNxERgGHDhsHHx6e+y6gSqw/ulStXQqVSPfDxySef1HeJRERmsavvAurK5MmT4evrW6k9KCioHqohIqq+Jya4e/fujY4dO9Z3GURENWb1p0qqSggBHx8f9O3bt9KyoqIiaLVaxMTEYN++fQ899aJSqfDhhx8q6/76668YMWIEPD09oVar4evri1GjRqGkpOSRp3FUKhVWrlwJ4O75t4YNG+LixYsIDQ2Fk5MTPD09MXnyZFS8waPBYEBKSgratm0LjUYDnU6HmJgY3Lp1y6ifj4/PQ7f9KC+88AKefvppZGZmokuXLnB0dISvry+WLFli1K+kpAQTJ05EQEAAtFotnJyc0L17d+zdu7fSmAaDAR9//DHatWsHjUaDpk2b4uWXX8bRo0eVPg+r+f5zlJcuXYJKpcKcOXPw0UcfwdvbG46OjggODsaPP/5YadtnzpzBX//6V7i6ukKj0aBjx4748ssvTc592LBhJrc/bNiwSn137NiB7t27w8nJCc7OzggLC8NPP/1kctwHzevSpUtG/T7//HMEBATA0dERrq6uGDhwILKzs4363Ht/KpozZ06lMX18fCrVnpaWVuk1Baq+f5lizn78MHFxcWjYsCEKCwsrLRs0aBDc3d1RXl4OAPj3v/+NsLAw5WfQz88PU6ZMUZY/yL2f9X379hm139uv7v1s3mPO/lNTT8wRd35+Pm7cuAGVSgWtVgs7O+Opq1QqvPHGG5g1axZu3rwJV1dXZdnWrVuh1+vxxhtvoHXr1vjss8+UZcuWLcPp06fx0UcfKW3PPPMMAODq1avo1KkT8vLyEB0dDX9/f/z666/YuHEjCgsL8fzzzxuNNW3aNADA+PHjlbYuXboo/y4vL8fLL7+Mzp07Y9asWUhPT0dSUhLKysowefJkpV9MTAxWrlyJ4cOH491330VWVhYWLlyI48eP4+DBg7C3t1f6tm/fHv/3f/9n9FqsXr0au3btqtLreuvWLbzyyivo378/Bg0ahA0bNmDUqFFwcHDAm2++CQDQ6/X45JNPMGjQIIwcORIFBQX49NNPERoaiiNHjqB9+/bKeCNGjMDKlSvRu3dvvPXWWygrK8M333yDQ4cOGX1ievHFFzF06FCjWubOnWsyPFavXo2CggLExsaiqKgIH3/8MXr27ImTJ09Cp9MBAH766Sd07doVzZo1w7hx4+Dk5IQNGzYgPDwcX3zxBV577bVK46rVaqPvSN56661KfT777DNERUUhNDQUM2fORGFhIRYvXoxu3brh+PHjJr8Me+2119CvXz8AwDfffINly5YZLZ82bRoSExPRv39/vPXWW7h+/ToWLFiA559/HsePH0ejRo0qjWmusrIyo/3wfubsX6ZUdT9+mAEDBiA1NRXbtm3D66+/rrQXFhZi69atGDZsGGxtbQHc/Z6rYcOGiI+PR8OGDbFnzx5MnDgRer0es2fPruIr8nDV2X9qRFi5FStWCABGDzs7O9GqVSuxYMECUVpaqvQ9e/asACAWL15sNMZf/vIX4ePjIwwGQ6Xxo6KihLe3t8ltDx06VNjY2Ijvv/++0jJTYwUHB4vg4GCTY0VFRQkA4p133jEaIywsTDg4OIjr168LIYT45ptvBACxZs0ao/XT09MrtXt7e4uwsLBK24qNjRVV2TWCg4MFADF37lylrbi4WLRv3164ubmJkpISIYQQZWVlori42GjdW7duCZ1OJ958802lbc+ePQKAePfddytt6/7XC4CIjY2t1CcsLMzovcjKyhIAhKOjo/jll1+U9sOHDwsAYsyYMUpbr169RLt27URRUZHRNrt06SJatWpVaVuDBw8WDRs2NGpzcnISUVFRyvOCggLRqFEjMXLkSKN+OTk5QqvVVmovLS0VAMSkSZOUtnv7b1ZWlhBCiEuXLglbW1sxbdo0o3VPnjwp7OzsjNqDg4NF27ZtK9U+e/ZsozGFuLsv3F/7okWLhFqtFj169DB6Tc3Zv0yp6n78KAaDQTRr1kxEREQYtW/YsEEAEAcOHFDaCgsLK60fExMjGjRoYPR+V/xZ3rt3rwAg9u7da7Tuvf1qxYoVSpu5+09NPTGnSlJTU7Fr1y7s2rULK1euhK+vL9555x2MHj1a6dO6dWsEBgZizZo1StvNmzexY8cOREZGVun0wT0GgwFbtmxBnz59TJ5bN2es+8XFxRmNERcXh5KSEnz99dcA7n681Wq1ePHFF3Hjxg3lERAQgIYNG5o8PVETdnZ2iImJUZ47ODggJiYG165dQ2ZmJgDA1tYWDg4OAO6+Ljdv3kRZWRk6duyIY8eOKet+8cUXUKlUSEpKqrSd6r5eABAeHo5mzZopzzt16oTAwEBs374dwN33eM+ePejfvz8KCgqU1+z3339HaGgozp8/j19//dVozKKiImg0modud9euXcjLy8OgQYOM3gtbW1sEBgZWei9KSkoA3D2Sf5BNmzbBYDCgf//+RmO6u7ujVatWlcYsLy836nfjxg2TpxfuV1hYiMmTJyMuLg7Nmzc3Wmap/etR+/GjqFQqvP7669i+fTtu376ttK9fvx7NmjVDt27dlDZHR0fl3/fe3+7du6OwsBBnzpyp0vYepjr7T009MadKOnXqZBSgAwcORPv27bF06VJMmzYNWq0WADB06FDExcXh8uXL8Pb2RlpaGkpLSzFkyBCztnf9+nXo9XqT5xiry8bGBi1atDBqa926NQAo5yvPnz+P/Px8uLm5mRzj2rVrFqsHADw9PeHk5PTAmjp37gwAWLVqFebOnYszZ86gtLRU6Xv/lT4XLlyAp6en0WkqS2jVqlWlttatW2PDhg0AgJ9//hlCCCQmJiIxMdHkGNeuXTMK/xs3bij7zIOcP38eANCzZ0+Ty11cXIye5+XlAQAaNmz40DGFECbnBKDSaYozZ86gadOmD62zonnz5qGoqAgffPAB4uPjK22/pvtXVfbjqhgwYABSUlLw5ZdfYvDgwbh9+za2b9+OmJgYo1/0P/30EyZMmIA9e/ZAr9cbjZGfn1/l7T1IdfafmnpigrsiW1tb9OrVCz/++CPOnDmDwMBAAHcDfcyYMVizZg0++OADfP755+jYsSP+9Kc/1XPFVWMwGODm5mb0qeF+5v4QW8Lnn3+OYcOGITw8HO+99x7c3Nxga2uL5ORkXLhwoc7rqchgMAAA/vGPfyA0NNRkn5YtWxo9v3Tp0gPDs+K4n332Gdzd3Sstr/g9S05ODgCY7Hv/mCqVCjt27FDO4d6vYuj7+Phg+fLlRm1paWmVzpvfc+PGDcyePRsJCQkmf4E+TvtX586d4ePjgw0bNmDw4MHYunUr/vvf/2LAgAFKn7y8PAQHB8PFxQWTJ0+Gn58fNBoNjh07hvfff195j0x50Ke8il9qVmf/qaknNriB/70BNjb/O2Pk6uqKsLAwrFmzBpGRkTh48CBSUlLMHrtp06ZwcXExefVCdRkMBly8eFE5OgGAc+fOAYDyJZefnx++/vprdO3a1egjYm25evUq7ty5Y3TUXbGmjRs3okWLFti0aZPRD0PFUyJ+fn7YuXNnpS+Ha+reke/9zp07p9R37+jP3t4eISEhjxzv+vXruHLlCgYNGvTQfn5+fgAANze3Ko176tQpAECbNm0eOqYQAr6+vkb7wYM4OTlV2vaJEyce2H/q1KlwdnbG3//+9wduv6b7V1X246rq378/Pv74Y+j1eqxfvx4+Pj7Kpzzg7pUhv//+OzZt2oTnn39eac/Kynrk2I0bNwbwv09C91y+fNnoubn7jyU8Mee4KyotLcXOnTthb28Pf39/o2VDhgzBqVOn8N5778HW1hYDBw40e3wbGxuEh4dj69atRpey3SOq+X80L1y40GiMhQsXwt7eHr169QJwd0cuLy/HlClTKq1bVlZWaSesqbKyMixdulR5XlJSgqVLl6Jp06YICAgAAOXI8P45Hz58GBkZGUZjRUREQAiBSZMmVdpOdV8vANiyZYvROcYjR47g8OHD6N27N4C7wfrCCy9g6dKl+O233yqtf/36daPnaWlpAGDy0tH7hYaGwsXFBdOnTzc6PfSgcdevXw8PD4+HBne/fv1ga2uLSZMmVXpNhBD4/fffH1rTw1y6dAmLFy/Ghx9++MBQttT+9aj9uKoGDBiA4uJirFq1Cunp6ejfv7/RclP7XklJCRYtWvTIsb29vWFra4sDBw4YtVdc19z9xxKemCPuHTt2KF9EXLt2DWvXrsX58+cxZswYODs7G/UNCwtDkyZNkJaWht69ez/wfN6jTJ8+Hf/5z38QHByM6OhotGnTBr/99hvS0tLw7bffmn3ZlkajQXp6OqKiohAYGIgdO3Zg27Zt+OCDD5SPqMHBwYiJiUFycjJOnDiBl156Cfb29jh//jzS0tLw8ccf469//Wu15mOKp6cnZs6ciUuXLqF169ZYv349Tpw4gWXLlinnW1999VVs2rQJr732GsLCwpCVlYUlS5bgqaeeMvpiqUePHhgyZAjmz5+P8+fP4+WXX4bBYMA333yDHj16GH2hZY6WLVuiW7duGDVqFIqLi5GSkoImTZpg7NixSp/U1FR069YN7dq1w8iRI9GiRQvk5uYiIyMDv/zyC3744Qel34QJE9C0aVNcuHDB6FRPWVkZLl68iF27duHFF1+Ei4sLFi9ejCFDhqBDhw4YOHAgmjZtiitXrmDbtm3o2rUrFi5ciKNHjyIxMRHp6elYsmTJQ7+I9fPzw9SpU5GQkIBLly4hPDwczs7OyMrKwubNmxEdHY1//OMf1Xqd9u/fjzZt2mD48OEP7GOJ/asq+3FVdejQAS1btsT48eNRXFxsdJoEuHs5bePGjREVFYV3330XKpUKn332WZUOBLRaLV5//XUsWLAAKpUKfn5++Oqrr0yex6/q/mMxFr9O5TFj6nJAjUYj2rVrJ+bNmyfKy8tNrvf2228LAGLt2rUPHf9hlwMKIcTly5fF0KFDRdOmTYVarRYtWrQQsbGxlS6PE+LRlwM6OTmJCxcuiJdeekk0aNBA6HQ6kZSUZHIOy5YtEwEBAcLR0VE4OzuLdu3aibFjx4qrV68qfSxxOWDbtm3F0aNHRVBQkNBoNMLb21ssXLjQqJ/BYBDTp08X3t7eQq1Wi2effVZ89dVXJl+7srIyMXv2bOHv7y8cHBxE06ZNRe/evUVmZqbSB2ZeDjh79mwxd+5c4eXlJdRqtejevbv44YcfKq1/4cIFMXToUOHu7i7s7e1Fs2bNxKuvvio2btxotO1HPSq+h3v37hWhoaFCq9UKjUYj/Pz8xLBhw8TRo0eFEELMnDlTPPfccyYvpat4OeA9X3zxhejWrZtwcnISTk5Owt/fX8TGxoqzZ89Wen8qetDlgADE5s2bjfo+aP+uyv5lirn7cVWMHz9eABAtW7Y0ufzgwYOic+fOwtHRUXh6eoqxY8eKnTt3VrrUz9Rcr1+/LiIiIkSDBg1E48aNRUxMjPjxxx8rXQ4oRNX2H0ux+uCurtGjRwtnZ2dx586d+i5FCPG/Hf5x8qBgeFzcH9yWUvGHvaIVK1Y88JcvPZ77sYye2HPcD1NUVITPP/8cERERaNCgQX2XQ0Rk5Ik5x10V165dw9dff42NGzfi999/f+A36/TkioyMVP5M3hQ/Pz+8+OKLdViR9bl586byx0im2Nra1stlrY8TBvd9Tp06hcjISLi5uWH+/PlG99AgAu5ek/4w3bt3R/fu3euoGuvUr18/7N+//4HLvb29zfpDHWukEqIG11kREVlYZmbmQ+806OjoiK5du9ZhRY8fBjcRkWT45SQRkWSkPMdtMBhw9epVODs71+iucUREjwshBAoKCuDp6Wl0Gw5TpAzuq1evwsvLq77LICKyuOzsbPzxj398aB8pg/ven6hnZ2dXujUmEZGM9Ho9vLy8Kt2CwxQpg/ve6REXFxcGNxFZlaqc/uWXk0REkmFwExFJhsFNRCQZBjcRkWQY3EREkmFwExFJhsFNRCQZBjcRkWSk/AMcIrKcSapJdb7NJJFU59u0JjziJiKSDI+4icjq1fWnitr+RMEjbiIiyTC4iYgkw+AmIpIMg5uISDIMbiIiyTC4iYgkw+AmIpIMr+Mm6VjbNblE5uIRNxGRZBjcRESSYXATEUmGwU1EJBkGNxGRZBjcRESSYXATEUmG13ETPQKvG6fHjdlH3AcOHECfPn3g6ekJlUqFLVu2GC0XQmDixInw8PCAo6MjQkJCcP78eaM+N2/eRGRkJFxcXNCoUSOMGDECt2/frtFEiIieFGYfcd+5cwd//vOf8eabb6Jfv36Vls+aNQvz58/HqlWr4Ovri8TERISGhuLUqVPQaDQAgMjISPz222/YtWsXSktLMXz4cERHR2Pt2rU1nxGR5Orj/4AkuZgd3L1790bv3r1NLhNCICUlBRMmTEDfvn0BAKtXr4ZOp8OWLVswcOBAnD59Gunp6fj+++/RsWNHAMCCBQvwyiuvYM6cOfD09KzBdIiIrJ9Fv5zMyspCTk4OQkJClDatVovAwEBkZGQAADIyMtCoUSMltAEgJCQENjY2OHz4sCXLISKyShb9cjInJwcAoNPpjNp1Op2yLCcnB25ubsZF2NnB1dVV6VNRcXExiouLled6vd6SZRMRSUWKq0qSk5MxaRLP+z2ueE6WqG5Z9FSJu7s7ACA3N9eoPTc3V1nm7u6Oa9euGS0vKyvDzZs3lT4VJSQkID8/X3lkZ2dbsmwiIqlY9Ijb19cX7u7u2L17N9q3bw/g7mmNw4cPY9SoUQCAoKAg5OXlITMzEwEBAQCAPXv2wGAwIDAw0OS4arUaarXakqUSUT3ip7SaMTu4b9++jZ9//ll5npWVhRMnTsDV1RXNmzfH6NGjMXXqVLRq1Uq5HNDT0xPh4eEAgDZt2uDll1/GyJEjsWTJEpSWliIuLg4DBw7kFSVERFVgdnAfPXoUPXr0UJ7Hx8cDAKKiorBy5UqMHTsWd+7cQXR0NPLy8tCtWzekp6cr13ADwJo1axAXF4devXrBxsYGERERmD9/vgWmQ0Rk/VRCCFHfRZhLr9dDq9UiPz8fLi4u9V3OE48fe4mMVee2BebkGm8yRUQkGQY3EZFkGNxERJJhcBMRSYbBTUQkGQY3EZFkGNxERJJhcBMRSUaKuwNaEv//QCKSHY+4iYgkw+AmIpIMg5uISDIMbiIiyTC4iYgkw+AmIpLME3c54JOA98cmsm484iYikgyDm4hIMgxuIiLJMLiJiCTD4CYikgyDm4hIMgxuIiLJMLiJiCTD4CYikgz/crKW8a8YicjSeMRNRCQZBjcRkWQY3EREkrF4cJeXlyMxMRG+vr5wdHSEn58fpkyZAiGE0kcIgYkTJ8LDwwOOjo4ICQnB+fPnLV0KEZFVsnhwz5w5E4sXL8bChQtx+vRpzJw5E7NmzcKCBQuUPrNmzcL8+fOxZMkSHD58GE5OTggNDUVRUZGlyyEisjoWv6rku+++Q9++fREWFgYA8PHxwb/+9S8cOXIEwN2j7ZSUFEyYMAF9+/YFAKxevRo6nQ5btmzBwIEDLV0SEZFVsfgRd5cuXbB7926cO3cOAPDDDz/g22+/Re/evQEAWVlZyMnJQUhIiLKOVqtFYGAgMjIyTI5ZXFwMvV5v9CAielJZ/Ih73Lhx0Ov18Pf3h62tLcrLyzFt2jRERkYCAHJycgAAOp3OaD2dTqcsqyg5ORmTJvF6aCIioBaOuDds2IA1a9Zg7dq1OHbsGFatWoU5c+Zg1apV1R4zISEB+fn5yiM7O9uCFRMRycXiR9zvvfcexo0bp5yrbteuHS5fvozk5GRERUXB3d0dAJCbmwsPDw9lvdzcXLRv397kmGq1Gmq12tKlEhFJyeJH3IWFhbCxMR7W1tYWBoMBAODr6wt3d3fs3r1bWa7X63H48GEEBQVZuhwiIqtj8SPuPn36YNq0aWjevDnatm2L48ePY968eXjzzTcBACqVCqNHj8bUqVPRqlUr+Pr6IjExEZ6enggPD7d0OUREVsfiwb1gwQIkJibi7bffxrVr1+Dp6YmYmBhMnDhR6TN27FjcuXMH0dHRyMvLQ7du3ZCeng6NRmPpcoiIrI5K3P8njZLQ6/XQarXIz8+Hi4uLWevybn1EVNuSRJLZ65iTa7xXCRGRZBjcRESSYXATEUmGwU1EJBkGNxGRZBjcRESSYXATEUmGwU1EJBkGNxGRZBjcRESSYXATEUmGwU1EJBkGNxGRZBjcRESSYXATEUmGwU1EJBkGNxGRZBjcRESSYXATEUmGwU1EJBkGNxGRZBjcRESSYXATEUmGwU1EJBkGNxGRZBjcRESSYXATEUmGwU1EJJlaCe5ff/0Vb7zxBpo0aQJHR0e0a9cOR48eVZYLITBx4kR4eHjA0dERISEhOH/+fG2UQkRkdSwe3Ldu3ULXrl1hb2+PHTt24NSpU5g7dy4aN26s9Jk1axbmz5+PJUuW4PDhw3ByckJoaCiKioosXQ4RkdWxs/SAM2fOhJeXF1asWKG0+fr6Kv8WQiAlJQUTJkxA3759AQCrV6+GTqfDli1bMHDgQEuXRERkVSx+xP3ll1+iY8eOeP311+Hm5oZnn30Wy5cvV5ZnZWUhJycHISEhSptWq0VgYCAyMjJMjllcXAy9Xm/0ICJ6Ulk8uC9evIjFixejVatW2LlzJ0aNGoV3330Xq1atAgDk5OQAAHQ6ndF6Op1OWVZRcnIytFqt8vDy8rJ02URE0rB4cBsMBnTo0AHTp0/Hs88+i+joaIwcORJLliyp9pgJCQnIz89XHtnZ2RasmIhILhYPbg8PDzz11FNGbW3atMGVK1cAAO7u7gCA3Nxcoz65ubnKsorUajVcXFyMHkRETyqLB3fXrl1x9uxZo7Zz587B29sbwN0vKt3d3bF7925luV6vx+HDhxEUFGTpcoiIrI7FryoZM2YMunTpgunTp6N///44cuQIli1bhmXLlgEAVCoVRo8ejalTp6JVq1bw9fVFYmIiPD09ER4ebulyiIisjsWD+7nnnsPmzZuRkJCAyZMnw9fXFykpKYiMjFT6jB07Fnfu3EF0dDTy8vLQrVs3pKenQ6PRWLocIiKroxJCiPouwlx6vR5arRb5+flmn++epJpUS1UREd2VJJLMXsecXOO9SoiIJMPgJiKSDIObiEgyDG4iIskwuImIJMPgJiKSDIObiEgyDG4iIskwuImIJMPgJiKSDIObiEgyDG4iIskwuImIJMPgJiKSDIObiEgyDG4iIskwuImIJMPgJiKSDIObiEgyDG4iIskwuImIJMPgJiKSDIObiEgyDG4iIskwuImIJMPgJiKSDIObiEgyDG4iIsnUenDPmDEDKpUKo0ePVtqKiooQGxuLJk2aoGHDhoiIiEBubm5tl0JEZBVqNbi///57LF26FM8884xR+5gxY7B161akpaVh//79uHr1Kvr161ebpRARWY1aC+7bt28jMjISy5cvR+PGjZX2/Px8fPrpp5g3bx569uyJgIAArFixAt999x0OHTpUW+UQEVmNWgvu2NhYhIWFISQkxKg9MzMTpaWlRu3+/v5o3rw5MjIyaqscIiKrYVcbg65btw7Hjh3D999/X2lZTk4OHBwc0KhRI6N2nU6HnJwck+MVFxejuLhYea7X6y1aLxGRTCx+xJ2dnY2///3vWLNmDTQajUXGTE5OhlarVR5eXl4WGZeISEYWD+7MzExcu3YNHTp0gJ2dHezs7LB//37Mnz8fdnZ20Ol0KCkpQV5entF6ubm5cHd3NzlmQkIC8vPzlUd2dralyyYikobFT5X06tULJ0+eNGobPnw4/P398f7778PLywv29vbYvXs3IiIiAABnz57FlStXEBQUZHJMtVoNtVpt6VKJiKRk8eB2dnbG008/bdTm5OSEJk2aKO0jRoxAfHw8XF1d4eLignfeeQdBQUHo3LmzpcshIrI6tfLl5KN89NFHsLGxQUREBIqLixEaGopFixbVRylERNKpk+Det2+f0XONRoPU1FSkpqbWxeaJiKwK71VCRCQZBjcRkWQY3EREkmFwExFJhsFNRCQZBjcRkWQY3EREkmFwExFJhsFNRCQZBjcRkWQY3EREkmFwExFJhsFNRCQZBjcRkWQY3EREkmFwExFJhsFNRCQZBjcRkWQY3EREkmFwExFJhsFNRCQZBjcRkWQY3EREkmFwExFJhsFNRCQZBjcRkWQY3EREkmFwExFJhsFNRCQZiwd3cnIynnvuOTg7O8PNzQ3h4eE4e/asUZ+ioiLExsaiSZMmaNiwISIiIpCbm2vpUoiIrJLFg3v//v2IjY3FoUOHsGvXLpSWluKll17CnTt3lD5jxozB1q1bkZaWhv379+Pq1avo16+fpUshIrJKdpYeMD093ej5ypUr4ebmhszMTDz//PPIz8/Hp59+irVr16Jnz54AgBUrVqBNmzY4dOgQOnfubOmSiIisSq2f487PzwcAuLq6AgAyMzNRWlqKkJAQpY+/vz+aN2+OjIwMk2MUFxdDr9cbPYiInlS1GtwGgwGjR49G165d8fTTTwMAcnJy4ODggEaNGhn11el0yMnJMTlOcnIytFqt8vDy8qrNsomIHmu1GtyxsbH48ccfsW7duhqNk5CQgPz8fOWRnZ1toQqJiORj8XPc98TFxeGrr77CgQMH8Mc//lFpd3d3R0lJCfLy8oyOunNzc+Hu7m5yLLVaDbVaXVulEhFJxeJH3EIIxMXFYfPmzdizZw98fX2NlgcEBMDe3h67d+9W2s6ePYsrV64gKCjI0uUQEVkdix9xx8bGYu3atfj3v/8NZ2dn5by1VquFo6MjtFotRowYgfj4eLi6usLFxQXvvPMOgoKCeEUJEVEVWDy4Fy9eDAB44YUXjNpXrFiBYcOGAQA++ugj2NjYICIiAsXFxQgNDcWiRYssXQoRkVWyeHALIR7ZR6PRIDU1FampqZbePBGR1eO9SoiIJMPgJiKSDIObiEgyDG4iIskwuImIJMPgJiKSDIObiEgyDG4iIskwuImIJMPgJiKSDIObiEgyDG4iIskwuImIJMPgJiKSDIObiEgyDG4iIskwuImIJMPgJiKSDIObiEgyDG4iIskwuImIJMPgJiKSDIObiEgyDG4iIskwuImIJMPgJiKSDIObiEgyDG4iIsnUW3CnpqbCx8cHGo0GgYGBOHLkSH2VQkQklXoJ7vXr1yM+Ph5JSUk4duwY/vznPyM0NBTXrl2rj3KIiKRSL8E9b948jBw5EsOHD8dTTz2FJUuWoEGDBvjnP/9ZH+UQEUnFrq43WFJSgszMTCQkJChtNjY2CAkJQUZGhsl1iouLUVxcrDzPz88HAOj1erO3X4Qis9chIjJHdbLp3jpCiEf2rfPgvnHjBsrLy6HT6YzadTodzpw5Y3Kd5ORkTJo0qVK7l5dXrdRIRFQTM7Qzqr1uQUEBtFrtQ/vUeXBXR0JCAuLj45XnBoMBN2/eRJMmTaBSqao8jl6vh5eXF7Kzs+Hi4lIbpdY7a58j5yc/a59jdecnhEBBQQE8PT0f2bfOg/sPf/gDbG1tkZuba9Sem5sLd3d3k+uo1Wqo1WqjtkaNGlW7BhcXF6vcYe5n7XPk/ORn7XOszvwedaR9T51/Oeng4ICAgADs3r1baTMYDNi9ezeCgoLquhwiIunUy6mS+Ph4REVFoWPHjujUqRNSUlJw584dDB8+vD7KISKSSr0E94ABA3D9+nVMnDgROTk5aN++PdLT0yt9YWlparUaSUlJlU67WBNrnyPnJz9rn2NdzE8lqnLtCRERPTZ4rxIiIskwuImIJMPgJiKSDIObiEgyVhfc5t4uNi0tDf7+/tBoNGjXrh22b99eR5VWjznzW758Obp3747GjRujcePGCAkJkeL2udW95e+6deugUqkQHh5euwXWkLnzy8vLQ2xsLDw8PKBWq9G6dWur2k8BICUlBX/605/g6OgILy8vjBkzBkVFj+d9hQ4cOIA+ffrA09MTKpUKW7ZseeQ6+/btQ4cOHaBWq9GyZUusXLmyZkUIK7Ju3Trh4OAg/vnPf4qffvpJjBw5UjRq1Ejk5uaa7H/w4EFha2srZs2aJU6dOiUmTJgg7O3txcmTJ+u48qoxd36DBw8Wqamp4vjx4+L06dNi2LBhQqvVil9++aWOK686c+d4T1ZWlmjWrJno3r276Nu3b90UWw3mzq+4uFh07NhRvPLKK+Lbb78VWVlZYt++feLEiRN1XHnVmTvHNWvWCLVaLdasWSOysrLEzp07hYeHhxgzZkwdV14127dvF+PHjxebNm0SAMTmzZsf2v/ixYuiQYMGIj4+Xpw6dUosWLBA2NraivT09GrXYFXB3alTJxEbG6s8Ly8vF56eniI5Odlk//79+4uwsDCjtsDAQBETE1OrdVaXufOrqKysTDg7O4tVq1bVVok1Vp05lpWViS5duohPPvlEREVFPdbBbe78Fi9eLFq0aCFKSkrqqsQaM3eOsbGxomfPnkZt8fHxomvXrrVapyVUJbjHjh0r2rZta9Q2YMAAERoaWu3tWs2pknu3iw0JCVHaHnW72IyMDKP+ABAaGvrA/vWpOvOrqLCwEKWlpXB1da2tMmukunOcPHky3NzcMGLEiLoos9qqM78vv/wSQUFBiI2NhU6nw9NPP43p06ejvLy8rso2S3Xm2KVLF2RmZiqnUy5evIjt27fjlVdeqZOaa1tt5IwUdwesiurcLjYnJ8dk/5ycnFqrs7qqM7+K3n//fXh6elbaiR4X1Znjt99+i08//RQnTpyogwprpjrzu3jxIvbs2YPIyEhs374dP//8M95++22UlpYiKSmpLso2S3XmOHjwYNy4cQPdunWDEAJlZWX429/+hg8++KAuSq51D8oZvV6P//73v3B0dDR7TKs54qaHmzFjBtatW4fNmzdDo9HUdzkWUVBQgCFDhmD58uX4wx/+UN/l1AqDwQA3NzcsW7YMAQEBGDBgAMaPH48lS5bUd2kWs2/fPkyfPh2LFi3CsWPHsGnTJmzbtg1Tpkyp79IeW1ZzxF2d28W6u7ub1b8+VWd+98yZMwczZszA119/jWeeeaY2y6wRc+d44cIFXLp0CX369FHaDAYDAMDOzg5nz56Fn59f7RZthuq8hx4eHrC3t4etra3S1qZNG+Tk5KCkpAQODg61WrO5qjPHxMREDBkyBG+99RYAoF27drhz5w6io6Mxfvx42NjIfXz5oJxxcXGp1tE2YEVH3NW5XWxQUJBRfwDYtWvXY3l72ereDnfWrFmYMmUK0tPT0bFjx7ootdrMnaO/vz9OnjyJEydOKI+//OUv6NGjB06cOPHY/Q9J1XkPu3btip9//ln5hQQA586dg4eHx2MX2kD15lhYWFgpnO/9ohJWcCulWsmZan+t+Rhat26dUKvVYuXKleLUqVMiOjpaNGrUSOTk5AghhBgyZIgYN26c0v/gwYPCzs5OzJkzR5w+fVokJSU99pcDmjO/GTNmCAcHB7Fx40bx22+/KY+CgoL6msIjmTvHih73q0rMnd+VK1eEs7OziIuLE2fPnhVfffWVcHNzE1OnTq2vKTySuXNMSkoSzs7O4l//+pe4ePGi+M9//iP8/PxE//7962sKD1VQUCCOHz8ujh8/LgCIefPmiePHj4vLly8LIYQYN26cGDJkiNL/3uWA7733njh9+rRITU3l5YAVLViwQDRv3lw4ODiITp06iUOHDinLgoODRVRUlFH/DRs2iNatWwsHBwfRtm1bsW3btjqu2DzmzM/b21sAqPRISkqq+8LNYO57eL/HPbiFMH9+3333nQgMDBRqtVq0aNFCTJs2TZSVldVx1eYxZ46lpaXiww8/FH5+fkKj0QgvLy/x9ttvi1u3btV94VWwd+9ekz9X9+YUFRUlgoODK63Tvn174eDgIFq0aCFWrFhRoxp4W1ciIslYzTluIqInBYObiEgyDG4iIskwuImIJMPgJiKSDIObiEgyDG4iIskwuImIJMPgJiKSDIObiEgyDG4iIskwuImIJPP/oOr6XdRiFpIAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 400x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.rcParams[\"figure.figsize\"] = (4,3)\n",
    "plt.hist(p_values, bins=10, color='purple')\n",
    "plt.title(f\"Бутстреп распределение p_value\");"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.2. Мощность"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_tweedie(mu, shape, p=1.5, phi=25):\n",
    "    tw = tweedie_gen()\n",
    "    return tw(p=p, mu=mu, phi=phi).rvs(shape)\n",
    "\n",
    "\n",
    "def bootstrap_power(week: pd.DataFrame, metric_col: str, effect: float, alpha=0.05) -> (np.array, bool):\n",
    "    new_df = groups_splitter(week, columns=['user_id', metric_col, 'group'])\n",
    "\n",
    "    vec_a = new_df[(new_df['group'] == 'A')][metric_col]\n",
    "    vec_b = new_df[(new_df['group'] == 'B')][metric_col]\n",
    "\n",
    "    # Effect\n",
    "    tw_values = generate_tweedie(effect * vec_b.mean(), vec_b.size)\n",
    "    vec_b += tw_values\n",
    "\n",
    "    sample_a_mean = calc_bootstrap_mean(vec_a.to_numpy(), size=1000)\n",
    "    sample_b_mean = calc_bootstrap_mean(vec_b.to_numpy(), size=1000)\n",
    "    diff_mean = sample_a_mean - sample_b_mean\n",
    "\n",
    "    # Compute the confidence interval (e.g., 95% CI)\n",
    "    left_side, right_side = np.percentile(diff_mean, [100 * alpha / 2., 100 * (1 - alpha / 2.)])\n",
    "    # Do not reject the equality of means when it's necessary to reject\n",
    "    is_this_second_type_error = left_side <= 0 <= right_side\n",
    "\n",
    "    return is_this_second_type_error\n",
    "\n",
    "\n",
    "def test_power(week, metric_column, effect):\n",
    "    second_type_errors = Parallel(n_jobs=-1)(delayed(bootstrap_power)(week, metric_column, effect) for _ in range(N))\n",
    "\n",
    "    beta = sum(second_type_errors) / len(second_type_errors)    \n",
    "    print(f\"II type error: {beta:.2f}\")\n",
    "    \n",
    "    return beta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAEiCAYAAAD9DXUdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA15UlEQVR4nO3de1xUdf4/8BcgM4A4A4owoIAoCqJ4Q8XJUkiWUclytW9qruH9qwt9U7yirtfdKFtTNy/UukoXydTvaq0ohly0FG8kKaiURqHmgKkwgnL//P7wx/l64qLcb6/n4zGPnHPec877HGBenbuREEKAiIjo/zNu7AaIiKhpYTAQEZEMg4GIiGQYDEREJMNgICIiGQYDERHJMBiIiEiGwUBERDIMBiIikmEwENXQzz//DCMjI0RERNR6WgkJCTAyMkJCQkKtp0VUWwwGarZOnTqF1atXIzs7u0af37ZtW518qRO1NAwGarZOnTqFNWvWMBiI6hiDgagFys/PR2lpaWO3Qc0Ug4GapdWrV2PRokUAABcXFxgZGcHIyAg///wziouLsW7dOnTr1g1KpRJdunTBsmXLUFBQIH2+S5cuSE1NxfHjx6XP+vj4AADu3buHhQsXwtPTE5aWllCpVBg1ahS+//77Oun95s2bGDt2LNq2bQtbW1vMnz9f1tuTPU6dOrXccB8fH6lX4P+OT+zZswcrVqxAp06dYGFhAYPBUCf9UuvTprEbIKqJcePG4YcffsDnn3+OjRs3wsbGBgDQsWNHzJw5Ex9//DFeffVVLFiwAGfOnEFYWBiuXLmCAwcOAAA2bdqEN998E5aWlli+fDkAwM7ODgDw008/4eDBg/iv//ovuLi4IDMzEx9++CGGDx+Oy5cvw8HBocZ9P3r0CCNGjEBGRgb+53/+Bw4ODvj0008RFxdXyzUCrFu3DgqFAgsXLkRBQQEUCkWtp0mtlCBqpt577z0BQKSnp0vDkpOTBQAxc+ZMWe3ChQsFABEXFycN69Wrlxg+fHi56ebn54uSkhLZsPT0dKFUKsXatWtlwwCIXbt2PXPPmzZtEgDE3r17pWF5eXnC1dVVABDx8fHScGdnZxEYGFhuGsOHD5f1HR8fLwCIrl27iocPHz5zL0SV4a4kalEOHz4MAAgJCZENX7BgAQAgKirqqdNQKpUwNn78p1FSUoK7d+/C0tISbm5u+O6772rdn729PV599VVpmIWFBWbPnl2r6QJAYGAgzM3Naz0dIgYDtSi//PILjI2N4erqKhuu0WhgZWWFX3755anTKC0txcaNG9G9e3colUrY2NigY8eOuHjxInJycmrdn6urK4yMjGTD3dzcajVd4PGxFqK6wGCgFun3X7zV8fbbbyMkJATDhg3DZ599hqNHjyImJga9evVq0DN9KluGkpKSCodza4HqCg8+U7NV0Rens7MzSktL8eOPP6Jnz57S8MzMTGRnZ8PZ2bnKzwPA/v374evri3/961+y4dnZ2dJB7ppydnZGSkoKhBCy+aelpZWrtba2rvAajV9++QVdu3atVR9EVeEWAzVbbdu2BQDZl+fo0aMBPD7r6Envv/8+ACAgIED2+Yq+eE1MTCCEkA3bt28fbt26VeueR48ejV9//RX79++Xhj18+BAfffRRudpu3brh9OnTKCwslIYdOnQIN27cqHUfRFXhFgM1W15eXgCA5cuXY+LEiTA1NcWYMWMQGBiIjz76CNnZ2Rg+fDjOnj2Ljz/+GGPHjoWvr6/s89u3b8df//pXuLq6wtbWFi+++CJeeuklrF27FtOmTcNzzz2HS5cuYffu3XXyf+mzZs3Cli1b8MYbbyApKQn29vb49NNPYWFhUa525syZ2L9/P0aOHInXXnsN169fx2effYZu3brVug+iKjX2aVFEtbFu3TrRqVMnYWxsLJ26WlRUJNasWSNcXFyEqampcHR0FKGhoSI/P1/2Wb1eLwICAkS7du0EAOkU0Pz8fLFgwQJhb28vzM3NxdChQ0ViYmK500RrcrqqEEL88ssv4uWXXxYWFhbCxsZGvPXWWyI6Orrc6apCCLFhwwbRqVMnoVQqxdChQ8X58+crPV1137591eqDqDJGQvxum5mIiFo1HmMgIiIZHmMgqgOFhYW4d+9elTVqtZqnlFKzwGAgqgOnTp2SHdiuyK5duyq8KR5RU1OtXUnbt29Hnz59oFKpoFKpoNVqceTIEWl8fn4+goKC0KFDB1haWmL8+PHIzMyUTSMjIwMBAQGwsLCAra0tFi1ahOLiYllNQkICBgwYAKVSCVdX1wrvmb9161Z06dIFZmZm8Pb2xtmzZ6uzKER1qm/fvoiJianypdPpGrtNomdTnSPVX331lYiKihI//PCDSEtLE8uWLROmpqYiJSVFCCHEnDlzhKOjo4iNjRXnz58XQ4YMEc8995z0+eLiYtG7d2/h5+cnLly4IA4fPixsbGxEaGioVPPTTz8JCwsLERISIi5fviw++OADYWJiIqKjo6WaPXv2CIVCIXbu3ClSU1PFrFmzhJWVlcjMzKzdoXgiIhK1Pl3V2tpa7NixQ2RnZwtTU1PZKXNXrlwRAERiYqIQQojDhw8LY2NjodfrpZrt27cLlUolCgoKhBBCLF68WPTq1Us2jwkTJgidTie9Hzx4sAgKCpLel5SUCAcHBxEWFlbbxSEiavVqfIyhpKQE+/btQ15eHrRaLZKSklBUVAQ/Pz+pxt3dHU5OTkhMTMSQIUOQmJgIT09P6b73AKDT6TB37lykpqaif//+SExMlE2jrGbevHkAHh/kS0pKQmhoqDTe2NgYfn5+SExMrLLngoIC2QNRSktLce/ePXTo0KFW99YhIqorQgg8ePAADg4O0l1+G1q1g+HSpUvQarXIz8+HpaUlDhw4AA8PDyQnJ0OhUMDKykpWb2dnB71eDwDQ6/WyUCgbXzauqhqDwYBHjx7h/v37KCkpqbDm6tWrVfYeFhaGNWvWVHeRiYga3I0bN9C5c+dGmXe1g8HNzQ3JycnIycnB/v37ERgYiOPHj9dHb3UuNDRUdp/+nJwcODk54caNG1CpVJV+7tfNL8BhgP2zzeSFvbVtk4haMYPBAEdHR7Rr167Reqh2MCgUCule915eXjh37hw2b96MCRMmoLCwENnZ2bKthszMTGg0GgCP74n/+7OHys5aerLm92cyZWZmQqVSwdzcHCYmJjAxMamwpmwalVEqlVAqleWGl51lVZkHZiZQtTWtctpPTOzZ6oiIqtCYu7drvQOrtLQUBQUF8PLygqmpKWJjY6VxaWlpyMjIgFarBQBotVpcunQJWVlZUk1MTAxUKhU8PDykmienUVZTNg2FQgEvLy9ZTWlpKWJjY6UaIiKquWptMYSGhmLUqFFwcnLCgwcPEBkZiYSEBBw9ehRqtRozZsxASEgI2rdvD5VKhTfffBNarRZDhgwBAPj7+8PDwwNTpkzB+vXrodfrsWLFCgQFBUn/Jz9nzhxs2bIFixcvxvTp0xEXF4e9e/fKHskYEhKCwMBADBw4EIMHD8amTZuQl5eHadOm1eGqISJqnaoVDFlZWXjjjTdw+/ZtqNVq9OnTB0ePHsUf/vAHAMDGjRthbGyM8ePHo6CgADqdDtu2bZM+b2JigkOHDmHu3LnQarVo27YtAgMDsXbtWqnGxcUFUVFRmD9/PjZv3ozOnTtjx44dsouDJkyYgDt37mDlypXQ6/Xo168foqOjyx2QJiKi6mvVd1c1GAxQq9XIycmp8hjDrfcGoNOgTs82UZ//1FF3RPWvpKQERUVFjd1Gq2JqagoTE5NKxz/r91J94r2SiFohIQT0en2FT7Cj+mdlZQWNRtNkr59iMBC1QmWhYGtrCwsLiyb7BdXSCCHw8OFD6QQce/tnPA2+gTEYiFqZkpISKRQ6dOjQ2O20OmW3Xs/KyoKtrW2Vu5UaCx/UQ9TKlB1TqOg509QwytZ9Uz2+w2AgaqW4+6jxNPV1z2AgIiIZBgMREcnw4DMRST4f83mDzm/SfyY9c+3Tdr+sWrUKH374Id566y0sXbpUGr506VK8++67iI+Ph4+PjzTcx8cHjo6O+PTTT6vdd0vHLQYiahZu374tvTZt2gSVSiUbtnDhQvj4+CAhIUH2ufj4eDg6OsqG5+fn4/Tp03jxxRcbdiGaCW4xEFGz8OTdk9VqNYyMjMrdUdnX1xcLFixAcXEx2rRpgwcPHuDChQvYuHEj9u3bJ9UlJiaioKAAvr6+DdZ/c8ItBiJqMXx9fZGbm4tz584BAL755hv06NED48ePx5kzZ5Cfnw/g8VZEly5d0KVLl0bstuliMBBRi9G9e3d06tRJ2m2UkJCA4cOHQ6PRSI8ZLhvOrYXKMRiIqEV58jhDQkKCdMB5+PDhSEhIwKNHj3DmzBkGQxUYDETUovj6+uLkyZO4e/cuLly4gOHDhwN4HAzx8fE4deoUCgsLeeC5CgwGImpRfH19kZeXh/fffx/du3eHra0tAGDYsGE4e/Ysjhw5Iu1yoooxGIioRenatSucnJzwwQcfSFsLAODo6AgHBwd89NFH3I30FAwGImpxfH198eDBA9kFbcDj3UkPHjxgMDwFn+DGJ7hRK5Ofn4/09HS4uLjAzMyssdtplar6GTSFJ7hxi4GIiGQYDEREJMNgICIiGQYDERHJMBiIiEiGwUBERDIMBiIikmEwEBGRTLWCISwsDIMGDUK7du1ga2uLsWPHIi0tTVbj4+MDIyMj2WvOnDmymoyMDAQEBMDCwgK2trZYtGgRiouLZTUJCQkYMGAAlEolXF1dERERUa6frVu3okuXLjAzM4O3tzfOnj1bncUhIqIKVCsYjh8/jqCgIJw+fRoxMTEoKiqCv78/8vLyZHWzZs2SPXJv/fr10riSkhIEBASgsLAQp06dwscff4yIiAisXLlSqklPT0dAQAB8fX2RnJyMefPmYebMmTh69KhU88UXXyAkJASrVq3Cd999h759+0Kn0yErK6um64KIqBwfHx/MmzevsdtoUNV6tGd0dLTsfUREBGxtbZGUlIRhw4ZJwy0sLMo9cq/M119/jcuXL+PYsWOws7NDv379sG7dOixZsgSrV6+GQqFAeHg4XFxcsGHDBgBAz5498e2332Ljxo3Q6XQAgPfffx+zZs3CtGnTAADh4eGIiorCzp07ZQ8CJ6JqSBjTsPOrxi1kwsPDsWjRIty/fx9t2jz+6srNzYW1tTWGDh0qe6Zz2YN4rl27hm7dutV11zL//Oc/8cknnyAlJQUA4OXlhbfffhuDBw+u1/nWp1odY8jJyQEAtG/fXjZ89+7dsLGxQe/evREaGoqHDx9K4xITE+Hp6Qk7OztpmE6ng8FgQGpqqlTj5+cnm6ZOp5OevlRYWIikpCRZjbGxMfz8/KQaImpZyh7bef78eWnYN998A41GI3tsJ/D40Z1OTk71HgrA4xCaNGkS4uPjkZiYCEdHR/j7++PWrVv1Pu/6UuNgKC0txbx58zB06FD07t1bGv7666/js88+Q3x8PEJDQ/Hpp5/iT3/6kzRer9fLQgGA9F6v11dZYzAY8OjRI/z2228oKSmpsKZsGhUpKCiAwWCQvYioeXBzc4O9vX25LYNXXnkFLi4uOH36tGy4r68vSktLERYWBhcXF5ibm6Nv377Yv3+/bLopKSkYNWoULC0tYWdnhylTpuC3336rtI+oqCio1Wrs3r0bwOP/Ef7zn/+Mfv36wd3dHTt27EBpaSliY2PrdgU0oBoHQ1BQEFJSUrBnzx7Z8NmzZ0On08HT0xOTJ0/GJ598ggMHDuD69eu1bra2wsLCoFarpZejo2Njt0RE1eDr64v4+HjpfXx8PHx8fKSnswGQPbozLCwMn3zyCcLDw5Gamor58+fjT3/6E44fPw4AyM7Oxosvvoj+/fvj/PnziI6ORmZmJl577bUK5x8ZGYlJkyZh9+7dmDx5coU1Dx8+RFFRUbk9Kc1JtY4xlAkODsahQ4dw4sQJdO7cucpab29vAJD29Wk0mnJnD2VmZgKAdFxCo9FIw56sUalUMDc3h4mJCUxMTCqsqezYBgCEhoYiJCREem8wGBgORM2Ir68v5s2bh+LiYjx69Eh6dGdRURHCw8MBPN4VXVBQAB8fH3h4eODYsWPQarUAHj/E59tvv8WHH36I4cOHY8uWLejfvz/efvttaR47d+6Eo6MjfvjhB/To0UMavnXrVixfvhz/+c9/ZA8A+r0lS5bAwcGh3O7w5qRawSCEwJtvvokDBw4gISEBLi4uT/1McnIyAMDe3h4AoNVq8be//Q1ZWVnSI/diYmKgUqng4eEh1Rw+fFg2nZiYGOmHq1Ao4OXlhdjYWIwdOxYApE234ODgSntRKpVQKpXVWWQiakJ8fHyQl5eHc+fO4f79++jRowc6duyI4cOHY9q0acjPz0dCQgK6du2K3NxcPHz4EH/4wx9k0ygsLET//v0BAN9//z3i4+NhaWlZbl7Xr1+XgmH//v3IysrCyZMnMWjQoEr7e+edd7Bnzx4kJCQ062ddVCsYgoKCEBkZiS+//BLt2rWT9uer1WqYm5vj+vXriIyMxOjRo9GhQwdcvHgR8+fPx7Bhw9CnTx8AgL+/Pzw8PDBlyhSsX78eer0eK1asQFBQkPSlPWfOHGzZsgWLFy/G9OnTERcXh7179yIqKkrqJSQkBIGBgRg4cCAGDx6MTZs2IS8vTzpLiYhaHldXV3Tu3Bnx8fG4f/++9H/uDg4OcHR0xKlTpxAfH48XX3wRubm5AB4fE/j9853Lvmtyc3MxZswYvPvuu+XmVfY/swDQv39/fPfdd9i5cycGDhwIIyOjcvV///vf8c477+DYsWPS911zVa1g2L59OwCUe1zerl27MHXqVCgUChw7dkz6knZ0dMT48eOxYsUKqdbExASHDh3C3LlzodVq0bZtWwQGBmLt2rVSjYuLC6KiojB//nxs3rwZnTt3xo4dO6RTVQFgwoQJuHPnDlauXAm9Xo9+/fohOjq63AFpImpZfH19kZCQgPv372PRokXS8GHDhuHIkSM4e/Ys5s6dCw8PDyiVSmRkZFS662fAgAH43//9X3Tp0kU6BbYi3bp1w4YNG+Dj4wMTExNs2bJFNn79+vX429/+hqNHj2LgwIF1s6CNqNq7kqri6OgoHdSpirOzc7ldRb/n4+ODCxcuVFkTHBxc5a4jImp5fH19ERQUhKKiItkX/vDhwxEcHIzCwkL4+vqiXbt2WLhwIebPn4/S0lI8//zzyMnJwcmTJ6FSqRAYGIigoCD885//xKRJk7B48WK0b98e165dw549e7Bjxw6YmJhI0+/Ro4d0sLtNmzbYtGkTAODdd9/FypUrERkZiS5dukh7UiwtLSvcRdUc1OjgMxFRY/H19cWjR4/g7u4u20MwfPhwPHjwQDqtFQDWrVuHjh07IiwsDD/99BOsrKwwYMAALFu2DMDjXVAnT57EkiVL4O/vj4KCAjg7O2PkyJEwNi5/0qabmxvi4uKkLYcNGzZg+/btKCwsxKuvviqrXbVqFVavXl1/K6IeGYmnbQa0YM/60O1b7w1Ap0GdKh0vU40rOYkaQ1UPoqeGUdXP4Fm/l+oT765KREQyDAYiIpJhMBARkQyDgYiIZBgMREQkw2AgaqVKS0sbu4VWq6mve17HQNTKKBQKGBsb49dff0XHjh2hUCgqvMUD1T0hBAoLC3Hnzh0YGxtDoVA0dksVYjAQtTLGxsZwcXHB7du38euvvzZ2O62ShYUFnJycKryIrilgMBC1QgqFAk5OTiguLkZJSUljt9OqmJiYoE2bNk16K43BQNRKGRkZwdTUFKampo3dCjUxTXM7hoiIGg2DgYiIZBgMREQkw2AgIiIZBgMREckwGIiISIbBQEREMgwGIiKSYTAQEZEMg4GIiGQYDEREJMNgICIiGQYDERHJMBiIiEimWsEQFhaGQYMGoV27drC1tcXYsWORlpYmq8nPz0dQUBA6dOgAS0tLjB8/HpmZmbKajIwMBAQEwMLCAra2tli0aBGKi4tlNQkJCRgwYACUSiVcXV0RERFRrp+tW7eiS5cuMDMzg7e3N86ePVudxSEiogpU63kMx48fR1BQEAYNGoTi4mIsW7YM/v7+uHz5Mtq2bQsAmD9/PqKiorBv3z6o1WoEBwdj3LhxOHnyJACgpKQEAQEB0Gg0OHXqFG7fvo033ngDpqamePvttwEA6enpCAgIwJw5c7B7927ExsZi5syZsLe3h06nAwB88cUXCAkJQXh4OLy9vbFp0ybodDqkpaXB1ta2LtdR9SSMebY6n//Ubx9ERDVkJIQQNf3wnTt3YGtri+PHj2PYsGHIyclBx44dERkZiVdffRUAcPXqVfTs2ROJiYkYMmQIjhw5gpdeegm//vor7OzsAADh4eFYsmQJ7ty5A4VCgSVLliAqKgopKSnSvCZOnIjs7GxER0cDALy9vTFo0CBs2bIFwOOHazs6OuLNN9/E0qVLn6l/g8EAtVqNnJwcqFSqSutuvTcAnQZ1qtE6qhSDgYgq8KzfS/WpVscYcnJyAADt27cHACQlJaGoqAh+fn5Sjbu7O5ycnJCYmAgASExMhKenpxQKAKDT6WAwGJCamirVPDmNspqyaRQWFiIpKUlWY2xsDD8/P6mGiIhqpsaP9iwtLcW8efMwdOhQ9O7dGwCg1+uhUChgZWUlq7Wzs4Ner5dqngyFsvFl46qqMRgMePToEe7fv4+SkpIKa65evVppzwUFBSgoKJDeGwyGaiwxEVHrUOMthqCgIKSkpGDPnj112U+9CgsLg1qtll6Ojo6N3RIRUZNTo2AIDg7GoUOHEB8fj86dO0vDNRoNCgsLkZ2dLavPzMyERqORan5/llLZ+6fVqFQqmJubw8bGBiYmJhXWlE2jIqGhocjJyZFeN27cqN6CExG1AtUKBiEEgoODceDAAcTFxcHFxUU23svLC6ampoiNjZWGpaWlISMjA1qtFgCg1Wpx6dIlZGVlSTUxMTFQqVTw8PCQap6cRllN2TQUCgW8vLxkNaWlpYiNjZVqKqJUKqFSqWQvIiKSq9YxhqCgIERGRuLLL79Eu3btpGMCarUa5ubmUKvVmDFjBkJCQtC+fXuoVCq8+eab0Gq1GDJkCADA398fHh4emDJlCtavXw+9Xo8VK1YgKCgISqUSADBnzhxs2bIFixcvxvTp0xEXF4e9e/ciKipK6iUkJASBgYEYOHAgBg8ejE2bNiEvLw/Tpk2rq3VDRNQqVSsYtm/fDgDw8fGRDd+1axemTp0KANi4cSOMjY0xfvx4FBQUQKfTYdu2bVKtiYkJDh06hLlz50Kr1aJt27YIDAzE2rVrpRoXFxdERUVh/vz52Lx5Mzp37owdO3ZI1zAAwIQJE3Dnzh2sXLkSer0e/fr1Q3R0dLkD0kREVD21uo6hueN1DETU1DT76xiIiKjlYTAQEZEMg4GIiGQYDEREJMNgICIiGQYDERHJMBiIiEiGwUBERDIMBiIikmEwEBGRDIOBiIhkGAxERCTDYCAiIhkGAxERyTAYiIhIhsFAREQyDAYiIpJhMBARkQyDgYiIZBgMREQkw2AgIiIZBgMREckwGIiISIbBQEREMgwGIiKSYTAQEZEMg4GIiGSqHQwnTpzAmDFj4ODgACMjIxw8eFA2furUqTAyMpK9Ro4cKau5d+8eJk+eDJVKBSsrK8yYMQO5ubmymosXL+KFF16AmZkZHB0dsX79+nK97Nu3D+7u7jAzM4OnpycOHz5c3cUhIqLfqXYw5OXloW/fvti6dWulNSNHjsTt27el1+effy4bP3nyZKSmpiImJgaHDh3CiRMnMHv2bGm8wWCAv78/nJ2dkZSUhPfeew+rV6/GRx99JNWcOnUKkyZNwowZM3DhwgWMHTsWY8eORUpKSnUXiYiInmAkhBA1/rCREQ4cOICxY8dKw6ZOnYrs7OxyWxJlrly5Ag8PD5w7dw4DBw4EAERHR2P06NG4efMmHBwcsH37dixfvhx6vR4KhQIAsHTpUhw8eBBXr14FAEyYMAF5eXk4dOiQNO0hQ4agX79+CA8Pf6b+DQYD1Go1cnJyoFKpKq279d4AAECnQZ2eabrPxOc/dTctImoxnvV7qT7VyzGGhIQE2Nraws3NDXPnzsXdu3elcYmJibCyspJCAQD8/PxgbGyMM2fOSDXDhg2TQgEAdDod0tLScP/+fanGz89PNl+dTofExMRK+yooKIDBYJC9iIhIrk1dT3DkyJEYN24cXFxccP36dSxbtgyjRo1CYmIiTExMoNfrYWtrK2+iTRu0b98eer0eAKDX6+Hi4iKrsbOzk8ZZW1tDr9dLw56sKZtGRcLCwrBmzZq6WMzaSxjz7LXcuiCiBlTnwTBx4kTp356enujTpw+6deuGhIQEjBgxoq5nVy2hoaEICQmR3hsMBjg6OjZiR0RETU+9n67atWtX2NjY4Nq1awAAjUaDrKwsWU1xcTHu3bsHjUYj1WRmZspqyt4/raZsfEWUSiVUKpXsRUREcvUeDDdv3sTdu3dhb28PANBqtcjOzkZSUpJUExcXh9LSUnh7e0s1J06cQFFRkVQTExMDNzc3WFtbSzWxsbGyecXExECr1db3IhERtWjVDobc3FwkJycjOTkZAJCeno7k5GRkZGQgNzcXixYtwunTp/Hzzz8jNjYWr7zyClxdXaHT6QAAPXv2xMiRIzFr1iycPXsWJ0+eRHBwMCZOnAgHBwcAwOuvvw6FQoEZM2YgNTUVX3zxBTZv3izbDfTWW28hOjoaGzZswNWrV7F69WqcP38ewcHBdbBaiIhar2oHw/nz59G/f3/0798fABASEoL+/ftj5cqVMDExwcWLF/Hyyy+jR48emDFjBry8vPDNN99AqVRK09i9ezfc3d0xYsQIjB49Gs8//7zsGgW1Wo2vv/4a6enp8PLywoIFC7By5UrZtQ7PPfccIiMj8dFHH6Fv377Yv38/Dh48iN69e9dmfRARtXq1uo6huWvU6xiqg2clEbUaLfY6BiIiar4YDEREJMNgqIZb5241dgtERPWOwUBERDIMBiIikmEwEBGRDIOBiIhk6vwmetRM8O6uRFQJbjEQEZEMg4GIiGQYDEREJMNgICIiGR58bmmqc1CZiKgC3GIgIiIZBkM18X5JRNTSMRiIiEiGxxiaAx43IKIGxC0GIiKSYTAQEZEMg4GIiGQYDEREJMNgICIiGQYDERHJMBiIiEiGwUBERDIMBiIikql2MJw4cQJjxoyBg4MDjIyMcPDgQdl4IQRWrlwJe3t7mJubw8/PDz/++KOs5t69e5g8eTJUKhWsrKwwY8YM5ObmymouXryIF154AWZmZnB0dMT69evL9bJv3z64u7vDzMwMnp6eOHz4cHUXp0Z4vyQiasmqfUuMvLw89O3bF9OnT8e4cePKjV+/fj3+8Y9/4OOPP4aLiwv+8pe/QKfT4fLlyzAzMwMATJ48Gbdv30ZMTAyKioowbdo0zJ49G5GRkQAAg8EAf39/+Pn5ITw8HJcuXcL06dNhZWWF2bNnAwBOnTqFSZMmISwsDC+99BIiIyMxduxYfPfdd+jdu3dt1gn93rPekoPPhiZqEYyEEKLGHzYywoEDBzB27FgAj7cWHBwcsGDBAixcuBAAkJOTAzs7O0RERGDixIm4cuUKPDw8cO7cOQwcOBAAEB0djdGjR+PmzZtwcHDA9u3bsXz5cuj1eigUCgDA0qVLcfDgQVy9ehUAMGHCBOTl5eHQoUNSP0OGDEG/fv0QHh7+TP0bDAao1Wrk5ORApVJVWnfrvQHlhnUa1OmZ5tGqMBiIau1Zv5fqU50eY0hPT4der4efn580TK1Ww9vbG4mJiQCAxMREWFlZSaEAAH5+fjA2NsaZM2ekmmHDhkmhAAA6nQ5paWm4f/++VPPkfMpqyuZDREQ1U6d3V9Xr9QAAOzs72XA7OztpnF6vh62trbyJNm3Qvn17WY2Li0u5aZSNs7a2hl6vr3I+FSkoKEBBQYH03mAwVGfxiIhahVZ1VlJYWBjUarX0cnR0bOyWiIianDoNBo1GAwDIzMyUDc/MzJTGaTQaZGVlycYXFxfj3r17spqKpvHkPCqrKRtfkdDQUOTk5EivGzduVHcRiYhavDoNBhcXF2g0GsTGxkrDDAYDzpw5A61WCwDQarXIzs5GUlKSVBMXF4fS0lJ4e3tLNSdOnEBRUZFUExMTAzc3N1hbW0s1T86nrKZsPhVRKpVQqVSyFxERyVU7GHJzc5GcnIzk5GQAjw84JycnIyMjA0ZGRpg3bx7++te/4quvvsKlS5fwxhtvwMHBQTpzqWfPnhg5ciRmzZqFs2fP4uTJkwgODsbEiRPh4OAAAHj99dehUCgwY8YMpKam4osvvsDmzZsREhIi9fHWW28hOjoaGzZswNWrV7F69WqcP38ewcHBtV8rREStWLUPPp8/fx6+vr7S+7Iv68DAQERERGDx4sXIy8vD7NmzkZ2djeeffx7R0dHSNQwAsHv3bgQHB2PEiBEwNjbG+PHj8Y9//EMar1ar8fXXXyMoKAheXl6wsbHBypUrpWsYAOC5555DZGQkVqxYgWXLlqF79+44ePAgr2EgIqqlWl3H0NzxOoY6xusYiGqtxV3HQEREzR+DoYZ4vyQiaqkYDEREJMNgICIiGQYDERHJMBiIiEiGwUBERDIMBiIikmEwEBGRDIOBiIhk6vRBPdTKPeuzoQHePoOoCeMWAxERyTAYaoG3xSCilojBQEREMgwGIiKSYTAQEZEMg4GIiGQYDEREJMNgICIiGQYDERHJ8Mpnahy8SpqoyeIWQy3xIjciamkYDEREJMNgICIiGR5joKbvWY9H8FgEUZ3gFkMd4HEGImpJ6jwYVq9eDSMjI9nL3d1dGp+fn4+goCB06NABlpaWGD9+PDIzM2XTyMjIQEBAACwsLGBra4tFixahuLhYVpOQkIABAwZAqVTC1dUVERERdb0oREStUr3sSurVqxeOHTv2fzNp83+zmT9/PqKiorBv3z6o1WoEBwdj3LhxOHnyJACgpKQEAQEB0Gg0OHXqFG7fvo033ngDpqamePvttwEA6enpCAgIwJw5c7B7927ExsZi5syZsLe3h06nq49FouaAp8AS1Yl6CYY2bdpAo9GUG56Tk4N//etfiIyMxIsvvggA2LVrF3r27InTp09jyJAh+Prrr3H58mUcO3YMdnZ26NevH9atW4clS5Zg9erVUCgUCA8Ph4uLCzZs2AAA6NmzJ7799lts3LiRwUBEVEv1cozhxx9/hIODA7p27YrJkycjIyMDAJCUlISioiL4+flJte7u7nByckJiYiIAIDExEZ6enrCzs5NqdDodDAYDUlNTpZonp1FWUzYNIiKquTrfYvD29kZERATc3Nxw+/ZtrFmzBi+88AJSUlKg1+uhUChgZWUl+4ydnR30ej0AQK/Xy0KhbHzZuKpqDAYDHj16BHNz8wp7KygoQEFBgfTeYDDUalmpGeNuJ6JK1XkwjBo1Svp3nz594O3tDWdnZ+zdu7fSL+yGEhYWhjVr1jRqD0RETV29n65qZWWFHj164Nq1a9BoNCgsLER2drasJjMzUzomodFoyp2lVPb+aTUqlarK8AkNDUVOTo70unHjRm0Xj4ioxan3YMjNzcX169dhb28PLy8vmJqaIjY2VhqflpaGjIwMaLVaAIBWq8WlS5eQlZUl1cTExEClUsHDw0OqeXIaZTVl06iMUqmESqWSveoKr2UgopaizoNh4cKFOH78OH7++WecOnUKf/zjH2FiYoJJkyZBrVZjxowZCAkJQXx8PJKSkjBt2jRotVoMGTIEAODv7w8PDw9MmTIF33//PY4ePYoVK1YgKCgISqUSADBnzhz89NNPWLx4Ma5evYpt27Zh7969mD9/fl0vDhFRq1Pnxxhu3ryJSZMm4e7du+jYsSOef/55nD59Gh07dgQAbNy4EcbGxhg/fjwKCgqg0+mwbds26fMmJiY4dOgQ5s6dC61Wi7Zt2yIwMBBr166ValxcXBAVFYX58+dj8+bN6Ny5M3bs2MFTVal+8JYc1MoYCSFEYzfRWAwGA9RqNXJycqrcrXTrvQHPNL1OgzrVVWvUHDEYqA486/dSfeJN9IjqCk+BpRaCN9EjIiIZBkMd4plJRNQScFcSUWPgAW1qwrjFQEREMtxiIGrKeECbGgG3GIiISIZbDHXs1rlbvJ6BGge3LqiOcIuBiIhkuMVA1BrxrCiqAoOBiCrH3VOtEnclERGRDLcY6gEPQFOrxN1TLQaDgYgaFndPNXkMBiJquhgijYLHGIiISIZbDPWExxmIGlh1ti6eVSvdCuEWAxERyXCLgYioMq10K4TBUI+4O4mIynla2OQVNUwfVeCuJCIikmEwEBGRDIOhnvE50ETU3DAYiIhIhsHQAG6du8UtByJqNhgMREQkw2BoQNxqIKLmoNkHw9atW9GlSxeYmZnB29sbZ8+ebeyWqsRwIKKmrlkHwxdffIGQkBCsWrUK3333Hfr27QudToesrKzGbq1KDAciasqadTC8//77mDVrFqZNmwYPDw+Eh4fDwsICO3fubOzWnooHpImoqWq2t8QoLCxEUlISQkNDpWHGxsbw8/NDYmJihZ8pKChAQUGB9D4nJwcAYDAYqpzXg/ySOui4Yle/yQAAOAywx6/f3YbDAPt6mxcRNX2Gh8UAACFEo/XQbIPht99+Q0lJCezs7GTD7ezscPXq1Qo/ExYWhjVr1pQb7ujoWC89Vs/F3/2XiFqzu3fvQq1WN8q8m20w1ERoaChCQkKk96Wlpbh37x46dOgAIyOjCj9jMBjg6OiIGzduQKVSNVSrdaY599+cewead//NuXegefefk5MDJycntG/fvtF6aLbBYGNjAxMTE2RmZsqGZ2ZmQqPRVPgZpVIJpVIpG2ZlZfVM81OpVM3uF+xJzbn/5tw70Lz7b869A827f2PjxjsE3GwPPisUCnh5eSE2NlYaVlpaitjYWGi12kbsjIioeWu2WwwAEBISgsDAQAwcOBCDBw/Gpk2bkJeXh2nTpjV2a0REzVazDoYJEybgzp07WLlyJfR6Pfr164fo6OhyB6RrQ6lUYtWqVeV2QTUXzbn/5tw70Lz7b869A827/6bQu5FozHOiiIioyWm2xxiIiKh+MBiIiEiGwUBERDIMBiIikmEwPEVj39Z79erVMDIykr3c3d2l8fn5+QgKCkKHDh1gaWmJ8ePHl7voLyMjAwEBAbCwsICtrS0WLVqE4uJiWU1CQgIGDBgApVIJV1dXRERE1KjfEydOYMyYMXBwcICRkREOHjwoGy+EwMqVK2Fvbw9zc3P4+fnhxx9/lNXcu3cPkydPhkqlgpWVFWbMmIHc3FxZzcWLF/HCCy/AzMwMjo6OWL9+fble9u3bB3d3d5iZmcHT0xOHDx+uVe9Tp04t97MYOXJkk+g9LCwMgwYNQrt27WBra4uxY8ciLS1NVtOQvyvV/bt5lv59fHzKrf85c+Y0ev/bt29Hnz59pIvptFotjhw5Io1vyuu9UoIqtWfPHqFQKMTOnTtFamqqmDVrlrCyshKZmZkN1sOqVatEr169xO3bt6XXnTt3pPFz5swRjo6OIjY2Vpw/f14MGTJEPPfcc9L44uJi0bt3b+Hn5ycuXLggDh8+LGxsbERoaKhU89NPPwkLCwsREhIiLl++LD744ANhYmIioqOjq93v4cOHxfLly8W///1vAUAcOHBANv6dd94RarVaHDx4UHz//ffi5ZdfFi4uLuLRo0dSzciRI0Xfvn3F6dOnxTfffCNcXV3FpEmTpPE5OTnCzs5OTJ48WaSkpIjPP/9cmJubiw8//FCqOXnypDAxMRHr168Xly9fFitWrBCmpqbi0qVLNe49MDBQjBw5UvazuHfvnqymsXrX6XRi165dIiUlRSQnJ4vRo0cLJycnkZubK9U01O9KTf5unqX/4cOHi1mzZsnWf05OTqP3/9VXX4moqCjxww8/iLS0NLFs2TJhamoqUlJSmvx6rwyDoQqDBw8WQUFB0vuSkhLh4OAgwsLCGqyHVatWib59+1Y4Ljs7W5iamop9+/ZJw65cuSIAiMTERCHE4y87Y2NjodfrpZrt27cLlUolCgoKhBBCLF68WPTq1Us27QkTJgidTler3n//5VpaWio0Go147733ZMugVCrF559/LoQQ4vLlywKAOHfunFRz5MgRYWRkJG7duiWEEGLbtm3C2tpa6l8IIZYsWSLc3Nyk96+99poICAiQ9ePt7S3++7//u0a9C/E4GF555ZVKP9NUehdCiKysLAFAHD9+XAjRsL8rdfF38/v+hXgcDG+99Valn2lK/VtbW4sdO3Y0u/VehruSKlF2W28/Pz9p2NNu611ffvzxRzg4OKBr166YPHkyMjIe36o7KSkJRUVFsh7d3d3h5OQk9ZiYmAhPT0/ZRX86nQ4GgwGpqalSzZPTKKup6+VMT0+HXq+XzUutVsPb21vWr5WVFQYOHCjV+Pn5wdjYGGfOnJFqhg0bBoVCIes3LS0N9+/fr9dlSkhIgK2tLdzc3DB37lzcvXtXGteUei+7pXzZjdga6nelrv5uft9/md27d8PGxga9e/dGaGgoHj58KI1rCv2XlJRgz549yMvLg1arbXbrvUyzvvK5PtXktt71wdvbGxEREXBzc8Pt27exZs0avPDCC0hJSYFer4dCoSh3I0A7Ozvo9XoAgF6vr3AZysZVVWMwGPDo0SOYm5vXybKUza+ieT3Zi62trWx8mzZt0L59e1mNi4tLpctkbW1d6TKVTaMmRo4ciXHjxsHFxQXXr1/HsmXLMGrUKCQmJsLExKTJ9F5aWop58+Zh6NCh6N27tzTthvhduX//fq3/birqHwBef/11ODs7w8HBARcvXsSSJUuQlpaGf//7343e/6VLl6DVapGfnw9LS0scOHAAHh4eSE5Objbr/UkMhiZu1KhR0r/79OkDb29vODs7Y+/evXX2hU3PZuLEidK/PT090adPH3Tr1g0JCQkYMWJEI3YmFxQUhJSUFHz77beN3UqNVNb/7NmzpX97enrC3t4eI0aMwPXr19GtW7eGblPGzc0NycnJyMnJwf79+xEYGIjjx483ak+1wV1JlajJbb0bgpWVFXr06IFr165Bo9GgsLAQ2dnZspone9RoNBUuQ9m4qmpUKlWdhk/Z/KpapxqNptwzu4uLi3Hv3r06Waa6/Nl17doVNjY2uHbtWpPpPTg4GIcOHUJ8fDw6d+4sDW+o35Xa/t1U1n9FvL29AUC2/hurf4VCAVdXV3h5eSEsLAx9+/bF5s2bm816/z0GQyWa6m29c3Nzcf36ddjb28PLywumpqayHtPS0pCRkSH1qNVqcenSJdkXVkxMDFQqFTw8PKSaJ6dRVlPXy+ni4gKNRiObl8FgwJkzZ2T9ZmdnIykpSaqJi4tDaWmp9EWg1Wpx4sQJFBUVyfp1c3ODtbV1gy3TzZs3cffuXdjb2zd670IIBAcH48CBA4iLiyu3u6qhfldq+nfztP4rkpycDACy9d9Y/f9eaWkpCgoKmvx6r1S1D1e3Inv27BFKpVJERESIy5cvi9mzZwsrKyvZ2QP1bcGCBSIhIUGkp6eLkydPCj8/P2FjYyOysrKEEI9PhXNychJxcXHi/PnzQqvVCq1WK32+7FQ4f39/kZycLKKjo0XHjh0rPBVu0aJF4sqVK2Lr1q01Pl31wYMH4sKFC+LChQsCgHj//ffFhQsXxC+//CKEeHy6qpWVlfjyyy/FxYsXxSuvvFLh6ar9+/cXZ86cEd9++63o3r277JTP7OxsYWdnJ6ZMmSJSUlLEnj17hIWFRblTPtu0aSP+/ve/iytXrohVq1Y99ZTPqnp/8OCBWLhwoUhMTBTp6eni2LFjYsCAAaJ79+4iPz+/0XufO3euUKvVIiEhQXY658OHD6WahvpdqcnfzdP6v3btmli7dq04f/68SE9PF19++aXo2rWrGDZsWKP3v3TpUnH8+HGRnp4uLl68KJYuXSqMjIzE119/3eTXe2UYDE/xwQcfCCcnJ6FQKMTgwYPF6dOnG3T+EyZMEPb29kKhUIhOnTqJCRMmiGvXrknjHz16JP785z8La2trYWFhIf74xz+K27dvy6bx888/i1GjRglzc3NhY2MjFixYIIqKimQ18fHxol+/fkKhUIiuXbuKXbt21ajf+Ph4AaDcKzAwUAjx+JTVv/zlL8LOzk4olUoxYsQIkZaWJpvG3bt3xaRJk4SlpaVQqVRi2rRp4sGDB7Ka77//Xjz//PNCqVSKTp06iXfeeadcL3v37hU9evQQCoVC9OrVS0RFRdW494cPHwp/f3/RsWNHYWpqKpydncWsWbPK/dE1Vu8V9Q1A9nNsyN+V6v7dPK3/jIwMMWzYMNG+fXuhVCqFq6urWLRokew6hsbqf/r06cLZ2VkoFArRsWNHMWLECCkUhGja670yvO02ERHJ8BgDERHJMBiIiEiGwUBERDIMBiIikmEwEBGRDIOBiIhkGAxERCTDYCAiIhkGAxERyTAYiIhIhsFAREQyDAYiIpL5f5KWyQe68wPJAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 400x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "effect = 0.03\n",
    "tw = generate_tweedie(effect * week2.total_dur.mean(), week2.total_dur.size)\n",
    "\n",
    "plt.hist(tw, bins=100, color='purple', alpha=0.7, label='TW')\n",
    "plt.hist(week2.total_dur, bins=200, color='orange', alpha=0.7, label='Week2')\n",
    "plt.legend()\n",
    "plt.title('total_dur')\n",
    "plt.ylim([0, 30000])\n",
    "plt.xlim([-1000, 30000])\n",
    "plt.tight_layout()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Эффект 1%:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "II type error: 0.68\n",
      "CPU times: total: 2min 38s\n",
      "Wall time: 14min 51s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "effect = 0.01\n",
    "metric_column=\"total_dur\"\n",
    "\n",
    "beta = test_power(week2, metric_column, effect)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Эффект 3%:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "II type error: 0.01\n",
      "CPU times: total: 2min 37s\n",
      "Wall time: 15min 3s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "effect = 0.03\n",
    "metric_column=\"total_dur\"\n",
    "\n",
    "beta = test_power(week2, metric_column, effect)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Эффект 5%:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "II type error: 0.00\n",
      "CPU times: total: 2min 37s\n",
      "Wall time: 15min 1s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "effect = 0.05\n",
    "metric_column=\"total_dur\"\n",
    "\n",
    "beta = test_power(week2, metric_column, effect)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Эффект в 1% плохо детектируется\n",
    "- Эффект в 3% детектируется в 99% случаев\n",
    "- Эффект в 5% детектируется в более чем 99% случаев\n",
    "\n",
    "Стоит учесть что здесь я добавлял Tweedie эффект, а не увеличивал просмотры топ-10% пользователей, как в итоговом A/B тесте из домашнего задания.\n",
    "Поэтому разные эффекты могут давать разные $\\alpha$ и $\\beta$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYEAAAEmCAYAAACEQCxyAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA6S0lEQVR4nO3deVxU9f7H8dcMDMMii6hsCogrbmBZKWm5AeZukl69VlbezBt5UyzNcklzSUvzVmZlXr3dX9bVFr22qGSp11RCS9wXDEVlMTUYARmGmfP7w+skAcqM4BmYz/Px4KFztnn7cTifOWfOnK9GURQFIYQQTkmrdgAhhBDqkSYghBBOTJqAEEI4MWkCQgjhxKQJCCGEE5MmIIQQTkyagBBCODFpAkII4cRc1Q7gCCwWC1lZWXh7e6PRaNSOI4QQt0xRFC5fvkxISAhabeXv96UJAFlZWYSGhqodQwghqt2ZM2do0qRJpfOlCQDe3t7A1WL5+PhUeT2TycTmzZuJj49Hp9PVVLxaR+pSOalNxaQulbO3NgaDgdDQUOv+rTLSBMB6CsjHx8fmJuDp6YmPj4+8cK8jdamc1KZiUpfK3WptbnaKWz4YFkIIJyZNQAghnJg0ASGEcGKqNoHt27czcOBAQkJC0Gg0rFu3rsx8RVGYMWMGwcHBeHh4EBsby4kTJ8osc+nSJUaNGoWPjw9+fn6MGTOGgoKC2/ivEEKI2kvVJlBYWEh0dDRLly6tcP7ChQt58803effdd0lJScHLy4s+ffpQXFxsXWbUqFEcOnSI5ORkvvzyS7Zv387YsWNv1z9BCCFqNVWvDurbty99+/atcJ6iKCxZsoRp06YxePBgAD788EMCAwNZt24dI0aM4MiRI2zcuJHU1FTuuusuAN566y369evH66+/TkhIyG37twghRG3ksJeIZmRkkJOTQ2xsrHWar68vnTt3ZteuXYwYMYJdu3bh5+dnbQAAsbGxaLVaUlJSePDBByvcttFoxGg0Wh8bDAbg6qVYJpOpyhmvLWvLOs5A6lI5qU3FHKUuZouCyWzBZL76Z6n18XXTzNctY7n6Z+l183//83/rl1owXbfda8uWWiyUmK/Ov/55Ss0KJdc9d0mpheZuGuJsrE1Va+mwTSAnJweAwMDAMtMDAwOt83JycggICCgz39XVFX9/f+syFZk/fz6zZs0qN33z5s14enranDU5OdnmdZyB1KVydb02igLm635KLeUfWxQovTbNoqFU0XDg399enVbB8ubrljVfv+4fli/7XJryz/fHbV83TcExbxvTIEBj82umqKioSss5bBOoSVOnTiUpKcn6+No36+Lj423+slhycjJxcXHyBZfrSF0qZ2ttFEX537vT3985lv7hXWmJueJ3khW+67zuney1P8uuf3Wdcs9jqeBd7g3eEZdalNtQzZqn0YCbixZXF83VP7UadC5adP+bdvXvv//pqi37uMxydq6rUSz8cnCvzb9P185w3IzDNoGgoCAAcnNzCQ4Otk7Pzc2lY8eO1mXOnz9fZr3S0lIuXbpkXb8ier0evV5fbrpOp7Nrp2XvenWd1KWsnPxiZqw/xMkzLqw6+xOlFsqcari28726g/1956rUjf2pdWdaduf3v2laDYUFl2no74ebq0uZ+dYdolaLm+vvf9e5atBp/7eM9e8aXF20uP1vmqtWW+75rt/Bul2Xwa3cDlqLi1b9IwOTyUThSdt/n6q6rMM2gYiICIKCgtiyZYt1p28wGEhJSeGvf/0rADExMeTl5bF37146deoEwHfffYfFYqFz585qRReiHItFIWnNPnaevAho4HK+3dty1WrK7Kgq3sFpcbv27tK17N+t70j/t5N0c/39He71f//jjtrtf9vV/eHvleW4tjN2ddHgqtXc8PYFJpOJr7/+mn79Ossbh9tM1SZQUFBAenq69XFGRgb79u3D39+fsLAwJkyYwJw5c2jZsiURERFMnz6dkJAQhgwZAkCbNm144IEHePLJJ3n33XcxmUw888wzjBgxQq4MEg7lX7tPs/PkRTx0WoaGm+h2Tyfc3XRl3nlW9k65zGkFrRatA7w7FXWHqk1gz5499OzZ0/r42nn60aNHs2rVKiZPnkxhYSFjx44lLy+Pbt26sXHjRtzd3a3rfPTRRzzzzDP07t0brVZLQkICb7755m3/twhRmYwLhcz/5ggAk/u0wv/iQWLbBMg7XuEQVG0CPXr0QLnBCU+NRsPs2bOZPXt2pcv4+/uzevXqmognxC0zWxQmrdlHsclC1xYN+PPdoWzceFDtWEJYyb2DhKhB72//hZ8y8/DWu7LwoWg5lSMcjjQBIWrI0RwDbyQfB2DGwLY09vNQOZEQ5UkTEKIGlJRamLQmjRKzhdg2ATzUqfLh/YRQkzQBIWrA29+d4FCWgfqeOuYN7XDT0Z2EUIs0ASGqWdqZPJZuPQnAnCEdCPB2v8kaQqhHmoAQ1ajYZGbS2jTMFoWB0SH0jwq++UpCqEiagBDV6PVNx0g/X0Ajbz2zB7VTO44QNyVNQIhqkvLLRVb8kAHAgoQO1PdyUzmREDcnTUCIalBoLOW5T9NQFPjTXaH0igy8+UpCOABpAkJUg7lfH+HMpSs09vNg2oA2ascRosqkCQhxi7Yd/5XVKZkAvDYsCm93uSeQqD2kCQhxC/KLTEz5dD8Aj93blHubN1Q5kRC2kSYgxC14ecMhcgzFNGvoxZQHItWOI4TNpAkIYaeNB7P54udzaDXw+vBoPNxc1I4khM2kCQhhhwsFRl764uotocd1b86dYfVVTiSEfaQJCGEjRVF46YsDXCwsITLIm2djW6odSQi7SRMQwkbr9p1j06FcdC4aFg/viN5VTgOJ2kuagBA2yM6/woz1hwB4tndL2ob4qJxIiFsjTUCIKlIUhSmfHeBycSnRoX6M695c7UhC3DJpAkJU0eofM9l+/Ff0rloWDYvG1UV+fUTt5/Cv4suXLzNhwgTCw8Px8PDg3nvvJTU11TpfURRmzJhBcHAwHh4exMbGcuLECRUTi7oo82IRc786AsDkByJpEVBP5URCVA+HbwJ/+ctfSE5O5l//+hcHDhwgPj6e2NhYzp07B8DChQt58803effdd0lJScHLy4s+ffpQXFyscnJRV5gtCs+tTaOoxEznCH8ev7ep2pGEqDYO3QSuXLnCZ599xsKFC7n//vtp0aIFL7/8Mi1atGDZsmUoisKSJUuYNm0agwcPJioqig8//JCsrCzWrVundnxRR6z8IYMfT13Cy82F14dFo9XKUJGi7nBVO8CNlJaWYjabcXcvOzyfh4cHO3bsICMjg5ycHGJjY63zfH196dy5M7t27WLEiBEVbtdoNGI0Gq2PDQYDACaTCZPJVOV815a1ZR1nUJfqkn6+gIWbjgEwtW9rgrx1t/Tvqku1qU5Sl8rZW5uqLu/QTcDb25uYmBheeeUV2rRpQ2BgIB9//DG7du2iRYsW5OTkABAYWPbe7YGBgdZ5FZk/fz6zZs0qN33z5s14enranDM5OdnmdZxBba+L2QJvHHShpFRDGz8L9XL38/XX+6tl27W9NjVF6lI5W2tTVFRUpeUcugkA/Otf/+KJJ56gcePGuLi4cOeddzJy5Ej27t1r9zanTp1KUlKS9bHBYCA0NJT4+Hh8fKp+3bfJZCI5OZm4uDh0Orl98DV1pS5vf3+SM4Un8fVwZfmT9xLoc+sDxteV2lQ3qUvl7K3NtTMcN+PwTaB58+Zs27aNwsJCDAYDwcHB/OlPf6JZs2YEBQUBkJubS3Dw7wN65+bm0rFjx0q3qdfr0ev15abrdDq7XoD2rlfX1ea6HDyXz9KtvwAwe3B7mjTwrtbt1+ba1CSpS+VsrU1Vl3XoD4av5+XlRXBwML/99hubNm1i8ODBREREEBQUxJYtW6zLGQwGUlJSiImJUTGtqM2MpWYmrUmj1KLQt30Qg6JD1I4kRI1x+COBTZs2oSgKrVu3Jj09neeff57IyEgef/xxNBoNEyZMYM6cObRs2ZKIiAimT59OSEgIQ4YMUTu6qKXeSD7BsdzLNKznxpwh7dFo5GogUXc5fBPIz89n6tSpnD17Fn9/fxISEpg7d671UGfy5MkUFhYyduxY8vLy6NatGxs3bix3RZEQVbH39CXe334SgLkPdqBBvfKnDYWoSxy+CQwfPpzhw4dXOl+j0TB79mxmz559G1OJuqiopJRJa9KwKDD0zsb0aRekdiQhalyt+UxAiJq24JujnLpYRLCvOzMHtlM7jhC3hTQBIYAf0i/wz12nAViQEIWvh1yhIpyDNAHh9AzFJp5fmwbAw13CuL9VI5UTCXH7SBMQTu+VDYfJyi8mvIEnU/u2UTuOELeVNAHh1L49nMvavWfRaOD1YdF46R3+WgkhqpU0AeG0LhWW8MLnBwB48r5m3N3UX+VEQtx+0gSE05q+/iAXCoy0DKhHUlwrteMIoQppAsIpbUjL4qv92bhoNSwe3hF3nYvakYRQhTQB4XTOG4qZvv4gAM/0bEGHJr4qJxJCPdIEhFNRFIUXPj9AXpGJ9o19eKZXC7UjCaEqaQLCqazdc5bvjp7HzUXL4uEd0bnIr4BwbvIbIJzGmUtFzP7yMACT4lvRKrB6xwgQojaSJiCcgsWiMPnT/RQYS7krvD5/ua+Z2pGEcAjSBIRT+HDXKXb9chEPnQuvD4vGRStjBAgB0gSEE/jl1wJe3XgUgBf7RdK0oZfKiYRwHNIERJ1WarYwaW0axSYL97VsyMNdwtWOJIRDkSYg6rT3//sLP2fm4a13ZUFClAwVKcQfSBMQddaRbANvJB8HYOagdoT4eaicSAjHI01A1EklpRaS1qRhMivEtQ0k4c7GakcSwiFJExB10lvfneBItoH6njrmPdhBTgMJUQmHbgJms5np06cTERGBh4cHzZs355VXXkFRFOsyiqIwY8YMgoOD8fDwIDY2lhMnTqiYWqht35k83tl6EoC5D3agkbde5URCOC6HbgILFixg2bJlvP322xw5coQFCxawcOFC3nrrLesyCxcu5M033+Tdd98lJSUFLy8v+vTpQ3FxsYrJhVqKTWaS1uzDbFEY3DGEfh2C1Y4khENz6GGUdu7cyeDBg+nfvz8ATZs25eOPP+bHH38Erh4FLFmyhGnTpjF48GAAPvzwQwIDA1m3bh0jRoxQLbtQx8KNx/jl10ICvPXMGtRO7ThCODyHbgL33nsv77//PsePH6dVq1akpaWxY8cOFi9eDEBGRgY5OTnExsZa1/H19aVz587s2rWr0iZgNBoxGo3WxwaDAQCTyYTJZKpyvmvL2rKOM1CrLikZl/jHDxkAzB3SFi+dxuH+b+Q1UzGpS+XsrU1Vl3foJvDCCy9gMBiIjIzExcUFs9nM3LlzGTVqFAA5OTkABAYGllkvMDDQOq8i8+fPZ9asWeWmb968GU9PT5tzJicn27yOM7iddSk2w4I0F0BDTICFovRUvk6/bU9vM3nNVEzqUjlba1NUVFSl5Ry6CaxZs4aPPvqI1atX065dO/bt28eECRMICQlh9OjRdm936tSpJCUlWR8bDAZCQ0OJj4/Hx8enytsxmUwkJycTFxeHTqezO09do0Zdpq0/xCXjOZr4ufPO2Hup56ADxstrpmJSl8rZW5trZzhuxjF/U/7n+eef54UXXrCe1unQoQOnT59m/vz5jB49mqCgIAByc3MJDv79A8Dc3Fw6duxY6Xb1ej16ffkrRnQ6nV0vQHvXq+tuV12+P3aef+85B8BrwzpSv57jfylMXjMVk7pUztbaVHVZh746qKioCK22bEQXFxcsFgsAERERBAUFsWXLFut8g8FASkoKMTExtzWrUEdeUQlTPt0PwBNdI4hp3kDlRELULg59JDBw4EDmzp1LWFgY7dq14+eff2bx4sU88cQTAGg0GiZMmMCcOXNo2bIlERERTJ8+nZCQEIYMGaJueHFbzPzPIc5fNtKskReTH2itdhwhah2HbgJvvfUW06dP5+mnn+b8+fOEhITw1FNPMWPGDOsykydPprCwkLFjx5KXl0e3bt3YuHEj7u7uKiYXt8PXB7JZvy8LrQYWDYvGXeeidiQhah2HbgLe3t4sWbKEJUuWVLqMRqNh9uzZzJ49+/YFE6r79bKRaesOAvB0jxbcEVZf5URC1E4O/ZmAEBVRFIUXvzjApcIS2gT78LfeLdWOJEStJU1A1Dqf/3SO5MO56Fw0LB4ejZurvIyFsJf89ohaJSvvCi9vOATAhNhWtAmu+vc6hBDlSRMQtYaiKEz5bD+Xi0u5I8yPp+5vpnYkIWo9aQKi1vi/lEz+e+IC7joti4ZF4+oiL18hbtUt/xYVFhZW+evJQtjr9MVC5n11BIApD0TSrFE9lRMJUTfY3QQOHz7MXXfdhbe3N/Xr16dDhw7s3bu3OrMJAYDZojBpTRpXTGZimjVgdExTtSMJUWfY3QSeeuopnnnmGQoKCrh48SJDhw7l0Ucfrc5sQgCwYscv7Dn9G/X0rix8KAqtVoaKFKK6VLkJDB48mHPnzlkf//rrrwwaNAhPT0/8/Pzo168fubm5NRJSOK/juZd5fdNxAKYPaEOov+23+hZCVK7K3xh++OGH6dWrF4mJiYwfP55nnnmGdu3a0b17d0wmE9999x2TJk2qyazCyZjMFpLW7KPEbKFXZADD7wpVO5IQdU6VjwSGDRvGjz/+yOHDh+nSpQtdu3Zl8+bNdO3alfvuu4/Nmzczbdq0mswqnMzS79M5eM6Ar4eOV4d2QKOR00BCVDeb7h3k6+vLu+++y44dOxg9ejRxcXG88sordo3GJcSNHDibz9vfXR0a7JUh7QnwkRsCClETbPpg+NKlS+zdu9d6JZCPjw933HEHX3/9dU3lE06o2GQmac0+Si0K/TsEMzAq+OYrCSHsUuUmsHr1apo0aUL//v0JDw/nm2++YebMmaxfv56FCxcyfPhw+WBYVIs3ko9z4nwBDevpeWVIezkNJEQNqnITmDp1Kv/4xz/Iyclhy5YtTJ8+HYDIyEi2bt1KXFycjOYlbtmeU5d4/7+/ADB/aAf8vdxUTiRE3VblJlBQUEDr1ldHbmrevHm5keyffPJJdu/eXb3phFMpNJYyaW0aigIPdWpCXNtAtSMJUedV+YPh0aNH079/f3r06MGePXt45JFHyi0TEBBQreGEc3n1m6OcvlhEiK87Mwa2VTuOEE6hyk1g8eLF9OzZk6NHj/LYY48RHx9fk7mEk/nviV/51+7TACx8KBofd53KiYRwDjZdIjpw4EAGDhxYU1mEk8q/YmLyp/sBeDQmnG4tG6qcSAjnIffiFaqbveEw2fnFNG3gyQt9I9WOI4RTcfgm0LRpUzQaTbmfxMREAIqLi0lMTKRBgwbUq1ePhIQEuVS1Ftl8KIfPfjqLVgOLhkfj6WbTwakQ4hY5fBNITU0lOzvb+pOcnAxcvY0FwMSJE9mwYQNr165l27ZtZGVlMXToUDUjiyq6WGDkxS8OAPDk/c3oFO6vciIhnI/Dv+1q1KhRmcevvvoqzZs3p3v37uTn57NixQpWr15Nr169AFi5ciVt2rRh9+7ddOnSRY3IogoURWHauoNcKCihVWA9kuJaqR1JCKfk8E3geiUlJfzf//0fSUlJaDQa9u7di8lkIjY21rpMZGQkYWFh7Nq1q9ImYDQaMRqN1sfXRkYzmUyYTKYq57m2rC3rOIOq1GXD/my+OZiDq1bDwqHt0SoWTCbL7YqoGnnNVEzqUjl7a1PV5au1CWi1Wnr06MFrr71Gp06dqnPTAKxbt468vDwee+wxAHJycnBzc8PPz6/McoGBgeTk5FS6nfnz5zNr1qxy0zdv3mzXzfCunaISZVVWl/wSeHWfC6AhLqSU0/t2cHrfbY2mOnnNVEzqUjlba/PHL/RWplqbwD/+8Q9OnTpFYmJijXx7eMWKFfTt25eQkJBb2s7UqVNJSkqyPjYYDISGhhIfH4+Pj0+Vt2MymUhOTiYuLg6dTq5rv+ZGdVEUhSf/9TNF5gt0aOzDor/cg86JBoyX10zFpC6Vs7c2VR37vVqbwLV36C+//HJ1bhaA06dP8+233/L5559bpwUFBVFSUkJeXl6Zo4Hc3FyCgoIq3ZZer0ev15ebrtPp7HoB2rteXVdRXT75MZNtJy7g5qpl8fCOeLqX/39wBvKaqZjUpXK21qaqy9r0FsxkMuHq6srBgwdtWa1arFy5koCAAPr372+d1qlTJ3Q6HVu2bLFOO3bsGJmZmXIzOwd05lIRr3x5GIDn41vTMtBb5URCCJuOBHQ6HWFhYZjN5prKUyGLxcLKlSsZPXo0rq6/R/b19WXMmDEkJSXh7++Pj48P48ePJyYmRq4McjAWi8Jza9MoLDFzT1N/nugWoXYkIQR2fE/gpZde4sUXX+TSpUs1kadC3377LZmZmTzxxBPl5r3xxhsMGDCAhIQE7r//foKCgsqcMhKOYdXOU6RkXMLTzYXXhkXhopUxAoRwBDZ/JvD222+Tnp5OSEgI4eHheHl5lZn/008/VVu4a+Lj41EUpcJ57u7uLF26lKVLl1b784rqkX6+gAUbjwLwYr82hDfwuskaQojbxeYmMGTIkBqIIeqqUrOFSWvTMJZauK9lQ0Z1DlM7khDiOjY3gZkzZ9ZEDlFHvbf9F9LO5OHt7srCh6JkqEghHIxdF2jn5eXxwQcfMHXqVOtnAz/99BPnzp2r1nCidjuSfZkl3x4HYNagdgT7eqicSAjxRzYfCezfv5/Y2Fh8fX05deoUTz75JP7+/nz++edkZmby4Ycf1kROUcuUWmDyZwcwmRX6tAvkwTsaqx1JCFEBm48EkpKSeOyxxzhx4gTu7u7W6f369WP79u3VGk7UXhvPajmaW4C/lxtzH+wgp4GEcFA2N4HU1FSeeuqpctMbN258w/v1COfx85k8vj13dac/78H2NKznnN8KFqI2sLkJ6PX6Cu9Jcfz48XK3fRbO50qJmSmfHURBw+DoYB5oH6x2JCHEDdjcBAYNGsTs2bOttynVaDRkZmYyZcoUEhISqj2gqF0WbjpKxsUifHUK0/vLUJFCODqbm8CiRYsoKCggICCAK1eu0L17d1q0aIG3tzdz586tiYyilth58gIrfzgFwMjmFnw95EZgQjg6m68O8vX1JTk5mR07drB//34KCgq48847ywzsIpzP5WITz6/dD8CIu5vQxvWUuoGEEFVicxMoLi7G3d2dbt260a1bt5rIJGqhuV8d4VzeFUL9PZjSpxXbt5xSO5IQogpsbgJ+fn7cc889dO/enZ49exITE4OHh3wJyJl9dzSXT1LPoNHA6w9FU09fq0YtFcKp2fyZwLfffssDDzxASkoKgwYNon79+nTr1o2XXnpJhoZzQr8VljDlswMAjOkaQedmDVROJISwhc1NoFu3brz44ots3ryZvLw8vv/+e1q0aMHChQt54IEHaiKjcGAz/3OIXy8bad7Ii+f6tFY7jhDCRnYdtx8/fpytW7daf4xGIwMGDKBHjx7VHE84sq/2Z/OftCxctBoWD++Iu85F7UhCCBvZ3AQaN27MlStX6NGjBz169GDKlClERcndIZ3N+cvFTFt39TRQYo/mRIf6qRtICGEXm08HNWrUiKKiInJycsjJySE3N5crV67URDbhoBRF4cXPD/JbkYm2wT4806ul2pGEEHayuQns27ePnJwcXnjhBYxGIy+++CINGzbk3nvv5aWXXqqJjMLBfLr3LN8eycXNRcviP0Xj5mrXHcmFEA7Ars8E/Pz8GDRoEF27duXee+9l/fr1fPzxx6SkpMi3huu4c3lXmL3hMAAT41oRGeSjciIhxK2wuQl8/vnn1g+EDx8+jL+/P926dWPRokV07969JjIKB2GxKEz+NI3LxlLuCPNj7P3N1I4khLhFNh/Hjxs3jqysLMaOHcvPP//M+fPn+fzzz/nb3/5GdHR0tQc8d+4cDz/8MA0aNMDDw4MOHTqwZ88e63xFUZgxYwbBwcF4eHgQGxvLiRMnqj2HgP9LOc0P6Rdx12lZPLwjLlq5GECI2s7mI4Hz58/XRI4K/fbbb3Tt2pWePXvyzTff0KhRI06cOEH9+vWtyyxcuJA333yTf/7zn0RERDB9+nT69OnD4cOHywx6I25NxoVC5n99FICpfdsQ0dBL5URCiOpg12cCZrOZdevWceTIEQDatm3L4MGDcXGp3uvEFyxYQGhoKCtXrrROi4iIsP5dURSWLFnCtGnTGDx4MAAffvghgYGBrFu3jhEjRlRrHmdltig8tzaNKyYz9zZvwCNdwtWOJISoJjY3gfT0dPr168e5c+do3frqN0Tnz59PaGgoX331Fc2bN6+2cP/5z3/o06cPw4YNY9u2bTRu3Jinn36aJ598EoCMjAxycnLK3MHU19eXzp07s2vXrkqbgNFoxGg0Wh9fGyTHZDJZx0moimvL2rJObfT+fzPYe/o3vPQuzBvSFrO5FLO58uWdpS72kNpUTOpSOXtrU9XlNYqiKLZsuF+/fiiKwkcffYS/vz8AFy9e5OGHH0ar1fLVV1/ZFPRGrp3OSUpKYtiwYaSmpvLss8/y7rvvMnr0aHbu3EnXrl3JysoiOPj3EayGDx+ORqPh3//+d4Xbffnll5k1a1a56atXr8bT07Pa8tcFWUXw+n4XzIqGkc3NdAmw6eUihFBJUVERf/7zn8nPz8fHp/Kr+GxuAl5eXuzevZsOHTqUmZ6WlkbXrl0pKCiwL3EF3NzcuOuuu9i5c6d12t/+9jdSU1PZtWuX3U2goiOB0NBQLly4cMNi/ZHJZCI5OZm4uDh0uro3gEpJqYVh76dwOPsyPVs35L1Rd1Tpm+F1vS63QmpTMalL5eytjcFgoGHDhjdtAjafDtLr9Vy+fLnc9IKCAtzc3Gzd3A0FBwfTtm3bMtPatGnDZ599BkBQUBAAubm5ZZpAbm4uHTt2rHS7er0evb784Oc6nc6uF6C96zm6t7Ye53D2Zfw8dSxIiLb5/7eu1qU6SG0qJnWpnK21qeqyNl8iOmDAAMaOHUtKSgqKoqAoCrt372bcuHEMGjTI1s3dUNeuXTl27FiZacePHyc8/OoHkxEREQQFBbFlyxbrfIPBQEpKCjExMdWaxdnsP5vH0u/TAZgzpD0BPnKllRB1kc1N4M0336R58+bExMTg7u6Ou7s7Xbt2pUWLFvz973+v1nATJ05k9+7dzJs3j/T0dFavXs37779PYmIicHWQ+wkTJjBnzhz+85//cODAAR599FFCQkIYMmRItWZxJsUmM0lr0jBbFAZEBTMgKkTtSEKIGmLXyGLr16/nxIkTHDlyBI1GQ5s2bWjRokW1h7v77rv54osvmDp1KrNnzyYiIoIlS5YwatQo6zKTJ0+msLCQsWPHkpeXR7du3di4caN8R+AWLNp8jPTzBTTy1vPK4PZqxxFC1CC7xwFs2bKldcdfk7eRHjBgAAMGDKh0vkajYfbs2cyePbvGMjiTHzMu8cGODABeHdqB+l7V+zmPEMKx2HX7xxUrVtC+fXvr6aD27dvzwQcfVHc2cZsVGkt5bm0aigLD72pC7zaBakcSQtQwm48EZsyYweLFixk/frz1w9ddu3YxceJEMjMz5R15LTbv6yNkXiqisZ8H0we0vfkKQohaz+YmsGzZMpYvX87IkSOt0wYNGkRUVBTjx4+XJlBLbTv+Kx+lZALw2kNReLvLZXpCOAObTweZTCbuuuuuctM7depEaWlptYQSt1d+kYkpn+4H4LF7m3Jvi4YqJxJC3C42N4FHHnmEZcuWlZv+/vvvl7lqR9QeszYcIsdQTERDL6Y8EKl2HCHEbWTX1UErVqxg8+bNdOnSBYCUlBQyMzN59NFHSUpKsi63ePHi6kkpaszGgzl8/vM5tBp4fVg0Hm7VeydYIYRjs7kJHDx4kDvvvBOAkydPAtCwYUMaNmzIwYMHrcvV5GWjonpcKDDy0hcHAHiqe3M6hde/yRpCiLrG5ibw/fff10QOcZspisK0Lw5ysbCEyCBvJsS2VDuSEEIFdn1PQNR+6/dlsfFQDq5aDYuGR6N3ldNAQjgjaQJOKCe/mBnrr566e7Z3S9qF+KqcSAihFmkCTkZRFCZ/th9DcSnRTXz5a4/qGwlOCFH7SBNwMh//eIbtx39F76pl0fCOuLrIS0AIZyZ7ACeSebGIOV8dBuD5Pq1pEVBP5URCCLVJE3ASFovCc2vTKCoxc0+EP090jVA7khDCAUgTcBL/+CGDH09dwtPNhUXDotFq5XscQghpAk4h/fxlFm66OkzntP5tCfX3VDmREMJRSBOo40rNFpLWpFFSaqF7q0aMvCdU7UhCCAciTaCOW7b1JPvP5uPj7sqChCi5nYcQogxpAnXYwXP5/H3LCQBmD25PkK+MuyyEKMuhm8DLL7+MRqMp8xMZ+futjouLi0lMTKRBgwbUq1ePhIQEcnNzVUzsOIylZiatSaPUovBAuyAGdwxRO5IQwgE5dBMAaNeuHdnZ2dafHTt2WOdNnDiRDRs2sHbtWrZt20ZWVhZDhw5VMa3jWPLtCY7lXqaBlxtzH2wvp4GEEBWyazyB28nV1ZWgoKBy0/Pz81mxYgWrV6+mV69eAKxcuZI2bdqwe/du61gHzmjv6Uu8t+3qbb7nDe1Ag3p6lRMJIRyVwx8JnDhxgpCQEJo1a8aoUaPIzLw6Du7evXsxmUzExsZal42MjCQsLIxdu3apFVd1RSWlTFqThkWBoXc0pk+78g1UCCGucegjgc6dO7Nq1Spat25NdnY2s2bN4r777uPgwYPk5OTg5uaGn59fmXUCAwPJycm54XaNRiNGo9H62GAwAFfHTzaZTFXOd21ZW9apafO/Psqpi0UE+uh5qW8rVbI5Yl0chdSmYlKXytlbm6our1EURbE5lUry8vIIDw9n8eLFeHh48Pjjj5fZmQPcc8899OzZkwULFlS6nZdffplZs2aVm7569Wo8PWvvF6mO5Wt45/DVcQH+2sZMpF+t+a8VQlSzoqIi/vznP5Ofn4+Pj0+lyzn0kcAf+fn50apVK9LT04mLi6OkpIS8vLwyRwO5ubkVfoZwvalTp5YZC9lgMBAaGkp8fPwNi/VHJpOJ5ORk4uLi0Ol0Nv97qtPlYhML3t4FFPPne5qQNLCtalkcqS6ORmpTMalL5eytzbUzHDdTq5pAQUEBJ0+e5JFHHqFTp07odDq2bNlCQkICAMeOHSMzM5OYmJgbbkev16PXl/+wVKfT2fUCtHe96vTq+sNk5RcT5u/JS/3bodOp/1/rCHVxVFKbikldKmdrbaq6rPp7iht47rnnGDhwIOHh4WRlZTFz5kxcXFwYOXIkvr6+jBkzhqSkJPz9/fHx8WH8+PHExMQ43ZVB3x7OZc2es2g08PqwaLz0Dv3fKoRwIA69tzh79iwjR47k4sWLNGrUiG7durF7924aNWoEwBtvvIFWqyUhIQGj0UifPn145513VE59e/1WWMILnx8A4Mn7mnFPhL/KiYQQtYlDN4FPPvnkhvPd3d1ZunQpS5cuvU2JHM/09Qe5UGCkRUA9kuJaqR1HCFHLOPz3BETlNqRl8eX+bFy0GhYPj8Zd56J2JCFELSNNoJY6byhm+vqDADzTswVRTfzUDSSEqJWkCdRCiqIw9fMD5BWZaBfiwzO9WqgdSQhRS0kTqIXW7jnLlqPncXPRsnh4R3Qu8t8ohLCP7D1qmbO/FTH7y8MATIpvResgb5UTCSFqM2kCtYjFojD50/0UGEvpFF6fv9zXTO1IQohaTppALfLhrlPsPHkRD50Li4ZF46KVMQKEELdGmkAt8cuvBby68SgAL/aLpGlDL5UTCSHqAmkCtYDZojBpbRrFJgvdWjRkVOdwtSMJIeoIaQK1wHvbT/JzZh7eelcWPhSFVk4DCSGqiTQBB3c0x8AbyccBmDmoHSF+HionEkLUJdIEHFhJqYWkf6dhMivEtgkk4c7GakcSQtQx0gQc2FvfneBwtoH6njrmDW2PRiOngYQQ1UuagIPadyaPd7aeBGDugx0I8HZXOZEQoi6SJuCAik1mJq3Zh9miMCg6hH4dgtWOJISoo6QJOKDXNh3j5K+FBHjrmT24ndpxhBB1mDQBB7P7l4v844cMABYkROHn6aZyIiFEXSZNwIEUGEt5bm0aigIj7g6lZ2SA2pGEEHWcNAEHMverI5z97QqN/Tx4qX8bteMIIZyANAEHsfXYeT7+MROA14dF4+2uUzmREMIZ1Kom8Oqrr6LRaJgwYYJ1WnFxMYmJiTRo0IB69eqRkJBAbm6ueiHtkF9kYspn+wF4vGtTYpo3UDmREMJZ1JomkJqaynvvvUdUVFSZ6RMnTmTDhg2sXbuWbdu2kZWVxdChQ1VKaZ+Z/zlIrsFIs4ZeTO4TqXYcIYQTqRVNoKCggFGjRrF8+XLq169vnZ6fn8+KFStYvHgxvXr1olOnTqxcuZKdO3eye/duFRNX3TcHslm3LwutBhYNj8bDzUXtSEIIJ1IrmkBiYiL9+/cnNja2zPS9e/diMpnKTI+MjCQsLIxdu3bd7pg2+/WykZfWHQTgrz2ac0dY/ZusIYQQ1ctV7QA388knn/DTTz+Rmppabl5OTg5ubm74+fmVmR4YGEhOTk6l2zQajRiNRutjg8EAgMlkwmQyVTnbtWVtWecaRVF48fP9XCosITKwHn+9P8Ku7TiiW6lLXSe1qZjUpXL21qaqyzt0Ezhz5gzPPvssycnJuLtX371z5s+fz6xZs8pN37x5M56enjZvLzk52eZ1Un/VkJzugotGYVBgHls2b7R5G47Onro4C6lNxaQulbO1NkVFRVVaTqMoimJPoNth3bp1PPjgg7i4/H6e3Gw2o9Fo0Gq1bNq0idjYWH777bcyRwPh4eFMmDCBiRMnVrjdio4EQkNDuXDhAj4+PlXOZzKZSE5OJi4uDp2u6pd0ZucX0//tnVwuLiUptgV/7V63Boy3ty7OQGpTMalL5eytjcFgoGHDhuTn599wv+bQRwK9e/fmwIEDZaY9/vjjREZGMmXKFEJDQ9HpdGzZsoWEhAQAjh07RmZmJjExMZVuV6/Xo9fry03X6XR2vQBtWU9RFF5af5jLxaV0DPXj6Z4tcXWpFR/N2MzeejoDqU3FpC6Vs7U2VV3WoZuAt7c37du3LzPNy8uLBg0aWKePGTOGpKQk/P398fHxYfz48cTExNClSxc1It/URymZ/PfEBfSuWhYNj66zDUAIUTs4dBOoijfeeAOtVktCQgJGo5E+ffrwzjvvqB2rQqcvFjLv6yMATHkgkuaN6qmcSAjh7GpdE9i6dWuZx+7u7ixdupSlS5eqE6iKzBaF59amUVRipkszfx67t6nakYQQonZ8T6Au+MeODFJP/YaXmwuvPRSNVitDRQoh1CdN4DY4nnuZ1zYfA2D6gLaE+tt+GaoQQtQEaQI1zGS2MGlNGiWlFnq0bsSf7g5VO5IQQlhJE6hh73x/kgPn8vH10LEgIQqNRk4DCSEchzSBGnTgbD5vfXcCgNmD2xHoU33fehZCiOogTaCGFJvMTFq7j1KLQr8OQQyKDlE7khBClCNNoIa88e1xjucW0LCeG3OGdJDTQEIIhyRNoAbsOXWJ97f/AsD8oVH4e7mpnEgIISomTaCaFZWUMmltGooCCXc2Ia5toNqRhBCiUtIEqtmr3xzl9MUiQnzdmTmordpxhBDihqQJVKMdJy7w4a7TACx8KBofd7kbohDCsUkTqCaGYhPPf5oGwCNdwunWsqHKiYQQ4uakCVST2RsOk51fTHgDT6b2i1Q7jhBCVIk0gWqw5ch5Pt17Fo0GFg2LxtOt1t2cVQjhpKQJ3KICE7y0/jAAY+9rxl1N/VVOJIQQVSdN4BYoisLaX7RcLCyhVWA9Jsa1UjuSEELYRJrALfjyQA77Lmlx1WpYNKwj7joXtSMJIYRNpAnYKddQzKwvrw4V+XT3ZnRo4qtyIiGEsJ00ATt9tT+b/CulhHopjOseoXYcIYSwi1zGYqcnukUQ6K3jzOG96FyklwohaieH3nstW7aMqKgofHx88PHxISYmhm+++cY6v7i4mMTERBo0aEC9evVISEggNzf3tuWLbxtIsIwUKYSoxRy6CTRp0oRXX32VvXv3smfPHnr16sXgwYM5dOgQABMnTmTDhg2sXbuWbdu2kZWVxdChQ1VOLYQQtYdDnw4aOHBgmcdz585l2bJl7N69myZNmrBixQpWr15Nr169AFi5ciVt2rRh9+7ddOnSRY3IQghRqzh0E7ie2Wxm7dq1FBYWEhMTw969ezGZTMTGxlqXiYyMJCwsjF27dt2wCRiNRoxGo/WxwWAAwGQyYTKZqpzp2rK2rOMMpC6Vk9pUTOpSOXtrU9XlHb4JHDhwgJiYGIqLi6lXrx5ffPEFbdu2Zd++fbi5ueHn51dm+cDAQHJycm64zfnz5zNr1qxy0zdv3oynp+0n+ZOTk21exxlIXSontamY1KVyttamqKioSss5fBNo3bo1+/btIz8/n08//ZTRo0ezbdu2W9rm1KlTSUpKsj42GAyEhoYSHx+Pj49PlbdjMplITk4mLi4OnU5uG32N1KVyUpuKSV0qZ29trp3huBmHbwJubm60aNECgE6dOpGamsrf//53/vSnP1FSUkJeXl6Zo4Hc3FyCgoJuuE29Xo9ery83XafT2fUCtHe9uk7qUjmpTcWkLpWztTZVXdahrw6qiMViwWg00qlTJ3Q6HVu2bLHOO3bsGJmZmcTExKiYUAghag+HPhKYOnUqffv2JSwsjMuXL7N69Wq2bt3Kpk2b8PX1ZcyYMSQlJeHv74+Pjw/jx48nJibG5iuDFEUBqn74dI3JZKKoqAiDwSDvXq4jdamc1KZiUpfK2Vuba/uza/u3SikO7IknnlDCw8MVNzc3pVGjRkrv3r2VzZs3W+dfuXJFefrpp5X69esrnp6eyoMPPqhkZ2fb/DxnzpxRAPmRH/mRnzr3c+bMmRvu/zSKcrM2UfdZLBaysrLw9vZGo9FUeb1rHyifOXPGpg+U6zqpS+WkNhWTulTO3tooisLly5cJCQlBq638zL9Dnw66XbRaLU2aNLF7/Wu3tRBlSV0qJ7WpmNSlcvbUxtfX96bL1LoPhoUQQlQfaQJCCOHEpAncAr1ez8yZMyv8zoEzk7pUTmpTMalL5Wq6NvLBsBBCODE5EhBCCCcmTUAIIZyYNAEhhHBi0gSEEMKJSROww/bt2xk4cCAhISFoNBrWrVundiSHMH/+fO6++268vb0JCAhgyJAhHDt2TO1YqrvZWNniqldffRWNRsOECRPUjqK6l19+GY1GU+YnMjKyRp5LmoAdCgsLiY6OZunSpWpHcSjbtm0jMTGR3bt3k5ycjMlkIj4+nsLCQrWjqepmY2ULSE1N5b333iMqKkrtKA6jXbt2ZGdnW3927NhRI88jt42wQ9++fenbt6/aMRzOxo0byzxetWoVAQEB7N27l/vvv1+lVOq70VjZ7dq1UymV4ygoKGDUqFEsX76cOXPmqB3HYbi6ut50bJTqIEcCosbk5+cD4O/vr3ISx2E2m/nkk0+sY2ULSExMpH///mXGCxdw4sQJQkJCaNasGaNGjSIzM7NGnkeOBESNsFgsTJgwga5du9K+fXu146iusrGynd0nn3zCTz/9RGpqqtpRHErnzp1ZtWoVrVu3Jjs7m1mzZnHfffdx8OBBvL29q/W5pAmIGpGYmMjBgwdr7DxmbVPZWNnO3AjOnDnDs88+S3JyMu7u7mrHcSjXn26Oioqic+fOhIeHs2bNGsaMGVOtzyVNQFS7Z555hi+//JLt27ff0i2665LKxsp+7733VE6mnr1793L+/HnuvPNO6zSz2cz27dt5++23MRqNuLi4qJjQcfj5+dGqVSvS09OrfdvSBES1URSF8ePH88UXX7B161YiIiLUjuSwro2V7cx69+7NgQMHykx7/PHHiYyMZMqUKdIArlNQUMDJkyd55JFHqn3b0gTsUFBQUKYjZ2RksG/fPvz9/QkLC1MxmboSExNZvXo169evx9vbm5ycHODqwBYeHh4qp1PPjcbKdmbe3t7lPi/y8vKiQYMGTv850nPPPcfAgQMJDw8nKyuLmTNn4uLiwsiRI6v9uaQJ2GHPnj307NnT+jgpKQmA0aNHs2rVKpVSqW/ZsmUA9OjRo8z0lStX8thjj93+QA7i/PnzPProo2RnZ+Pr60tUVBSbNm0iLi5O7WjCQZ09e5aRI0dy8eJFGjVqRLdu3di9ezeNGjWq9ueSW0kLIYQTk+8JCCGEE5MmIIQQTkyagBBCODFpAkII4cSkCQghhBOTJiCEEE5MmoAQQjgxaQJC3MAPP/xAhw4d0Ol0DBkypNJpQtRW0gSEuIGkpCQ6duxIRkaG9dvgFU27FadOnUKj0bBv3z6b13399dcJCAggICCARYsWlZmXkpJCp06dKC0tveWMou6S20YIcQMnT55k3LhxZe6GWtE0Nezfv58ZM2bw5ZdfoigKAwYMID4+ng4dOlBaWsq4ceN4//33cXWVX3NROTkSEE7LYrEwf/58IiIi8PDwIDo6mk8//RT4/d35xYsXeeKJJ9BoNKxatarCaQAHDx6kb9++1KtXj8DAQB555BEuXLhQ5rkWLlxIixYt0Ov1hIWFMXfuXADr3VbvuOMONBpNuXsvVebo0aNERUXRq1cvevfuTVRUFEePHgXgtdde4/777+fuu++upmqJOksRwknNmTNHiYyMVDZu3KicPHlSWblypaLX65WtW7cqpaWlSnZ2tuLj46MsWbJEyc7OVgoKCspNKyoqUn777TelUaNGytSpU5UjR44oP/30kxIXF6f07NnT+lyTJ09W6tevr6xatUpJT09X/vvf/yrLly9XFEVRfvzxRwVQvv32WyU7O1u5ePFilfIfPnxYqV+/vnL69Gnl1KlTip+fn3L48GElPT1dadmypWIwGGqkbqJukSYgnFJxcbHi6emp7Ny5s8z0MWPGKCNHjrQ+9vX1VVauXFlmmT9Oe+WVV5T4+Pgyy5w5c0YBlGPHjikGg0HR6/XWnf4fZWRkKIDy888/2/zvWLZsmdKqVSulVatWyrJlyxRFUZTevXsrX3zxhbJ27VqlXbt2SseOHZVt27bZvG3hHORkoXBK6enpFBUVlbudc0lJCXfccYdN20pLS+P777+nXr165eadPHmSvLw8jEYjvXv3vqXMFRk3bhzjxo2zPv7nP/+Jt7c3MTExtG7dmtTUVM6ePcuIESPIyMhAr9dXewZRu0kTEE6poKAAgK+++orGjRuXmWfrjrKgoICBAweyYMGCcvOCg4P55Zdf7A9qgwsXLjBr1iy2b99OSkoKrVq1omXLlrRs2RKTycTx48fp0KHDbckiag9pAsIptW3bFr1eT2ZmJt27d7+lbd1555189tlnNG3atMIrcVq2bImHhwdbtmzhL3/5S7n5bm5uwNXxdW/FxIkTmThxIk2aNCE1NRWTyWSdV1paesvbF3WTNAHhlLy9vXnuueeYOHEiFouFbt26kZ+fzw8//ICPjw+jR4+u8rYSExNZvnw5I0eOZPLkyfj7+5Oens4nn3zCBx98gLu7O1OmTGHy5Mm4ubnRtWtXfv31Vw4dOsSYMWMICAjAw8ODjRs30qRJE9zd3fH19bXp35OcnMzx48f55z//CcDdd9/N0aNH+eabbzhz5gwuLi60bt3apm0KJ6H2hxJCqMVisShLlixRWrdureh0OqVRo0ZKnz59ynyIWpUPhhVFUY4fP648+OCDip+fn+Lh4aFERkYqEyZMUCwWi6IoimI2m5U5c+Yo4eHhik6nU8LCwpR58+ZZ11++fLkSGhqqaLVapXv37oqiKMr333+vAEpGRsYN/x1FRUVKq1atyn2wvHz5ciUwMFAJCwtTvvzyS5tqI5yHDC8phINauXIl8+bN4/Dhw+h0OrXjiDpKviwmhIP6+uuvmTdvnjQAUaPkSEAIIZyYHAkIIYQTkyYghBBOTJqAEEI4MWkCQgjhxKQJCCGEE5MmIIQQTkyagBBCODFpAkII4cSkCQghhBOTJiCEEE7s/wHqxml4B3n4gAAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 400x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot([1, 3, 5], [32, 99, 100])\n",
    "plt.grid()\n",
    "plt.ylabel(\"power, %\")\n",
    "plt.xlabel(\"effect, %\");"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.3. Оценка MDE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "source": [
    "$$ MDE = \\frac{(z_{1 - \\frac{\\alpha}{2}} + z_{1 - \\beta}) * \\sqrt{\\sigma_{control}^2 + \\sigma_{experiment}^2}}{\\sqrt{N}} $$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "from scipy.stats import norm\n",
    "\n",
    "\n",
    "def get_mde(metric_vec, alpha=0.05, beta=0.2):\n",
    "\n",
    "    metric_mean, metric_std, metric_n = metric_vec.mean(), metric_vec.std(), metric_vec.count()\n",
    "\n",
    "    z_alpha = norm.ppf(1 - (alpha / 2), loc=0, scale=1)\n",
    "    z_beta = norm.ppf(1 - beta, loc=0, scale=1)\n",
    "\n",
    "    mde = (z_alpha + z_beta)*metric_std / np.sqrt(metric_n)\n",
    "    \n",
    "    return mde*100/metric_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "first_type_errors = [0.05, 0.1]\n",
    "second_type_errors = [0.68, 0.5, 0.2, 0.1, 0.05, 0.01]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0.68</th>\n",
       "      <th>0.5</th>\n",
       "      <th>0.2</th>\n",
       "      <th>0.1</th>\n",
       "      <th>0.05</th>\n",
       "      <th>0.01</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>alpha</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0.05</th>\n",
       "      <td>0.5%</td>\n",
       "      <td>0.7%</td>\n",
       "      <td>0.9%</td>\n",
       "      <td>1.1%</td>\n",
       "      <td>1.2%</td>\n",
       "      <td>1.4%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0.10</th>\n",
       "      <td>0.4%</td>\n",
       "      <td>0.5%</td>\n",
       "      <td>0.8%</td>\n",
       "      <td>1.0%</td>\n",
       "      <td>1.1%</td>\n",
       "      <td>1.3%</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       0.68   0.5   0.2   0.1  0.05  0.01\n",
       "alpha                                    \n",
       "0.05   0.5%  0.7%  0.9%  1.1%  1.2%  1.4%\n",
       "0.10   0.4%  0.5%  0.8%  1.0%  1.1%  1.3%"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mde_week2 = []\n",
    "\n",
    "for a in first_type_errors:\n",
    "    mdes = {str(b): f\"{get_mde(week2.total_dur, alpha=a, beta=b):.1f}%\" for b in second_type_errors}\n",
    "    mdes[\"alpha\"] = a\n",
    "    mde_week2.append(mdes)\n",
    "    \n",
    "pd.DataFrame(mde_week2).set_index(\"alpha\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Вывод**:\n",
    "- на большей выборке можем детектировать меньший эффект ☝️\n",
    "- с уменьшением $\\alpha$ и $\\beta$ - увеличивается минимальный детектируемый эффект"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. A/B тест"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "week2_test = week2.copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Зануляем все просмотры меньше 500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAEiCAYAAAD9DXUdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAArOElEQVR4nO3deVhTd74/8HcCJOCSgAsJVNS0OijulRbjNrUyYmU6dfR3Ryt1K3UFR6R1axWXdoaKdZdqbavY56qo96nLqEURFK+KqCgVUNF28OJtDbRFiFhlSb6/PxzO9Qh1DAIJ+H49z3k053zOOZ+vIG/OkhyFEEKAiIjoX5T2boCIiBwLg4GIiGQYDEREJMNgICIiGQYDERHJMBiIiEiGwUBERDIMBiIikmEwEBGRDIOBiIhk7BoMJ06cwOuvvw5vb28oFArs3btXtlwIgaioKHh5ecHNzQ2BgYG4fv26rKawsBAhISHQaDRwd3dHaGgoSkpKZDWXLl3CgAED4OrqCh8fH8TExFTpZffu3ejUqRNcXV3RrVs3HDp0qNbHS0TUENg1GO7evYsePXogNja22uUxMTFYu3YtNm7ciLS0NDRt2hRBQUG4f/++VBMSEoLs7GwkJibiwIEDOHHiBCZPniwtN5vNGDJkCNq1a4f09HQsX74cixcvxqZNm6Sa06dP480330RoaCguXryI4cOHY/jw4cjKyqq7wRMROSrhIACIPXv2SK+tVqvQ6/Vi+fLl0ryioiKhVqvFjh07hBBCXL58WQAQ586dk2q++eYboVAoxA8//CCEEOLTTz8VHh4eorS0VKqZO3eu8PX1lV7/5S9/EcHBwbJ+AgICxJQpU2p1jEREDYHDXmPIzc2FyWRCYGCgNE+r1SIgIACpqakAgNTUVLi7u8Pf31+qCQwMhFKpRFpamlQzcOBAqFQqqSYoKAg5OTm4ffu2VPPwfiprKvdDRPQscbZ3A7/FZDIBAHQ6nWy+TqeTlplMJnh6esqWOzs7o0WLFrIag8FQZRuVyzw8PGAymR67n+qUlpaitLRUem21WlFYWIiWLVtCoVDYMlQioscSQuDOnTvw9vaGUln3v887bDA4uujoaCxZssTebRDRM+TmzZto06ZNne/HYYNBr9cDAPLz8+Hl5SXNz8/PR8+ePaWagoIC2XoVFRUoLCyU1tfr9cjPz5fVVL7+dzWVy6szf/58REZGSq+Li4vRtm1bvP7xXri4Nn2iMVotFfhywstwdq67L0NFRQVC485C6fRk+6iPnojINmazGT4+PmjevHm97M9h//cbDAbo9XokJSVJQWA2m5GWloZp06YBAIxGI4qKipCeno7evXsDAJKTk2G1WhEQECDVfPDBBygvL4eLiwsAIDExEb6+vvDw8JBqkpKSEBERIe0/MTERRqPxN/tTq9VQq9VV5ru4NoWL25MHg0ajqfNgcHFralMw1HVPRFQz9XWa2q4Xn0tKSpCRkYGMjAwADy44Z2RkIC8vDwqFAhEREfjoo4+wf/9+ZGZmYty4cfD29sbw4cMBAJ07d8bQoUMxadIknD17FqdOnUJ4eDhGjx4Nb29vAMCYMWOgUqkQGhqK7Oxs7Ny5E2vWrJH9tj9z5kwkJCRgxYoVuHr1KhYvXozz588jPDy8vv9JiIjszq6/Fp4/fx6DBg2SXlf+sB4/fjzi4uIwZ84c3L17F5MnT0ZRURH69++PhIQEuLq6Suts27YN4eHhGDx4MJRKJUaOHIm1a9dKy7VaLY4cOYKwsDD07t0brVq1QlRUlOy9Dn379sX27duxYMECvP/+++jYsSP27t2Lrl271sO/AhGRY1EIIYS9m2gMzGYztFotRqxOtOlU0vZJxjo/lTTm81SbTiXVdU9EZJvKny/FxcXQaDR1vj+HfR8DERHZB4OBiIhkGAxERCTDYCAiIhkGAxERyTAYiIhIhsFAREQyDAYiIpJhMBARkQyDgYiIZBgMREQkw2AgIiIZBgMREckwGIiISIbBQEREMgwGIiKSYTAQEZEMg4GIiGQYDEREJMNgICIiGQYDERHJMBiIiEiGwUBERDIMBiIikmEwEBGRDIOBiIhkGAxERCTDYCAiIhkGAxERyTAYiIhIhsFAREQyDAYiIpJhMBARkYxDB4PFYsHChQthMBjg5uaGF154AR9++CGEEFKNEAJRUVHw8vKCm5sbAgMDcf36ddl2CgsLERISAo1GA3d3d4SGhqKkpERWc+nSJQwYMACurq7w8fFBTExMvYyRiMjROHQwLFu2DBs2bMD69etx5coVLFu2DDExMVi3bp1UExMTg7Vr12Ljxo1IS0tD06ZNERQUhPv370s1ISEhyM7ORmJiIg4cOIATJ05g8uTJ0nKz2YwhQ4agXbt2SE9Px/Lly7F48WJs2rSpXsdLROQInO3dwOOcPn0ab7zxBoKDgwEA7du3x44dO3D27FkAD44WVq9ejQULFuCNN94AAHz11VfQ6XTYu3cvRo8ejStXriAhIQHnzp2Dv78/AGDdunUYNmwYPvnkE3h7e2Pbtm0oKyvD5s2boVKp0KVLF2RkZGDlypWyACEiehY49BFD3759kZSUhGvXrgEAvv32W5w8eRKvvfYaACA3NxcmkwmBgYHSOlqtFgEBAUhNTQUApKamwt3dXQoFAAgMDIRSqURaWppUM3DgQKhUKqkmKCgIOTk5uH37dp2Pk4jIkTj0EcO8efNgNpvRqVMnODk5wWKx4G9/+xtCQkIAACaTCQCg0+lk6+l0OmmZyWSCp6enbLmzszNatGghqzEYDFW2UbnMw8OjSm+lpaUoLS2VXpvN5qcZKhGRw3DoI4Zdu3Zh27Zt2L59Oy5cuICtW7fik08+wdatW+3dGqKjo6HVaqXJx8fH3i0REdUKhw6G2bNnY968eRg9ejS6deuGsWPHYtasWYiOjgYA6PV6AEB+fr5svfz8fGmZXq9HQUGBbHlFRQUKCwtlNdVt4+F9PGr+/PkoLi6Wpps3bz7laImIHINDB8Ovv/4KpVLeopOTE6xWKwDAYDBAr9cjKSlJWm42m5GWlgaj0QgAMBqNKCoqQnp6ulSTnJwMq9WKgIAAqebEiRMoLy+XahITE+Hr61vtaSQAUKvV0Gg0somIqDFw6GB4/fXX8be//Q0HDx7EjRs3sGfPHqxcuRJ//vOfAQAKhQIRERH46KOPsH//fmRmZmLcuHHw9vbG8OHDAQCdO3fG0KFDMWnSJJw9exanTp1CeHg4Ro8eDW9vbwDAmDFjoFKpEBoaiuzsbOzcuRNr1qxBZGSkvYZORGQ3Dn3xed26dVi4cCGmT5+OgoICeHt7Y8qUKYiKipJq5syZg7t372Ly5MkoKipC//79kZCQAFdXV6lm27ZtCA8Px+DBg6FUKjFy5EisXbtWWq7VanHkyBGEhYWhd+/eaNWqFaKionirKhE9kxTi4bcRU42ZzWZotVqMWJ0IF7emT7SO1VKB7ZOMcHauu3yuqKjAmM9ToXR6sn3UR09EZJvKny/FxcX1ctraoU8lERFR/WMwEBGRDIOBiIhkGAxERCTDYCAiIhkGAxERyTAYiIhIhsFAREQyDAYiIpJhMBARkQyDgYiIZBgMREQkw2AgIiIZBgMREckwGIiISIbBQEREMgwGIiKSYTAQEZEMg4GIiGQYDEREJMNgICIiGQYDERHJMBiIiEiGwUBERDIMBiIikmEwEBGRDIOBiIhkGAxERCTDYCAiIhkGAxERyTAYiIhIhsFAREQyDAYiIpJx+GD44Ycf8NZbb6Fly5Zwc3NDt27dcP78eWm5EAJRUVHw8vKCm5sbAgMDcf36ddk2CgsLERISAo1GA3d3d4SGhqKkpERWc+nSJQwYMACurq7w8fFBTExMvYzP0QghUFFRYfMkhLB360RUS5zt3cDj3L59G/369cOgQYPwzTffoHXr1rh+/To8PDykmpiYGKxduxZbt26FwWDAwoULERQUhMuXL8PV1RUAEBISglu3biExMRHl5eWYOHEiJk+ejO3btwMAzGYzhgwZgsDAQGzcuBGZmZl4++234e7ujsmTJ9tl7PYirBa89UUqlE5P/q0hrBbsmNIfzs4O/e1ERE/Iof8nL1u2DD4+PtiyZYs0z2AwSH8XQmD16tVYsGAB3njjDQDAV199BZ1Oh71792L06NG4cuUKEhIScO7cOfj7+wMA1q1bh2HDhuGTTz6Bt7c3tm3bhrKyMmzevBkqlQpdunRBRkYGVq5c+cwFAwAolE42BYO1Dnshovrn0KeS9u/fD39/f/zHf/wHPD090atXL3z++efS8tzcXJhMJgQGBkrztFotAgICkJqaCgBITU2Fu7u7FAoAEBgYCKVSibS0NKlm4MCBUKlUUk1QUBBycnJw+/btuh4mEZFDcehg+Oc//4kNGzagY8eOOHz4MKZNm4a//vWv2Lp1KwDAZDIBAHQ6nWw9nU4nLTOZTPD09JQtd3Z2RosWLWQ11W3j4X08qrS0FGazWTYRETUGDn0qyWq1wt/fH3//+98BAL169UJWVhY2btyI8ePH27W36OhoLFmyxK49EBHVBYc+YvDy8oKfn59sXufOnZGXlwcA0Ov1AID8/HxZTX5+vrRMr9ejoKBAtryiogKFhYWymuq28fA+HjV//nwUFxdL082bN2syRCIih+PQwdCvXz/k5OTI5l27dg3t2rUD8OBCtF6vR1JSkrTcbDYjLS0NRqMRAGA0GlFUVIT09HSpJjk5GVarFQEBAVLNiRMnUF5eLtUkJibC19dXdgfUw9RqNTQajWwiImoMHDoYZs2ahTNnzuDvf/87vvvuO2zfvh2bNm1CWFgYAEChUCAiIgIfffQR9u/fj8zMTIwbNw7e3t4YPnw4gAdHGEOHDsWkSZNw9uxZnDp1CuHh4Rg9ejS8vb0BAGPGjIFKpUJoaCiys7Oxc+dOrFmzBpGRkfYaOhGR3dQoGJ5//nn88ssvVeYXFRXh+eeff+qmKr300kvYs2cPduzYga5du+LDDz/E6tWrERISItXMmTMHM2bMwOTJk/HSSy+hpKQECQkJ0nsYAGDbtm3o1KkTBg8ejGHDhqF///7YtGmTtFyr1eLIkSPIzc1F79698e677yIqKuqZvFWViEghavCWVaVSWe3dPvn5+Wjbti1KS0trrcGGwmw2Q6vVYsTqRLi4NX2idayWCmyfZKzTN4ZVVFRgzOdP/oa1irL7UCid4OTs8sT7qI9xED3LKn++FBcX18tpa5v+J+/fv1/6++HDh6HVaqXXFosFSUlJaN++fa01R0RE9c+mYKg8b69QKKrcLuri4oL27dtjxYoVtdYcERHVP5uCwWp98OEHBoMB586dQ6tWreqkKaqeEAIWi8WmdSoqKuqoGyJqrGp0Ujg3N7e2+6AnYLFY8OZnJ6FQOj35OuVlUNpwvYCIqMZXC5OSkpCUlISCggLpSKLS5s2bn7oxqp7NH3Bn4REDEdmmRsGwZMkSLF26FP7+/vDy8oJCoajtvoiIyE5qFAwbN25EXFwcxo4dW9v9EBGRndXoDW5lZWXo27dvbfdCREQOoEbB8M4770hPPyMiosalRqeS7t+/j02bNuHo0aPo3r07XFzkd72sXLmyVpojIqL6V6NguHTpEnr27AkAyMrKki3jhWgiooatRsFw7Nix2u6DiIgchEN/7DYREdW/Gh0xDBo06LGnjJKTk2vcEBER2VeNgqHy+kKl8vJyZGRkICsry+7PYiYioqdTo2BYtWpVtfMXL16MkpKSp2qIiIjsq1avMbz11lv8nCQiogauVoMhNTVV9khNIiJqeGp0KmnEiBGy10II3Lp1C+fPn8fChQtrpTEiIrKPGgXDw4/0BB48A9rX1xdLly7FkCFDaqUxIiKyjxoFw5YtW2q7DyIichA1flAPAKSnp+PKlSsAgC5duqBXr1610hQ1bjV5RCkAODk58SNXiOpBjYKhoKAAo0ePxvHjx+Hu7g4AKCoqwqBBgxAfH4/WrVvXZo/UyNTkEaXCasGOKf3h7PxUv8sQ0ROo0V1JM2bMwJ07d5CdnY3CwkIUFhYiKysLZrMZf/3rX2u7R2qEKh9R+qQTFEpUVFTYNAkh7D1MogapRr9+JSQk4OjRo+jcubM0z8/PD7Gxsbz4THVCWC1464vUJ37eNY8wiGquRv9rrFZrlWcwAICLiwusVutTN0VUncqjjCfB70KimqvRqaRXX30VM2fOxI8//ijN++GHHzBr1iwMHjy41pojIqL6V6NgWL9+PcxmM9q3b48XXngBL7zwAgwGA8xmM9atW1fbPRIRUT2q0akkHx8fXLhwAUePHsXVq1cBAJ07d0ZgYGCtNkcNgxACFRUVT1xvSy0R1T+bgiE5ORnh4eE4c+YMNBoN/vCHP+APf/gDAKC4uBhdunTBxo0bMWDAgDpplhyTrReGLeVlUDpXvUZFRI7BplNJq1evxqRJk6DRaKos02q1mDJlClauXFlrzVHDYcvtpwolHxxI5Mhs+h/67bffYujQob+5fMiQIUhPT3/qpoiIyH5sCob8/Pxqb1Ot5OzsjJ9++umpmyIiIvuxKRiee+45ZGVl/ebyS5cuwcvL66mbIiIi+7EpGIYNG4aFCxfi/v37VZbdu3cPixYtwh//+Mdaa+5RH3/8MRQKBSIiIqR59+/fR1hYGFq2bIlmzZph5MiRyM/Pl62Xl5eH4OBgNGnSBJ6enpg9e3aVO2OOHz+OF198EWq1Gh06dEBcXFydjYOIyJHZFAwLFixAYWEhfve73yEmJgb79u3Dvn37sGzZMvj6+qKwsBAffPBBnTR67tw5fPbZZ+jevbts/qxZs/CPf/wDu3fvRkpKCn788UfZg4QsFguCg4NRVlaG06dPY+vWrYiLi0NUVJRUk5ubi+DgYAwaNAgZGRmIiIjAO++8g8OHD9fJWIiIHJlNt6vqdDqcPn0a06ZNw/z586UPKVMoFAgKCkJsbCx0Ol2tN1lSUoKQkBB8/vnn+Oijj6T5xcXF+PLLL7F9+3a8+uqrAB48K6Jz5844c+YM+vTpgyNHjuDy5cs4evQodDodevbsiQ8//BBz587F4sWLoVKpsHHjRhgMBqxYsQLAg/dknDx5EqtWrUJQUFCtj4eIyJHZfN9gu3btcOjQIfz8889IS0vDmTNn8PPPP+PQoUMwGAx10SPCwsIQHBxc5Q106enpKC8vl83v1KkT2rZti9TUVAAPnkPdrVs3WWAFBQXBbDYjOztbqnl020FBQdI2qlNaWgqz2SybiIgagxp/9KSHhwdeeuml2uylWvHx8bhw4QLOnTtXZZnJZIJKpZKeCVFJp9PBZDJJNY8exVS+/nc1ZrMZ9+7dg5ubW5V9R0dHY8mSJTUeFxGRo3LodxrdvHkTM2fOxLZt2+Dq6mrvdmTmz5+P4uJiabp586a9WyIiqhUOHQzp6ekoKCjAiy++CGdnZzg7OyMlJQVr166Fs7MzdDodysrKUFRUJFsvPz8fer0eAKDX66vcpVT5+t/VaDSaao8WAECtVkOj0cgmIqLGwKGDYfDgwcjMzERGRoY0+fv7IyQkRPq7i4sLkpKSpHVycnKQl5cHo9EIADAajcjMzERBQYFUk5iYCI1GAz8/P6nm4W1U1lRug4joWeLQj7dq3rw5unbtKpvXtGlTtGzZUpofGhqKyMhItGjRAhqNBjNmzIDRaESfPn0APPiYDj8/P4wdOxYxMTEwmUxYsGABwsLCoFarAQBTp07F+vXrMWfOHLz99ttITk7Grl27cPDgwfodMBGRA3DoYHgSq1atglKpxMiRI1FaWoqgoCB8+umn0nInJyccOHAA06ZNg9FoRNOmTTF+/HgsXbpUqjEYDDh48CBmzZqFNWvWoE2bNvjiiy94qyoRPZMaXDAcP35c9trV1RWxsbGIjY39zXUqb7F9nFdeeQUXL16sjRaJiBo0h77GQERE9Y/BQEREMgwGIiKSYTAQEZFMg7v4TPQkhBBVPlr9STg5OUGhUDzxPiwWS51tn8heGAzUKAmrBW99kQql05N/iwurBTum9Iez85OtY7FY8OZnJ6FQOtXJ9onshd+h1GgplE42BYO1jvdRk+0T2QOvMRARkQyPGIj+xdbrEjW5hkHUEDAYiP7F1usSlvIyKJ1d6rgrovrHYCB6iE3XDCw8YqDGicFAVE/q4xZaotrAYCCqJ/VxCy1RbeB3G1E9qo9baImeFm9XJSIiGQYDERHJ8FQSkQOryQVrXqymp8VgIHJgtl6w5sVqqg387iFycLZcsLbwlliqBQwGokaEt8RSbeB3AlEjw1ti6WnxriQiIpJhMBARkQxPJRE943hLLD2KwUD0jKuPW2L5fOyGhcFARHX+iFI+H7th4b86EdmkJqeeKioq+HzsBoTBQEQ2qcl7JWx92t2z/OyK6k671fdjZBkMRGQzm98rYePT7p7lN+pVd9qt/N7deu2hYf8LElGjZWv41OTjQBz1COPRsdvy71AbGAxE1Cg44t1VQggAsCl86vu0UXUYDETUaDja3VWW8jIolMo6vR5TFxgMRPRMqpe7qyy21VeuY28OHQzR0dH4+uuvcfXqVbi5uaFv375YtmwZfH19pZr79+/j3XffRXx8PEpLSxEUFIRPP/0UOp1OqsnLy8O0adNw7NgxNGvWDOPHj0d0dLTsEPL48eOIjIxEdnY2fHx8sGDBAkyYMKE+h0tE9ag+7q5qqBz6s5JSUlIQFhaGM2fOIDExEeXl5RgyZAju3v2/K/SzZs3CP/7xD+zevRspKSn48ccfMWLECGm5xWJBcHAwysrKcPr0aWzduhVxcXGIioqSanJzcxEcHIxBgwYhIyMDEREReOedd3D48OF6HS8R1a/K3+afdFIoHfpHZq1x6COGhIQE2eu4uDh4enoiPT0dAwcORHFxMb788kts374dr776KgBgy5Yt6Ny5M86cOYM+ffrgyJEjuHz5Mo4ePQqdToeePXviww8/xNy5c7F48WKoVCps3LgRBoMBK1asAAB07twZJ0+exKpVqxAUFFTv4yYisqcGFX/FxcUAgBYtWgAA0tPTUV5ejsDAQKmmU6dOaNu2LVJTUwEAqamp6Natm+zUUlBQEMxmM7Kzs6Wah7dRWVO5jeqUlpbCbDbLJiKixqDBBIPVakVERAT69euHrl27AgBMJhNUKhXc3d1ltTqdDiaTSap5OBQql1cue1yN2WzGvXv3qu0nOjoaWq1Wmnx8fJ56jEREjqDBBENYWBiysrIQHx9v71YAAPPnz0dxcbE03bx5094tERHVCoe+xlApPDwcBw4cwIkTJ9CmTRtpvl6vR1lZGYqKimRHDfn5+dDr9VLN2bNnZdvLz8+XllX+WTnv4RqNRgM3N7dqe1Kr1VCr1U89NiIiR+PQRwxCCISHh2PPnj1ITk6GwWCQLe/duzdcXFyQlJQkzcvJyUFeXh6MRiMAwGg0IjMzEwUFBVJNYmIiNBoN/Pz8pJqHt1FZU7kNIqJniUMfMYSFhWH79u3Yt28fmjdvLl0T0Gq1cHNzg1arRWhoKCIjI9GiRQtoNBrMmDEDRqMRffr0AQAMGTIEfn5+GDt2LGJiYmAymbBgwQKEhYVJv/FPnToV69evx5w5c/D2228jOTkZu3btwsGDB+02diIie3HoI4YNGzaguLgYr7zyCry8vKRp586dUs2qVavwxz/+ESNHjsTAgQOh1+vx9ddfS8udnJxw4MABODk5wWg04q233sK4ceOwdOlSqcZgMODgwYNITExEjx49sGLFCnzxxRe8VZWInkkOfcRQ+QFUj+Pq6orY2FjExsb+Zk27du1w6NChx27nlVdewcWLF23ukYiosXHoIwYiIqp/DAYiIpJhMBARkQyDgYiIZBgMREQkw2AgIiIZBgMREckwGIiISIbBQEREMgwGIiKSYTAQEZEMg4GIiGQYDEREJMNgICIiGQYDERHJMBiIiEiGwUBERDIMBiIikmEwEBGRDIOBiIhkGAxERCTDYCAiIhkGAxERyTAYiIhIhsFAREQyDAYiIpJhMBARkQyDgYiIZBgMREQkw2AgIiIZBgMREckwGIiISIbBQEREMgyGR8TGxqJ9+/ZwdXVFQEAAzp49a++WiIjqFYPhITt37kRkZCQWLVqECxcuoEePHggKCkJBQYG9WyMiqjcMhoesXLkSkyZNwsSJE+Hn54eNGzeiSZMm2Lx5s71bIyKqNwyGfykrK0N6ejoCAwOleUqlEoGBgUhNTbVjZ0RE9cvZ3g04ip9//hkWiwU6nU42X6fT4erVq1XqS0tLUVpaKr0uLi5+MP9uMayWiifap7BaUFhYCGfnJ/syVFRUoOyuGQql0xPVA4ClogwKhRJKpyfbh6313EfD7on7aBj7KL9/FwAghHji7TwNBkMNRUdHY8mSJVXmH/zg/9m0nX2za6sjImrsfvnlF2i12jrfD4PhX1q1agUnJyfk5+fL5ufn50Ov11epnz9/PiIjI6XXRUVFaNeuHfLy8urlC1ffzGYzfHx8cPPmTWg0Gnu3U+s4voatsY+vuLgYbdu2RYsWLeplfwyGf1GpVOjduzeSkpIwfPhwAIDVakVSUhLCw8Or1KvVaqjV6irztVpto/zGrKTRaDi+Bozja9iUyvq5LMxgeEhkZCTGjx8Pf39/vPzyy1i9ejXu3r2LiRMn2rs1IqJ6w2B4yKhRo/DTTz8hKioKJpMJPXv2REJCQpUL0kREjRmD4RHh4eHVnjr6d9RqNRYtWlTt6aXGgONr2Di+hq2+x6cQ9XX/ExERNQh8gxsREckwGIiISIbBQEREMgyGWtIQPq77xIkTeP311+Ht7Q2FQoG9e/fKlgshEBUVBS8vL7i5uSEwMBDXr1+X1RQWFiIkJAQajQbu7u4IDQ1FSUmJrObSpUsYMGAAXF1d4ePjg5iYmLoeGqKjo/HSSy+hefPm8PT0xPDhw5GTkyOruX//PsLCwtCyZUs0a9YMI0eOrPKGxry8PAQHB6NJkybw9PTE7NmzUVEh/4iT48eP48UXX4RarUaHDh0QFxdX18PDhg0b0L17d+k+faPRiG+++aZRjK06H3/8MRQKBSIiIqR5DXmMixcvhkKhkE2dOnVy3LEJemrx8fFCpVKJzZs3i+zsbDFp0iTh7u4u8vPz7d2azKFDh8QHH3wgvv76awFA7NmzR7b8448/FlqtVuzdu1d8++234k9/+pMwGAzi3r17Us3QoUNFjx49xJkzZ8R///d/iw4dOog333xTWl5cXCx0Op0ICQkRWVlZYseOHcLNzU189tlndTq2oKAgsWXLFpGVlSUyMjLEsGHDRNu2bUVJSYlUM3XqVOHj4yOSkpLE+fPnRZ8+fUTfvn2l5RUVFaJr164iMDBQXLx4URw6dEi0atVKzJ8/X6r55z//KZo0aSIiIyPF5cuXxbp164STk5NISEio0/Ht379fHDx4UFy7dk3k5OSI999/X7i4uIisrKwGP7ZHnT17VrRv3150795dzJw5U5rfkMe4aNEi0aVLF3Hr1i1p+umnnxx2bAyGWvDyyy+LsLAw6bXFYhHe3t4iOjrajl093qPBYLVahV6vF8uXL5fmFRUVCbVaLXbs2CGEEOLy5csCgDh37pxU88033wiFQiF++OEHIYQQn376qfDw8BClpaVSzdy5c4Wvr28dj0iuoKBAABApKSlCiAdjcXFxEbt375Zqrly5IgCI1NRUIcSD4FQqlcJkMkk1GzZsEBqNRhrPnDlzRJcuXWT7GjVqlAgKCqrrIVXh4eEhvvjii0Y1tjt37oiOHTuKxMRE8fvf/14KhoY+xkWLFokePXpUu8wRx8ZTSU+psXxcd25uLkwmk2wcWq0WAQEB0jhSU1Ph7u4Of39/qSYwMBBKpRJpaWlSzcCBA6FSqaSaoKAg5OTk4Pbt2/U0mv/7tNvKz5ZJT09HeXm5bHydOnVC27ZtZePr1q2b7A2NQUFBMJvNyM7Olmoe3kZlTX1+rS0WC+Lj43H37l0YjcZGNbawsDAEBwdX6aMxjPH69evw9vbG888/j5CQEOTl5QFwzLExGJ7S4z6u22Qy2akr21X2+rhxmEwmeHp6ypY7OzujRYsWsprqtvHwPuqa1WpFREQE+vXrh65du0r7VqlUcHd3r9KbLb3/Vo3ZbMa9e/fqYjiSzMxMNGvWDGq1GlOnTsWePXvg5+fXKMYGAPHx8bhw4QKio6OrLGvoYwwICEBcXBwSEhKwYcMG5ObmYsCAAbhz545Djo3vfKZGJywsDFlZWTh58qS9W6lVvr6+yMjIQHFxMf7rv/4L48ePR0pKir3bqhU3b97EzJkzkZiYCFdXV3u3U+tee+016e/du3dHQEAA2rVrh127dsHNzc2OnVWPRwxPydaP63ZUlb0+bhx6vb7K868rKipQWFgoq6luGw/voy6Fh4fjwIEDOHbsGNq0aSPN1+v1KCsrQ1FRUZXebOn9t2o0Gk2d/wdXqVTo0KEDevfujejoaPTo0QNr1qxpFGNLT09HQUEBXnzxRTg7O8PZ2RkpKSlYu3YtnJ2dodPpGvwYH+bu7o7f/e53+O677xzy68dgeEoPf1x3pcqP6zYajXbszDYGgwF6vV42DrPZjLS0NGkcRqMRRUVFSE9Pl2qSk5NhtVoREBAg1Zw4cQLl5eVSTWJiInx9feHh4VFn/QshEB4ejj179iA5ORkGg0G2vHfv3nBxcZGNLycnB3l5ebLxZWZmysIvMTERGo0Gfn5+Us3D26isscfX2mq1orS0tFGMbfDgwcjMzERGRoY0+fv7IyQkRPp7Qx/jw0pKSvD999/Dy8vLMb9+Nl+upiri4+OFWq0WcXFx4vLly2Ly5MnC3d1ddgeBI7hz5464ePGiuHjxogAgVq5cKS5evCj+53/+Rwjx4HZVd3d3sW/fPnHp0iXxxhtvVHu7aq9evURaWpo4efKk6Nixo+x21aKiIqHT6cTYsWNFVlaWiI+PF02aNKnz21WnTZsmtFqtOH78uOyWwF9//VWqmTp1qmjbtq1ITk4W58+fF0ajURiNRml55S2BQ4YMERkZGSIhIUG0bt262lsCZ8+eLa5cuSJiY2Pr5XbHefPmiZSUFJGbmysuXbok5s2bJxQKhThy5EiDH9tvefiuJCEa9hjfffddcfz4cZGbmytOnTolAgMDRatWrURBQYFDjo3BUEvWrVsn2rZtK1QqlXj55ZfFmTNn7N1SFceOHRMAqkzjx48XQjy4ZXXhwoVCp9MJtVotBg8eLHJycmTb+OWXX8Sbb74pmjVrJjQajZg4caK4c+eOrObbb78V/fv3F2q1Wjz33HPi448/rvOxVTcuAGLLli1Szb1798T06dOFh4eHaNKkifjzn/8sbt26JdvOjRs3xGuvvSbc3NxEq1atxLvvvivKy8tlNceOHRM9e/YUKpVKPP/887J91JW3335btGvXTqhUKtG6dWsxePBgKRQa+th+y6PB0JDHOGrUKOHl5SVUKpV47rnnxKhRo8R3333nsGPjp6sSEZEMrzEQEZEMg4GIiGQYDEREJMNgICIiGQYDERHJMBiIiEiGwUBERDIMBiIikmEwENWjCRMmYPjw4TVe/5VXXpE97pKoLjAY6JlXkx+2/AFNjRmDgegZ8/An3xJVh8FAz7QJEyYgJSUFa9asgUKhgEKhwI0bN5CSkoKXX34ZarUaXl5emDdvHioqKh67jsViQWhoKAwGA9zc3ODr64s1a9bUuLe7d+9i3LhxaNasGby8vLBixYoqNQqFAnv37pXNc3d3R1xcHADgxo0bUCgU2LlzJ37/+9/D1dUV27Ztq3FP9GzgE9zombZmzRpcu3YNXbt2xdKlSwE8eKbysGHDMGHCBHz11Ve4evUqJk2aBFdXVyxevLjadVq3bg2r1Yo2bdpg9+7daNmyJU6fPo3JkyfDy8sLf/nLX2zubfbs2UhJScG+ffvg6emJ999/HxcuXEDPnj1t3ta8efOwYsUK9OrVq1E+IY1qF4OBnmlarRYqlQpNmjSRnoT1wQcfwMfHB+vXr4dCoUCnTp3w448/Yu7cuYiKiqp2HQBwcnLCkiVLpNcGgwGpqanYtWuXzcFQUlKCL7/8Ev/5n/+JwYMHAwC2bt0qeyqdLSIiIjBixIgarUvPHp5KInrElStXYDQaoVAopHn9+vVDSUkJ/vd///ex68bGxqJ3795o3bo1mjVrhk2bNiEvL8/mHr7//nuUlZVJT8YDgBYtWsDX19fmbQGAv79/jdajZxODgaiWxMfH47333kNoaCiOHDmCjIwMTJw4EWVlZXW2T4VCgUcfqVLdxeWmTZvWWQ/U+DAY6JmnUqlgsVik1507d0ZqaqrsB+6pU6fQvHlz6VTOo+tU1vTt2xfTp09Hr1690KFDB3z//fc16umFF16Ai4sL0tLSpHm3b9/GtWvXZHWtW7fGrVu3pNfXr1/Hr7/+WqN9ElViMNAzr3379khLS8ONGzfw888/Y/r06bh58yZmzJiBq1evYt++fVi0aBEiIyOhVCqrXcdqtaJjx444f/48Dh8+jGvXrmHhwoU4d+5cjXpq1qwZQkNDMXv2bCQnJyMrKwsTJkyQ9l/p1Vdfxfr163Hx4kWcP38eU6dOhYuLy1P/m9CzjcFAz7z33nsPTk5O8PPzQ+vWrVFeXo5Dhw7h7Nmz6NGjB6ZOnYrQ0FAsWLDgN9fJy8vDlClTMGLECIwaNQoBAQH45ZdfMH369Br3tXz5cgwYMACvv/46AgMD0b9/f/Tu3VtWs2LFCvj4+GDAgAEYM2YM3nvvPTRp0qTG+yQCAD7zmYiIZHjEQEREMgwGIjvIy8tDs2bNfnOqyS2uRLWFp5KI7KCiogI3btz4zeXt27eHszPff0r2wWAgIiIZnkoiIiIZBgMREckwGIiISIbBQEREMgwGIiKSYTAQEZEMg4GIiGQYDEREJPP/Ae3FZ1NHl6azAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 400x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "week2_test.loc[week2_test[\"total_dur\"] < 500, \"total_dur\"] = 0\n",
    "\n",
    "sns.histplot(week2_test.total_dur, bins=1000)\n",
    "plt.axis(xmin=0, xmax=5000, ymax=10000)\n",
    "plt.tight_layout();"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "Модель показала себя лучше, чем в группе B на 2.5%, причем эффект распространился на 10% самых смотрящих пользователей.\n",
    "\n",
    "=> Поэтому докину эффект 2.5% для топ-10 смотрящих пользователей в группу B"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "salt = \"kiontestmodel20210805\"\n",
    "\n",
    "week2_groups = groups_splitter(week2_test, [\"user_id\", \"total_dur\", \"group\"], user_salt=salt)\n",
    "week2_groups.total_dur = week2_groups.total_dur.astype(float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>total_dur</th>\n",
       "      <th>group</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>223506</th>\n",
       "      <td>803922</td>\n",
       "      <td>0.0</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61999</th>\n",
       "      <td>231833</td>\n",
       "      <td>0.0</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62000</th>\n",
       "      <td>231838</td>\n",
       "      <td>0.0</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57132</th>\n",
       "      <td>214158</td>\n",
       "      <td>223396.0</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76042</th>\n",
       "      <td>281449</td>\n",
       "      <td>223477.0</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24122</th>\n",
       "      <td>1085683</td>\n",
       "      <td>223485.0</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        user_id  total_dur group\n",
       "223506   803922        0.0     B\n",
       "61999    231833        0.0     A\n",
       "62000    231838        0.0     B\n",
       "57132    214158   223396.0     B\n",
       "76042    281449   223477.0     A\n",
       "24122   1085683   223485.0     B"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# отсортирую по total_dur\n",
    "week2_groups = week2_groups.sort_values(\"total_dur\")\n",
    "\n",
    "pd.concat([week2_groups.head(3), week2_groups.tail(3)], axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "group_b = week2_groups[week2_groups.group == \"B\"]\n",
    "top_10_watchers_group_b = group_b.index[int(len(group_b) * 0.9) :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>total_dur</th>\n",
       "      <th>group</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>223506</th>\n",
       "      <td>803922</td>\n",
       "      <td>0.000</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61999</th>\n",
       "      <td>231833</td>\n",
       "      <td>0.000</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62000</th>\n",
       "      <td>231838</td>\n",
       "      <td>0.000</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57132</th>\n",
       "      <td>214158</td>\n",
       "      <td>228980.900</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76042</th>\n",
       "      <td>281449</td>\n",
       "      <td>223477.000</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24122</th>\n",
       "      <td>1085683</td>\n",
       "      <td>229072.125</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        user_id   total_dur group\n",
       "223506   803922       0.000     B\n",
       "61999    231833       0.000     A\n",
       "62000    231838       0.000     B\n",
       "57132    214158  228980.900     B\n",
       "76042    281449  223477.000     A\n",
       "24122   1085683  229072.125     B"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Effect for B group\n",
    "effect = 1.025\n",
    "week2_groups.loc[top_10_watchers_group_b, \"total_dur\"] *= effect\n",
    "\n",
    "pd.concat([week2_groups.head(3), week2_groups.tail(3)], axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(18.421093750000004, 445.92226562499997)\n",
      "H_0 отвергнута: True\n"
     ]
    }
   ],
   "source": [
    "metric_column = \"total_dur\"\n",
    "alpha = 0.05\n",
    "\n",
    "vec_a = week2_groups[(week2_groups['group'] == 'A')][metric_column]\n",
    "vec_b = week2_groups[(week2_groups['group'] == 'B')][metric_column]\n",
    "\n",
    "sample_a_mean = calc_bootstrap_mean(vec_a.to_numpy(), size=1000)\n",
    "sample_b_mean = calc_bootstrap_mean(vec_b.to_numpy(), size=1000)\n",
    "diff_mean = sample_b_mean - sample_a_mean\n",
    "\n",
    "left_side, right_side = np.percentile(diff_mean, [100 * alpha / 2., 100 * (1 - alpha / 2.)])\n",
    "print((left_side, right_side))\n",
    "print(\"H_0 отвергнута:\", not left_side < 0 < right_side)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Вывод**:\n",
    "\n",
    "С вероятностью 95% математическое ожидание разностей средних групп A и B лежит правее нуля, поэтому мы отвергаем нулевую гипотезу ```H_0``` об отсутствии изменеий между группами и принимаем альтернитивную гипотезу ```H_a``` о наличии эффекта между группами A и B.\n",
    "\n",
    "При $\\alpha = 0.05$ и $\\beta = 0.2$ минимальный задетектированный эффект составил ```0.9%``` (вычисленный по теоритической формуле).\n",
    "\n",
    "Эмпирический MDE (при тех же $\\alpha = 0.05$ и $\\beta = 0.2$) лежит в диапазоне от ```1% до 3%```, так как для эффекта в 1% мощность равна 32%, а для эффекта в 3% мощность уже равна 99%, зачит мощность в 80% можно получить добавив эффект больше 1% и меньше 3%."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "source": [
    "# ДЗ 1:\n",
    "\n",
    "### Дизайн теста\n",
    "1) Из interactions взять данные за 2 последние недели (2021-08-09 по 2021-08-22)\n",
    "\n",
    "2) Посчитайте на этих данных корректность на бутстрепе и сравните ее с результатами за 1 неделю (посчитано на семинаре) (0.5 балла)\n",
    "\n",
    "3) На двух неделях посчитайте мощность. Для этого нужно в одну из групп докинуть эффект. Докиньте эффект в 1%, 3% и 5% и сравните полученную мощность (1.5 балла)\n",
    "\n",
    "4) Посчитайте MDE, который можно зафиксировать на 2х неделях. В качестве alpha и beta подставьте ваши вычисленные ошибки 1 и 2 рода. Учитывайте что у нас формула для MDE работает для t-test или z-test (1 балл)\n",
    "\n",
    "### A/B тест\n",
    "1) Представим что у нас прошел тест, используем те же самые данные за 2 недели\n",
    "\n",
    "2) Занулите для всех пользователей total_dur, у которых total_dur < 500. Их НЕ УБИРАЕМ, а просто обрабатываем эти значения, принимая за нулевые, но пользователей также учитываем в эксперименте (0.5 балла)\n",
    "\n",
    "3) Разбейте их самостоятельно на две равные группы, используйте функцию groups_splitter и соль = 'kiontestmodel20210805' (0.5 балла)\n",
    "\n",
    "4) Оказалось, что модель в группе В показала себя лучше, чем в группе А на 2.5%, причем эффект распространился неравномерно и преимущественно на 10% самых смотрящих пользователей, докиньте такой эффект самостоятельно. Нужно метрику total_dur увеличить на 2.5% для 10% пользователей с самым продолжительным смотрением. (1 балл)\n",
    "\n",
    "5) Посчитайте результат такого теста и сделайте выводы (2 балла)\n",
    "\n",
    "Еще 1 балл можно получить за хорошее и подробное объяснение каждого шага"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "source": [
    "СОВЕТ: Т.к бутстрэп считается долго, то для дз отлаживаем код на малом количестве итераций, а результаты уже показываем на 1000+"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}