1 分词.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 分词"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.1最大正向匹配法"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1、读取字典文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['AT&T 3 nz\\n', 'B超 3 n\\n', 'c# 3 nz\\n', 'C# 3 nz\\n']\n",
      "['AT&T', 'B超', 'c#', 'C#']\n",
      "349046\n"
     ]
    }
   ],
   "source": [
    "with open('../data/字典.txt', encoding='utf-8') as f:\n",
    "    txt = f.readlines()\n",
    "    \n",
    "print(txt[:4])\n",
    "dictionary = [i.split()[0] for i in txt]\n",
    "print(dictionary[:4])\n",
    "print(len(dictionary))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2、最大正向匹配法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentence = '今天我来到北京清华大学。'\n",
    "max_len = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "我来到北京清华大学。\n",
      "来到北京清华大学。\n",
      "北京清华大学。\n",
      "清华大学。\n",
      "。\n",
      "\n",
      "['今天', '我', '来到', '北京', '清华大学', '。']\n"
     ]
    }
   ],
   "source": [
    "words = []\n",
    "while len(sentence) != 0:\n",
    "    tmp = sentence[:max_len]\n",
    "    while tmp not in dictionary and len(tmp) != 1:\n",
    "        tmp = tmp[: -1]\n",
    "    words.append(tmp)\n",
    "    sentence = sentence[len(tmp):]\n",
    "    print(sentence)\n",
    "\n",
    "print(words)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "3、自定义函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fenci(sentence, dic, max_len=5):\n",
    "    words = []\n",
    "    while len(sentence) != 0:\n",
    "        tmp = sentence[:max_len]\n",
    "        while tmp not in dic and len(tmp) != 1:  # 要注意添加len(tmp) != 1这个添加，不然会陷入死循环，最后一个符号出不来\n",
    "            tmp = tmp[: -1]      \n",
    "        words.append(tmp)\n",
    "        sentence = sentence[len(tmp):]\n",
    "    print(words)\n",
    "    \n",
    "    return words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['今天', '我们', '学习', '文本', '挖掘', '。']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['今天', '我们', '学习', '文本', '挖掘', '。']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fenci(sentence='今天我们学习文本挖掘。', dic=dictionary, max_len=4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.2 最大逆向匹配法"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "读取字典"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../data/字典.txt', encoding='utf-8') as f:\n",
    "    txt = f.readlines()\n",
    "dictionary = [i.split()[0] for i in txt]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "自定义实现"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['今天', '我们', '学习', '文本', '挖掘', '。']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence = '今天我来到北京清华大学。'\n",
    "max_len = 5\n",
    "\n",
    "def fenci2(sentence, dic, max_len):\n",
    "    words = []\n",
    "    while(len(sentence) != 0):\n",
    "        tmp = sentence[-max_len: ]\n",
    "        while tmp not in dic and len(tmp) != 1:\n",
    "            tmp = tmp[1:]\n",
    "        words = [tmp] + words\n",
    "        sentence = sentence[:-len(tmp)]\n",
    "    \n",
    "    return words\n",
    "\n",
    "fenci2(sentence='今天我们学习文本挖掘。', dic=dictionary, max_len=4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.3 隐式马可夫模型HMM"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "数据导入"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "p_start = {'good':0.63, 'normal':0.17, 'bad':0.2}\n",
    "p_emit = {\n",
    "    'good':{'working':0.05, 'traval':0.35, 'shopping':0.35, 'running':0.25},\n",
    "    'normal':{'working':0.25, 'traval':0.25, 'shopping':0.25, 'running':0.25},\n",
    "    'bad':{'working':0.6, 'traval':0.2, 'shopping':0.05, 'running':0.15}\n",
    "}\n",
    "p_trans = {\n",
    "    'good':{'good':0.5, 'normal':0.375, 'bad':0.125},\n",
    "    'normal':{'good':0.25, 'normal':0.125, 'bad':0.625},\n",
    "    'bad':{'good':0.25, 'normal':0.375, 'bad':0.375}\n",
    "}\n",
    "# 不容易构建、保存，但是索引比较容易"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "假设观察到K连续3天的行为分布是：工作、购物、旅行。那么K三天的心情是什么样子的？"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-- 穷举法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'good': 0.0315, 'normal': 0.0425, 'bad': 0.12}]\n",
      "{'good-good-good': 0.0009646874999999999, 'normal-good-good': 0.0006507812499999999, 'bad-good-good': 0.0018374999999999997, 'good-normal-good': 0.0002583984375, 'normal-normal-good': 0.0001162109375, 'bad-normal-good': 0.000984375, 'good-bad-good': 1.72265625e-05, 'normal-bad-good': 0.0001162109375, 'bad-bad-good': 0.000196875, 'good-good-normal': 0.000516796875, 'normal-good-normal': 0.0003486328125, 'bad-good-normal': 0.000984375, 'good-normal-normal': 9.228515625e-05, 'normal-normal-normal': 4.150390625e-05, 'bad-normal-normal': 0.0003515625, 'good-bad-normal': 1.845703125e-05, 'normal-bad-normal': 0.00012451171875, 'bad-bad-normal': 0.00021093750000000002, 'good-good-bad': 0.0001378125, 'normal-good-bad': 9.296875e-05, 'bad-good-bad': 0.0002625, 'good-normal-bad': 0.000369140625, 'normal-normal-bad': 0.000166015625, 'bad-normal-bad': 0.00140625, 'good-bad-bad': 1.4765625000000001e-05, 'normal-bad-bad': 9.960937500000001e-05, 'bad-bad-bad': 0.00016875000000000004}\n",
      "连续状态为['working', 'shopping', 'traval']后，心情可能是：bad-good-good\n"
     ]
    }
   ],
   "source": [
    "obs = ['working', 'shopping', 'traval']\n",
    "states = ['good', 'normal', 'bad']\n",
    "\n",
    "V = [{}]  # 记录条路径及相应的概率\n",
    "\n",
    "# 初始化\n",
    "for y in states:\n",
    "    V[0][y] = p_start[y] * p_emit[y][obs[0]]\n",
    "print(V)\n",
    "\n",
    "\n",
    "for t in range(1, len(obs)): \n",
    "    V.append({})\n",
    "    for y in states:  # t时刻的心情状态\n",
    "        for i, j in V[-2].items():\n",
    "            pa = i.split('-')[-1]  # t-1时刻的心情状态\n",
    "            V[t][i+'-'+y] = j * p_emit[y][obs[t]] * p_trans[pa][y]\n",
    "\n",
    "print(V[-1])\n",
    "(prob, path) = max((j, i) for i, j in V[-1].items())\n",
    "print(f'连续状态为{obs}后，心情可能是：{path}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['good', 'normal', 'bad'], ['good', 'good', 'good', 'normal', 'normal', 'normal', 'bad', 'bad', 'bad'], ['good', 'good', 'good', 'good', 'good', 'good', 'good', 'good', 'good', 'normal', 'normal', 'normal', 'normal', 'normal', 'normal', 'normal', 'normal', 'normal', 'bad', 'bad', 'bad', 'bad', 'bad', 'bad', 'bad', 'bad', 'bad']]\n",
      "[{'good': 0.0315, 'normal': 0.0425, 'bad': 0.12}, {'good-good': 0.0055125, 'normal-good': 0.00371875, 'bad-good': 0.010499999999999999, 'good-normal': 0.002953125, 'normal-normal': 0.001328125, 'bad-normal': 0.01125, 'good-bad': 0.000196875, 'normal-bad': 0.001328125, 'bad-bad': 0.0022500000000000003}, {'good-good-good': 0.0009646874999999999, 'normal-good-good': 0.0006507812499999999, 'bad-good-good': 0.0018374999999999997, 'good-normal-good': 0.0002583984375, 'normal-normal-good': 0.0001162109375, 'bad-normal-good': 0.000984375, 'good-bad-good': 1.72265625e-05, 'normal-bad-good': 0.0001162109375, 'bad-bad-good': 0.000196875, 'good-good-normal': 0.000516796875, 'normal-good-normal': 0.0003486328125, 'bad-good-normal': 0.000984375, 'good-normal-normal': 9.228515625e-05, 'normal-normal-normal': 4.150390625e-05, 'bad-normal-normal': 0.0003515625, 'good-bad-normal': 1.845703125e-05, 'normal-bad-normal': 0.00012451171875, 'bad-bad-normal': 0.00021093750000000002, 'good-good-bad': 0.0001378125, 'normal-good-bad': 9.296875e-05, 'bad-good-bad': 0.0002625, 'good-normal-bad': 0.000369140625, 'normal-normal-bad': 0.000166015625, 'bad-normal-bad': 0.00140625, 'good-bad-bad': 1.4765625000000001e-05, 'normal-bad-bad': 9.960937500000001e-05, 'bad-bad-bad': 0.00016875000000000004}]\n"
     ]
    }
   ],
   "source": [
    "print(tmp)\n",
    "print(V)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-- 维特比算法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# obs = ['working', 'shopping', 'traval']\n",
    "# states = ['good', 'normal', 'bad']\n",
    "\n",
    "# V = [{}]\n",
    "# path = {}\n",
    "\n",
    "# # 初始化\n",
    "# for y in states:\n",
    "#     V[0][y] = p_start[y] * p_emit[y][obs[0]]\n",
    "#     path[y] = [y]\n",
    "# print(path)\n",
    "# print(V)\n",
    "\n",
    "# # 从第二天开始\n",
    "# for t in range(1, 3):\n",
    "#     V.append({})\n",
    "#     newpath = {}\n",
    "#     for y in states:\n",
    "#         em_p = p_emit[y][obs[t]]\n",
    "#         (prob, state) = max([(V[t - 1][y0] * p_trans[y0][y] * em_p, y0) for y0 in states])\n",
    "#         V[t][y] = prob\n",
    "#         newpath[y] = path[state] + [y]\n",
    "#     print(newpath)\n",
    "#     path = newpath\n",
    "\n",
    "# (prob, state) = max((V[len(obs) - 1][y], y) for y in states)  # 确定三条路径的最优解\n",
    "\n",
    "# print(f'最后一天的心情为：{state}, 概率为：{prob}')\n",
    "# print(f'连续状态为{obs}后，心情可能是：{path[state]}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " [{'good': 0.0315, 'normal': 0.0425, 'bad': 0.12}]\n",
      "\n",
      " [{'good': ['good'], 'normal': ['normal'], 'bad': ['bad']}]\n",
      "\n",
      " [{'good': 0.0315, 'normal': 0.0425, 'bad': 0.12}, {'good': 0.010499999999999999, 'normal': 0.01125, 'bad': 0.00225}, {'good': 0.0018374999999999997, 'normal': 0.000984375, 'bad': 0.00140625}]\n",
      "\n",
      " [{'good': ['good'], 'normal': ['normal'], 'bad': ['bad']}, {'good': ['bad', 'good'], 'normal': ['bad', 'normal'], 'bad': ['bad', 'bad']}, {'good': ['bad', 'good', 'good'], 'normal': ['bad', 'good', 'normal'], 'bad': ['bad', 'normal', 'bad']}]\n",
      "\n",
      "最后一天的心情为：good, 概率为：0.0018374999999999997\n",
      "\n",
      "连续状态为['working', 'shopping', 'traval']后，心情可能是：['bad', 'good', 'good']\n"
     ]
    }
   ],
   "source": [
    "obs = ['working', 'shopping', 'traval']\n",
    "states = ['good', 'normal', 'bad']\n",
    "V = [{}]\n",
    "path = [{}]\n",
    "\n",
    "# 初始化\n",
    "for y in states:\n",
    "    V[0][y] = p_start[y] * p_emit[y][obs[0]]\n",
    "    path[0][y] = [y]\n",
    "    \n",
    "print('\\n', V)\n",
    "print('\\n', path)\n",
    "\n",
    "for t in range(1, len(obs)):\n",
    "    V.append({})\n",
    "    newpath = {}\n",
    "    for y in states:\n",
    "        em_p = p_emit[y][obs[t]]\n",
    "        (prob, state) = max((V[-2][y0]*p_trans[y0][y]*em_p, y0) for y0 in states)\n",
    "        V[-1][y] = prob\n",
    "        newpath[y] = path[-1][state] + [y]\n",
    "    path.append(newpath)\n",
    "    \n",
    "print('\\n', V)\n",
    "print('\\n', path)\n",
    "\n",
    "(prob, state) = max((V[-1][y], y) for y in states)  # 确定三条路径的最优解\n",
    "\n",
    "print(f'\\n最后一天的心情为：{state}, 概率为：{prob}')\n",
    "print(f'\\n连续状态为{obs}后，心情可能是：{path[-1][state]}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'bad': 0.12, 'good': 0.0315, 'normal': 0.0425},\n",
       " {'bad': 0.00225, 'good': 0.010499999999999999, 'normal': 0.01125},\n",
       " {'bad': 0.00140625, 'good': 0.0018374999999999997, 'normal': 0.000984375}]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "V"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'good': ['good'], 'normal': ['normal'], 'bad': ['bad']}, {'good': ['bad', 'good'], 'normal': ['bad', 'normal'], 'bad': ['bad', 'bad']}, {'good': ['bad', 'good', 'good'], 'normal': ['bad', 'good', 'normal'], 'bad': ['bad', 'normal', 'bad']}]\n"
     ]
    }
   ],
   "source": [
    "print(path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2、 HMM应用--分词  \n",
    "jieba程序的路径：C:\\Users\\45543\\AppData\\Local\\Continuum\\Anaconda3\\Lib\\site-packages\\jieba  \n",
    "中文字符集合：[\\u4E00-\\u9FD5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 导入父级目录下的Python文件\n",
    "import sys\n",
    "sys.path.append(r'C:\\Users\\45543\\Desktop\\NLP\\data')  # 这种方法属于一次性的，只对当前的python解释器进程有效，关掉python重启后就失效了\n",
    "from hmm.prob_start import P as start_P\n",
    "from hmm.prob_trans import P as trans_P\n",
    "from hmm.prob_emit import P as emit_P"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'B': -0.26268660809250016, 'E': -3.14e+100, 'M': -3.14e+100, 'S': -1.4652633398537678}\n",
      "\n",
      " {'B': {'E': -0.51082562376599, 'M': -0.916290731874155}, 'E': {'B': -0.5897149736854513, 'S': -0.8085250474669937}, 'M': {'E': -0.33344856811948514, 'M': -1.2603623820268226}, 'S': {'B': -0.7211965654669841, 'S': -0.6658631448798212}}\n"
     ]
    }
   ],
   "source": [
    "print(start_P)\n",
    "print('\\n', trans_P)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'B': -6.8985414338214115, 'E': -3.14e+100, 'M': -3.14e+100, 'S': -8.598245859008907}] {'B': ['B'], 'E': ['E'], 'M': ['M'], 'S': ['S']}\n",
      "-68.04528930719714 E\n",
      "{'B': ['B', 'E', 'S', 'S', 'S', 'B', 'E', 'B', 'M', 'E', 'B'], 'E': ['B', 'E', 'S', 'S', 'S', 'B', 'E', 'B', 'M', 'M', 'E'], 'M': ['B', 'E', 'S', 'S', 'S', 'B', 'E', 'B', 'M', 'M', 'M'], 'S': ['B', 'E', 'S', 'S', 'S', 'B', 'E', 'B', 'M', 'E', 'S']}\n",
      "\n",
      "今天我来到北京清华大学  \n",
      "标注结果为：\n",
      "  ['B', 'E', 'S', 'S', 'S', 'B', 'E', 'B', 'M', 'M', 'E']  \n",
      "分词结果为：\n",
      "  ['今天', '我', '来', '到', '北京', '清华大学']\n",
      "\n"
     ]
    }
   ],
   "source": [
    "obs = sentence = '今天我来到北京清华大学'\n",
    "states = 'BEMS'\n",
    "\n",
    "V = [{}]\n",
    "path = {}\n",
    "\n",
    "PrevStatus = {\n",
    "    'B': 'ES',\n",
    "    'M': 'MB',\n",
    "    'S': 'SE',\n",
    "    'E': 'BM'   # t时刻状态:t-1时刻状态\n",
    "}\n",
    "\n",
    "for y in states:\n",
    "    V[0][y] = start_P[y] + emit_P[y][obs[0]]\n",
    "    path[y] = [y]\n",
    "print(V, path)\n",
    "\n",
    "for t in range(1, len(obs)):\n",
    "    V.append({})\n",
    "    newpath = {}\n",
    "    for y in states:\n",
    "        em_p = emit_P[y][obs[t]]\n",
    "        \n",
    "        (prob, state) = max((em_p + V[t-1][y0] + trans_P[y0][y], y0) for y0 in PrevStatus[y])  # y0是上一个状态\n",
    "        V[t][y] = prob\n",
    "        newpath[y] = path[state] + [y]\n",
    "    path = newpath\n",
    "\n",
    "(prob, state) = max((V[len(obs) - 1][y], y) for y in 'ES')  # 注意'ES'，句子最后一个状态只能是这两种\n",
    "print(prob, state)\n",
    "print(path)\n",
    "pos_list = path[state]\n",
    "\n",
    "res = []\n",
    "for i, char in enumerate(obs):    \n",
    "    sign = pos_list[i]\n",
    "    if sign == 'B':\n",
    "        begin = i\n",
    "    elif sign == 'E':\n",
    "        res.append(obs[begin: i+1])\n",
    "    elif sign == 'S':\n",
    "        res.append(obs[i])\n",
    "    \n",
    "print(f'''\n",
    "{obs}  \n",
    "标注结果为：\n",
    "  {path[state]}  \n",
    "分词结果为：\n",
    "  {res}\n",
    "''')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pat = path[state]\n",
    "res = []\n",
    "for i, (word, char) in enumerate(zip(obs, pat)):\n",
    "    if char == 'S':\n",
    "        res.append(word)\n",
    "    elif char == 'B':\n",
    "        begin = i\n",
    "    elif char == 'E':\n",
    "        res.append(obs[begin:(i+1)])\n",
    "print(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def viterbi(obs, states, start_P, trans_P, emit_P):\n",
    "    V = [{}]\n",
    "    path = {}\n",
    "\n",
    "    PrevStatus = {\n",
    "        'B': 'ES',\n",
    "        'M': 'MB',\n",
    "        'S': 'SE',\n",
    "        'E': 'BM'   # t时刻状态:t-1时刻状态\n",
    "    }\n",
    "\n",
    "    for y in states:\n",
    "        V[0][y] = start_P[y] + emit_P[y][obs[0]]\n",
    "        path[y] = [y]\n",
    "    print(V, path)\n",
    "\n",
    "    for t in range(1, len(obs)):\n",
    "        V.append({})\n",
    "        newpath = {}\n",
    "        for y in states:\n",
    "            em_p = emit_P[y][obs[t]]\n",
    "\n",
    "            (prob, state) = max((em_p + V[t-1][y0] + trans_P[y0][y], y0) for y0 in PrevStatus[y])  # y0是上一个状态\n",
    "            V[t][y] = prob\n",
    "            newpath[y] = path[state] + [y]\n",
    "        path = newpath\n",
    "    (prob, state) = max((V[len(obs) - 1][y], y) for y in 'ES')\n",
    "    \n",
    "    return (prob, path[state])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'B': -6.8985414338214115, 'M': -3.14e+100, 'E': -3.14e+100, 'S': -8.598245859008907}] {'B': ['B'], 'M': ['M'], 'E': ['E'], 'S': ['S']}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(-68.04528930719714, ['B', 'E', 'S', 'S', 'S', 'B', 'E', 'B', 'M', 'M', 'E'])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "viterbi(sentence, 'BMES', start_P, trans_P, emit_P)  # 调用自定义函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step     2\n",
      "step   200\n"
     ]
    }
   ],
   "source": [
    "print(f'step{2:>6}')\n",
    "print(f'step{200:>6}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.4 jieba分词"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from the default dictionary ...\n",
      "Loading model from cache C:\\Users\\45543\\AppData\\Local\\Temp\\jieba.cache\n",
      "Loading model cost 0.758 seconds.\n",
      "Prefix dict has been built succesfully.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['今天', '我', '来到', '北京', '清华大学']\n",
      "['今天', '我', '来到', '北京', '清华', '华大', '大学', '清华大学']\n"
     ]
    }
   ],
   "source": [
    "import jieba\n",
    "print(jieba.lcut('今天我来到北京清华大学'))\n",
    "print(jieba.lcut_for_search('今天我来到北京清华大学'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "对《鹿鼎记》进行分词"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['\\ufeff', '《鹿鼎记》', '作者：金庸', '正文', '第一回', '纵横钩党清流祸', '峭茜风期月旦评']\n"
     ]
    }
   ],
   "source": [
    "with open('../data/鹿鼎记.txt', 'r', encoding='utf-8') as f:\n",
    "    txt = f.read()\n",
    "    \n",
    "txt = txt.split()\n",
    "print(txt[:7])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "import jieba\n",
    "\n",
    "# 导入自定义字典\n",
    "jieba.load_userdict('../data/coal_dict.txt')\n",
    "\n",
    "words = [jieba.lcut(t) for t in txt]\n",
    "\n",
    "from tkinter import _flatten\n",
    "words = list(_flatten(words))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['\\ufeff',\n",
       " '《',\n",
       " '鹿鼎记',\n",
       " '》',\n",
       " '作者',\n",
       " '：',\n",
       " '金庸',\n",
       " '正文',\n",
       " '第一回',\n",
       " '纵横',\n",
       " '钩',\n",
       " '党',\n",
       " '清流',\n",
       " '祸',\n",
       " '峭茜',\n",
       " '风期',\n",
       " '月旦评',\n",
       " '北风',\n",
       " '如刀',\n",
       " '，',\n",
       " '满地',\n",
       " '冰霜',\n",
       " '。']"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words[:23]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}