tinyllama_fine-tuning_Taylor_Swift.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
        "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
        "id": "kIdWjnQBCR6a"
      },
      "source": [
        "## Fine Tuning TinyLlama\n",
        "\n",
        "Here, the purpose is to make teach the model the common words in and the style of Taylor Swift's song lyrics.\n",
        "\n",
        "So there is no separate instruction here in the dataset.\n",
        "\n",
        "Data - https://github.com/shaynak/taylor-swift-lyrics/blob/main/songs.csv"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "execution": {
          "iopub.execute_input": "2023-09-13T18:30:54.478962Z",
          "iopub.status.busy": "2023-09-13T18:30:54.478274Z",
          "iopub.status.idle": "2023-09-13T18:31:51.481873Z",
          "shell.execute_reply": "2023-09-13T18:31:51.480565Z",
          "shell.execute_reply.started": "2023-09-13T18:30:54.478926Z"
        },
        "id": "P-COdKCFCR6d",
        "outputId": "21e3ae12-8dd5-4c47-df3a-4d3ff65dc1f0",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "!pip install trl transformers accelerate peft bitsandbytes einops wandb -Uqqq"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2023-09-13T18:31:51.485617Z",
          "iopub.status.busy": "2023-09-13T18:31:51.485224Z",
          "iopub.status.idle": "2023-09-13T18:32:05.194582Z",
          "shell.execute_reply": "2023-09-13T18:32:05.193554Z",
          "shell.execute_reply.started": "2023-09-13T18:31:51.485579Z"
        },
        "id": "YzWYnxSaCR6e",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "import glob\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import re\n",
        "from peft import get_peft_model, PeftConfig, PeftModel, LoraConfig, prepare_model_for_kbit_training\n",
        "from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig, TrainingArguments, GenerationConfig\n",
        "from trl import SFTTrainer\n",
        "from datasets import Dataset\n",
        "import glob\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Load all .csv files from a directory which contains the Taylor Swift Lyrics"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PX26zB2dCR6g",
        "outputId": "061250cf-a572-4195-c3d2-11e60713bb6b"
      },
      "outputs": [],
      "source": [
        "import logging\n",
        "\n",
        "def find_csv_files(path, file_extension=\"*.csv\"):\n",
        "    \"\"\"\n",
        "    Find all CSV files in the given directory.\n",
        "\n",
        "    :param path: str, the directory to search in\n",
        "    :param file_extension: str, pattern to match files\n",
        "    :return: list of file paths\n",
        "    \"\"\"\n",
        "    try:\n",
        "        files = glob.glob(f\"{path}/{file_extension}\")\n",
        "        if not files:\n",
        "            logging.warning(f\"No files found in {path} with extension {file_extension}\")\n",
        "        return files\n",
        "    except Exception as e:\n",
        "        logging.error(f\"Error finding files in {path}: {e}\")\n",
        "        return []\n",
        "\n",
        "def read_csv_files(file_paths, column_name='Lyrics'):\n",
        "    \"\"\"\n",
        "    Read and concatenate CSV files into a single DataFrame.\n",
        "\n",
        "    :param file_paths: list of file paths\n",
        "    :param column_name: str, name of the column to extract\n",
        "    :return: DataFrame\n",
        "    \"\"\"\n",
        "    df_list = []\n",
        "    for file in file_paths:\n",
        "        try:\n",
        "            df = pd.read_csv(file)\n",
        "            if column_name in df.columns:\n",
        "                df_list.append(df)\n",
        "            else:\n",
        "                logging.warning(f\"Column {column_name} not found in {file}\")\n",
        "        except Exception as e:\n",
        "            logging.error(f\"Error reading {file}: {e}\")\n",
        "    return pd.concat(df_list, ignore_index=True) if df_list else pd.DataFrame()\n",
        "\n",
        "def concatenate_lyrics(df, column_name='Lyrics'):\n",
        "    \"\"\"\n",
        "    Concatenate lyrics from a DataFrame into a single string.\n",
        "\n",
        "    :param df: DataFrame containing the lyrics\n",
        "    :param column_name: str, the column containing lyrics\n",
        "    :return: str, concatenated lyrics\n",
        "    \"\"\"\n",
        "    if column_name in df.columns:\n",
        "        return '\\n'.join(df[column_name])\n",
        "    logging.warning(f\"Column {column_name} not found in DataFrame\")\n",
        "    return \"\"\n",
        "\n",
        "def load_and_concatenate_lyrics(path, file_extension=\"*.csv\", column_name='Lyrics'):\n",
        "    \"\"\"\n",
        "    Load CSV files from the specified path, concatenate them,\n",
        "    and return the combined lyrics.\n",
        "\n",
        "    :param path: str, the directory containing the CSV files\n",
        "    :param file_extension: str, pattern to match files\n",
        "    :param column_name: str, name of the column to extract\n",
        "    :return: str, concatenated lyrics from all CSV files\n",
        "    \"\"\"\n",
        "    files = find_csv_files(path, file_extension)\n",
        "    if not files:\n",
        "        return \"\"\n",
        "\n",
        "    df = read_csv_files(files, column_name)\n",
        "    if df.empty:\n",
        "        return \"\"\n",
        "\n",
        "    return concatenate_lyrics(df, column_name)\n",
        "\n",
        "path = '../data_input/' # For local machine\n",
        "\n",
        "# path = '/content/'        # From Colab\n",
        "\n",
        "lyrics = load_and_concatenate_lyrics(path)\n",
        "\n",
        "print(lyrics[:200])  # Printing first 200 characters of the lyrics"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "execution": {
          "iopub.execute_input": "2023-09-13T18:32:05.339495Z",
          "iopub.status.busy": "2023-09-13T18:32:05.338568Z",
          "iopub.status.idle": "2023-09-13T18:32:05.349315Z",
          "shell.execute_reply": "2023-09-13T18:32:05.347798Z",
          "shell.execute_reply.started": "2023-09-13T18:32:05.339459Z"
        },
        "id": "pM6JRJhzCR6h",
        "outputId": "b5128243-ad75-4422-9a58-0ba7a0b9b28e",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "# List of all unique characters\n",
        "# This is just to verify the contents of the lyrics\n",
        "print(' '.join(sorted(set(lyrics))))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2023-09-13T18:32:05.350857Z",
          "iopub.status.busy": "2023-09-13T18:32:05.350579Z",
          "iopub.status.idle": "2023-09-13T18:32:05.396171Z",
          "shell.execute_reply": "2023-09-13T18:32:05.395192Z",
          "shell.execute_reply.started": "2023-09-13T18:32:05.350822Z"
        },
        "id": "rmjOMy18CR6i",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "import re\n",
        "\n",
        "def replace_characters(text, replacement_dict):\n",
        "    \"\"\"Replace characters in a string based on a dictionary mapping.\"\"\"\n",
        "    return text.translate(str.maketrans(replacement_dict))\n",
        "\n",
        "\"\"\" See more detailed notes on this replace_characters() method at the\n",
        "end of this notebook \"\"\"\n",
        "\n",
        "def remove_patterns(text, pattern_list):\n",
        "    \"\"\"Remove all instances of patterns in a list from a string.\"\"\"\n",
        "    for pattern in pattern_list:\n",
        "        text = re.sub(pattern, '', text)\n",
        "    return text\n",
        "\n",
        "def clean_lyrics(lyrics):\n",
        "    \"\"\"Clean lyrics by replacing and removing specific characters and patterns.\"\"\"\n",
        "    replace_with_space = ['\\u2005', '\\u200b', '\\u205f', '\\xa0', '-']\n",
        "    replace_letters = {'í':'i', 'é':'e', 'ï':'i', 'ó':'o', ';':',', '‘':'\\'', '’':'\\'', ':':',', 'е':'e'}\n",
        "    remove_list = ['\\)', '\\(', '–','\"','”', '\"', '\\[.*\\]', '.*\\|.*', '—']\n",
        "\n",
        "    # Replace specific letters\n",
        "    lyrics = replace_characters(lyrics, replace_letters)\n",
        "\n",
        "    # Replace specific strings with a space\n",
        "    for string in replace_with_space:\n",
        "        lyrics = lyrics.replace(string, ' ')\n",
        "\n",
        "    # Remove unwanted patterns\n",
        "    lyrics = remove_patterns(lyrics, remove_list)\n",
        "\n",
        "    return lyrics\n",
        "\n",
        "# Example usage\n",
        "cleaned_lyrics = clean_lyrics(lyrics)\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CxQ-jIS-CR6i",
        "outputId": "41d39c18-2c0c-40f9-f280-f4ba87e16b23"
      },
      "outputs": [],
      "source": [
        "# This is just to verify the cleaned text\n",
        "print(''.join(sorted(set(cleaned_lyrics))))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 174
        },
        "id": "K3e69_oiCR6j",
        "outputId": "3f922ad1-dd71-4a32-de5b-5708925b8262"
      },
      "outputs": [],
      "source": [
        "cleaned_lyrics"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2023-09-13T18:32:05.397924Z",
          "iopub.status.busy": "2023-09-13T18:32:05.397597Z",
          "iopub.status.idle": "2023-09-13T18:32:05.420652Z",
          "shell.execute_reply": "2023-09-13T18:32:05.419761Z",
          "shell.execute_reply.started": "2023-09-13T18:32:05.397893Z"
        },
        "id": "7M_WcnhgCR6j",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "from datasets import Dataset\n",
        "\n",
        "def create_train_test_datasets(cleaned_lyrics, train_ratio=0.95, segment_length=500):\n",
        "    \"\"\"\n",
        "    Splits the cleaned lyrics into training and testing datasets.\n",
        "    The training data is further segmented into smaller chunks.\n",
        "\n",
        "    Args:\n",
        "    cleaned_lyrics (list): The preprocessed lyrics data.\n",
        "    train_ratio (float, optional): Ratio of data to be used for training. Default is 0.95.\n",
        "    segment_length (int, optional): Length of each segment in the training data. Default is 500.\n",
        "\n",
        "    Returns:\n",
        "    Tuple[Dataset, list]: A tuple containing the training dataset and the test dataset.\n",
        "    \"\"\"\n",
        "    split_point = int(len(cleaned_lyrics) * train_ratio)\n",
        "    train_data = cleaned_lyrics[:split_point]\n",
        "    test_data = cleaned_lyrics[split_point:]\n",
        "\n",
        "    # Segmenting the training data\n",
        "    train_data_segments = [train_data[i:i + segment_length]\n",
        "                           for i in range(0, len(train_data), segment_length)]\n",
        "    # In above using `range(start, end, step)` syntax\n",
        "    train_dataset = Dataset.from_dict({'text': train_data_segments})\n",
        "\n",
        "    return train_dataset, test_data\n",
        "\n",
        "# Example usage\n",
        "train_dataset, test_data = create_train_test_datasets(cleaned_lyrics)\n",
        "\n",
        "\n",
        "# 557"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AB8pDCdMCR6k",
        "outputId": "c7e527ef-e37c-4490-ba27-828cdd943422"
      },
      "outputs": [],
      "source": [
        "print(len(train_dataset))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_MfUDDEWCR6k",
        "outputId": "d99586cb-5ef6-4e20-ef70-1fa98a14d592"
      },
      "outputs": [],
      "source": [
        "# Check the dataset structure\n",
        "# Should output below\n",
        "\"\"\"\n",
        "Dataset({\n",
        "    features: ['text'],\n",
        "    num_rows: 557\n",
        "})\n",
        "\"\"\"\n",
        "train_dataset\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 504
        },
        "id": "z4TaWEu7CR6l",
        "outputId": "70adf089-fc6b-4f0b-83d7-c40e66bc568f"
      },
      "outputs": [],
      "source": [
        "from pprint import pprint\n",
        "pprint(train_dataset[0])\n",
        "\n",
        "\"\"\"\n",
        "{'text': \"Knew he was a killer first time that I saw him\\nWondered how many girls he had loved and left haunted\\nBut if he's a ghost, then I can be a phantom\\nHoldin' him for ransom, some\\nSome boys are tryin' too hard, he don't try at all though\\nYounger than my exes, but he act like such a man, so\\nI see nothing better, I keep him forever\\nLike a vendetta ta\\nI, I, I see how this is gon' go\\nTouch me and you'll never be alone\\nI Island breeze and lights down low\\nNo one has to know\\nIn the middle of the night, in m\"} \"\"\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 113,
          "referenced_widgets": [
            "f2f42be2169e4d5c947b4dbb67dacce6",
            "a08fe5cd982f4cb8a6883034889a3c3a",
            "4ca3d43ad62c4592a992616979d11884",
            "d8c0b35b2362424bbeb612e676dcb681",
            "b85be3155e0a4f27a9b5dbb9b9820baf",
            "1a32d7e9048c4fffbe8f90acb2cdcf29",
            "038a7e2a3ab440a68ea7cdaa15e3cba8",
            "700aad1c80db4668813eece0ad22b3de",
            "e7a1d24bba074ebda46e6ab631b4f056",
            "cfdcab19276247a18468c916abbb081c",
            "5eb5507229ec4b96a7c652558d25e424",
            "67fc10cfdc614afe9491a6c0576a842b",
            "3d6cc4ecf25748dabc610f64d5220ec5",
            "077636025d5d488cab0fc6b8f6be6300",
            "943605cb14d34c8fbc54916e2c1f46d7",
            "03cc0c1245d5430e98d66c0facf01cdd",
            "afd90946c85549fc87462a3cf3b17477",
            "f8b09584f0ae4f54bfadb4010ddc01e3",
            "91e472c9b4574db1b5d19a8982a27e1f",
            "a32458fd5e9d4d35823a5bbd83dcf45e",
            "1ece89281458475ba780c46681a2c2e6",
            "0d4dd76f91d54ed5ac149397e7a4f468",
            "b2ad3d3a05ab44fdb3a687ea7e5c6a8c",
            "e8965ae93ea94589b4560877de3f0120",
            "a7a25b1dbb2d480ab93e39540b4f6d8e",
            "aab5e78bc4c442f7b898dfec3e102eac",
            "a20d8dd7fe2f408e865a7aa15220ad3f",
            "61c1f646137a4e699ae72839241f4193",
            "ca42c802d3494e448741a360fa8a394c",
            "81576b568e9c4a3f95e542b28de49d88",
            "98dcbdb86b734bb3a782493670874799",
            "e2f8f90db43d4c4e8c7c86992e8a5d45",
            "d44cb02b71ac4a1e8993e6764ce0a062"
          ]
        },
        "execution": {
          "iopub.execute_input": "2023-09-13T18:34:04.854830Z",
          "iopub.status.busy": "2023-09-13T18:34:04.854443Z",
          "iopub.status.idle": "2023-09-13T18:34:09.453336Z",
          "shell.execute_reply": "2023-09-13T18:34:09.452093Z",
          "shell.execute_reply.started": "2023-09-13T18:34:04.854799Z"
        },
        "id": "_DTE99sqCR6l",
        "outputId": "3e77b725-ff25-4662-be91-85f1efed60ba",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "def load_quantized_model(model_identifier: str, compute_dtype: torch.dtype) -> AutoModelForCausalLM:\n",
        "    \"\"\"\n",
        "    Loads a quantized model using the BitsAndBytesConfig with specified parameters.\n",
        "\n",
        "    Args:\n",
        "    - model_identifier (str): The name of the model to be loaded.\n",
        "    - compute_dtype (torch.dtype): The data type for computation in the quantized model.\n",
        "\n",
        "    Returns:\n",
        "    - AutoModelForCausalLM: The loaded quantized model.\n",
        "    \"\"\"\n",
        "    quantization_config = BitsAndBytesConfig(\n",
        "        load_in_4bit=True,\n",
        "        bnb_4bit_quant_type=\"nf4\",\n",
        "        bnb_4bit_use_double_quant=True,\n",
        "        bnb_4bit_compute_dtype=compute_dtype,\n",
        "    )\n",
        "\n",
        "    model =  AutoModelForCausalLM.from_pretrained(\n",
        "        model_identifier,\n",
        "        quantization_config=quantization_config,\n",
        "        device_map=\"auto\",\n",
        "        trust_remote_code=True,\n",
        "    )\n",
        "    return model\n",
        "\n",
        "# Load the model\n",
        "# model_name = \"PY007/TinyLlama-1.1B-step-50K-105b\"\n",
        "\n",
        "model_name = \"TinyLlama/TinyLlama-1.1B-Chat-v1.0\"\n",
        "\n",
        "model = load_quantized_model(model_name, torch.bfloat16)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 145,
          "referenced_widgets": [
            "39de801da3484b00bf04b9866fbf985f",
            "d264f3543e28409db57bc6e0f2a53139",
            "809802cf4d5440e0b6d4862658b7bd24",
            "98833d77318747ae9a859f63e7e8dbed",
            "84a70d99d9fd4acf8b203b7dd8f93027",
            "43c9e45258de46a384d1403610f7eb56",
            "ac2c81ef433947ffa85c8ba470a64449",
            "8f0dcd30b3594aa093cae7311d222e6f",
            "141bab2715aa4a3d89cf64a06977b04b",
            "34ec523aba564ebab2de39665162a37f",
            "1e86c217408749ad95a73b7d4f936a35",
            "e785aed1edfb40fb83197559e133cd1b",
            "8556f4a6a12a424aafa136c6a58dc2f5",
            "5f316346e25d4b4a84325e68267bb6b9",
            "753fe787608b4324a54145ad4b4b9be5",
            "f6a192d5b1c94ab1a636d0204ad68f38",
            "3a90a980b11a446c85d8447a7368d22d",
            "dc26a6d266e2474d9f74907c390e8268",
            "9c5dd2dc66434100a30e04973d07ac47",
            "a4639106f1634771bf318fb8577e3394",
            "4a3989426af448cdba8e4616176e60c0",
            "e2fa31d586724fefb910420f78a4ada0",
            "ed6da7550bab4630b0186b13d78872aa",
            "523fc795232d4ab39a2c7565e967c978",
            "9cf2f40fbb874da29932676abb0e6170",
            "aa2b5f3761004fa1a99d5a356bf316da",
            "9a3323f0a3824463b8f2b62e34f02b1d",
            "9f27bb62517c43b69a20aba2fe4efebf",
            "1ea3d30adfc148d58cf0a1b029b7e09b",
            "35b60b750e5e4d418920800ff31f4256",
            "00aab10751ee43b795c683c8721edceb",
            "5c03f2c8b47b40d3b47c74656caead38",
            "737ded9421c041b78a0a6b611eb65348",
            "e9108e89dcc44b588d177e94f0be02df",
            "65051076f1a14ac9ac0cacdfe60148d7",
            "87ec26d90c434b5eb0e06bc5c94ebbe3",
            "6a7c8c715b044db4821705878b4cb54c",
            "8908a01c37a1416bb4ed8b4fd049e1c9",
            "1031996b9bb046d6a3468b77544e2cec",
            "291618408c20410f86922bc9e7c554fd",
            "003143dc734a43fb83d73af5bdc0d128",
            "3f8071c1b6ba47e5911d4d0c4c6d5cda",
            "dfe295100d3c49b197ebeb2662d25175",
            "883f20b0438b4244bcaad0054903b4df"
          ]
        },
        "execution": {
          "iopub.execute_input": "2023-09-13T18:34:09.456008Z",
          "iopub.status.busy": "2023-09-13T18:34:09.455390Z",
          "iopub.status.idle": "2023-09-13T18:34:09.630490Z",
          "shell.execute_reply": "2023-09-13T18:34:09.629261Z",
          "shell.execute_reply.started": "2023-09-13T18:34:09.455942Z"
        },
        "id": "Dbx9MGMzCR6l",
        "outputId": "a11e1e51-34b8-40bd-b822-19b350546226",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "# Creating tokenizer and defining the pad token\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)\n",
        "tokenizer.pad_token = tokenizer.eos_token"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 29,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "execution": {
          "iopub.execute_input": "2023-09-13T18:34:09.634524Z",
          "iopub.status.busy": "2023-09-13T18:34:09.631970Z",
          "iopub.status.idle": "2023-09-13T18:34:27.510143Z",
          "shell.execute_reply": "2023-09-13T18:34:27.509186Z",
          "shell.execute_reply.started": "2023-09-13T18:34:09.634491Z"
        },
        "id": "jUpY2fPjCR6l",
        "outputId": "9694071a-e622-4aa8-abae-503a24c45bd4",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "# Generating lyrics with the base model. The repetition penalty in the generation config prevents the model from continually repeating the same string.\n",
        "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
        "\n",
        "def generate_lyrics(query, model):\n",
        "    encoding = tokenizer(query, return_tensors=\"pt\").to(device)\n",
        "    generation_config = GenerationConfig(max_new_tokens=250, pad_token_id = tokenizer.eos_token_id,repetition_penalty=1.3, eos_token_id = tokenizer.eos_token_id)\n",
        "    outputs = model.generate(input_ids=encoding.input_ids, generation_config=generation_config)\n",
        "    text_output = tokenizer.decode(outputs[0],skip_special_tokens=True)\n",
        "    print('INPUT\\n', query, '\\n\\nOUTPUT\\n', text_output[len(query):])\n",
        "\n",
        "generate_lyrics(test_data[200:700], model)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2023-09-13T18:34:27.518158Z",
          "iopub.status.busy": "2023-09-13T18:34:27.515727Z",
          "iopub.status.idle": "2023-09-13T18:34:28.963773Z",
          "shell.execute_reply": "2023-09-13T18:34:28.962709Z",
          "shell.execute_reply.started": "2023-09-13T18:34:27.518121Z"
        },
        "id": "Xbmv0OBBCR6l",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "# Setting arguments for low-rank adaptation\n",
        "\n",
        "model = prepare_model_for_kbit_training(model)\n",
        "\n",
        "lora_alpha = 32 # The weight matrix is scaled by lora_alpha/lora_rank, so I set lora_alpha = lora_rank to remove scaling\n",
        "lora_dropout = 0.05\n",
        "lora_rank = 32\n",
        "\n",
        "lora_config = LoraConfig(\n",
        "    lora_alpha=lora_alpha,\n",
        "    lora_dropout=lora_dropout,\n",
        "    r=lora_rank,\n",
        "    bias=\"none\",  # setting to 'none' for only training weight params instead of biases\n",
        "    task_type=\"CAUSAL_LM\")\n",
        "\n",
        "peft_model = get_peft_model(model, lora_config)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "execution": {
          "iopub.execute_input": "2023-09-13T18:34:28.971384Z",
          "iopub.status.busy": "2023-09-13T18:34:28.969063Z",
          "iopub.status.idle": "2023-09-13T18:34:28.982056Z",
          "shell.execute_reply": "2023-09-13T18:34:28.980879Z",
          "shell.execute_reply.started": "2023-09-13T18:34:28.971346Z"
        },
        "id": "8RQrjst8CR6m",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "# Setting training arguments\n",
        "\n",
        "output_dir = \"hf-username/tinylama_taylor_swift\" # Model repo on your hugging face account where you want to save your model\n",
        "per_device_train_batch_size = 3\n",
        "gradient_accumulation_steps = 2\n",
        "optim = \"paged_adamw_32bit\"\n",
        "save_strategy=\"steps\"\n",
        "save_steps = 10\n",
        "logging_steps = 10\n",
        "learning_rate = 2e-3\n",
        "max_grad_norm = 0.3 # Sets limit for gradient clipping\n",
        "max_steps = 200     # Number of training steps\n",
        "warmup_ratio = 0.03 # Portion of steps used for learning_rate to warmup from 0\n",
        "lr_scheduler_type = \"cosine\" # I chose cosine to avoid learning plateaus\n",
        "\n",
        "training_arguments = TrainingArguments(\n",
        "    output_dir=output_dir,\n",
        "    per_device_train_batch_size=per_device_train_batch_size,\n",
        "    gradient_accumulation_steps=gradient_accumulation_steps,\n",
        "    optim=optim,\n",
        "    save_steps=save_steps,\n",
        "    logging_steps=logging_steps,\n",
        "    learning_rate=learning_rate,\n",
        "    max_grad_norm=max_grad_norm,\n",
        "    max_steps=max_steps,\n",
        "    warmup_ratio=warmup_ratio,\n",
        "    lr_scheduler_type=lr_scheduler_type,\n",
        "    # push_to_hub=True,\n",
        "    report_to='none'\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 49,
          "referenced_widgets": [
            "5b33396d5ba4481cb0d715e2eb52928b",
            "a35b9e3cd74d446981641e7952ef12ca",
            "c88cc16340f04ade93303bdfc04582a3",
            "262ccbf79ff44cf1832565a7c7460e1d",
            "e28e001c022145c3aeae29476fa91f84",
            "51b2dce41d2449d29457fb1320a9ce15",
            "78395a5926fe42f3b77c539dafbf37d1",
            "64b621720c1c4c148fe70a74667deb1d",
            "01581d0543fa44bc99a9de31e660b75c",
            "066ad979fd6641d08df2779ef69c3c1d",
            "80d92ea66e28406ea85077b42ae981f8"
          ]
        },
        "execution": {
          "iopub.execute_input": "2023-09-13T18:34:28.984399Z",
          "iopub.status.busy": "2023-09-13T18:34:28.983634Z",
          "iopub.status.idle": "2023-09-13T18:34:29.430658Z",
          "shell.execute_reply": "2023-09-13T18:34:29.429642Z",
          "shell.execute_reply.started": "2023-09-13T18:34:28.984361Z"
        },
        "id": "DtF-H4YpCR6m",
        "outputId": "6fecdb9a-e144-4316-8026-e048382d7bfb",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "trainer = SFTTrainer(\n",
        "    model=peft_model,\n",
        "    train_dataset=train_dataset,\n",
        "    peft_config=lora_config,\n",
        "    max_seq_length=500,\n",
        "    dataset_text_field='text',\n",
        "    tokenizer=tokenizer,\n",
        "    args=training_arguments\n",
        ")\n",
        "peft_model.config.use_cache = False"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 147
        },
        "execution": {
          "iopub.execute_input": "2023-09-13T18:34:29.435585Z",
          "iopub.status.busy": "2023-09-13T18:34:29.435093Z",
          "iopub.status.idle": "2023-09-13T18:40:20.272515Z",
          "shell.execute_reply": "2023-09-13T18:40:20.271435Z",
          "shell.execute_reply.started": "2023-09-13T18:34:29.435549Z"
        },
        "id": "rLK8g0WRCR6m",
        "outputId": "9d258486-84b0-46fe-c6a1-ac53b95eaff2",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "trainer.train()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "###############################################\n",
        "## Explanatory Notes\n",
        "###############################################\n",
        "\n",
        "```py\n",
        "def replace_characters(text, replacement_dict):\n",
        "    \"\"\"Replace characters in a string based on a dictionary mapping.\"\"\"\n",
        "    return text.translate(str.maketrans(replacement_dict))\n",
        "\n",
        "```\n",
        "\n",
        "\n",
        "- `str.maketrans(replacement_dict)`: This function creates a translation table. The `replacement_dict` is a dictionary where each key-value pair represents a character to be replaced and its corresponding replacement character. \n",
        "\n",
        "📌 This translation table maps Unicode ordinals (or character code points) of the keys to ordinals of the values. The `str.maketrans` function is typically used to create a mapping table.\n",
        "\n",
        "- `text.translate(translation_table)`: This method returns a copy of `text` where each character has been mapped through the translation table. Characters mapped to `None` are deleted. In this context, it applies the character replacements defined in `replacement_dict` to the string `text`.\n",
        "\n",
        "In essence, this combination is a highly efficient way to perform multiple character replacements in a string in a single pass, making it more suitable for tasks like cleaning or formatting large text data, compared to using multiple `replace` calls.\n",
        "\n",
        "-----------------------\n",
        "\n",
        "Simple Example\n",
        "\n",
        "```py\n",
        "\n",
        "dict = {\"a\": \"123\", \"b\": \"456\", \"c\": \"789\"}\n",
        "string = \"abc\"\n",
        "print(string.maketrans(dict))\n",
        "# {97: '123', 98: '456', 99: '789'}\n",
        "\n",
        "```\n",
        "\n",
        "\n",
        "📌 The `maketrans` method when used with a single argument, such as a dictionary, the keys must be single characters and the values are their corresponding replacements.\n",
        "\n",
        "📌 In above, `string.maketrans(dict)` creates a translation table from the dictionary `dict`. The dictionary keys ('a', 'b', 'c') are single characters, and their ASCII values are 97, 98, and 99 respectively.\n",
        "\n",
        "📌 The translation table is a dictionary where each key is the ASCII value of the single-character keys from `dict`, and the values are the corresponding values from `dict`.\n",
        "\n",
        "📌 Thus, the output `{97: '123', 98: '456', 99: '789'}` represents a mapping where 'a' (ASCII 97) is replaced by '123', 'b' (ASCII 98) by '456', and 'c' (ASCII 99) by '789'. This table is used for translating or replacing characters in a string using the `translate` method.\n",
        "\n",
        "-------\n",
        "\n",
        "📌 The `maketrans` function must map characters to their replacements, and in Python, characters are often managed internally using their ASCII values. This representation allows for efficient storage and manipulation of characters in memory.\n",
        "\n",
        "📌 When you pass a dictionary to `maketrans`, it converts the keys from characters to their ASCII values because the **string translation process in Python operates at the byte level**. Using ASCII values directly is more efficient for this process.\n",
        "\n",
        "📌 Therefore, in your code `string.maketrans(dict)`, the dictionary keys 'a', 'b', and 'c' are automatically converted to their ASCII values 97, 98, and 99, respectively. This conversion forms a mapping table where each ASCII value (representing a character in the original string) is mapped to the corresponding replacement string as defined in your dictionary.\n",
        "\n",
        "📌 This translation table is then used by the `translate` method of string objects to replace characters in the string based on the provided mappings. For example, if you use this table to translate a string, every occurrence of 'a' will be replaced by '123', 'b' by '456', and 'c' by '789'."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "#######################################################3\n",
        "\n",
        "## Explaining the below block\n",
        "\n",
        "```py\n",
        "train_data_segments = [train_data[i:i + segment_length]\n",
        "                           for i in range(0, len(train_data), segment_length)]\n",
        "\n",
        "```\n",
        "\n",
        "📌 The line `for i in range(0, len(train_data), segment_length)` in the list comprehension serves a specific purpose in segmenting the training data. \n",
        "\n",
        "This line iterates over the `train_data` list in steps of `segment_length`. For each iteration, it starts at index `i` and ends at `i + segment_length`, creating a new segment of the training data. By specifying `segment_length` as the step size in the `range` function, it ensures that each segment is of the specified length, except possibly the last segment which might be shorter if the total length of `train_data` is not a multiple of `segment_length`.\n",
        "\n",
        "The usage of `range(start, end, step)` here is an efficient way to iterate through the `train_data` while creating segments of a defined length, which is crucial for managing memory and computational efficiency."
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "T4",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.13"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "003143dc734a43fb83d73af5bdc0d128": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "00aab10751ee43b795c683c8721edceb": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "01581d0543fa44bc99a9de31e660b75c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "038a7e2a3ab440a68ea7cdaa15e3cba8": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "03cc0c1245d5430e98d66c0facf01cdd": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "066ad979fd6641d08df2779ef69c3c1d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "077636025d5d488cab0fc6b8f6be6300": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_91e472c9b4574db1b5d19a8982a27e1f",
            "max": 2200119864,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_a32458fd5e9d4d35823a5bbd83dcf45e",
            "value": 2200119864
          }
        },
        "0d4dd76f91d54ed5ac149397e7a4f468": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1031996b9bb046d6a3468b77544e2cec": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "141bab2715aa4a3d89cf64a06977b04b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "1a32d7e9048c4fffbe8f90acb2cdcf29": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1e86c217408749ad95a73b7d4f936a35": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1ea3d30adfc148d58cf0a1b029b7e09b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1ece89281458475ba780c46681a2c2e6": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "262ccbf79ff44cf1832565a7c7460e1d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_066ad979fd6641d08df2779ef69c3c1d",
            "placeholder": "​",
            "style": "IPY_MODEL_80d92ea66e28406ea85077b42ae981f8",
            "value": " 769/769 [00:00&lt;00:00, 1681.54 examples/s]"
          }
        },
        "291618408c20410f86922bc9e7c554fd": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "34ec523aba564ebab2de39665162a37f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "35b60b750e5e4d418920800ff31f4256": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "39de801da3484b00bf04b9866fbf985f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_d264f3543e28409db57bc6e0f2a53139",
              "IPY_MODEL_809802cf4d5440e0b6d4862658b7bd24",
              "IPY_MODEL_98833d77318747ae9a859f63e7e8dbed"
            ],
            "layout": "IPY_MODEL_84a70d99d9fd4acf8b203b7dd8f93027"
          }
        },
        "3a90a980b11a446c85d8447a7368d22d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3d6cc4ecf25748dabc610f64d5220ec5": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_afd90946c85549fc87462a3cf3b17477",
            "placeholder": "​",
            "style": "IPY_MODEL_f8b09584f0ae4f54bfadb4010ddc01e3",
            "value": "model.safetensors: 100%"
          }
        },
        "3f8071c1b6ba47e5911d4d0c4c6d5cda": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "43c9e45258de46a384d1403610f7eb56": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4a3989426af448cdba8e4616176e60c0": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4ca3d43ad62c4592a992616979d11884": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_700aad1c80db4668813eece0ad22b3de",
            "max": 608,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e7a1d24bba074ebda46e6ab631b4f056",
            "value": 608
          }
        },
        "51b2dce41d2449d29457fb1320a9ce15": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "523fc795232d4ab39a2c7565e967c978": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9f27bb62517c43b69a20aba2fe4efebf",
            "placeholder": "​",
            "style": "IPY_MODEL_1ea3d30adfc148d58cf0a1b029b7e09b",
            "value": "tokenizer.json: 100%"
          }
        },
        "5b33396d5ba4481cb0d715e2eb52928b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a35b9e3cd74d446981641e7952ef12ca",
              "IPY_MODEL_c88cc16340f04ade93303bdfc04582a3",
              "IPY_MODEL_262ccbf79ff44cf1832565a7c7460e1d"
            ],
            "layout": "IPY_MODEL_e28e001c022145c3aeae29476fa91f84"
          }
        },
        "5c03f2c8b47b40d3b47c74656caead38": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5eb5507229ec4b96a7c652558d25e424": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5f316346e25d4b4a84325e68267bb6b9": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9c5dd2dc66434100a30e04973d07ac47",
            "max": 499723,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_a4639106f1634771bf318fb8577e3394",
            "value": 499723
          }
        },
        "61c1f646137a4e699ae72839241f4193": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "64b621720c1c4c148fe70a74667deb1d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "65051076f1a14ac9ac0cacdfe60148d7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1031996b9bb046d6a3468b77544e2cec",
            "placeholder": "​",
            "style": "IPY_MODEL_291618408c20410f86922bc9e7c554fd",
            "value": "special_tokens_map.json: 100%"
          }
        },
        "67fc10cfdc614afe9491a6c0576a842b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_3d6cc4ecf25748dabc610f64d5220ec5",
              "IPY_MODEL_077636025d5d488cab0fc6b8f6be6300",
              "IPY_MODEL_943605cb14d34c8fbc54916e2c1f46d7"
            ],
            "layout": "IPY_MODEL_03cc0c1245d5430e98d66c0facf01cdd"
          }
        },
        "6a7c8c715b044db4821705878b4cb54c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_dfe295100d3c49b197ebeb2662d25175",
            "placeholder": "​",
            "style": "IPY_MODEL_883f20b0438b4244bcaad0054903b4df",
            "value": " 551/551 [00:00&lt;00:00, 44.1kB/s]"
          }
        },
        "700aad1c80db4668813eece0ad22b3de": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "737ded9421c041b78a0a6b611eb65348": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "753fe787608b4324a54145ad4b4b9be5": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4a3989426af448cdba8e4616176e60c0",
            "placeholder": "​",
            "style": "IPY_MODEL_e2fa31d586724fefb910420f78a4ada0",
            "value": " 500k/500k [00:00&lt;00:00, 27.6MB/s]"
          }
        },
        "78395a5926fe42f3b77c539dafbf37d1": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "809802cf4d5440e0b6d4862658b7bd24": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8f0dcd30b3594aa093cae7311d222e6f",
            "max": 1289,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_141bab2715aa4a3d89cf64a06977b04b",
            "value": 1289
          }
        },
        "80d92ea66e28406ea85077b42ae981f8": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "81576b568e9c4a3f95e542b28de49d88": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "84a70d99d9fd4acf8b203b7dd8f93027": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8556f4a6a12a424aafa136c6a58dc2f5": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3a90a980b11a446c85d8447a7368d22d",
            "placeholder": "​",
            "style": "IPY_MODEL_dc26a6d266e2474d9f74907c390e8268",
            "value": "tokenizer.model: 100%"
          }
        },
        "87ec26d90c434b5eb0e06bc5c94ebbe3": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_003143dc734a43fb83d73af5bdc0d128",
            "max": 551,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_3f8071c1b6ba47e5911d4d0c4c6d5cda",
            "value": 551
          }
        },
        "883f20b0438b4244bcaad0054903b4df": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8908a01c37a1416bb4ed8b4fd049e1c9": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8f0dcd30b3594aa093cae7311d222e6f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "91e472c9b4574db1b5d19a8982a27e1f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "943605cb14d34c8fbc54916e2c1f46d7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1ece89281458475ba780c46681a2c2e6",
            "placeholder": "​",
            "style": "IPY_MODEL_0d4dd76f91d54ed5ac149397e7a4f468",
            "value": " 2.20G/2.20G [00:17&lt;00:00, 174MB/s]"
          }
        },
        "98833d77318747ae9a859f63e7e8dbed": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_34ec523aba564ebab2de39665162a37f",
            "placeholder": "​",
            "style": "IPY_MODEL_1e86c217408749ad95a73b7d4f936a35",
            "value": " 1.29k/1.29k [00:00&lt;00:00, 89.2kB/s]"
          }
        },
        "98dcbdb86b734bb3a782493670874799": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "9a3323f0a3824463b8f2b62e34f02b1d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9c5dd2dc66434100a30e04973d07ac47": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9cf2f40fbb874da29932676abb0e6170": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_35b60b750e5e4d418920800ff31f4256",
            "max": 1842767,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_00aab10751ee43b795c683c8721edceb",
            "value": 1842767
          }
        },
        "9f27bb62517c43b69a20aba2fe4efebf": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a08fe5cd982f4cb8a6883034889a3c3a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1a32d7e9048c4fffbe8f90acb2cdcf29",
            "placeholder": "​",
            "style": "IPY_MODEL_038a7e2a3ab440a68ea7cdaa15e3cba8",
            "value": "config.json: 100%"
          }
        },
        "a20d8dd7fe2f408e865a7aa15220ad3f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a32458fd5e9d4d35823a5bbd83dcf45e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a35b9e3cd74d446981641e7952ef12ca": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_51b2dce41d2449d29457fb1320a9ce15",
            "placeholder": "​",
            "style": "IPY_MODEL_78395a5926fe42f3b77c539dafbf37d1",
            "value": "Map: 100%"
          }
        },
        "a4639106f1634771bf318fb8577e3394": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a7a25b1dbb2d480ab93e39540b4f6d8e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_81576b568e9c4a3f95e542b28de49d88",
            "max": 124,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_98dcbdb86b734bb3a782493670874799",
            "value": 124
          }
        },
        "aa2b5f3761004fa1a99d5a356bf316da": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5c03f2c8b47b40d3b47c74656caead38",
            "placeholder": "​",
            "style": "IPY_MODEL_737ded9421c041b78a0a6b611eb65348",
            "value": " 1.84M/1.84M [00:00&lt;00:00, 20.2MB/s]"
          }
        },
        "aab5e78bc4c442f7b898dfec3e102eac": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e2f8f90db43d4c4e8c7c86992e8a5d45",
            "placeholder": "​",
            "style": "IPY_MODEL_d44cb02b71ac4a1e8993e6764ce0a062",
            "value": " 124/124 [00:00&lt;00:00, 6.75kB/s]"
          }
        },
        "ac2c81ef433947ffa85c8ba470a64449": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "afd90946c85549fc87462a3cf3b17477": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b2ad3d3a05ab44fdb3a687ea7e5c6a8c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_e8965ae93ea94589b4560877de3f0120",
              "IPY_MODEL_a7a25b1dbb2d480ab93e39540b4f6d8e",
              "IPY_MODEL_aab5e78bc4c442f7b898dfec3e102eac"
            ],
            "layout": "IPY_MODEL_a20d8dd7fe2f408e865a7aa15220ad3f"
          }
        },
        "b85be3155e0a4f27a9b5dbb9b9820baf": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c88cc16340f04ade93303bdfc04582a3": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_64b621720c1c4c148fe70a74667deb1d",
            "max": 769,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_01581d0543fa44bc99a9de31e660b75c",
            "value": 769
          }
        },
        "ca42c802d3494e448741a360fa8a394c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "cfdcab19276247a18468c916abbb081c": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d264f3543e28409db57bc6e0f2a53139": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_43c9e45258de46a384d1403610f7eb56",
            "placeholder": "​",
            "style": "IPY_MODEL_ac2c81ef433947ffa85c8ba470a64449",
            "value": "tokenizer_config.json: 100%"
          }
        },
        "d44cb02b71ac4a1e8993e6764ce0a062": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d8c0b35b2362424bbeb612e676dcb681": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cfdcab19276247a18468c916abbb081c",
            "placeholder": "​",
            "style": "IPY_MODEL_5eb5507229ec4b96a7c652558d25e424",
            "value": " 608/608 [00:00&lt;00:00, 9.25kB/s]"
          }
        },
        "dc26a6d266e2474d9f74907c390e8268": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "dfe295100d3c49b197ebeb2662d25175": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e28e001c022145c3aeae29476fa91f84": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e2f8f90db43d4c4e8c7c86992e8a5d45": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e2fa31d586724fefb910420f78a4ada0": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e785aed1edfb40fb83197559e133cd1b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_8556f4a6a12a424aafa136c6a58dc2f5",
              "IPY_MODEL_5f316346e25d4b4a84325e68267bb6b9",
              "IPY_MODEL_753fe787608b4324a54145ad4b4b9be5"
            ],
            "layout": "IPY_MODEL_f6a192d5b1c94ab1a636d0204ad68f38"
          }
        },
        "e7a1d24bba074ebda46e6ab631b4f056": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "e8965ae93ea94589b4560877de3f0120": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_61c1f646137a4e699ae72839241f4193",
            "placeholder": "​",
            "style": "IPY_MODEL_ca42c802d3494e448741a360fa8a394c",
            "value": "generation_config.json: 100%"
          }
        },
        "e9108e89dcc44b588d177e94f0be02df": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_65051076f1a14ac9ac0cacdfe60148d7",
              "IPY_MODEL_87ec26d90c434b5eb0e06bc5c94ebbe3",
              "IPY_MODEL_6a7c8c715b044db4821705878b4cb54c"
            ],
            "layout": "IPY_MODEL_8908a01c37a1416bb4ed8b4fd049e1c9"
          }
        },
        "ed6da7550bab4630b0186b13d78872aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_523fc795232d4ab39a2c7565e967c978",
              "IPY_MODEL_9cf2f40fbb874da29932676abb0e6170",
              "IPY_MODEL_aa2b5f3761004fa1a99d5a356bf316da"
            ],
            "layout": "IPY_MODEL_9a3323f0a3824463b8f2b62e34f02b1d"
          }
        },
        "f2f42be2169e4d5c947b4dbb67dacce6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a08fe5cd982f4cb8a6883034889a3c3a",
              "IPY_MODEL_4ca3d43ad62c4592a992616979d11884",
              "IPY_MODEL_d8c0b35b2362424bbeb612e676dcb681"
            ],
            "layout": "IPY_MODEL_b85be3155e0a4f27a9b5dbb9b9820baf"
          }
        },
        "f6a192d5b1c94ab1a636d0204ad68f38": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f8b09584f0ae4f54bfadb4010ddc01e3": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}