strength-programmer - 3_Finetuninng_Gemma_2b_it.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "collapsed_sections": [
        "giYslB0Tvihi",
        "dbYvR1G9vmoy"
      ],
      "toc_visible": true,
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "77edf79fa9bc4e45ab252e2532b497d8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "VBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "VBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "VBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b15e131e0ab24754bee0bf29c73ba5b0",
              "IPY_MODEL_fb057c9c5ccb4cf3921e7e87cfb3ab72",
              "IPY_MODEL_0006298be43b46639350b8b4cb55a088",
              "IPY_MODEL_1d1720fb2c89420f8b039402ec41e2ed"
            ],
            "layout": "IPY_MODEL_635cd679a9ef488a82b69ecde6859119"
          }
        },
        "dddf14daad59443ebe59fa0392b1ffde": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e2003e53be0c4fcd8656dfc8e8676f50",
            "placeholder": "​",
            "style": "IPY_MODEL_17e76d31e72a4cc8a9b28e671f1861ac",
            "value": "<center> <img\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.svg\nalt='Hugging Face'> <br> Copy a token from <a\nhref=\"https://huggingface.co/settings/tokens\" target=\"_blank\">your Hugging Face\ntokens page</a> and paste it below. <br> Immediately click login after copying\nyour token or it might be stored in plain text in this notebook file. </center>"
          }
        },
        "8757b2675f7a4b8b8d88a11730953ccc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "PasswordModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "PasswordModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "PasswordView",
            "continuous_update": true,
            "description": "Token:",
            "description_tooltip": null,
            "disabled": false,
            "layout": "IPY_MODEL_e1b8dd5242cc41329fdb2dbba00cf0b0",
            "placeholder": "​",
            "style": "IPY_MODEL_af4cebfd1736479ca49b248ac7f3180e",
            "value": ""
          }
        },
        "53c8b9a71ed942ba9d40b6f3cfd15d27": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "CheckboxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "CheckboxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "CheckboxView",
            "description": "Add token as git credential?",
            "description_tooltip": null,
            "disabled": false,
            "indent": true,
            "layout": "IPY_MODEL_b15ca3349017492f9bc42fce4c4e2666",
            "style": "IPY_MODEL_317932e503a04f42a0df4837e6084e79",
            "value": true
          }
        },
        "5a51d060eb99492d9e0418eef49986aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ButtonModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ButtonModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ButtonView",
            "button_style": "",
            "description": "Login",
            "disabled": false,
            "icon": "",
            "layout": "IPY_MODEL_57669b50fef6448eaa978ea68485a2cd",
            "style": "IPY_MODEL_da3b4cb230fb4f5f8c0acee68f13a932",
            "tooltip": ""
          }
        },
        "e38921b61f8b4912a498619775852551": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f4cc5f12ddf84103866801f813a0e9c8",
            "placeholder": "​",
            "style": "IPY_MODEL_293f1f89d2884108aa0d137fa79f08f6",
            "value": "\n<b>Pro Tip:</b> If you don't already have one, you can create a dedicated\n'notebooks' token with 'write' access, that you can then easily reuse for all\nnotebooks. </center>"
          }
        },
        "635cd679a9ef488a82b69ecde6859119": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": "center",
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": "flex",
            "flex": null,
            "flex_flow": "column",
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "e2003e53be0c4fcd8656dfc8e8676f50": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "17e76d31e72a4cc8a9b28e671f1861ac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e1b8dd5242cc41329fdb2dbba00cf0b0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "af4cebfd1736479ca49b248ac7f3180e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b15ca3349017492f9bc42fce4c4e2666": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "317932e503a04f42a0df4837e6084e79": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "57669b50fef6448eaa978ea68485a2cd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "da3b4cb230fb4f5f8c0acee68f13a932": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ButtonStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ButtonStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "button_color": null,
            "font_weight": ""
          }
        },
        "f4cc5f12ddf84103866801f813a0e9c8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "293f1f89d2884108aa0d137fa79f08f6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3d6465df2f41404ab212d3979422b71f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "LabelModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "LabelView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e38c5521f27946d088a7124e149f54f6",
            "placeholder": "​",
            "style": "IPY_MODEL_feb123f826854717b896aec111e34454",
            "value": "Connecting..."
          }
        },
        "e38c5521f27946d088a7124e149f54f6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "feb123f826854717b896aec111e34454": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b15e131e0ab24754bee0bf29c73ba5b0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "LabelModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "LabelView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bc2b7557e38e466bbeadce7a5031727f",
            "placeholder": "​",
            "style": "IPY_MODEL_46ecd2cb502146fb9780c8dd662175a5",
            "value": "Token is valid (permission: fineGrained)."
          }
        },
        "fb057c9c5ccb4cf3921e7e87cfb3ab72": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "LabelModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "LabelView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bf2b1b434a9b4f05b0289ed63fb3cd7a",
            "placeholder": "​",
            "style": "IPY_MODEL_18a45f5cb0dc4bf7a9f2b75ccb254774",
            "value": "Your token has been saved in your configured git credential helpers (store)."
          }
        },
        "0006298be43b46639350b8b4cb55a088": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "LabelModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "LabelView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d9ae3029bd1b496ab52c596f7ba28fe3",
            "placeholder": "​",
            "style": "IPY_MODEL_18855a606b014cf59adbec29d292197c",
            "value": "Your token has been saved to /root/.cache/huggingface/token"
          }
        },
        "1d1720fb2c89420f8b039402ec41e2ed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "LabelModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "LabelView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_de56c083ba61475a8b01e7bf36a5531d",
            "placeholder": "​",
            "style": "IPY_MODEL_4d1f39060b854f2e938a9c012c318ef6",
            "value": "Login successful"
          }
        },
        "bc2b7557e38e466bbeadce7a5031727f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "46ecd2cb502146fb9780c8dd662175a5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bf2b1b434a9b4f05b0289ed63fb3cd7a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "18a45f5cb0dc4bf7a9f2b75ccb254774": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d9ae3029bd1b496ab52c596f7ba28fe3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "18855a606b014cf59adbec29d292197c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "de56c083ba61475a8b01e7bf36a5531d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4d1f39060b854f2e938a9c012c318ef6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9fd74448ecdc4f9f91417ed2615e53f1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_4a17588d3d934e268606e165eeec9feb",
              "IPY_MODEL_7f6c7dc6d9f846e6bf2dd12e59d63e68",
              "IPY_MODEL_24028921209c41519b383e12e07954a2"
            ],
            "layout": "IPY_MODEL_1e588375ec214f8c92c5b343363e591b"
          }
        },
        "4a17588d3d934e268606e165eeec9feb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e32f3946859a46a7bb32ab3b325620d2",
            "placeholder": "​",
            "style": "IPY_MODEL_717e95ab43e140419e263c9015593854",
            "value": "Loading checkpoint shards: 100%"
          }
        },
        "7f6c7dc6d9f846e6bf2dd12e59d63e68": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2f4d429754494762a4434e5815ae44f8",
            "max": 2,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_4bddfc0b39744e41a970a92962651b0a",
            "value": 2
          }
        },
        "24028921209c41519b383e12e07954a2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_67cf01d78ab34e6fbec3e63318ace902",
            "placeholder": "​",
            "style": "IPY_MODEL_8d306e09734549e08ddc791a9493c706",
            "value": " 2/2 [00:35&lt;00:00, 14.74s/it]"
          }
        },
        "1e588375ec214f8c92c5b343363e591b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e32f3946859a46a7bb32ab3b325620d2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "717e95ab43e140419e263c9015593854": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2f4d429754494762a4434e5815ae44f8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4bddfc0b39744e41a970a92962651b0a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "67cf01d78ab34e6fbec3e63318ace902": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8d306e09734549e08ddc791a9493c706": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6ac8f2f7f4664a26bb7e07c1cde83eae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_212c6a5679744ff891650f601cd4e6c9",
              "IPY_MODEL_82ceff906c2443e2b05c4b6b379a489c",
              "IPY_MODEL_83775177ca1b4c06a2a57dc2c6b41827"
            ],
            "layout": "IPY_MODEL_2de0ee6fe2a14eb89e4aa61b3992a0c9"
          }
        },
        "212c6a5679744ff891650f601cd4e6c9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_be734680b384438d9304c133ad53d47d",
            "placeholder": "​",
            "style": "IPY_MODEL_cfe81d1d55154c41a40c1171ed9ed765",
            "value": "Map: 100%"
          }
        },
        "82ceff906c2443e2b05c4b6b379a489c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7ecb6130e52744bd834ffee0342932ee",
            "max": 45,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_60961672e2774b1e9f45a6318e5251e2",
            "value": 45
          }
        },
        "83775177ca1b4c06a2a57dc2c6b41827": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4b5ab79df25d4126955dc8ba22091e5f",
            "placeholder": "​",
            "style": "IPY_MODEL_b1bfa85380784177ab5faec7614f1b3b",
            "value": " 45/45 [00:00&lt;00:00, 664.04 examples/s]"
          }
        },
        "2de0ee6fe2a14eb89e4aa61b3992a0c9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "be734680b384438d9304c133ad53d47d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cfe81d1d55154c41a40c1171ed9ed765": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7ecb6130e52744bd834ffee0342932ee": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "60961672e2774b1e9f45a6318e5251e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "4b5ab79df25d4126955dc8ba22091e5f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b1bfa85380784177ab5faec7614f1b3b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e87c74286f65461a9b21b92b3d8efd32": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b0dda4fb1e394b6daa2ae25bc9286860",
              "IPY_MODEL_4c6106a368874816a6b251221a50ac28",
              "IPY_MODEL_75c8656786d648b09844a1600a7e7e28"
            ],
            "layout": "IPY_MODEL_6c5ef84c98034728a5c54fe283fb3cbb"
          }
        },
        "b0dda4fb1e394b6daa2ae25bc9286860": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b129f9ed2f5445319d75694e7d3883df",
            "placeholder": "​",
            "style": "IPY_MODEL_7197f23c39b9405d8e6086c21c25f02a",
            "value": "Map: 100%"
          }
        },
        "4c6106a368874816a6b251221a50ac28": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4371a7716a1c481e886f0539ced5cd9e",
            "max": 5,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_5c519c6d3863421a937dd6104228df26",
            "value": 5
          }
        },
        "75c8656786d648b09844a1600a7e7e28": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b797e3464da3466ca368f9d0858255ce",
            "placeholder": "​",
            "style": "IPY_MODEL_8b8c8938d90d4262ba6b8fd752c032e9",
            "value": " 5/5 [00:00&lt;00:00, 119.27 examples/s]"
          }
        },
        "6c5ef84c98034728a5c54fe283fb3cbb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b129f9ed2f5445319d75694e7d3883df": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7197f23c39b9405d8e6086c21c25f02a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4371a7716a1c481e886f0539ced5cd9e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5c519c6d3863421a937dd6104228df26": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b797e3464da3466ca368f9d0858255ce": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8b8c8938d90d4262ba6b8fd752c032e9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9515e687b4874d0b8f42b6bc120511ef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_9d974183fe4e4d188abb537b765e3cc4",
              "IPY_MODEL_527d48bf918741c89d3828edd7845709",
              "IPY_MODEL_f37123994dfe42c49772bd839400b0a9"
            ],
            "layout": "IPY_MODEL_972afde01bb9485bae8b90fcc2f81367"
          }
        },
        "9d974183fe4e4d188abb537b765e3cc4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_25acb2292d0a42e0ae32a5a90de06068",
            "placeholder": "​",
            "style": "IPY_MODEL_87a766de8e644f7d8a0069946726155e",
            "value": "Loading checkpoint shards:   0%"
          }
        },
        "527d48bf918741c89d3828edd7845709": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "danger",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_769d9e2f800443d4acc608fbba2a1281",
            "max": 2,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_3357e57e0c984418a0d60f8ed81f4be0",
            "value": 0
          }
        },
        "f37123994dfe42c49772bd839400b0a9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_38690d18d1d74f3caf389b8cdb503d64",
            "placeholder": "​",
            "style": "IPY_MODEL_310f5bd29c684bd18a6504fd64df6e9f",
            "value": " 0/2 [00:05&lt;?, ?it/s]"
          }
        },
        "972afde01bb9485bae8b90fcc2f81367": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "25acb2292d0a42e0ae32a5a90de06068": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "87a766de8e644f7d8a0069946726155e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "769d9e2f800443d4acc608fbba2a1281": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3357e57e0c984418a0d60f8ed81f4be0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "38690d18d1d74f3caf389b8cdb503d64": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "310f5bd29c684bd18a6504fd64df6e9f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "54ebcbece75646b181f95d21d9cd6c19": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_0dc9ff1f3937441f957d73d9aa93d49f",
              "IPY_MODEL_dbfb7dd75d7e4990b011dd2f655bb7c2",
              "IPY_MODEL_3afa987700db41e7a570fced3c789f42"
            ],
            "layout": "IPY_MODEL_e83cdbb744b14f00b112d82cbeb6b87d"
          }
        },
        "0dc9ff1f3937441f957d73d9aa93d49f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_223592b76a8b4c78ab42f5578ffad70b",
            "placeholder": "​",
            "style": "IPY_MODEL_ef9b3255842747269119fdd87448c4d0",
            "value": "Loading checkpoint shards: 100%"
          }
        },
        "dbfb7dd75d7e4990b011dd2f655bb7c2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e46d9a945ec94fdca298d8bbda84a350",
            "max": 2,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_6f8e3a86ab3a49cda308b197faea4504",
            "value": 2
          }
        },
        "3afa987700db41e7a570fced3c789f42": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_edf90949427d4e7da4b987ef6b7fd7ad",
            "placeholder": "​",
            "style": "IPY_MODEL_44c343d856bd40a48d848f89d03b9396",
            "value": " 2/2 [00:27&lt;00:00, 11.58s/it]"
          }
        },
        "e83cdbb744b14f00b112d82cbeb6b87d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "223592b76a8b4c78ab42f5578ffad70b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ef9b3255842747269119fdd87448c4d0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e46d9a945ec94fdca298d8bbda84a350": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6f8e3a86ab3a49cda308b197faea4504": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "edf90949427d4e7da4b987ef6b7fd7ad": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "44c343d856bd40a48d848f89d03b9396": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9f01f00f98674ad7b4a7e298d34d5fda": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_613ce7bace0f4126b1c1808c670efe6f",
              "IPY_MODEL_31ee7bcfebe14211824d51387bb8c0fb",
              "IPY_MODEL_cabaae388e514feba2e780f7f025f462"
            ],
            "layout": "IPY_MODEL_97aa38a813aa414baf612248c70b87d5"
          }
        },
        "613ce7bace0f4126b1c1808c670efe6f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_73c40a53096342d3964b5c098cad8cb9",
            "placeholder": "​",
            "style": "IPY_MODEL_1c54bbf30b164c4897b8175d9a5204f8",
            "value": "Loading checkpoint shards: 100%"
          }
        },
        "31ee7bcfebe14211824d51387bb8c0fb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_aef0f473777b448dad4a3e586be50377",
            "max": 2,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_030d263dc79f49a197a23a32dc8593ea",
            "value": 2
          }
        },
        "cabaae388e514feba2e780f7f025f462": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_df3e1ebd8fe044b1b360d4caaae1cc86",
            "placeholder": "​",
            "style": "IPY_MODEL_6309eeea5a964b31a72faa7f762e573f",
            "value": " 2/2 [00:18&lt;00:00,  7.86s/it]"
          }
        },
        "97aa38a813aa414baf612248c70b87d5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "73c40a53096342d3964b5c098cad8cb9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1c54bbf30b164c4897b8175d9a5204f8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "aef0f473777b448dad4a3e586be50377": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "030d263dc79f49a197a23a32dc8593ea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "df3e1ebd8fe044b1b360d4caaae1cc86": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6309eeea5a964b31a72faa7f762e573f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "!pip install -q -U pyarrow==14.0.1\n",
        "!pip install -q -U fsspec==2023.10.0\n",
        "\n",
        "!pip install -q -U bitsandbytes==0.42.0\n",
        "!pip install -q -U peft==0.8.2\n",
        "!pip install -q -U trl==0.7.10\n",
        "!pip install -q -U accelerate==0.27.1\n",
        "!pip install -q -U datasets==2.17.0\n",
        "!pip install -q -U transformers==4.38.0\n",
        "!pip install python-dotenv"
      ],
      "metadata": {
        "id": "_CAX2is1UnI-",
        "outputId": "108694c7-1667-4cec-87a4-0ec2bb6af34c",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: python-dotenv in /usr/local/lib/python3.10/dist-packages (1.0.1)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from huggingface_hub import login\n",
        "\n",
        "login()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 162,
          "referenced_widgets": [
            "77edf79fa9bc4e45ab252e2532b497d8",
            "dddf14daad59443ebe59fa0392b1ffde",
            "8757b2675f7a4b8b8d88a11730953ccc",
            "53c8b9a71ed942ba9d40b6f3cfd15d27",
            "5a51d060eb99492d9e0418eef49986aa",
            "e38921b61f8b4912a498619775852551",
            "635cd679a9ef488a82b69ecde6859119",
            "e2003e53be0c4fcd8656dfc8e8676f50",
            "17e76d31e72a4cc8a9b28e671f1861ac",
            "e1b8dd5242cc41329fdb2dbba00cf0b0",
            "af4cebfd1736479ca49b248ac7f3180e",
            "b15ca3349017492f9bc42fce4c4e2666",
            "317932e503a04f42a0df4837e6084e79",
            "57669b50fef6448eaa978ea68485a2cd",
            "da3b4cb230fb4f5f8c0acee68f13a932",
            "f4cc5f12ddf84103866801f813a0e9c8",
            "293f1f89d2884108aa0d137fa79f08f6",
            "3d6465df2f41404ab212d3979422b71f",
            "e38c5521f27946d088a7124e149f54f6",
            "feb123f826854717b896aec111e34454",
            "b15e131e0ab24754bee0bf29c73ba5b0",
            "fb057c9c5ccb4cf3921e7e87cfb3ab72",
            "0006298be43b46639350b8b4cb55a088",
            "1d1720fb2c89420f8b039402ec41e2ed",
            "bc2b7557e38e466bbeadce7a5031727f",
            "46ecd2cb502146fb9780c8dd662175a5",
            "bf2b1b434a9b4f05b0289ed63fb3cd7a",
            "18a45f5cb0dc4bf7a9f2b75ccb254774",
            "d9ae3029bd1b496ab52c596f7ba28fe3",
            "18855a606b014cf59adbec29d292197c",
            "de56c083ba61475a8b01e7bf36a5531d",
            "4d1f39060b854f2e938a9c012c318ef6"
          ]
        },
        "id": "vCY8f_sSReRv",
        "outputId": "fb3baf90-78fc-4950-b3ea-1a7f494354b8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "77edf79fa9bc4e45ab252e2532b497d8"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# # If HF token is stored in .env:\n",
        "# import os\n",
        "# os.environ[\"HF_TOKEN\"] = os.getenv('HF_TOKEN')"
      ],
      "metadata": {
        "id": "XaYuwd90O64V"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Tech Stack Used:\n",
        "* Transformers\n",
        "* Peft\n",
        "* BitsAndBytes\n",
        "* Accelerate\n",
        "* TRL\n",
        "\n",
        "### Basic steps Involved in fine-tuning:\n",
        "1. Load the base model\n",
        "2. Train the base model\n",
        "3. Save the LoRA adapter\n",
        "4. Reload the base model at half/full precision\n",
        "5. Merge the LoRA weights with the base model"
      ],
      "metadata": {
        "id": "VoMnxvx2SZUk"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Load Model"
      ],
      "metadata": {
        "id": "t7GCLryCYtbW"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "- `accelerate`: a library provided by HF to manage distributed training of machine learning models across multiple devices like GPUs and TPUs. It simplifies the process of running models on different hardware setups.\n",
        "- `bitsandbytes` (imported as bnb): a lightweight library used for efficient and faster computation, especially for low-level operations in deep learning (like quantization)."
      ],
      "metadata": {
        "id": "w2ZA31ZkOx2p"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import accelerate\n",
        "import bitsandbytes as bnb\n",
        "\n",
        "print(f\"Accelerate version: {accelerate.__version__}\")\n",
        "print(f\"BitsAndBytes version: {bnb.__version__}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "e7BPQOMnT3br",
        "outputId": "d2336207-3fa3-415e-83a3-20366cb9676f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Accelerate version: 0.27.1\n",
            "BitsAndBytes version: 0.42.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Quantization config:\n",
        "- `load_in_4bit=True`: enables 4-bit quantization, significantly reducing the model's memory footprint while maintaining performance.\n",
        "- `bnb_4bit_use_double_quant=True`: uses a two-step quantization process (double quantization) to further improve accuracy.\n",
        "- `bnb_4bit_quant_type=\"nf4\"`: specifies the quantization type. In this case, nf4 refers to the Normal Float 4 format, which is effective for quantized deep learning models.\n",
        "- `bnb_4bit_compute_dtype=torch.bfloat16`: sets the computation precision to bfloat16, a lower-precision data type that accelerates computation without a significant loss in accuracy.\n",
        "\n",
        "Loading of model and tokenizer:\n",
        "- `AutoTokenizer`: loaded for the same model using AutoTokenizer.from_pretrained(model_id). The tokenizer converts text into a format that the model can process (like token IDs).\n",
        "- `add_eos_token=True`: ensures that an end-of-sequence token is added automatically during tokenization, used in language modeling tasks."
      ],
      "metadata": {
        "id": "RQU0g7SoPTTR"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig\n",
        "\n",
        "# Set the qunatization config\n",
        "bnb_config = BitsAndBytesConfig(\n",
        "    load_in_4bit=True,\n",
        "    bnb_4bit_use_double_quant=True,\n",
        "    bnb_4bit_quant_type=\"nf4\",\n",
        "    bnb_4bit_compute_dtype=torch.bfloat16\n",
        ")\n",
        "\n",
        "#Load the model and Tokenizer\n",
        "model_id = \"google/gemma-2b-it\"\n",
        "\n",
        "model = AutoModelForCausalLM.from_pretrained(model_id, quantization_config=bnb_config, device_map={\"\":0})\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_id, add_eos_token=True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 104,
          "referenced_widgets": [
            "9fd74448ecdc4f9f91417ed2615e53f1",
            "4a17588d3d934e268606e165eeec9feb",
            "7f6c7dc6d9f846e6bf2dd12e59d63e68",
            "24028921209c41519b383e12e07954a2",
            "1e588375ec214f8c92c5b343363e591b",
            "e32f3946859a46a7bb32ab3b325620d2",
            "717e95ab43e140419e263c9015593854",
            "2f4d429754494762a4434e5815ae44f8",
            "4bddfc0b39744e41a970a92962651b0a",
            "67cf01d78ab34e6fbec3e63318ace902",
            "8d306e09734549e08ddc791a9493c706"
          ]
        },
        "id": "ZI5ewi52PqvZ",
        "outputId": "8c176922-856e-4820-9049-0170e6ccd126"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1150: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "9fd74448ecdc4f9f91417ed2615e53f1"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Dataset Preparation"
      ],
      "metadata": {
        "id": "4nM2mYDXX0bt"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Loading the Dataset"
      ],
      "metadata": {
        "id": "sBeQ6XdKVldM"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Let's use the [mamachang/medical-reasoning](https://huggingface.co/datasets/mamachang/medical-reasoning) dataset from HF."
      ],
      "metadata": {
        "id": "rnzIjU18Vupx"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from datasets import load_dataset\n",
        "\n",
        "dataset = load_dataset(\"HafthorJulian/random_trivia_julian\")"
      ],
      "metadata": {
        "id": "hJPnzeRvVk2u"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "dataset"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dASvTsrZ4WHd",
        "outputId": "86ebbdda-529e-4d3c-8097-955769400712"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DatasetDict({\n",
              "    train: Dataset({\n",
              "        features: ['question', 'question_id', 'question_source', 'entity_pages', 'search_results', 'answer'],\n",
              "        num_rows: 50\n",
              "    })\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Convert HF dataset to pandas Dataframe\n",
        "\n",
        "df = dataset[\"train\"].to_pandas()\n",
        "df.sample(10)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "NUwwWC2uTuvE",
        "outputId": "51396e09-b1cd-470e-fb26-231abd17ac0c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                             question question_id  \\\n",
              "40  Which series had the characters Felix Unger an...       tc_60   \n",
              "26  In which branch of the arts does Allegra Kent ...       tc_41   \n",
              "32  Which is the largest of the Japanese Volcano I...       tc_48   \n",
              "34  Who was the first woman to make a solo flight ...       tc_52   \n",
              "8              What is Bruce Willis' real first name?       tc_16   \n",
              "47                What was Kevin Kline's first movie?       tc_70   \n",
              "33  Ezzard Charles was a world champion in which s...       tc_50   \n",
              "3   From which country did Angola achieve independ...        tc_8   \n",
              "4               Which city does David Soul come from?        tc_9   \n",
              "0   Which American-born Sinclair won the Nobel Pri...        tc_1   \n",
              "\n",
              "                  question_source  \\\n",
              "40  http://www.triviacountry.com/   \n",
              "26  http://www.triviacountry.com/   \n",
              "32  http://www.triviacountry.com/   \n",
              "34  http://www.triviacountry.com/   \n",
              "8   http://www.triviacountry.com/   \n",
              "47  http://www.triviacountry.com/   \n",
              "33  http://www.triviacountry.com/   \n",
              "3   http://www.triviacountry.com/   \n",
              "4   http://www.triviacountry.com/   \n",
              "0   http://www.triviacountry.com/   \n",
              "\n",
              "                                         entity_pages  \\\n",
              "40  {'doc_source': ['TagMe'], 'filename': ['The_Od...   \n",
              "26  {'doc_source': ['TagMe'], 'filename': ['Allegr...   \n",
              "32  {'doc_source': ['TagMe', 'TagMe'], 'filename':...   \n",
              "34  {'doc_source': ['Search'], 'filename': ['Charl...   \n",
              "8   {'doc_source': ['TagMe'], 'filename': ['Bruce_...   \n",
              "47  {'doc_source': ['TagMe'], 'filename': ['Kevin_...   \n",
              "33  {'doc_source': ['TagMe', 'TagMe'], 'filename':...   \n",
              "3   {'doc_source': ['TagMe', 'TagMe', 'Search'], '...   \n",
              "4   {'doc_source': ['TagMe'], 'filename': ['David_...   \n",
              "0   {'doc_source': [], 'filename': [], 'title': []...   \n",
              "\n",
              "                                       search_results  \\\n",
              "40  {'description': ['The Odd Couple . 30min ... O...   \n",
              "26  {'description': ['... Allegra Kent, Balanchine...   \n",
              "32  {'description': ['Japan's Volcano Island is a ...   \n",
              "34  {'description': ['Amelia Earhart was the first...   \n",
              "8   {'description': ['Walter Bruce Willis was born...   \n",
              "47  {'description': ['Kevin Kline's Best Movies .....   \n",
              "33  {'description': ['Explore David Stoppa's board...   \n",
              "3   {'description': ['Angola; Angola from past to ...   \n",
              "4   {'description': ['David Soul, Actor: Starsky a...   \n",
              "0   {'description': ['The Nobel Prize in Literatur...   \n",
              "\n",
              "                                               answer  \n",
              "40  {'aliases': ['The Odd Couple', 'The Female Odd...  \n",
              "26  {'aliases': ['Ballet', 'Ballet competitions', ...  \n",
              "32  {'aliases': ['Mount Surabachi', 'Iō-tō', 'Iwō ...  \n",
              "34  {'aliases': ['Amelia airheart', 'Amelia Airhar...  \n",
              "8   {'aliases': ['Walter (TV Series)', 'Walter', '...  \n",
              "47  {'aliases': ['Sophie's Choice', 'Sophie's Choi...  \n",
              "33  {'aliases': ['Prize fight', 'Prize fighting', ...  \n",
              "3   {'aliases': ['Portogało', 'Republic of Portuga...  \n",
              "4   {'aliases': ['Chi-Beria', 'Sayre language acad...  \n",
              "0   {'aliases': ['(Harry) Sinclair Lewis', 'Harry ...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-7398cb58-0db0-4851-9c8b-a6dc699af084\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>question</th>\n",
              "      <th>question_id</th>\n",
              "      <th>question_source</th>\n",
              "      <th>entity_pages</th>\n",
              "      <th>search_results</th>\n",
              "      <th>answer</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>40</th>\n",
              "      <td>Which series had the characters Felix Unger an...</td>\n",
              "      <td>tc_60</td>\n",
              "      <td>http://www.triviacountry.com/</td>\n",
              "      <td>{'doc_source': ['TagMe'], 'filename': ['The_Od...</td>\n",
              "      <td>{'description': ['The Odd Couple . 30min ... O...</td>\n",
              "      <td>{'aliases': ['The Odd Couple', 'The Female Odd...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>In which branch of the arts does Allegra Kent ...</td>\n",
              "      <td>tc_41</td>\n",
              "      <td>http://www.triviacountry.com/</td>\n",
              "      <td>{'doc_source': ['TagMe'], 'filename': ['Allegr...</td>\n",
              "      <td>{'description': ['... Allegra Kent, Balanchine...</td>\n",
              "      <td>{'aliases': ['Ballet', 'Ballet competitions', ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32</th>\n",
              "      <td>Which is the largest of the Japanese Volcano I...</td>\n",
              "      <td>tc_48</td>\n",
              "      <td>http://www.triviacountry.com/</td>\n",
              "      <td>{'doc_source': ['TagMe', 'TagMe'], 'filename':...</td>\n",
              "      <td>{'description': ['Japan's Volcano Island is a ...</td>\n",
              "      <td>{'aliases': ['Mount Surabachi', 'Iō-tō', 'Iwō ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>34</th>\n",
              "      <td>Who was the first woman to make a solo flight ...</td>\n",
              "      <td>tc_52</td>\n",
              "      <td>http://www.triviacountry.com/</td>\n",
              "      <td>{'doc_source': ['Search'], 'filename': ['Charl...</td>\n",
              "      <td>{'description': ['Amelia Earhart was the first...</td>\n",
              "      <td>{'aliases': ['Amelia airheart', 'Amelia Airhar...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>What is Bruce Willis' real first name?</td>\n",
              "      <td>tc_16</td>\n",
              "      <td>http://www.triviacountry.com/</td>\n",
              "      <td>{'doc_source': ['TagMe'], 'filename': ['Bruce_...</td>\n",
              "      <td>{'description': ['Walter Bruce Willis was born...</td>\n",
              "      <td>{'aliases': ['Walter (TV Series)', 'Walter', '...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>47</th>\n",
              "      <td>What was Kevin Kline's first movie?</td>\n",
              "      <td>tc_70</td>\n",
              "      <td>http://www.triviacountry.com/</td>\n",
              "      <td>{'doc_source': ['TagMe'], 'filename': ['Kevin_...</td>\n",
              "      <td>{'description': ['Kevin Kline's Best Movies .....</td>\n",
              "      <td>{'aliases': ['Sophie's Choice', 'Sophie's Choi...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33</th>\n",
              "      <td>Ezzard Charles was a world champion in which s...</td>\n",
              "      <td>tc_50</td>\n",
              "      <td>http://www.triviacountry.com/</td>\n",
              "      <td>{'doc_source': ['TagMe', 'TagMe'], 'filename':...</td>\n",
              "      <td>{'description': ['Explore David Stoppa's board...</td>\n",
              "      <td>{'aliases': ['Prize fight', 'Prize fighting', ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>From which country did Angola achieve independ...</td>\n",
              "      <td>tc_8</td>\n",
              "      <td>http://www.triviacountry.com/</td>\n",
              "      <td>{'doc_source': ['TagMe', 'TagMe', 'Search'], '...</td>\n",
              "      <td>{'description': ['Angola; Angola from past to ...</td>\n",
              "      <td>{'aliases': ['Portogało', 'Republic of Portuga...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Which city does David Soul come from?</td>\n",
              "      <td>tc_9</td>\n",
              "      <td>http://www.triviacountry.com/</td>\n",
              "      <td>{'doc_source': ['TagMe'], 'filename': ['David_...</td>\n",
              "      <td>{'description': ['David Soul, Actor: Starsky a...</td>\n",
              "      <td>{'aliases': ['Chi-Beria', 'Sayre language acad...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Which American-born Sinclair won the Nobel Pri...</td>\n",
              "      <td>tc_1</td>\n",
              "      <td>http://www.triviacountry.com/</td>\n",
              "      <td>{'doc_source': [], 'filename': [], 'title': []...</td>\n",
              "      <td>{'description': ['The Nobel Prize in Literatur...</td>\n",
              "      <td>{'aliases': ['(Harry) Sinclair Lewis', 'Harry ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-7398cb58-0db0-4851-9c8b-a6dc699af084')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-7398cb58-0db0-4851-9c8b-a6dc699af084 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-7398cb58-0db0-4851-9c8b-a6dc699af084');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-a586795d-7ed8-4f4f-b637-196bc7014d96\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-a586795d-7ed8-4f4f-b637-196bc7014d96')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-a586795d-7ed8-4f4f-b637-196bc7014d96 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "summary": "{\n  \"name\": \"df\",\n  \"rows\": 10,\n  \"fields\": [\n    {\n      \"column\": \"question\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 10,\n        \"samples\": [\n          \"Which city does David Soul come from?\",\n          \"In which branch of the arts does Allegra Kent work?\",\n          \"What was Kevin Kline's first movie?\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"question_id\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 10,\n        \"samples\": [\n          \"tc_9\",\n          \"tc_41\",\n          \"tc_70\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"question_source\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"http://www.triviacountry.com/\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"entity_pages\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"search_results\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"answer\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "`generate_prompt` function generates a prompt for training a model based on a given data point, which typically includes a task instruction, optional context (input), and the corresponding expected response (output).\n",
        "\n",
        "- **Inputs**:\n",
        "  - `data_point`: A dictionary containing information such as `instruction`, `input`, and `output` related to a specific task.\n",
        "  - If `input` is provided, the function incorporates it into the prompt. Otherwise, the prompt only includes the instruction and output.\n",
        "  \n",
        "- **Output**:\n",
        "  - The function returns a formatted string, simulating a user-model interaction in a conversational structure.\n",
        "  - The user part includes the instruction (and possibly inputs), while the model part includes the expected response.\n",
        "\n",
        "The function is then applied to the dataset to generate a new column `prompt`, which contains the formatted prompts for each data point. This column can be used to train or fine-tune a model for specific tasks.\n"
      ],
      "metadata": {
        "id": "T0iIxsQrW4HS"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def generate_prompt(data_point):\n",
        "    # Generate prompt\n",
        "    prefix_text = 'Below is an instruction that describes a task. Write a response that ' \\\n",
        "                  'appropriately completes the request.\\n\\n'\n",
        "\n",
        "    # Samples with additional context info\n",
        "    if data_point['question']:\n",
        "        text = f\"\"\"<start_of_turn>user {prefix_text} here are the inputs {data_point[\"question\"]} <end_of_turn>\\n<start_of_turn>model{data_point[\"answer\"]} <end_of_turn>\"\"\"\n",
        "    # Without additional context info\n",
        "    else:\n",
        "        text = f\"\"\"<start_of_turn>user {prefix_text} <end_of_turn>\\n<start_of_turn>model{data_point[\"answer\"]} <end_of_turn>\"\"\"\n",
        "    return text\n",
        "\n",
        "# Add the \"prompt\" column in the dataset\n",
        "text_column = [generate_prompt(data_point) for data_point in dataset[\"train\"]]\n",
        "dataset = dataset[\"train\"].add_column(\"prompt\", text_column)\n",
        "dataset"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Y5jCuDlkWNF2",
        "outputId": "49840527-3427-4027-99b9-23bc68b817b5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Dataset({\n",
              "    features: ['question', 'question_id', 'question_source', 'entity_pages', 'search_results', 'answer', 'prompt'],\n",
              "    num_rows: 50\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(dataset[0]['prompt'])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QZ34RmsVW_2r",
        "outputId": "014d7e4a-42db-4f66-f797-023678fff9f4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<start_of_turn>user Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
            "\n",
            " here are the inputs Which American-born Sinclair won the Nobel Prize for Literature in 1930? <end_of_turn>\n",
            "<start_of_turn>model{'aliases': ['(Harry) Sinclair Lewis', 'Harry Sinclair Lewis', 'Lewis, (Harry) Sinclair', 'Grace Hegger', 'Sinclair Lewis'], 'normalized_aliases': ['grace hegger', 'lewis harry sinclair', 'harry sinclair lewis', 'sinclair lewis'], 'matched_wiki_entity_name': '', 'normalized_matched_wiki_entity_name': '', 'normalized_value': 'sinclair lewis', 'type': 'WikipediaEntity', 'value': 'Sinclair Lewis'} <end_of_turn>\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dataset = dataset.shuffle(seed=1234)  # Shuffle dataset\n",
        "# dataset = dataset.map(lambda samples: tokenizer(samples[\"prompt\"]), batched=True)"
      ],
      "metadata": {
        "id": "dij1cN-1XDKy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Train-Test Split"
      ],
      "metadata": {
        "id": "cWqdPd_ZXx47"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "dataset = dataset.train_test_split(test_size=0.1)\n",
        "train_data = dataset[\"train\"]\n",
        "test_data = dataset[\"test\"]\n",
        "\n",
        "print(train_data)\n",
        "print(test_data)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "V6oGcuYuXvDr",
        "outputId": "d5c4d0fa-9ee1-4af0-b80d-a2ad505ad188"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Dataset({\n",
            "    features: ['question', 'question_id', 'question_source', 'entity_pages', 'search_results', 'answer', 'prompt'],\n",
            "    num_rows: 45\n",
            "})\n",
            "Dataset({\n",
            "    features: ['question', 'question_id', 'question_source', 'entity_pages', 'search_results', 'answer', 'prompt'],\n",
            "    num_rows: 5\n",
            "})\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Low-Rank Adaptation for Large Language Models\n",
        "\n",
        "Low-Rank Adaptation (LoRA) is a technique for fine-tuning large language models (LLMs) in a more efficient way. Instead of adjusting all the model’s parameters, it focuses on a small subset by freezing the original model and training only adapter layers, which are broken down into low-rank matrices. The PEFT library simplifies this process by letting users easily choose which model layers should incorporate these adapter weights."
      ],
      "metadata": {
        "id": "W5f6wVsUYfbP"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Automated selection of target modules\n",
        "import bitsandbytes as bnb\n",
        "\n",
        "def find_all_linear_names(model):\n",
        "  cls = bnb.nn.Linear4bit #if args.bits == 4 else (bnb.nn.Linear8bitLt if args.bits == 8 else torch.nn.Linear)\n",
        "  lora_module_names = set()\n",
        "  for name, module in model.named_modules():\n",
        "    if isinstance(module, cls):\n",
        "      names = name.split('.')\n",
        "      lora_module_names.add(names[0] if len(names) == 1 else names[-1])\n",
        "    if 'lm_head' in lora_module_names: # needed for 16-bit\n",
        "      lora_module_names.remove('lm_head')\n",
        "  return list(lora_module_names)\n",
        "\n",
        "\n",
        "modules = find_all_linear_names(model)\n",
        "print(modules)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SYweyn2nbPXb",
        "outputId": "0f260bd7-ff60-49cb-d546-aa8325df456d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['o_proj', 'v_proj', 'up_proj', 'q_proj', 'k_proj', 'gate_proj', 'down_proj']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**What are modules?**\n",
        "\n",
        "A module refers to a component or building block of a neural network. Modules can represent layers or more complex components like the whole network.\n",
        "\n",
        "Think of a module as a piece of a neural network that performs some kind of computation. These modules can be combined to form more complex models.\n",
        "- A linear module might represent a fully connected layer, where the input data is multiplied by weights and a bias is added.\n",
        "\n",
        "![image.png](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAA0cAAAGdCAIAAABW4p+0AAAAAXNSR0IArs4c6QAAAERlWElmTU0AKgAAAAgAAYdpAAQAAAABAAAAGgAAAAAAA6ABAAMAAAABAAEAAKACAAQAAAABAAADR6ADAAQAAAABAAABnQAAAAA2fIvUAABAAElEQVR4AeydTchmyVn+M2EYQgjBnpY/g4QwnRGJijKYnhATF4HJNOJKYiadTdCAnRhQ0E1ib1zoYjBGEtFN0osowY9JB6ILdZgkRIKo2O3HRnQRbRmiuOkeCSIhhMz/N3Nl7pSnzkedOt/Pc53F+9apc1fVXVdV3XWdu+rUc98LL7zwCl9GwAgYASNgBIyAETACB0fglQfX3+obASNgBIyAETACRsAIvIjA/f/20mUwjIARMAJGwAgYASNgBA6HwBteur6t9oc+9KHDVcAKGwEjYASMgBEwAkbACIAARI7ddLq8AusuYQSMgBEwAkbACBiBU0Dg/qjE5z//+Qg7YASMgBEwAkbACBgBI7BnBJ544omGet9hde94xzsaz3xrBIyAETACRsAIGAEjcBQEvAJ7lJaynkbACBgBI2AEjIAR6EPArK4PHT8zAkbACBgBI2AEjMBREDCrO0pLWU8jYASMgBEwAkbACPQhYFbXh46fGQEjYASMgBEwAkbgKAiY1R2lpaynETACRsAIGAEjYAT6EDCr60PHz4yAETACRsAIGAEjcBQEzOqO0lLW0wgYASNgBIyAETACfQiY1fWh42dGwAgYASNgBIyAETgKAmZ1R2kp62kEjIARMAJGwAgYgT4EzOr60PEzI2AEjIARMAJGwAgcBQGzuqO0lPU0AkbACBgBI2AEjEAfAmZ1fej4mREwAkbACBgBI2AEjoKAWd1RWsp6GgEjYASMgBEwAkagD4H7+x762YoIfOELX1ixNBd1eATe8Y53HL4OroARMAJGwAjMioBZ3axwTsjsiSeemJDaSc8OgRdeeOHs6uwKGwEjYASMQC8CXoHthccPjYARMAJGwAgYASNwEATsq9tXQ73//e9/8skn96WTtdkTAjdv3vzkJz+5J42sixEwAkbACOwFAbO6vbSE9Piu7/ou75faV5PsTJvPf/7zO9PI6hgBI2AEjMBeEPAK7F5awnoYASNgBIyAETACRmAKAmZ1U9BzWiNgBIyAETACRsAI7AUBs7q9tIT1MAJGwAgYASNgBIzAFATM6qag57RGwAgYASNgBIyAEdgLAmZ1e2kJ62EEjIARMAJGwAgYgSkImNVNQc9pjYARMAJGwAgYASOwFwTM6vbSEtbDCBgBI2AEjIARMAJTEDCrm4Ke0xoBI2AEjIARMAJGYC8ImNXtpSWshxEwAkbACBgBI2AEpiBgVjcFPac1AkbACBgBI2AEjMBeEPAvhu2lJTbR4wtf+EL6C1QXL15817ve9YY3vGETZVyoETACRsAIGAEjMAUBs7op6B0y7b/927999rOfvXXrFpTuv//7vxt1+PCHPwyr47do3/SmN7373e/md2kbAr41AkbACBgBI2AE9omAWd0+22URrT7zmc9cv34dVpfm/iM/8iMPPvigYnik65Of/CQxH/jAB6B3v/7rv45MmsThUQgAKTT67t27pHrkkUfe//73j0puYSNgBIyAETAChQiY1RUCdWwxfHLXrl2DW6ga8sY9/vjjV65caXjjREHCk4c/D6fdhz70IbjdsSHYSHt8n2DIX1jdRz/60W9961uf+MQn/u7v/m4jdVysETACRsAInDICZnWn3LqqGy46vG5abGXbHPysZ+ccj+BwSggdISE87yMf+QjhGzdu2Gk3qrtA5oDuX//1X0GV5Wwx5r//+78flYmFjYARMAJGwAgUIuBvYAuBOqQYTO7JJ5+8evUqAXxyTz/99M2bN3soXaOSLL/iVRLJg4vg24OjNGR824UAmGshOwCHFrP8Sit0JXG8ETACRsAIGIEpCJjVTUFv12nxrl26dEmrruJnuIvGagwXxLfHd7JQE2gKzqcnnniCwNh8zlAeF2kDKMBk+bWiFc4QPVfZCBgBI2AEKhAwq6sA7QBJ4BN46fib0rJqvVOnHWSRLXrVWZ1PQhZez6eyrqkRMAJGwAjsAQGzuj20wvw6QLxE6b74xS/GPrkpxYgd4moiE/x/OKKm5HYOab1/7hxa2XU0AkbACOwKAX8tsavmmEcZKJcWXjnHZN7vG9gWxmosmfMVRf797DzaHzwXyPTt27epxL1791QVvJsKcILMvM1xcKisvhEwAkbACMyMgFndzIBunh2sAsqFGiybzuKla9SILf/QFErBHci3F42nvuVDV50CQ0BoxKEwUDqzOvcQI2AEjIARWA4Bs7rlsN0m51h71Wrp7Epoyz/f1eKx4xtPn6nbQBjeph9h47MSeenS32RrCPvWCBgBI2AEjMCMCJjVzQjm9lnF2iuULg7UGKsWTqbwMynt5cuXIXORD19x4qWD1fFJLB7B6oIiwzMPgLYxPPM+4OobASNgBGZBwKxuFhj3kgkb6VCFo4Yrjs/AscRaYWwCy6v0wgsvRGSsw0IfY4UxnjpQiAB8DvRweabYFqa1mBEwAkbACBiBBgJmdQ1ADnzLR5fysVXQLMgcK4Y45FhR5S+/Epb+OGx+SAcyMEh8dXjsKoo7MMozqR58bqb8nI0RMAJGwAgYgVeY1Z1OJ5CbjbW8iuU8bcLjlyTK07L2CnawE67yVKcD97Sa0Fhgzs91sENxWk5ObQSMgBEwAkbg2wj4vLrT6Qq3bt2iMiJbY2uFyw0v3ShyxmcBku9ZtB2rxvnI6yuTcIieT8VdUyNgBIyAEVgOAbO65bBdO2exK9w/YwvWebmPPPLI2IRikP7GcyxuljcCRsAIGAEjsAQCZnVLoLpBnlA6zpCjYA4HHlv8V77yFZJUHKXG9jsS2lc3FnDLGwEjYASMgBFYAgGzuiVQ3SBPOcxgZukRJIV6sJ0OyVHLr8pZvjrYpH8dqxBqixkBI2AEjIARWA4Bs7rlsN0g5yn7tCpYXSSJX8faoM4u0ggYASNgBIyAEXgJAbM6d4QZEMBXd+HCBQ460SrwDDk6CyNgBIyAETACRmAkAmZ1IwGzeBsCf/qnfwqf+8hHPnLp0iVzOxACjdu3bwsqTn5pw8xxRsAIGAEjYARmRsCsbmZAt8ru4sWLFL0ygYjtdL/1W7/1oQ99iC195nZ8OwKv5TsSoFBn4HhnYvgBia36hss1AkbACBiBM0HArO5EGppfCaMmL54IvKJnSF+/srvu0Ucf5Rcm7ty5Y27HFyRAwa9x8CNguggTowPqTqS3uRpGwAgYASOwSwTM6nbZLOOVglpxkY7zhMemxuWmtGMT6txjEUrS4qsztxuLoeWNgBEwAkbACMyFgFndXEhun4/OGRHTGqUNX7DWsToxSJ1aFyWa2wUUDhgBI2AEjIARWBMBs7o10V6kLPZvsRLK9epXv5oCCIwtBl9dxRHEUVDrucfmdmNbwfJGwAgYASNgBCYicP/E9E6+NAKQNn1NyYY5dmhRHAHtxA9eRSQs6nOf+9zHP/5xkTz57Up0+8xnPoNYxc+FlZx7LG53/fr1p556is8F0I3vZAmwyYxInpZoaBkjYASMgBEwAkagBAGzuhKUNpZ58sknReN69GBD29vf/nbIHFTvAx/4AD8XUcKZyPbmzZtkW84CpQPuPfgZ4atXr/ZopUfmdl0QxactBOoWwbtydrwRMAJGwAicIQJmdXtvdCgRni1RqC5dP/GJT+gTS7gdu9ygCPjGCEse35ioW2OZFWYmbx8fro6iFHDBa9eukTkZkrZLq0a8uV0KCI2C55W/ioS4Q6zxmPpT2RQlh42AETACRmAcAjErv3wOg/9vg4CajebIi3/++ed7HG/wgDRJNCgrpIqH88EY8hzgZKTlaZq8JBxF4BEskc9lqBGZhEoEuCUyl3RMA4EAvxHvWyNgBIyAETg3BHLm4K8lxpHgTaQhPW95y1taixYtSx/hopNPjnVYrdsiA8PLOROcLJx8aQ79Ydx7chxCLxrOv/6E6VNqhJ6t59vFomQq77ARMAJGwAgYASMwiIBZ3SBEGwuwSMfC3DPPPJPrAamKZdb06Y0bN7jVOmwaPz0MTYQskk9X0aOKaOV2VJYizO1GIWlhI2AEjIARMAIgYFa3327A16kpxeH3G1Jd4VVf/OIXIUZppMJBufTBaS5QFwOl45evxLdEHOvyaaRqcDueisia2zWA8q0RMAJGwAgYgX4EzOr68dnmKauc/Hgon5eKQrGEys76L33pS8HhCMCr4jbXkuVRfdYKNyr5hDbPoRGDSnyHod39U9ZeG9nGrbldQOGAETACRsAIGIE6BMzq6nBbKtVzzz0Hn+PS16kwM7bEsfuNb1ThPdA7CiaAlw6HXL8SfPeq3/LiFyAuXbqkc+n6k7Q+lYsOlUQxoXSty76tacdGituxBVBVJrn9dmMxtLwRMAJGwAicLwJM0qr8uX05srf6Nrqg+FxDSegOvGfUV6tPP/00SZT5u9/97vybiUYRjVu+qAj6SKD6o9dGtoW3we2kv3yWhWlPVcwD9lRb1vUyAkbACIxFQJMj80IktK+uwaY2uNV6axQsPoeLTkuoEU8Afgavkscuje8Jw+T41FROO9x1OO3YbMdJdT1J9Ait2EXHqquE6TQpwxtMPouAaFxwO/vtZkHVmRgBI2AEjMDJIuBX/2C46wdy6vae97xnITVYNg2nHb2ZJV04E4Qp9d7B2xATBYwev76LrhWB4HZS7Gz9dh6wrd3DkUbACBiBM0RAE6J9dcFYNgvIP5fun5Mqr3/96xfSSc42yJB+RoJNcri++JbiwoUL97184ZnDP8c+POkAn4Pkre+ia0XAfrtWWBxpBIyAETACRiAQ8ApsQLFSIOdzeOy4VigePofHi89puQjgk0u9d1IAGfgfu/Hw4cHnwjO0gnolRZjblaBkGSNgBIyAEThPBPw7sOu1O+4xXF86HIRS2TaHYyzfPLeCQlp+hSFRFtvm7t27p0KJ51pBgYlFoDkXSIKnnI6EiQHPQ+g/sfpObgSMgBEwAkagFQH76lphmTkS5sFaJ0cKi9LBPHCG4Z/bhNI16sYyK2roOhYlgsbJ6Si1wTY9tLlRTd8aASNgBIyAETh5BMzqlm3inM9pDZRPU5ct+GxyN7c7m6Z2RY2AETACRmAAAa/ADgBU/bix3oo/ifVBKEh1hk7YgwDAcm2+Jkujs28SD2KoygcoZvCBhgNGwAgYASOwKAJmdfPDaz43P6ZlObZyOz4KYfmbJeayPGqkOAiQL0vgc60HAfLLb5TOSjcB/tYU4DRGwAgYASNgBAoQ8ApsAUjFIl3rrbCN4jzWFoSLxNVKStZWaHJ5oJ3ut+Oglhd/gu3lH2HLs8eHyq+i5fGDMaSCL3IsDHQtznbms+Jv71J86Z8yAWEE8NtxjgzFDeZsASNgBIyAETACNQjE0RVneHzfjFWGRqTUjfVW9s+Nyl+NR3OMSlUn3HrasBRAcyqizX91me8qFRXRtxSqHUSLj1RSDXWmDC609DTmVKArnP4UG5mTs872y+WRpFlTLx1hmiCXLInxgC1ByTJGwAgYgXNAIGcOr/AkMbHhYQOBIfhW8DkpkLfNRMXy5KKeKctRoV1/IR9UbSzdycvdPAZuB+uKaqbcLuJZqC3UE0Di5zfwzJXT90ZXAdvCElOx6GxppMNGwAgYASNwhgjkzMGsrr4baJKOg3wJ4K2pzi5vm+qsWhOiW6hKWXA7+IFOGw55LVw2TidGsuHfCvljBRpHyci7FlSPAE7KwRqlLjqAqqC8qBHEGh3AfLDQVMCsLkXDYSNgBIzAOSOQMwezupr+kPM55tqKCT4tO2+b9OmUMLwhPFLwicLV1cYPS0yv4JQqzJi2we1SVjdI7FIX3RQGr/6jokd5+8DBrG7GzuCsjIARMAKHRiBnDmZ14xp0CT4nDfK2GadZh3TqoqtgZnC72BA2ZTdYh3abRcPtol5CPv52ragGnapwsLXWM3XalW+zCzVa83SkETACRsAInA8COXMwqytt/eX4nDTI26ZUsw651EWHQ4ilww7B4ehgEihJeDjBESTC8RZ8LgI5sYN16WnJKm157elUIpf8LUwVbVEobzEjYASMgBE4VQQ0MaXzsk82iam8M8ABFpxGcenSJQ6nIAxDAsE7d+6kbrDOxNs9ePLJJzlQg/KhL2g75SxcahpOO0A4gbM5OIOGE0+6GofzSjiCLn167do1blm/Boo0fmKYvnTjxg0y4UyZE0B1IhpObgSMgBEwAhMRMKvrA/CgfI4qQRF0+Bxup5s3b8Ie+upZ8EzLrzq9Jc5mK0i3U5FBcgaxi9P7UjCnI9lABGDlfjsBVBtV860RMAJGwAisjIBZXSfgzOWH88+pMtARKAJh6IJ4WGclRz6AI2rFUL6rkal3JI6vju1xXF06Qegff/xxkEzB7JHvyqckHooZqFJuSRLLGAEjYASMgBHIEfAvhuWYvCJ+TpRn+GYgRtevX5/dSdNS8ExRsVyI2jNl+Z1sWDHkNxLgOrDeQY/Xd5LtLMSXCg2N4HlcioynsNjbt28TCeuaWFmthkehDz74oJicYgLVp556amJBUYQDRsAIGAEjcHYIaPWHap/qXsJR9WIWj7PEwAQ+N/Y4sVHFpcLqeTRHGlkRjgaFmlQkL0kSRZR/uVmS7Q5l6A9ql+qaglXK3pRb/MX5F7UOVPmEIiLzQIjljxxjBIyAETACZ4VAzhzsq/v2DJv654iCz+GLShlezMR7DqywXEj1cSbheaIsPsiA7hzIi6m2wyeH/nw+Mqi5nHZ0hh5m1tMf2JxHvyItVOzixYtpJqB39+7dRx55JJKDKsKswPKhBiVGvANGwAgYASNgBAoROFlWp9kRT5uAYNGw6yPQ0+BzqiZMlADsoWIVD8RY/oNtvLQU+e21SGVLhqwYxrokkVoxRBL0wnsk4f3/RW34FljRJQikZKuhvJZN6TyN+JJbgYM3LsUtErZu0SOSL3MhyiHmgBEwAkbACBiBcgS2YXUQCG1XShW9fPkyt2l8Y+9RKtwfZsKW2yMVY5MZrCXldqfE56gpNEIspILSkRbuQrvgnuRKz3IjkqcpkoRhQpA5vsnQT9c3nh7ilnrRAbioC9wud91BcJGhLq0MbLCOaotRn5U88cQTsDolHMzfAkbACBgBI2AEmgiEo2XNpWh2DsEGUh8JG5iI5IqdTChWt5mpf/WK/Kkpf+EugQVJVts/14WzlKHWXQKD8YKOVcVByVxAoI06qThcULRanuGeY0Lz6AAEwA0Q0i4nckwvrauLRtaotOFaTtVo5LDJgG3o4FsjYASMgBHYAwI5c9jGV8cMimvkypUr+IfkB8Idok1OIlvMu3UOErxHeF/S2boRxivza7/2a1/96lcVz0R+xP1zjUrpVit3dbixl4uEqSOztYg0Uk2GN+vZZ59V0enTPYefe+65XL3cdSfyV4cn+ePqS99b8hLzmBfdpG94g3yuY9PmuTnGCBgBI2AEzg2BbVidUIbG4V5i1Ylb/CJyNcGx8EZUT6WssQ42oSjdUfgc+AAUZHQQE63cCc9BEFIBaAScpoJGoBIrhl/84hf7mXRa1v7DsDHQph9q+bVuU52qyRaCsfUFUsC8devW2ISWNwJGwAgYASOwJasDfeYw2BXTGBcfVMo7UrEtTA2Jw0kz8WC74luqIDGD2S4kAHPiwosD1ci3f6lQfeVAeJD85UrKXVpBBx977DEUg01WFJqrsVrMvXv3gKunOKpDb4TbIVPBzHpyHnwkj3VhNx7MzQJGwAgYASNwVghszOrAGg4nNsY8ymQ2ZS2vPC3z+uGaGe4lHxLELl81lqPupRW87+wXLKyjyHQFfRGZQzFyoOjC4jYXA6tWCguj4h0DkFUXsbrNtbUCRsAIGAEjYAQKEdie1Wkd9urVq1A6Pr1cx4WWT+qF3ia0LaQvrNwV8qTCotWioCTXJqn4vjJ2wnH4GQKFurV2jgrkIwnEbkrRrfqsGQmYvFRA6WYslDeHwg4wY6HOyggYASNgBM4Zge1ZHehDTTj/DA/KxOU8Dnqtbkv5uqqTz5WQrz24SnJDYS6Oa4EKL/HLYCU6HF2m4ZxrVIencGgtTzceldyyyJseEFOSBBkVd2iKXFhTixkBI2AEjMDsCOyC1TGTxTF1LHvF+Q5ja8skytLkYKqHHnro05/+dI8Y+hTqgCQTf09WejS4kWswhy4BFIAFQu9+7Md+rEvG8TkCJc45XjZwi5Yv6+elVJAzmpJ8pnyikavhGCNgBIyAETgTBHbB6mLxC4ICTYGZ1X0wwSTKbK15saf9Hn30UcR6BDZ5dN9991Huz/3cz/3UT/1UqgBodG3tp74QWe0DE51deb8gjSVVj7XUSOuXdABRKzZ96uvstFEWCtPQekmIhfWFCnK2RsAIGAEjcJIIvHLzWkFHmMmgcSwjyrcBt+viMYPaMgGzcNYv9swzzxSucvbns8TT1772teIc8beVMEHmODEYhyK4CTQ2JqJP0IIldMvzFIEG8Nhgl8scN0bMj85Z0RuVZOyWANoUuABzsA8fF1VrbgSMgBEwAsshsDGrY/KDYMkXwkwWTpFRv7OUogPF4fi0rkkRLx0X8lDJo3zhGP4w1Ga+h8bxWw43b95suHOCCuBbSgEpCdMKXYj1J9fqZEOT/iQHekpf4kLhQe9vXintKBhLdsUFS/yIeYmOMQJGwAgYASOwJavDCwJ7S88cZj7TDyIxvWlJsaKFmErv3LlDPpqSlQORUMZ/eOnSp45smYLYacGropTVksDqoFxUBwrFRaCLgYkNIFOhm36Ed2xCMcgT3gSmzx108ssocNQKo/gZXdGb6kaBbGEjYASMgBFoIiAWRez6v2gmdtX4FVFuQ0Vm0+laNfJXhnGGBWyvVWB6uWNzUK1pjkZCyGgjputWmxHhsl0CXfEUDSBdT7vigz6yENwlc9x4Eejv+77vU7uAbXldhAyMsDwJkmKQUHbWYXv65IYDdlR1LGwEjIARMAJLI5Azh22+lsAnwTQpzwS/8aX9/iiHXyoWYbnlVDnmsHgq7cf+bfVsUQoeJnLGKfj444/3LNqOLW52+SCggzlDC3BwgiFX6qfsT6hVv0ceeaRfLH+q5qOg8rLyTPYWg/cRTsYJO2CIblTtJ3/yJ//4j/+YXgq8aU115CG9K418Cft/A1Li6V3ltcNzTKHIs5zNFkleNvbcJ8vrZUkjYASMgBFYFYFzfvUPBskczFy+NKfuz1+tnvvq+lM1nophsPDXiO+5VQcYW32cSaLLExXuUWy1R9QF9xjsWTVKhx89JGraQJVbrlRYYSLBZBSeODtVNMQxSHyXF1ntRVmr4eOCjIARMAJGYJ8IaN5JJ+JtfHX5XLhJDDMoNEi/VCaPHVPpJprMVSguJTw9eNH4BiWmf261Myz9RQ3OQIF58AjHEkRkbMXZEMk+MLjIcQ9Axq8mZOQky5sATMSx4HY5qhWb7fIiFINXT2ByFrfoHa67/XuRu6rjeCNgBIyAEdgMgZj798lDV9AKcqOplDZg/l6hxNYi1ANSxt0qNhgZ27Niu1s0cd7JILUQl55dXK3FaQMfueHiahU4RGQPLAIq3daZozpXHVvBDJJNoNE6ofZcCjgfI2AEjIAROCgCOXN4hScJ2hICFPPoVsQub5u6Tta1Ygh5haakV4MuFBaXLhcWJtmtmLiakG/8xX+Zqg1WWt3OaVYqNjZMc+iNAk3StBQXHbJRogdsCpTDRsAIGIFzRiBnDmZ13+4P6TyK+2r9XpK3TbUOOsyWDEd9uVlYnDaTwUXqSGFhKeuIpY3eYHXh6QxNYGCSoe7TnZQUHfysFcxUt5TYRapQzAEjYASMgBE4TwRy5rDleXWNeXTbW2ZWvjrUPiodZbetPlNK5ztKVYRtdvrEdUpuaVq267EXjRg8miCWPjpimCq89a1vzTWHOckzlz6CzkLmSMIeOLbZaTtmKlAeBkM+wdYPnMDYWj93VYeUx0577Ci3vAhLGgEjYASMwDki4Ff/BsEXH6IrMIuv6Y5S56M5GvrU3aK5eAnkYBaPHRnGemVjubBOw81TUaNo63Tkg1hPu6c4IFnhtIsRR6GDzU1xInYIy2MXyTcH0AoYASNgBIzAtgjkzMErsC0tEhOn5tEWiQWi8raZWAgb6cLhBEPNlxTL85ePShquTHbLlRwlmbIluN2v/MqvqHb8HWRaFJQCAsfltocISjHwx8GZUjQaqETnVFWS/+Iv/qJULUlrGSNgBIyAEThhBDQdpNOWWV17czMBCyyIUeHs255RcWzeNsVJOwUhBMFQ65x25BAOrbocOpXb7gENSl0EeDgyVU2au1AvWBoEV5noL7egnfYW0IPwERn0uq6VU2L30EMPKZNCPS1mBIyAETACp4pAPqeY1XW2dfhjYADpVN2ZYNqDvG2m5fed1GzzD1YB8yivCwikCad4+76jzdah2BFIs1LBUEfMiacRUxJAvsHt1I6tfykR7liOf6pASuyUefrUYSNgBIyAEThDBHLmYFbX1w2YgOXU4e/Y+b4v37Znedu0SVXGwQliVxwFiV5Qo5yoUWXcVylTQTgcWpXF7yZZeC5ZyszZFShVawp1JvNYYFVr6i/IA2Be3NiyGsRubHLLGwEjYASMwIkhkDMHs7qBJmYyjql6UWKXt82AZuMfp763oB3UDg6nSxQ2HhGAkeTMb3zJ26dIeS2VnULg+itDzjC8uPqFxz4l81iBxe03NrnljcAOEWA8pjbHYSPQjwDWdYfdeCuVhBU+hVDgrH8xrL/r6Cmkh4Mn+D0xTpfgl52gOPhdShLuUIYTT7j0S1mwVX7JnsMy8qNPxPM4euPKlSs5z9thvQZVoo78xJlqSu9ftAVBbLlZiszf8573fPzjH6fKnL/DX940BqtvASNgBIyAETgTBMzqhhuaqRRix/lkHDPGGWMwoUNPpWyV05cB1AKiQ6Xu3r0rFB555BEYSeylG4bmCBJUMM6Wo8qq+xEUb9fxgQceiAcmdgGFA0dHALOT7hI5enWs/+wIMPPK4s2e84llaFZX1KAQO7y++OroVVz37t2LH2IvSr9XIdxyXHvVbga9YOEf/vCHyUjU/JQqS10g5fRGanfo14wZmtlZHB8BWN2iTvTjI3TuNeD9XObu3IEYqr9/W2IIoeQ5c6e223/2s59lTdZn/SfY7DEICxelgwCl+yP3qOt4nfAfi6Ri6ajp+AycwggYASNgBE4NAbO6cS3K26T8IrhJtNluXHpLr4IAhJt9gXqxY8kVAnRiy8qgKO+j6mVit0q3ciFGwAgYgb0jYFY3uoVgCRA75tSTIXZUBJ+WtgyOhmN/CajOpUuX+Itq+FbVWPtTcwaN6IQ3b97kL3mZ2M0AqLMwAkbACBwcAbO6mgaU+4fZFJ8QHjvW+2ty2UcalNcvzUPsqMs+lKrXAnKjxXFah5NcTn6nDouweCJN7Op7jFMaASNgBE4IAbO6ysaM2RRi98QTT2ixrzKvTZPh7Iny8W8dmqFCTNlhRouwLgnX4RiXqNoJB6IrUkd77E64oV01I2AEjMAgAmZ1gxB1CjCb3rlzh79IQCZYwewU3fEDuXl2rGCRajA5ji9RE3A4y+l9G9GPgoldPz5+agSMgBE4EwTM6iY1NJQoPkWUo2hSdlskvnr1ahA7+BDXFlpMKpNzlVl15cNkcmFxnDNookaT8j1UYhO7QzWXlTUCRsAILIKAWd1UWCEQeIYgE2Sk9S/8RlMzXTE9bAD9+aSAXWjwoRVLnqco7QvUtxHUgmuefA+Yi4ndARvNKhsBI2AE5kTApxDPg6bIBKyO6/bt27GBfZ7cF86FXWhipQuXM3/2oK2j2lKn6fzFHCdHETt9LwI4KH7ONPc47WZNt0FAP5/I70BG8Xw6dib7caPKDpwYAmZ1szUo0ycWAZKB34hp9VjEbjYUVswIqEVcoDJ883F6J9LVYWliV4ebU50PAvwENgsUuPnl429UnE0pbERhHBHgb+Opb1dAgPWuM9xFMxewZnVzIfliPvJ4idhxZFpsuZuzjAXywsaJg6L5IbgRYx7eLIsM5hxfYhOQ9osGsePnffWbKKmMw0bg3BDAbrD7Wa+CUXdMx+XLl+NWhwDwl4uvr3gqCxMCDiyKAN5T7Dl/j7gdaFFkyjM3qyvHqkgSEwAx0s/Jy2O387c9KB2vpKobHxzwCrtzhgSZA16GPTpDVk7+RLqibpcJpcSOmUyTUyblCCNwLghg6HhrjU3PeOM4kUo+uQYEqScPebgdDI8f/t65JW/U4nC3mHQmIJa8CBzxo739AO6vJeZvC3qkXF9YhPjdqvmLmSlHVI2cGE5sCozbHQZ4z4Yroyc0hfFvStfTRiJ24uixWt0j70dG4CQRwA7zHsi7KwHZjRdeeAFXEO+ErUSNfXUYFt5vn3/+eTm5eZPEkvN2dJL47KRSeENA2/uApzeHWd10DFtywFhgFGQydj6hataPOjz44IMR3ltAZ8dgmhn/kFGtd+9NyV3pY2K3q+awMusjgOONzTA69oj3bU4YLbcb2EboHfxP+1Jw2uHe45Vy/Vq4RCNQjoBZXTlW4yTFPILYwe3GpV9LGsVksygQeyeF1yq8tBy9beuQYTQMxlya/ozlTOzOuPHPveqpi078rPESWwIQXBCDI6cdS7H7X34pqZRlThgB76tbsHGxILiUtD93t8dMQOmwWSy84qXbJ6Xj5RjrHN9G2EU/tsuK2LFyDTnW20W5u2JsWZY3AjtBAMMbLjqMRry7Vqgnpx2OOoYP5oi/fGCxT2tZUbt5k4APsN+9e5ds+U7LpmZeeEtys6+uBKV6GcwBBkU9G2KHXWBmrc9umZQoyfvoPo2UdrSI0oEk1zIYnHiuNG784C9zUuMzwBOvvKt3fghgMeTax/bGEupEGOS0k528du3axNxOMjlMmjfwhx9+mNp99KMfxdTg2jzJmu65UmZ1a7QOXCQc+HKZrFHq8cuAfGAU4MHwTkyzX/umNClzUnBiE7spSDrt/hEQ68I/N+8HVRgiPoal+rBGGMz+cVhTQwCBSfP2qM9N3vnOdwqoNXVwWSBgVrdSN8C4aE5NnU8rlT1UjHYBxyrnkPhKz2EeXBTGyzFrxJCSlQo+3WKgxSZ2p9u8rtm3EYBehHcfHjYvLpgjvaJjNlXKvPkfNDfevbUCECvd0F8MDj9EedAaHVdt76tbr+3o4pgYmAo7D/ZzlB1DMV462QvMN2Kz28GxEGMgoJgoQ8J3vetdWIfNVRpbhd3Ky98puqy/itmtwlbMCIxCAKaltVe410KvgryiY50oCI+gDrEapeFJCvOtcWNzEUY73iGjyoAW040iMfU2QYHPLAH76maBsTQTXNOyAgwAiN0etjexshnao9Xm59Ux7Fl1FaXDLuPPN6WLBpolgA0Nawux20MnnKVezsQIgIDWXvGoTVl7xf6kF0apgW2swz711FONR+d5m/6W7nkisJ9a21e3dltgbiB2mB4sxR6cJeEwFxDbnlcX57/D5DDKfodbqHcKWHvsFoLX2W6FAK8oYmBiXaPU4J0Wisb3m6yltCbkJTOYotZhcQpyXb9+3W+eOfFtxRDcUj9Cq4wjJyJgVjcRwJrkInb46kTseMsJY1GT3bQ0mCTeSjUmMVvoNi2/+tTabEt6TCTEd0NN6utwnJQmdsdpK2taioAYA327wnpo1wcJMYMXL15Mc1C2nGCQ6oHRhkTCBXkXPdv3z1jeuXfvnsBhNlFgt0dlpY14kmGzum2aVcRFdoS3PcbGNnq8RKH4FoHXUwbhVm+cVB9Kp6VAjKm3qqzTGUzs1sHZpayGgChFxWkavNaSlhERmxNSnbv25xGPbw/7mQqfVZiJQy6JcHCGhwJLznVWaOyksmZ1mzUEFIpXQG1s2nxvU2Mddk1QMAewWzkLu6zqmvqcVVkmdmfV3KddWWyIXo+7SFhP9fWpZmMjf4+8HuG9g9WFd2pQ/vQE4G3hyBQOuq2oKW3HPKiGYHs3bcFP95oXViBpVlcB2pxJeDWE3uGuU6bf+MY35sx993lhiOMAP17yWPvYvcqnpmBK7LCkPjT/1Br4bOojVgEPqHhH5d0SOzw2oegjabFj5h8TOxr4Y/89BUyEkeRmddMxnJoDbIZfVtHW9T/6oz9673vfu5qBwA6mr1bsJuEkkbGmrbr+vJmp1oxnvnWteMOuLtoJUwSC2PG6vJ8zd1INHTYCgwjIlNWZEZgZ7zODRTQEMJVcpMWQrma0Gzr41gg0EDCrawCyzS3TqvjNf/3Xfy09rWKDWDW4desWlkgLFmmd8dZgp7CM7E3hHBb4Vvp0xrCWnskQa8gHa7aJM2JbkRUt/uM//uPPPPMMXeJtb3vbe97znv/3//5fRT5OUo6AfyWzHKsSSfnqKjbVkTnONr3blBSUyjBweDvFnKaRDucIaIE1lmtBe7nJJS/9rGLM6vbV3K961avkL1nCd8W3WnzxCqtL6wyditNMeKRL+/wgXtgsXInzUi4qqM9EUMOHDKdtsWGYdofSSYGvf/3rv/u7v7uhMmdSNIOrjkmcCT511QxrNjZ5HclQKmza2OLOSl47bfAX0DqQby4OkeErE2LOCod1KutTiNfBubQUfCSYCWwE+3BFrUpT9sqJSLH5lMkbQcYS0wn7Up9//nmGFu9PujhjRcesQLZkrRh+vPuO3UTco4uGN9kiwxYKHzLcg5UfGQEjYASOjgCTDjMIJxtormHSwU3AlDTjtHJ0iObV36xuXjyn5sayF7/ZJd8YrrL4imJKvrjoLl26xKormUDXRN34SqN1gRXCJ7LF2IPq6V0KNeB2ELIpapAWTXRKH5QRBeIb+InZOvmMCNDocbgDzYQhfsHX3AjgpZuxyZyVEdgzAtgTrljwwaqIz0Hs9qz2cXUzq9td29Hp4wxeer/224WWOPDKX3HCRUeAbHHO4Rsrd3oz9zCp66Mk+dimsEzS4iyUJlTQa0/RpnsL0DRqHRpLLHxvGlofI5AjgIkjUssR+dOFYmL1Y6H8TyBbprDGpHPlypUTqNduq2BWt8emwTxBpzSzQuMYFcyvKAq1IkyMbvtVZ5UzXHTiZzjn+pPkT9EEj5qcdhQKoWRpuKT0NCvkUVtklDe2cEamMg7vCgHerU3sdtUiVmYQAdk3LOegZENgrEFLk2szibapzLhnJi3i6OEGpaM6zz77LH/1i71Hr90O9Ter22GjfFulmFkxFrhM/uIv/oK/PMMGDf6kNDJ8kcDflJZVVzV12mHFRo1GdEBt2TuIAjYXlao1ccLVEIjupxacvv6+muYu6DwR0NevbPMYW33OvCUJnySPTcigYHSQ6s1vfjNrEby7kom5XT+MIMahB5iXCi9Df85+KgTM6nbdE+j6XKiI+UidZBgOWZMu7SFeCMCfWOuc5VxHsUMpwxa9QtOJ2vgLRQjw+Sl5l86O3xsCtJc9dntrFOvThYB2K2L3ZHC6xPJ4/YZp7lLKJRsx+iEE1h8efvhhjRQWZM3tGiiltzQNL/nQaLZ3p/EOz4iAWd2MYC6SFcbiYx/7GFl/85vfjAKwXD3uOiiXvo3gHJPYoxpppwRQhu8tyAHL1U8rkYF68vYscon5m4VcTlHeaSsQMLGrAM1JVkYArsAaAozq9a9/PUVrVbRcBy3aVpxCLPoIm+Sll5ECUzlPboeRl78TzLv2NSLDJhwoHaARlmuzvI0sWY6AWV05VttIMgA+/elP52V3ueuQh3Ihj61ZgkjhPMeEUUr/OiwDWGpAK/EX2tmet+BRYkzsjtJSp6qnSBtcDauii4ULLpY773vp4u2RWz4F42QoQGAf8CgoyBlHHWZtVCpsoOijVn5JSybnxu3UKHp7F3o0BG2UL0NrtYejFeC+cg0gIwBHwW7hYQRi4p/7433nNw4BNRXN0UimZYXWhsyFSasBwxBi8DSymutW6w6ohAnL82Tchs4EuM1lHFONwEIDNiZCAq26yQlBo9O18G20yjiyEAENEP4Wyp+22CAa0edbzaAieXvUYUy6Zb9HIWjq+XTvQvkQC0sbMWkg/HbSR4QvFTjzsBp9FOyDNuo8IVUHS8mAfXXCZKd/eenpeZvJ3XWx9grfqtgmIhRwoVNoevFWmgKE400WDfUa/nbeqvGxkxZ5+hnjcOwbcFqQw/tBwB67/bTFuWnCTpJ+M6IFAWTgCjJNbFBJTdOLnr0nnmD1QK4+/eV7Mrx9xJOQmFGoYnu1y4Vx0ZrwDP12rTh0RWqpJ22jLknHj0YgXoPOk+fup9ZquZRxoxtvn1gN2anWpm3Ii8khX1EvGJjen1oLIjLNE8VkZ1MFUg6H2qm8w3MhsNCALXwPTj129sJWt+mgd6o65yMmLEEjun1unTBEqfM4TBPZBhqE8x3GYoHkPHZZA3lZv0JLa79dNEQEaCaa0r66AKQ6oBGRTsQvOlQUW52pE86CQN42abbYBdYUcvcbxiUmV+35JZ+xRoqCNKmTG8OMLsEaKzG64GfEcKX6EEYfykIlxeuWGDJJjWwjlW8nIkBDqKtMzKeRnLZWtgQajxq3Qey05tV46tsSBEp4TEk+pyFTgsY//MM/PPDAA+qi6d9WaxNbRMrXYUchKYVT21uS3NwuRQk0aMdRL//lNiot6OTDGg7pBG1Wt5dGz9umVTMIE+2HQQnTFs3ZoFmtybsi5Q4cRQdTEumZPoAFlkFiFMIKjIJ9c1aHzm7uRguOvS3hMWPzPK58PxoNMhR2T4EuWiCDhp0cNbhKMIzXV7hjiXxDplEdLdQ2ZM7hFjsWHoHC+prVtQKVMwfvq2tYib3f4iDBrOCfw6bIcsXuulu3bqG9TOTYarBHhKk69wX25IMmkn/f+96HDkiSA5+7poyzJ/mJPWIrIVtz2KbD52BhfQbryLYS9vqQalByVwJMpSJ22kbZ2Ha5K1WtzHERoF9pdMi8cCbcq1/96rQ60Q/TSIX5VB/rRA46jD0XqIthmOtIKWxv3Xf9aIXawe3CAqiOdVrtPBUmEROXVhC7wYzDN8s71/yg6pnVHbThXoFNYVRA7yB5+gEWfaPAxwpjq8QYI0kFtxCDfPDBB8U1sVbnSelADxDYdq3jTEvwP7o1jwnVxK6kuS0zCgHYGO9IHGAuKiAmxC8N/vzP/3zkw3uFXi0iJg1giOiixNA/yafw1PQ0h0ZYKjHGCZA5rLEhMOr2rLgdlZW5g97RrDB1XBKsaTBrjALNwqUILLSg0+oqdGQPAmqwWE7tkWx9FP6h2GbXKtYaqW0oY9cNyUp2ExvXmu0ZRormljQi0IGPkKfpy7FaaMBG/xnVDWJaxUBXdLzyWp+YZP+a44lVdrA6KRr0Ino4JkX2kAC30bUI6BEdbzBbBBhfkRXetcinJG0qw6CAmkglujqMJH06MRx+u9YqT8x8J8m1LwUYp0BXZ6N2gsByaqjbpJOOfXXC5PB/1eOxOGHFyqvESEM4zFZ5QpljXl7l7StPaEnxITx8h4YCbqptAK0eOyJ5Rz90Ba38agjw2znyz/GrA/KHMVHhn2MtImwaAQYOVi42t/Wrx4IGOaiLsuTHckSF0w6tcNGpJ6MS1hIF+ssd9bTht6PuIIB/kXIJj8pqt8IgxmTBNS90u63vtoqZ1W2L/8ylT2EJFawukpSvPM5cYWe3NQIsRclSN4idbgtn360r4fK3REC/hfiXf/mXPXwu9OPsulGbdyGC7FTBaYdtxExdvXqVnXZwu0HCBI1j/ZdFQ7SidDo5fG65/nwO3C4a0YFFETCrWxReZ74BAphjKIUKJsArb+MFnUjsNVeXZSdeXz9g0COr1pogSeZ5Ea3CJxnJrMks2yB2gMb+ToHTBXI1GmRO6+iyL7Aaxj0kpG8wduBzKAO3oy/l/rmGnshwNSIHb3Ha8TEZviIkcdrB7S5cuIAHjtLTAR7DGa8eFzvA9HQJF12rzsHtKJFqos/p+e1aK+7IGREwq5sRTGe1JQJM8NhofW/Fqzm3WG29amPECWAiIxJ7zcUyR2rTpb0yIZ4kd+/e5S9XLoYwkuTAGzwFcZ4WYufps2T6SYndj/7oj4rSAZGmyYndAvCBGpCZaPnZT3AmrCuNYf4zyZsI9WrJ6RgaPrSafHV84tpYb51XGQgT21TYMyBuR+a8GFA63Uk/JstfqB62IjoSHZsF30VddK11RFWcgqBhbteKjyMHEKDfSGK53XzOuQQBtQLNUSKcy2hpAHOQPxqMUR8YFMsFsHdSe/ZDofKyBmPQAasNAqiE60jmmFTE6+WeGAw6MkSyb1q7bfib5owMwtQrIrVhkTyVMOIFWiqpb0eQLG/EyDyyHQyoXEoZlBwlEJo0qlmeCZDKY4du6VXXJykXak5l1aBphv1hdCBV2i7lVVhNUsSCv6uVuJ+C6Cc0kIYkTUkAPkdgZTTo56jR2mOxCZjTnXShHC7UJnI/DbqaJtNt1GqqrlmQ7CG9Igr1KcQBxcaBvG1GKQR3UQ4VBIsOQdpRxUl4CpWsKK4kiebLBldTBeEHqTUMGxHZipalw0OPoHrgkyZX2lxSk0QeH0U0ArkODYH8VnWpa688t4gJTQhE5NjAxz72MXXCxt+xeSKvdkzzAX+alerT6xDQxS0XwsESIgmRFWNhbJXr5FU7/tYlP24qhljQdJqMtmNIbosGCrzcm178v09sURKsopMHdPvUdiGtaB2N7t0200IV789WmNA9Quz+MIIOHBoBbCUXK1DsGqGBR9WFFS7SjkoiYZ17zFxbkXbRJK3VITLMYmvpYnWsyDSeslbLJjywZQudGJ4kWQRsSE75WqWR1eFugeiXfumXWtVmu7pm7tanaSQggzZLYxEJUWZRjOStbpU0W7ox7j36v9ZhyYT1WdqLlb7W/hBFOLACAnQP6LiahuJoF277x+MKWlEEOqS9aJ1Cx5aCksDFlyKcgawNwSwTEwBGIvcA49gaWX45BLyvbjls185ZtklMa1TZ7Aarm/aYQSkop0GjSt9cWHvm2OijQM7M4BOymyzKSFtV/PLly5srvxMFmGBgY13KaB7qehrx0C94WFA6ZiycbWDOW0orpYuECiDDzEcSrpinKZr+yd+GsG9XQwDwaVa6hyidmpX3InORsU0gbuf9dmNxOzd5s7rTaXGxq5gUyysGmymZNRsZRkFXrlxpPDrWrT5xuH37do/aInDhBJKkpyXhACGW87IHwH5eRQ6cN4H7QTng/YWZkWfdywapXlxLe3kZl8yhFFDGHvX8aAkEuvhcXbMuoeER8zS3O2KrramzWd2aaC9bln6XkDks+FZJeTr1g5fpEuFUhlmT2/BjpY8OHY5FosFalEsOZnVoAaYZPGrBolrr0kP7gJFvZuX+JCvyYcV2+sSPu46sYpkPysiKOaOjVT1HzouA+dy8eOa5mdvlmDhGCJjVnU5PYJxr4QnPROHshRgzKBDEilUhHLj35Flhz1Nhkp2LBQJ4ibpUbXg0+917XZmcarxYFOixxJbXEerW+rJBR8LHrLVv4GV1KRoiz6QihtXbOLQWBeLIlYqsnKQEgcPxOfVM+oauQstZAsUKMuZ2K4B8uCLM6g7XZH0K45ngMXaKTbUhh53V4V4sQqUXkZzPhI+EmW+UawTDd+3aNfJnGh77ZUZotcOAPvuQ06ihngicto4F7YAuNMR8S0fCLdf4ZE+w5O46OhILr5pH4YJBv+aFUWRRjBz6SInz5u/chMCB+BxdToeHs0bBJfOov3E6MSTvKC1rbneUllpJz5iV47NYBzZBQO1Nc0wsPRqU5SdlxWwKEWHkN7oU8xxTKU/HlhhF7ORIp1R/Ua4GjFKYR6kk+AiQACpi+JQyl4TzRWS4oxoIiBw3So9UeSBKzB91xQT4XQJ18aFJoFGXT5oKbkfvSl8Y8OSlAmJatAJ4pvELhVcurqcW6qWNDtkjv/NH9Jm0lWnNRkP3678mGjKGDUvYdYvNZNTPOCL6cZjlaeOFiipgMUY1R50alNt1wCTjDh2mw0gODYtdp+qJpRIm6aTj8+r20sR521RrptkLO8tIq86kK2EMrbQbdQmvH68ZosESUFXwpvpERVJzg9FHMoUODMGTKwWTsFiyjCY5cDEBaG4rt6QqjhLLzW5rXdJ61YXRXxARqMuhJxXmXu2S9plgxquRG1otyAfI9yi89KM1ecyidaG3qC7qPITLe3Iotg4aKJaqisIMWBY3Gi9m1IheynhXjfSXmHT4h+a7DaAtOstGqQo9VBtkGiCU10sFNeBKoUvD6APm1UNvURtVXuW9SUYXDcXM6gKKjQN521QrxBCdMbdUjZgXGcZp/B7CMseqeJAtDFZq3cKmpIabunAbVYCCMPcrB4nxN7fpjUkCeXEX0iLP08iwNYBpQ4xUUlg6lNg7UilJa7bVkStYTDWQNAQr1QK4cmyrazGYkKERmA+20WBu1QLr8Jhq9UoS0popSSJMTEnCXGYFNOKjGXod/KZEVbolvTT4Ch21JFVeuw1jqEJqZFT3vNsDSN0wBJ8YTRrOIsoAlV7oEDBKjBbP1RgEijyVnMCg8PkICBNAjiqb1QUUGwfytpmikDbYkWcJUSgsCBsR/pXqd7vCsjYXw+jIMPVr0hCrMFX9+edPj8vq0rowi9A5mRLW70hBKJlaUpXWDK/AY5arDuNC+stkEZ44yy6KBkMytKW/VagaI476pnPncgjPm3M/t+OpmBnEi3Bh0elcQHJgGQSWJAy9mEFIxSRVWJzEKEJdbrCsUdkeXViYpD3TrG4vbZq3zUTNwpbx/lQ+XLsKZSBpJj6oaeuq1+HiY46ZV/M1LWa8cqSWaN7q9OcWQ2Or6WFRHtNf9ylPgSugww4QngXA5dBIXXRTzCDvHuFtIrD+q8iUVou0vOGHDaf5IFhQ3rAnxBQSOxo98qHtxr7KVicnIUpyzdLrApajB4RJakvN6vbSpnnbTNQMJochU7a8GPGeVJch+aQjP+09dRk61RQEoi2mZJKnXc1i0p3kG+Dv9JeNvCIlMcxDGhfMSSXys8ssx2NmV1UZ0j2kc+A248y6EBrhlJ1i/VI8Y+ht2HVTferCDW73wAMPqE31F6rXk206FwDCWGdb5Ew+FU671WxU6HmIgBounZfN6vbScHnbzKIZpo3hp8wr3lZP4yV1FiR3kklMLfPqs5rFDP2rp4RZKh6TyoxbFMoVW4jHlCtQLrkon5MaS6ARLw+F/qdCQLCHMqfY0sIk+xRrcDtNEPrbQ+yC2RMY66LLcUjnphJrsJqNylXdc0zOHHxeXdqfTzDMD05wsqucdpzEdunSJU4P1qGv/bXluCYOt4sTYpmMU4bXn9ZPjUAXAjoOkBWcoHddkovG07eVP7160YKOmzlWQke46eQ2JnKmVa6Y2vdcNQ7U5FA6GBinrMdr7XSF4YjwIfKhG+tXeabnuUkOUDd6/qtf/eq8dM4djNGRPmXiUE+AgdENYhE2lRkVTucmDljlmNVRyS3chYBZXRcypxMv08ZQJIClE1fj7E3O1GUAExNVxY4zdDmm9b777sOg69cjtI+E5CHmgBGoQwDDLdsdewPq8pmeijlJOuxnbmbK1Kw5vXYTc6CNMA680UmfY/E56k6b6uXh+vXr08lHA0y4iHpO+U/4NHLYyS1E6n//939blcHyMzWkj+gSOtmeus/4PsaUdOPGDdqIaUhnvKeFOlyJQLTQnn2M56Cb2o/mWK6y+Mx5RSs3c/C5Esf4cgo75xyBhQYsL9/qgQTyQueKiXcD/ART8gwQpowXOV2odfWW0+oqyN3F3zQHVUoUKo1fMywTEXMJtmKFFepWNKprHWuvDXirM8wTRhFQnPzpIWKiCtHWeSBterURJIyEs1cwjE//dBNii9qo2Wu3dIZquNQS2leXd+aTjZGNxnBzMWIxSYzSRm2RoX8wzzF6mXo10zRkfGsE6hCgX5GQPsYLQ10OM6bSREWGu/rlN9xj6brnjPXtz0r+OVz4ctKErYgNiP3J9/M0XXtdyrrchAAAQABJREFUSCvMpl4J8AhqQWOhgpbLtrFK01oQzjM5a6mjAtQ6nzJa046KZCRqojnoOixrXOATV9S9EZ8ui4XMEoH7l8jUee4cAUw2xlr2mp537949KUw8186Vt3rHRYDOhvJBp7atCF0dcinLu60meemaIQCK/RLVcP3BH/zBb//2bytzJmOyevvb356XBZ/DRxIrbsCCpIxDLrzzGHDT2msd/2DpFooPIOST1hT0Ll++zNaUgIV1WHbsURZE5IivvujMFdVkFNy9e1dV1iAlzFOq/Hu/93tae6Xu1DqFZcYwa+WACfKUeMStrrgPg9/jnAtkiA8CTbh6LEeGJQGzuhKUTllmD16TU8bXdXsZgZhCHn/88ZfjNv6PkWUOYy7ZWI+O4kGMq4Lbwed+4Rd+Id7WlP0zzzzzxje+8c///M8ffvhhxZwSn1ONYFoEQKyCf8BlNTGTnHWM9P02iI5K0V82hEFEcMDABSuKS7PaKkxNVXQEck3wHFNH0IjtE7nM9Bi5PykLqGkFGOf0PFfLgTmUi6Ha6CeKR40gfOuoZFa3Ds4uxQicOwJBMh588MGdYMGCozTBHGOCd6JVQ42XqN0Ibgele+973/utb32rkQ+3//Iv//KDP/iD//RP/8TTk/HPpdUEK24hB2lkSRiCy9QLuSn/ZhYigjwl4ls6HKujvilt7YIIPidIcddR3y6x8ngcV7hRcYjmuQEm7kAE8GktxOoY5tQ6L7pc/x7J/Zg176vraSY/MgJGYDYEYmGFxazZMp2WUUxswTin5bdgaibXkv12//7v//7TP/3TrZROyvHZ4w/90A+dwP65HGuYChfxPZ6nPJVitG6Lu27UlC/6KN7TlfM+49kzRx+AyPZv9nr22WfRH0xmoa0wNsqljbqGm7z4y+FJ+3K2Fzo0nGr7bKNqreyrq4bOCY2AEahBYNTESQHY4kYxYZQJ5E/Ll4qC1a225aVRkbG3THhcsBZq3cpdPvjBD37zm9/sz/Z//ud/EKDuZBIbxfqTbPiU+jINsyTK1qv+noOk6lXhduUDMtK2QtpTd8nTCWEq0Zd65Hf1CJ3pAOoD7GZrrbs+JGp9NLYuonS0IHl2YQV3vHr1Kjkvt6gNi0UTLjoJ/YoS+zvV2GruQd6sbg+tYB2MgBHoRKBnV8qLHOf/bmwnlwpW11n2Lh+o1sy1TMmNGffLX/5yicrf8z3fIx5TIry5DPyDPsAFB2Um7iJtUHNUbQBSqDxFVCREEziBlin3z4+7oBDLgWmBLbVIWY4G12OPPdaVtjCeIshclK6r+ZQVrUChKyxqw8VRiREEsevpVIUV3JWYV2B31RxWxggYASNQhACTX74m23WubCPHr3/9642YQ9xCDjgbmYtArrAoCE/zR4MxpE3ZzKB8CIgLxu6CiD9cQK67CxcuQHGEJDFcVKSC76bVL6d0pFp5UVuuu5f61IudittU84OG7as7aMNZbSNwLgjkW6d5z9bEw3zT/+rfj5EyQYY9Riz68FVjv/xcT9GfrPirOUzZfuUrX6nInypwgUPut+vJjY1N/H5MKoCrhiuN6QoXAn7x4sVCyUImGvqEl0Wuu1BbU3LchnxhoC6hUrH/rIFnYaE7FIPccFGvRx99FPUIFLZja13IqtBLp+RikDTx2972tsYPmsVuPLp63acJGnetekanOgHXnVldaxM70ggYgZkRYJpXjjgARs2g+YoqZh0qQ27MN/nTCr3Rh48flWdF8rokzFJzlUg+XEDxyle+sudTiR49X/LLFJ3wQkE9+VQ/IttRxAgOp2VZttzBGyb6k6rVPuGE//mf/0ntRg3VBhpjKR3Jg0H+1V/9VSO3uO0hZyFTF5Drjhc8PtSF3tVlsnkqs7rNm+D/KCA79X+ifGMETgKBsNdjWd1ytddOLPLns1xOW12NGTAtQenwNwQm6ICvji9YKyrLvAut0RFrn/rUpzi+ZDAT+fZ6xAKZHhk9Kpxib9++vdDyFp+vwgj5lmJQ1eUE2Ke4mqN3llrwXtTVcHQkPp6A0yDzN3/zN9XFVVC6tCyGRsMhx5CRzvmjNGFPmORk0iPAuKDux90iqaqZ1fU0sR8ZASMwGwJho7Gtq/Gnfu2DZ7Cn6tOf/nS/8IxPWXiFiDA5peSJSZSXuvJS0DlfLXrf+95HPoOZ/OZv/uYP//AP94it2UBCgxKFRsMXSG/pqREUhOSahiXWP233VLnukboQrG5NxOpUTVPFYIxI3g3k9ZzinIvcJlI68sHx3ICUjqEdC/mjKLc/oJ6WyzCUGqv5ucyBYszq9tJY+eahvWhmPYzAHAjoDZtJ99atW3PkN0MeIhAoNkNeK2aRUplGsZiRz33uc4MuFsgfK86zzN8NBabfNuby1gxbKQiw4Ldb4fPJVCV1oelfiaZ5rhwGNznn5ip3CqWLF625lOnPh842l3MOM6LOwLtZzOYs5uooRNRY7X3DrK6/0dd7Osv2oPXUdUlGYDwCzB9YfMwctpv34/EZzJkCJ5A+8dMRWXNmvUxeUJlYae0p4a//+q+///u/v2sd9i1veQucj7rz3R8nh+2f0aqNVF+5J5mGW5kf1IquxTXWlmoyjn2fPdg2HsVmRDp249H+b1uZcao2ni0ICuCMGq1TKB2lQ4P4S0O3NnGq3pQw+c/unMOMyNeOz5hjgyhC/Yqf5ZCFYfDyyjF47OKUeimtTzaZjqFzMAJGoAgBLaAgKttdlGYxoaefflp573xK1gzEfMBUgQ+gxMH2z//8zzCb173udQEeX1HwI7Bf+tKX4HxkRZ5M1Rzlr4knxHYYiPOB2cB+584d/nbN94oPpjW2LhUEFwZJKbRISaOM1WdReboHwPK3R/PAWb8wUaLPREpHEXRO/kbRJYWOkiFnutDzzz/fX/dReUqY/kPOjCxuweGjH/0oZfHipMVuSCQlrkDpKN2+uormcxIjYARqEAhjLdtdk8VLabCPXNXJlXD/UzJ0Ex6sTWNjKwv/4yIVX2A8/PDDaXKmHyYbKB3EjvyZiuqKSPNcLoyGODl6yEcUTb3EVqGqo2qk/XwlRURZCmgvQfTqxtM93xZSWHogw4TRWvJB6HRKB2J6zVhoRbv8R37r2o5eB1B8GERyPsASwyMAiVS4LtuxqeyrG4uY5Y2AEahEANMmxxgTQLqyVpndhGRMHlJgh4466AWcFVcKk9AodtKKR4PSSYZJHb+XpnY4E83RmnYPkXCmcr4l8jH2nYGeQM8sLyVgEQWpO/c4MtlzQOxK7z/9es5C6cLPutCoXIFaUQQ9livKIhDhfgznempWNxeSzscIGIFhBOAQEprubBsurFsi1IhAt+xKTzgJGQ4HIylfaZ2iGTNN7KsDhP3gMKVSIlgwDPGtkqxwWMJamIZLhFMZtk+RlpgSP1aa8EBhwRJkq0vzWSgdme/ffd6FwK7ivQK7q+awMkbg9BFgqmDSZSbo2va+NAQULUcda5QVHpqF1JvulhurmIgd7ARAuEjOauzYTHYlD4ZUgW9BIKnw4/CR6NATeHPa3IhBoLXFcyypJa22xtOFopRdQTGLMpyh+MADD3zjG98AQDzHrXnSc0AvfUloFRuMhCKr+1Uw7MHMz0tAey+o8wu+jIAROFcE4uA0AgthgMlmTsVkR1mEFyqrJ1v2uGhqZxcz4R7J5R5p3tqk+q2VCkLJmuz6mMyLRiy/MrVFZXsmdUrnu5mQLAlEFwKuEvljyVA7hipdQmz1zW9+s9DrQgkQkAT2idUUFSErqHZrVmE3lrNRreXuPFKtk/Z2++p6xrsfGQEjMBUBXsH5tp+1FbnHeOlnHmXO4BUfj116ttPUksrSo4A0+dmf/VnNW2XpTlmKWRw/FsjgguIrClZmj4sMJIMZjn7FxZcWooxMzPRDbWOPhoTWIxy35QH6s7rQsX5Por+C1IhByvcfWgYN4T/8wz/EFcdQ5e+VK1fyjkFv4SS21AkaacsDeExpL+T5SnRiVuWFnqykCDLV2zkhtXpGwAgsh8AS78G8c8cbvwwoU4KqwKOYHrp8AEtUNjbzMZ2v75SKGs3rnYpsJwa0/kVL0TTTXS/lyiyBhuga/GD2Vo6RknpHyiu7N0kamop0sVvVMUYrHzEsoT9tJGtAT+jJP5C3ry5FSaY17Y3+WkKY+K8RMAKzIcCbPXvmcP/gkNOOcmUdK31Mt7FNBx8ALqLZyu7OCH8A7iieM4XgZQla2Z3ivJ7QOsyXwEKT4bFbp1EWgljti/9JLT5XKSBDdyU3aFC8IcyV+cr5AA4jlO9LcJK1tjU9Qb+uy2hVADfeEidNXrt2DWApLt4rVobixIrzCuxeGnRe67OXWlmPxRDY56QCjYOuwepa660ZUY94L8eIE4NBhwKSqsth0JrV2EgtISnV0mWN1W0/8jRKHGUHsaOPBRHfj5IlmtCXICIYVTok9IWeNn1dD0Kj7ooCJ7D2CiD43rTu2QoprQ/T0iNcQVqcBYHWddjWHEoiQVVrvl57LYGrSIbWklzq03N4fQSKWstCRuBlBObtotNXN5gD+ifO1uWbIA3MH8strKQ+AMLzQleR2xJrjhVqdCVhVS4Y9gpwLYdGzG70LvpnV30H41kijBPU5FIaTHIUgajXy3bl2/8Zy40qxDopCMy1ayIaiD7QKC6/nW6j8jxPIEYNBpJRl1cErBHlwCYINAaVb41APwLz9tLpFjP4WZfaXTNBmnAJDpHmn9q+eQEcldtyPGaUGj3CTOFB7JYGbVE06NjxskFBXd9X9kBBDvAY9eq6HHoy3/wRDf3QQw/lY5Za57oxhAMK6CBpc5nCmPTNgZ5W0i7TbVShbscSU9ulg9Ssbi8tmLfNXjSzHntCYKHXsFksZsqfGvNE/uqfgoofJeSZLUpMfJq8K0ylxBjIfEYHQ1dx5fGL8phyNfolmbPDkUPL9gtPebo0GlQkRg3dgM5WSEfoh42EU6q5w7Tg0DpmaZEubcFk+pgKVBmYhLvKasTPYqMaeZ7ArSxnCqNZ3V6aNW+bvWhmPfaEQBjEeZWay2KGeurP8Tc1Oq2ap24AUiFfOPW25sbck05XcMo1P+psVSmNXJrHpGVNDAeM6DylRXrUWAeNRgfDP0Qfo9s3FKOOSFLr8PDRG9FwrjeNRnEb3lKjcMf+xE/8RAxVAoODBWYcTrtRQ7XhohssKMVnLhuV5nkCYTVcamDN6vbSrHnb7EUz67EnBBi96irzKjWXxUx3sElP/S2ZFDHxmuCVhOPEqOwouw8mmpLTopmhF6Ij1U2wDo+pVq+RMLocJGAJJFdDA+WDpEYPgZ2ggK5gOfEUbjdlQ14Dyf3cMqyClmnbQ/jLgahET0Y0oAVQXSyZrFqJMp1qbF+ay0aV1O5AMmoC8AydzeoCio0DedtsrJCL3yUCMcXOq90sFrOL0rGQV64tmqRuEsYFt8w00LWuaYAJRmQuJiqNJmadsaSwXM8pkpoO+TslkzXTRssyec8O6fpo0McYRzmHU7ehF9FjYTmz13TNJuspK1qTmgJFSDLKiCl5AYskqdNO6PGXBo0rB5nhnBYaWQ0GSKUi6pIP5n9QAWGSsjqfbBJd0QEjYATqEeAIiTi4RNNG3EagJHfmA+YVcmPC4EwKkvCXWy4lv3z5crA3TkVp/GCAZMiEUy34W1KiZQYRkIsLSOPHJ/LZejCT/QiIc6AP/efZZ5+FvYVu8XMUEXNiAQajhhItyPksaTsy4jjOpvFO1V99yAQXxwbBtOgbOtIoP9iIAQvmjz32GHR5VP79pftpKwJmda2wONIIGIERCKSnwUHpRAJIzxSCEa9gV+TABZ/jLCv8cOkpqa00TroyZ3jmGNFsY0RpDvg0sz5MiL+08rvf/e4xGexRFrZBLU6gIiXg6mBIUS5GSn4Qt9Aoyaoh08OSkTx5otxAY/Nbs7rNm8AKGIFjIwDl4hhh1SGldPCAiRWDEcoZAL3j0hJMyvDIX84GZo7UhzexXCdvRQCoOaOYtqYtrl69CkuY3sStBTlydgQ0SGk4cmZM4ZabvQgyPCuWvASAs+RpVjcLjM7ECJwpAswW8t9Q/5TSCY65Zn3oHVeFz+9MW2WxakPsWK+kxWl3HLGslS/EDxarwTlmHL+KAeuiveYalecI5RHq7N+BPUIrWUcjsEsE+indLlW2UlMRgBngsRPD5vemRm2anFr2YumpCLsG8w1hixW4XsZUTY5VNZwp3XrQb1SSfXUbAe9izwAB1jvYFnb37l3qyg9pn5g9TSkdazonVrsz6J71VYQfsBqufffaeo8HiMj6HDdNyfK9+BwEiE2cJ7PNjiVy/RIu6Gr1/LhttGkHOVjh9tUdrMGs7lEQwJ6yA+nhhx9GYbk03vSmNx1F+UE9U0oHn/My3CBipycQC+4Qu1iFP1w1efVKXXR8Q3C4KrQqrI9axLkZoaybm9K1AnV6kfbVnV6bFtWIMc8n/aykaP9spOGVzp8sBRrVASgdTI5dR+wG49VfTjuYUHWGu0rYoHTM7rtSby5lqOa9e/cKN/NJmJOTGUFzKbD/fGh63lVw2qlL3Lx5kw6/f7VTDWmy9PY0wukI5Y0LV/pp1Mu1KEHArK4EpdOR4a2UpRP+Muxba8Uj6AiPfEhEKz4lkTBmvSLHDIcDgMkDf0ZJ8p3LpBMGPoDTo3QaIyydw8WZDgdZHVvRr1+/Hm9HNPpTTz11Mqt4g71RK+8idjA8XhSPxWvxYFEFDVgqy/vYYJV3LkBdtNmRqsGzBzvwzqtj9cYiYFY3FrGjyjPrMNSZsaICjHkGfDAPxTNnS4ZZjQsbt9xn8KHJiQWY5iF2aaWA+jTYD/W6du2aaneSlI5Wg3/jro5pPm3HPIwYYwQoIDQscjFkGGhsTieT85lNqX56lN3hmARjk80SmL4TOCNXqwR0VLh145DhvPc65iQRMKs7yWZtVgr3G/4DTcaMduabnmVWxHTeuqYo0sLzbCCamHbfs/Da/fDAT+gYOtKCOpwqpaNqcjVBU9JXoNZmg8BBCO7cuQNrRwD/HE476B3xh2M2rRUsjwQ0vHTaXYdtARZ6SHnyzSUxiVybqzFFAYYn3FSdlrrQA9Utp+TptEdEwF9LHLHVRuisoc4LHAEGOV944VFgp0WPCUOM+QkZ2Ik2ZPAWy1xFJiMKPmNR4Dq92p8JpRvVcHCXxtzJ2MHfQyYQu1FZnYAwxA6CK07MskChs/MEKr6HKmBzoNSidBhtttnQFfegmHVYHwGzuvUxX69ElgIvXbqEy40imWywuWO3+8DtYIGy1Djt4HYnSVlmaRJ4D1aViy32ylC3/D06aKZ0rT0E7tLYwIAYR9jwN49vzeHEImESsa8OcLgOUUG6N6+sOLoOykSx8+FE500Do30I2K3kQgh4BXYhYLfPNjZYYGoZ6mP5XFRAayss4MLqYCcQO14Ee1x9kfDcArhnZE/DTxPmFQy5DgpISumoBX1ptYqAJKULOvoenma6X9qTidTPwhLZ6pwgOQL02IsXL9Jpe1oBSW08aBTRU9lW6qYjJM7WsS1iR/VhSCJJa3aYnsbqeRSUiBdguoEWKHrkd/UIs6zOllLqXWloZVZGwL66lQFfqTj8Q4x2Cqtz0TW0xF5AUJgaNY3xCo7ta8j4FsYARFxsyRIauuVv0LvDoURDx5xHBfHErFAFyBwTFewKvxdMjlu2anFLl+ZDBAJoFZEvuYQ+gE86d4gqE72KcBY0CblyMWqEJDlAyCiIv4iFw3VUfcmcoQdKrYRvVFbHFcZcwOS0rw5iB5h7Nhd0pLRLMFqPgjyo0vlF6Ribsfx9FP2t51IIxHvJC742RUANTHNM1+L555/XpMJQn55bmgM77TDZqDqLnmnOpxQOR2Z/pWAPTCH87RdrPF1owMZkRiBKpCPRhdQzCXAbjxYN0M1QI/ow/EAoRfcjBpClKlppKxt/U62Qoa+m8LbWkSSCNJUM99Kofg4BpcRQLFWmEVYP4W8j/sRu42Wmv/Nsiwb9J51cG71oty2Sjk26+mpjc0NAusbvhirtoeicObw4PSt2D/qdsw5521SjEW2aTlTVuTUShqXWnNp46lsQ0CxFg3ahwfSf+nIIE9Ml3IiPxm3ET7xttZibULqoiGBszLKqPoil01goH2lFyxCOGAWY/2iXNLnS5pKqex7fyFC3NF9gpYGMAq2SityWx/QoNvuj4McNhp0WtDka6lQ0XI+SqcKbhzHsqKqehkHeXJ91FIhhTmCdEg9RirpBaqm8AitMTudvrL3SzI2ZZpZKkq2ssNdh6/BkxYRlRDl1xO1YAyJGm5Dq8lwiFe2rlSl6EUuKMYssUVZPnoKoIUBkvz4iE6z9NRJSKWIAnA3meiRJFl4bkqN+coBdfUDEZCPWSFYURCmNPM/wFkCAhcZiuTCW8veGA8QInoSeh1jExFCAJHiCKq8TQUn3hqr12QoBs7qtkF+kXO20IGtm4nCqzV4SEyEGhUlLWzpmz/+EM4RzY5SZQjSLsKSIXQZMqgyYNN9O6g4pEcvcltKNRUM0FBgVyJkZ1RHa4K/M9YV4bIUcW2LIky1vOwwNLkVGIGTOMwAseiugXaAje3t7UaPQMdBTfWPPzYSVYGyCpAZm+uXQntW2bmsi4G9g10R78bL4UhWyhW3iGK2KwkgL7YBqKC22IzVzQRPxlHDaKvZFb422LOVQM9PTNFjkSAJ64CxLjQMpnD0hsH7goJQOoPSJg76K7cJNxwvLkUZvl1jaz7sSFsbTglBGhka6B78w7amKiYLwqyRgQu+imnvo58dCGysBgHoJgYA2Dko8Vl2s7aIImNUtCu/ameu7VyhX67pVvzYc1yST0SUWrA4B3P4sWDApYlzM6roQy+MxzVjkRjwznKa64NMNgTVvP/axj/3Zn/0ZJdKFNlx4naXKom4lWSFZMWS6ct6tR6pL4RXiRey0CEtvp6un9mQFBXqKoPXlKUfJ/WiVKgwbFicmEtu7TyVThR3eEAGvwG4I/sxFh+Oh4j0Y1wKUjp3pcLXWLaKxYhVKwwIJR6ER70APAl2rcqJ6OsC2J/kKj0Tp5O6d0YO1guZRRPDmHpacuktJ2O/ei5wLA1r5bRRRmPaExehOvCdgZKgj7596k9lDfdlVifXDlKEV9G4PKqU6oJjYMJEYEFO6FByHcwTM6nJMjhoDIUN1prSKyRhLga8Cx1vMiA0U8ilKLjqcT15pamDVc9vlEBI57gK/J8MlHmn2zVt8ibIWylPUodX3LAInShGAwzZm1EQ+Qr6AmTHP08iKroWR0Wsnb5LQKQzItlXDfKU+3b1ZM4imUAI6Xq0r3ti3hdelr4+AWd36mC9VophBxWSMYcWuaSIsVw4ro7LY71+eypI5AppIYBhdnC9PMnsMC6/K8zWveQ0Up6IXza7SlAxF2ujV8a2rcmOM0Nvp6gF1MIzGdJ7O9GM1gbhQxNExHFvrcnkcTvpyU16ob37zm+VpZ5dsNBNmbfYiqjOkG8t3iJJQuoaq1dk64WkjYFZ3Iu0bPrP8jIbBGsp7kR8DMZhQro7GdDiYygINBESLN1xYwR+ghVcU+43f+I1dTR6tvhy9wDRgTG/pmVrsZo9p5ECAOZLa3bhxI4SBXRM5i1w8JWcu7S5ABn8SkV0MD7H77ruPqTcVEKFMi4iyHAgEgF0NhPXY3IBIE3SD6284DAMcAvRVDDLdjzAvHrxoxXtIKuawEWhBIE67ad1N5cjVEFDb0Bx1JYoZMD9VJJdR69pR15Oh1nzRPD0Stkf+HB4BRbzus69rsMrIIF/e7rMP2JjS1AMrusFgHccKoENUU+AQA1BEBrY4w9CcnIkMGkqA2yiOQcFcqBwkxt+8r5JzLMUCAvIklOsU+Z5G5KlA4y/6MPuGVqFDa0DF8bf16flE0oLRoG9+85u3rTjOsG0ViNLRJGChj0X8mQdiutmDjdpPW8gEpf3Evy2xl9bJ22aUZppg6uYJpc1nuxIFpLaHGVgBAkimr9SEiRH56AJTbKDraR6vxgL2/FFFTFA6Fl5PtSmhZTQNVz8+DbEeMpfmE6nIvzAJyc3qAkMYzP33v3gUA38h0xF/toFgunq7OFsc8oozxE7VRuWVLY8RJswLkcQnmwgT/33RRWEUpiDAVM01agWHpU++lwxqNaX0irSs72i5kKbnpMMPfvCDFZnsPwncmmtQz4ZYSRLybKQaLMUCDQRwr3L97d/+Lbvr+L6Elcdz/iCA5X5sgvoVuzNBpgGXb43AIAJmdYMQWcAILIIApArnBL6KRXIfyjQoHYLMH0Pifm4ElkLgta99LVnz92tf+xqvGbg8R70aTVGLDZFso6TEyITdbJscwAmd9SHD0QoOTEHArG4Kek5rBCoRYC7BRTfvgRrlqqSUDjVwMaJPeXJLGoHZEcAvxSIs/RBnFRRnUQc2X0bzNkVZrR9q4DJkRKAPgXW8ZZBLvryRMrgqF6377A3nDPeGgL+B3VuLVOqjL1jnPUx1UJU4NmL6z2gOlnVKAphvvBFQunzVW+svi1a2QenOecFrUZyd+SgEoHRsnFJvVBeF243KYVCYDPEF8tkydI2BJhbFGHxx58TLlzIRucSoXrhwgVXRwZynCFAWBUkZ+Jwp3RQwnRYE7Ks7kW5w5coVaoLZwkZgoEbVqtp6avWQ4mCTvGtikTlIImcqo5Q5eWHMN/MEi54NoGgF4sXOlwNhkNJpdllOAecMAvq9WkORIyBOQy/lwqq0vvnkqUpieAWF0oWtw2pxCBR/c29c6slDHv6HUeWomlyypNx+GaoZe1up7BJF9CvgpyeIwLyf1MVXGA6MRUB9K/2SZWwOsggVOWDauMYWh3yUyIec0h+mggJ1n9NWKHC4JPDgBplLbQqPBqGbMmDT0pk+U/Ti+7JUH4cXRaBu0KWtdhphcADnFI3wV2Fhyr8s7kKDMZUaqEbP70pFPAljuKFhhWntyZxH4SafpZr9ZZ3A07BRBE6gOnNVQTYq7ZxegV3Ubq+auYwjr5VjS03PUC1Py1us/DosZ/AiS6+ClOjV9tKlS7idCJfndg6S4MxRtz2wsE27h/NNhIjGitKZ2GJGmZitkxuB2RGgc4rY0WljdVKlMHxG/c4YjjfMkX47Dgt5586d8p7PYGSnBBxCH0TjtKPoOmvZgIhaUC8cdcTDOPHSFX5z3cjHt0YgR8ArsDkmB4jBssi4YPXu3r2LxgS++7u/WwFMRjk5QJis4l22vPLaVEdB8thh/lh+5YAMTBV5YgEJYEC9JhuQYrh5+4/bNQODlI6dkfEqvKZi51wWh9qcc/X7647poE/qPYS/7FjQWysfivLiip0p+U6WbSHic5gpDFHqeOsvPX1KuTi5KVFLsbAxii6nhmlWCjMYUUwGHJVKKpJn4hgj0IlAdPS5/IHOpw4BtRDN0UjOAgSRWBYu8afWtuRRMIY8k0ae6a3eiSvO/0QfNIEOprkR1rJF0EoC6ENkQ8y3dQgApjpAefKehdfyTCxpBBZCQJaEv3n+ja4bXAqrMmhSYqSQ8/RlXHQLpx0DEMVybUtisLeyjfwlXJLEMkIgXjsJGJNAQNNBOun7tyUCnI0DeduEQoPOeTmBkA9DlhodLFqrBSReJobkUVZhIN4vuwaYuV0hkmPFookLE6bzIpNiYSqLGYHVEOhhdeiAJWl9m02nsVxVur0s6rx9PpRBpbzQwZgYvJjc1EQPJrQACJjVtXaDnDmY1bUCtUFk3jahhNxpEsj/8s4XBgKjIwqYGp0wJXlaYkg+1lEXRKHfsKK/uV004lyBaM2SDKOlaOh5p7eS0i1jBEoQ6Gd15IAZef3rX98wXxgu4rvyFxGM190usYp4xpQ0GbR+aeaoGrtc0K1H8zSVwykCZnUpGhHOe6O/lmjYij3eMh+Lq7Uql34PL68+YmzdiGOW2OGLAYoLGxphxgn2ZexZ6mxtYeccKrFVpVWliEQfvHrsUKZEwtpv528pAp9FA+leOrpQ/7vBopo4cyMwBQG28D733HONHDAm7HVrROoW00fnJ6y1iFaZ6kg4GdaM5GyzUymDWbGFjq2B2uHHSEzftQbTWsAIjEZAHZRkQf0c2AQBtRzNkZcO93rjG9/Y2rSYrVw+2hTzkT+dGBOZo9WorKCPpIXbqSIEuPU76ygMEQ78+xMCbEBtL10/Vn66LQL9vrrw0OQGkB6eG5A6X9pYBOQLLPG6pRyu1VyPLfps5aMnjJ16ThuxnDl4BXYvLZ63DZrRfWXycotGDG6wVu2xdPk6bKvk2MjpFhPdzO3Gwp7Kl7A6QNasQycxpUvRc3iHCPSwOvb+xstJqw1kODRqFHyrET/qFsObXvnrcaElDAc5tSDDUTpYuIEAAKoPGMkUGWGSDgSzuhSfLcONtsGcxT4MHmH44gMFSfbP1mF0SEhWs1SMsSSyyN/8FXlUEeZ2o+BKhQdZnSldCpfD+0egh9WhPKaMPi/LI9OX/m2464JF5TxsEAcZpa6CKDSdOMktRmKXMcRES1WI5lxGeLAWJyxgVtfauOpjaec0q2sFaoPIaBvGf5gDIrEy8Woi80dkfp5IrnEYHQxfl1cvT9UaI3snDWd86VS2ZBg5o7PNX2sTRGQ0a8SkASC1ly4FxOH9I9DP6kJ/PupKDaOMBn/T+Uxvwv1vvJFhIyA1GD5kiMHE6sZFDBe3jSSyXfm6KsMwtdXcNhL6tgIB8Fej5w1RkdvJJBEm9M+okVldQLFxQG3z6KOPKsBf+FzDXqhbY3cKzQR2MDgTVqaOMFFovL9WZ9IDLnWhR4aeVByjXKdqTykn8wis1EPyGoFkULoS3p/n4BgjsD4CIkD8LSmaTo5VVBINBExH2MMumjWYsxY3xtLBVhJJVmEw07l2UAcL9CNgVteKj0ZB2tPM6lqBWjsSEqO20d+cz4VCNF6YsIjsCSAs00POnGU/6hAT0gaHIO1Eh1+PkjxSWeZ2/SjxNFqkIQmAQenKeX8jE98agfURGMXqQj1sJmNB/IkA8WJmGLqKd0INq7EJteCLDqFVvEhjyhrv5CHjQB0CZnWtuIkzaAhIwKyuFaj1IrEj6bLCQraA39sJwsSUTw9ghHRVEn6APUKreOMkScU+la78e+LN7XrA0SNNP4zkVNKULkXD4WMhUMfqoo6YJk1p2nmMsYpH5QFefbGQ5fKSxHprTpV5jLFJVusYzLEKH1rerK61+XJW59+BFSYb/OUQI8yQfuA5iodLccXtXAFsFqaTc+Y4M4kzlrg4bInMicQIpqXwG4s8TWMwVTKXaeRCYawhZXEBC3+BiAAXmHAGVbDMhUo/aLYc3MVpWGo1WpPzC4PBH7RGVtsIlCNAn+dCXrO+OGJ5ckliavjZ2bGpsEhcpP2TP/kT3oSxVOSAMh6DY5G0/JwIxOtFKw105BIINPxz2AV58mlXvXQuUajy1HvtoOGDFkAE0WrsksS8mqMA4ER3h9ttq8+8tavLLR+wscLOdILTri5bpzICWyEgc8TfiQrIUIzaZBIlkhbzErflAb2Ex3tUXSblxZ2zpH11ra2vbp8yB/vqgjOsEcCtgs9Jr3SUB2XhVnbhAx/4wAoavPhW+7JzjhPbIXkNzxxPr169GjIrqNRTBMhwAZf9dl0o0W10Zj1NZg9BF0qOPx8E2AFcV9lgZqOSK9UDDzxAKsxUvHSNyuQ0hJlK7t27N+gyaFSWOfH27dtjUzUy8W2KgFldisaCYf2+DQSFAMWkfG7BUnuz5ofCxv5WWG9+Sz00t+tCFkrnRZ8ucBxvBFZD4Ad+4Ad+//d//zypCZt28KLxbslKNKR2LAhPPvkkOeCFWq2xTr4g/w7s4k0s/xw/fspWNsK829H1cZJBVhYv+4QKAC6WX2NNFjbzyCOPQGswJSdUyxFVMaUbAZZFjcCSCNx///1j2cyS6qyaN85RfmocL11FqcyJULqKhE7Sg4BZXQ84Ux+18jl+6h5HfZ23f6pCx09vbhdtKC8dHSn9wDmeOmAEzg0BGdWVX/NUXLr999xgZ+8HjLbiWxNWbGOr3LmBtmh9zeoWgdd8bhFYX87U3E5IMI2xl+6cZ5SXe4T/G4FXaD8JyyBjscBcj00S8nI1velNb4oYBwoR4EwGHByFwhYrR8CsrhyrIknzuSKY5hA6T2737LPPCjxRup181zJHezoPIzAJAVErPgIbmwu79UnCjo6xCfE2iREeYoPy2NotKs9ngnj4bL6WANlfS8yGKsP7qaeeiu8hmHShHdevX9e6wGzFzJcRCmMB2ayWZollPJaFAmQuYD+H72Qxhf/4j/+o9sJLZ5uYdl2HzxwB7WzDrEG2Rg0N7Qmr8HlzigqYU9Z+jDwrwiCg6oMDGjZMOpFisdj5VrVJjgBroxcvXuwhXojxholntJF/SSfEwYkaXn4twapGJr7Ebj0KxZElCDR+EYGhAqpjTw5T45GwpMQpMoxzSum3egxmSBIjdkpB66eNbykEJmxvtfPt9CWHTqhPxyFTBWiD+dj+kKNH7dKccwHHGIEjIiA2xt/pyouZYbtGZcUIZWRVjFBpvoLRHqwO9iesOgFupZssBtae2jUimadyC09aMARATRMkJ22rmKY5mNkv//IvI8OFcAkUaIIwyqhS0nCwgggECyRQIn8mMgIwRd6/GDa16VMmUcfnpEHeNlM1y9LT8Col/qIwgz+94pECjPDDDaG0RagFTGu5KpCz7GADt9ZbcEY4zFnWPn0RQele9apXKfM+aT8zAsdBgHFBl+bvdJVl4sZmBcnA0I0tHXaikQgBGpt2dnmsCrZIpJbq8EIrKkY8Rh49iQEWWUI019Hl/E01QQbhlMMhrzoqoYQFcioW1olHaYatYQpFPh4p/7jtCbQq0yN/Jo8EYIq8WV190zfYA0OC0VKdXd421VnlCRmBepdSKQxvOkE6LNMkektL5RGeUrU089XCjdYJi5YrUFc7ACGh8NRfTCp9gHKxPnHx1isDGpLYzbHTAHkqOWl/5md+RuG8Io4xAkdEgLFJl+bvdOUZdxod5e46JWHkji1d45ohOTbhcvJCssHVZKawTqkND6BCGRkZhCNGAZAB0kiuhLmY5os8vpEbpTTUK7dmoTOBRrbnfCsAU+TN6mr6A11Tb0UClH5f54BJy87bJn06JYyBU+b8pe3Tsd2fLZWSmSAh9T3iWGq0FNVp1EKWApNUDgugkSo6AGlBuIsiB8IkAXzmALUFpq2wRFlbUpGWUshEOUTODhiBQyMgI8PfWWoRZCu1yWTOhaFm+MSFpEYxIysVLlEjRuXYN7SSzKtlqCPGgQqmOXBLJI/SSNk94iNStCyvDjZHBocqIyx4SR4JFWgtuiEDyADesHvKvCHZehs656W3yp9JpABMG92sblzTN1jCLHxOGuRtM06zNulZaBmUJbgIXacxJtuK3V0crSajI5AJh12IeKxVid5UX1ayrr1ILrNIclDNbWhDh5g8kJdVjdIbkr41AgdFQGOQv7PozxCTvUozJCzWomGrv4gRz4AaS+mQVxGFRmOWepVkQnWoGjVKhWUxUjR4GgxJr6OAJkzCMKY5qLJMdkRKDPlUgHBr0Q0ZmiDPXxk2JFtvQ+c8k1b5M4kUgGmjm9WVNv1yfE4a5G1TqlmHHAprNJIzTZ6Pw450LdGM/LCJvGyNNYItOW4RhS2Q6RHUhFMvJpEyWz2qgUPqopNB7JHvegSZi6ZhYuhqmpzSkaFsNNp2Ze54I3AsBDQq+TuX2owvjXEG+Fx5pvlIYYZw18hNhdcMS7F0gqd0WYwGvA2G1Lht6Kxs+RtiDQFuW4tOxVBDOZBJeqmlFNNvUZEJ4TTnMw8Lk7TRzeqGu8TSfE4a5G0zrFm3RLxNwkIYDN2CI57IOqAng3NEsp2JgoYMkABv/O0hdlicEE6HUF39mA/CaZevSpBnK6UjPlqhrlynMgJ7Q0DjcV6rosEF8Zr9FTReBQcd7evjLCQb1kkWowFvgyHFLWYnVzsaKMRyVFuLTrOSQJjQ1kBDyTQ54SidQOPROd8KybTR728F15FCIE5B0y1TPgeGhbdmzyjxI6GcJ4RRg4vwdxZVMWf83h8Xpw3x+31BL2bJfLVMMBxcVIGm5MykRrm0OMcv0dCNeG45CZ2/+Cxv3LgRnstcrDCGRuGXvihOP2XLSYcxW5AD6hGvrLCzrfoUFmQxI3CGCDBIGd0c3saPx3Oy41w2kIHJUAVPWOOxzvXs7wOYRAlA17okU6PHgXZj50EaIs0hSmEqIazZpOIg6MjHge8gEHPzObPdvO68DaRdkGk1fzvJU02JUZOkjLs6t+AHS7xNqsNgJZcGpLr6hQnDVfadwfByKH9hjWHSv0ZQWHQqFjnHC2hKxHs0STNx2AgcFwFRCv7OW4Vw7WCspltCnOsxVMlwb2uvgk5IomeKpNRuwBvghNmRPYSrpWkVFieWwZeNZEJsiLUW3ZBpvVWGrY8akbnODYHzvBWAaaP7F8Nenslf/s/bmDxScuTQWenNTK5jX01ezm/t/7yeLvo2ya9lAAWOwHAmrV3DOcoDpc9+9rNdOVG19HeH6BLxQply/a7ko+Kh4MqTQkGVXvf4448TIBN76UYhaWEjkCKA6YbMwUgYTVevXsVXpGGVyhSGsQC48GUEGK0zOv8KFVhBTPYcw5iaPsql7uAG59MMqHUDFhkaCx0kXEFJF1GEQLx/nCfPTWvNq4BeOAQc4XiPScUWCqvQlHHXFaQqLPo2Ge9MMJI6JTdPNbimCYByy/FSLnOGNV9IbQpS63MWnV6Luc29dCrdA3ahVnC2WyEgk8XfJRRIN7AyuCqcdjHiGJXT7fMSdYw8hWTDkRb6hxiBsOHpHIfNoY6Yu/BEEsDucaUxslH8JWeScwXnU+SoZRyZvlS3rjAFSZhAl8wZxguTtGf6a4kXuwG9RONBABFev9/kbVPRQWPttVp/BiRpuTB/DHKF9TfVJyzF7CuSaSnLhWliXYK99S8WitrNUlPyEYY0UAppoBelvOY1r0GZdIg2QAjJRrxvjcBBEWAk0uf5u5z+ctppmENBuA2a0lWolmggNEpFIEZrV5IN47EqYRmCb1EFIrmNiutdkci0XtyG5iADq1MOEuNvAyuyVZMpW4RJRQwJEeZp5FYSUCYlktRRwgRK5M9ERpikjXiyrI7uq40C1Jlux+tLa1cgMu2ghFvFVugfedtUFKoBnDZweSYQDjmlpEn+N82KcS7hxnthKnOsMMaIptcFgLre+c53TmkXUOp3CtLfAiXZ2de97nUUHZF5gKdSKX/kGCNwRARkgdOxsEQtGlyEQUSJjKaUqzFgIShENixh/5BcQttt8wxj2KNGQ4bbHuFZHmGcZfoIzJLhaWSSz1DbfAPLGny+DK+xjYqs4vOJjXTlb8RHTH+AnNlCka76kyH7ALiYYuEuoj5sFyDMX+VGKXwUObasfk1WfkqVtXGkYscbOOjzXuwX2wpzzVM8eQqGlEKSADBPcqwY7HiY8ugGVJBaEE9XqagOyel1vF089thj8XKc5pOOAj6J5ROwr371q634p6kcNgJGYCwCjGLYAOORgSarxV8u7ZZrzQ0rx4euGLrWwdua5DQiXzSFb3hDf10aMoPy/bn56YwIbMPqqAC+tNiunjOJe/fuMZbgKIiNqi1JmBTTyTJNzpBWzifG51RHLBQBDFDFABPZ5bVVlDcFTeEgOvEIsgJrAWquihIjn/UDKAxW2OuuyoZKgjScvhFfEqAUvUgU9mEwpO1gz8w9OdolJVrGCBiBfgR4sedChqHNQONv432VR3oNYwyeG5nrh85Pj4LANt/AMoHxwhRjBh6WTmN6Q3rwwQdjBJajKZ7RIw+VpDjN1hTKwOZKS+9Ju8kjvkiSB26wdCqCTF1dAAS0B1lOqgONyEVMsPP06Z7D8C3eGS5dusTf3KaH5sCup3xAF5HlAfUx3MblSdR2SlieypJGwAiMRYCxxqss77Gsur40CXz7D6tyzE04GmJ6Gpuz5Y3Atghsw+pUZ+ZUBURHUiCYTTX1ppGDYXlHBsUQYEhrENdxoJIi5pLhOM0LFy6A1eBkLwG+wB9btBJWnAAp9G7dujW2xD3IQ9rwpQEXF4GcOusLf5hunX1nownVHNXBtPZK58+V2QNi1sEInB4CDHAGaVynV0HX6NwQ2JLV4RySsyefU9myylQ6djYd5D3RuiyKMYzjdv8BIGLKF/9o1TbqfuXKlVaBnkhWpXmqtugRyx9xshqRUXQucIgYWFSr6473ePSv7idkO7YDR1nPPvvsIaCzkkbACBgBI7ArBDbbVycUWDBlQpXXBKd3QMOiXnjyInIwIO/IoBgC5D8447IEPChTUtaMMuIfgAYhBp+Uh8nfCS0YtYoq3URf0twKdRaDpPlYo3z44YcbqV75yld+61vfakTu4fa5557L1VAnhD3T6GDLrjuxVT50yIULY+hChZIhxp4eOifHnJ7S7xFF7RwwAkbACBiBRRHYmNUxdcFRmFBxngWrg7uwlgpxWa7mFDpj5vChQko0SBO//OUvo9vFixdDUo60VFvg4rstLhgA/CMcPKlMRbiwCmnOwSAbx5GnMocLB3UGZ5SPhlinImoF+v86xbkUI2AEjIAROCUENmZ10ALYGwRFW+LE5GB48JVgDOVws0BZKPz2t7/9/vtfrDuciVm8MFWXGMpzdT1N4wcXK//mpStN0hPGqcMFD4Dbff3rX++RXPoR1KfCL7W0Vl35DzY6RJmvHEC1KwfHGwEjYASMgBHYIQIbszoQ4XdFta8OMidWh+OHcAVYLAjCBeVi6UkOZeQrpx6B1kfl1C3/+KM1QyJTQinCxzrm937v96byyOTuulSAKnP9x3/8Rxq5cpivyeZyGa6gOVDru4RGWcBID4TMyWE2kdXRYSrcnw2VfGsEjIARMAJGoByB7VkdUynrsBA76AvTrRhM3aYizcp4/vrrz3QL80O4X6zxlBm6cJKu4zf33XcfJVJxGFJadJzDkkYSRhmtwEorACSy0GXYyKr6NlhpITLVBS2dUM45vVTMVRZtsfLq7VyaOx8jYASMgBE4KAJbfgMbkMUuNwgN+8SnTK4lTiO4CB9vDrr0Qr29BSBzfCPMpyFUNuiUqCRMIpjWCmqjBqXAXUKNFQqdsQiYPbs5QRL3at7rxMnKPa+5YhWwyGXb6krM83eMETACRsAIGIEUgV2wOiY/zalMaSy/1h36GrViGo4PLyKSAFM4C7ta2xWxW5MApZqMCmuaJwkkAxrHmZksH+e+TDDkQizky0sBhzqvkgCs802Wq7eEJDrTEwAzZcaNglSvCjwb+ZTfBik/IqTl1bSkETACRsAILITA9iuwqhjuOq0hQk3qGEYKEFM1m6KYtkU74HOcTwFxjFVXiuMRHjtcg9OLS4uePYzOcDiqM6gnPjxWnyvOBGbVu+5bBzGe8o9UZgenLkM4UwltwmEGnvSTivV6ITP2YGelosUH27qu4k5lBIyAETACp43AXlgdZA7WBbGbuEU9WosM4XZxmwYo6PLly1qE5S/kL3d9pfLbhvEnFSogdsVXsYXyIQZxafVuhkBrIE4z2TN6rZoXRsL8IFhQOs4EHltHvU7QzQrLktjEc49HlWVhI7ArBG7fvu2NB7tqkb0p0//V4N603VCfvbA6IJAjbez0WYcdvhC8dJxewZoXC77M3FoCrsttJ6niVyUgx+XVETkb61WiygDI3xKn107wqVCD2sGSqenYbqkDsce63OSr89xW0VJOcnQEMMLq/0eviPU3AtsisCNWJ2olbrcCKMy4uEbw1eFWwUGojw9WKHe5IoAOFoJlZH0ZFlKCJKQWjyaSY1kLpWjFfNSv1y9X94VyZuEeVgfxBVJtWywpSOCUE2vlqVMbCZ82US4B0DJnhcDYl5+zAseVzRGo2y+U53OqMXthdVA6WMKU7w0rWgg2I48d0zBzKjroW4qKrHaSBP1Zh6Ui165dizP5YCRUMB8JUDouNGf5tYQCRh3JXwvl8I+x3CUy2XkArs8Xvqy9gozq2+icFy5caF1gZSEJeVIBe3kdaYinnnoKeTZHltPH8vwtaQR2i0DXVpndKmzFjMCeEdjFN7AAxJTGZLa+l4LZl9la1ES7+piS99xg/bqBIac6IyMPk4RZXU0pHQxPGxQQhs/hsBxrVaGMsBCgOzoJzsGUWw7EIMcQ/a997Wuf+tSnEBPvT+XTvgoakDk9JR4879y5M4qcwZLFBW/cuJGW4rARMAJGwAgYgREIxDb5F9a9dEgYfykWYiGKsK4K/6e08DmxHMAHCv/n2So3ajOaY3ppIhxAukRFdEYd2sJdpqu6hxxAiUrRAUCsMXKgrWioMcJTOuoSCgerRo3B/LcasIOKWcAIGAEjYARWRiBnDi+uvil2ZVXCkwGLYr6EiKysQF5ceJ5QSXQzl1kuJm+b6rJQXgSF5bzqTFoTQoCU8x7aq1XD8khQosWBqMHk4pZuoNyotbprxJSXMigZjVVI6LcasIMVsYARMAJGwAisjEDOHDZbgY2vlHUGRDCqmFPXD+CtkRqoxAKcFFtfjeklQkHk/mEddtTurv6iWSLk2wgtFO6hvfq1HXxKFVj3BKIuyXChQWSjY8yIp8rV2itNpqXzLmUcbwSMgBEwAkZgEIHNWB17+fH3MJnBpVjYki9kUN2lBaQMszjchc9j2U21dIkL5U9F5IVic9gsDJUNZ5cuXRIg8I+dtNcU9CBtPY46OidX5E9YTjLwhNjRPeJRdYDdeDSNIIU1yglanZsTGgEjYASMgBHYbAV2ZS/lqOK0z0+dg+l2VNpqYRVXuAxXUgrrhrFaR+bVOZNPsB+YR8nerxL19iBD1VhUbbUC+RI8wsHzILV8ZDOlCnDKoHGjmibadErpTmsEjIARMAIngEDOHDbz1bVOpTuJZKbnG0bN9yyQ6WC2nehWrgakAeoA+ZBfTU47/EPlOSApF52WKeF2wDL2ZLtRxa0sDERvfetb80JhTrkzEmHAFKkCRs4KrnPapWnFDmOpN9fEMUbACBgBI2AEyhEwq2vHiimco+yC2MHt2uV2H4t7CdejuIj2C8JF4Gr9isM84LLsotMPb4AGPksWzQn0JzzQU1ZRadbf+Z3faehMHXu2uMHAwFMdQ0R51DJ9moRGIavw/zXU8K0RMAJGwAgYgbEI7OUU4rF6ryAvYgcHgt9wUaK2zK9Q9LxFUBG4CL4lSAx0DWJB/tA1+ATsJP19Kj5hgWdAU9IvRRCj4rnval4lV85N+yZVTfYgPvTQQ7/6q78qHbjtJ6+ABkp0DJCU4w1wQImNmPxoW54WGSDFz8dfbchDHkjN51ZudBdnBIyAETh9BOTFoZ4nsMC8UBWY5tUPmM7ZXLVQKSqC5lgof7JFeehdIZmAnVBxyMdy+myVM5wsuBeASA21MnyrXCvykdMuNRPEgHBcORumoCm9yAO2vIEsaQSMgBE4bQRy5mBfXTojt4dhNvzYAL4ZXDv4Y1iZDU7QnmCvsagNJxAtCO9R6pZDcegI3jv+5nxlr9UapxduV5oSnxlo0LKxTRB6x+9DjFpqByKIHQAC5q1bt+SKa+Ap5eTM44vXwt/nHVclSxsBI2AEjIAReAkBs7qijgATggQw5TNnc8BHbLkrSrxLITmTdqnagkpp2ZQCIGT8NlfKXGnfOr5OJpGPuHJagYsXL/KVSe6xS2UcNgJGwAgYASMwCwJmdaUwsnDG3KxjeOWxi7m8NAvLbYcAzjl+vlYf88JoW7/8gNhNVPA8ufJE0JzcCBgBI2AE5kLA38COQJI5W+4c7bXXJxQj0lt0IwTwsELERelwu/LhwnQCt1FVXKwRMAJGwAgYgU4EzOo6oWl9gH9Oy68QOxZkTexaUdpVJKuiUDptd2MjHfvndqWelTECRsAIGAEjMBcCZnWjkQxiR4Yi6sAAABL4SURBVEqIHVu1RmexswRUgc9B2Ms/eI7dzhQfVofDR/j4Q99G8GWDPnQdTmYJI2AEjIARMAIHRMCsrqbRtLNev6MFbxj14WRNeUumQX8uzlTDm0VFCCxZ2qp5B+eGiLeeQrKqNi7MCBgBI2AEjMDCCJjVVQIMsWPHvXw/rMPKIVSZ16bJOJIjysendRqsjorgetT6OG3Eorm/Qo1WdsAIGAEjYAROFQGzukktyz6tOP6NzVuQiUnZbZH4scceS4s9Afaj02e0kY7WoY38bUTaxA4bASNgBIzAqSJgVje1Zdl9D28gF2hE7MqfmumK6XFlaSkZ6kNFjs7q8M+JXlOdp59+2t9GrNiVXJQRMAJGwAhsjIDPq5uhAbQOyzcHInb6SHaGfFfJQkvJqxT1/9s7f9wokigOryUuYDiBZQlxAJOSIBwSGTsgIkAbk6zkgNgnIHJChMSfAyCfgRUSEQnyCbCPsPvhtyp6u3t6unu6e6rb3wSjmur68+orC356r1716JOkS4bRpsTHvVBwdOJOIAEJSEACORHQVzfMbsThLRQSQVh8RctLJh0G02ijgJ0Lokn7YAauFTQ3YjTSDiwBCUhAAvkSUNUNtjdx4wleIhTGycmJV9kNRnbdQGR4oKTjkmHktZcMrwPmcwlIQAISWCYBVd2Q+1q8QSNdqzHkBCOMFV6unZ2d3d3dOSpRLhkm3TVyIzgXyGcESA4pAQlIQAISmAEBVd3Am0QQlnN1BAEZl4Bg/lfZoeTCy4W8m919dRjvJcMD/wU7nAQkIAEJzJaAqm74rUPYEQSMFApkB1IJwTT8NAONWLyvjiFndF8dYPlgc9FFOhAVh5GABCQgAQnMj4Cqbqw9IxSYhF3ctTHWTJuNW7yvDj06i5tNUMleMrzZtttbAhKQgAQWSEBVN+Km3pzy+n2VXZ5uMO7p5YOYw+OFwfmrOo7QpYN0WI7NiNERd9GhJSABCUhAAjMhoKobd6Nw1yE7mKOoRcadsvvoXNX748cPbgM5Pj7u3nvSHkS0cXyij1FygPWS4UnpO5kEJCABCeRNQFU3+v4g7BBMqBDihigScjZHn3KhE3DJMAfpwIhDkZSUCHAvdK0uSwISkIAEJNCZgKquM7IeHeIquxB25GzicOoxyKhd8H6hlkadYpPBsS1dMmxuxCYk7SsBCUhAAgsmoKqbaHPRIpeXl3wzHw6nfIQdgoljavv7+9xXhzNsIhxdpkFxFi8ZDsdnlwFsKwEJSEACErgVBFR1020zvrr0ithfd3Lc3Mox3fQrZjo7O4srfHnOBXupvKL51NXF84gcpOMztQXOJwEJSEACEpgJAVXdpBsVwi7deJKDsCtl5l5dXU1KpHEyPJr4EfEmwi1dAdjYw4cSkIAEJCCB20tAVTf13iNQcDglYReqZWojCvNxzi/9wraDg4P0c7uFG2/m70uG43Ud2zXJ2SUgAQlIQAI5E7iTs3ELtg1hx1E2zrERYeTQGJHZbS0WfYkzDE8Yku709JTvbVmS5o3ciEgWPjo6Oj8/z8GqZJ4FCUhAAhKQQJ4EVHVb2xdu0EWs4JFC2O3t7W3Njj/++HUN8V9/bdGA4tTQIN014sJY5Y10RTiWJSABCUhAAg0EjMA2wBn9EX6ycJLhnRp9sjlM8OHDBy8ZnsNGaaMEJCABCeRIQFW35V3huBjh1xRh/Pbt25YN2t70xKNPTk4iN8JLhre3D84sAQlIQAJzJWAEdvs7F3cUkzaBKZ8/fybxM3IpRrUM8XRxcYF4KuXAYgz5ExOnJmAMki7u8AsaSeaOCsHBJSABCUhAAksioKrLYjfjduIwhZN2vJV1pPNkpCAQ8+Wb42u1K+cRt9bxiDSFhw8f8s3ruWpbDlWJrOQgXdiDnPVGuqHAOo4EJCABCdw2Aqq6vHb8wYMH379/R1fhvhpW3yCe0IuRWBprxh+GT64k2lBX0ebTzQcX2qgpC0zHQToWi0kI2XySNvL6s9AaCUhAAhKQQAsCqroWkCZs8vTp00ePHhGL5IMO+/jx4yCxSGQi75AI8YRfEDHXEGalGcFZXsyFrsMG+qLzuF6k6FAcBAlrRGgyFGtkpROHfQdZgoNIQAISkIAE8iFgtkQ+e/GfJbjowmWFlkp+rKKVKK3iz+YyEo34Ji43Coin9+/fI9fwijVIKJodHx/ThkBwWIJHjWN/DNI8V6en6LmQdHGQrsGeTsPaWAISkIAEJHBrCajqctx6FFWEXyNAyXeyMomhVNNQ4KIQbsLD5UYbTshdXl4i1xraVx9hCSowvHQ47dB2RWOq7dvUoC/xFOKoozFWkbExuBewjRm2kYAEJCABCSyMgKou0w2NvAHcZkVhh6RDDOHD47PWblxr6aIQXHS9g7nhSys67drMvsq8WE6MwJi9rVo1vvUSkIAEJCCBW0tAVZfv1iPs4io7nFuEYp8/fx7+LSzGhdZsN7IJ1xpt+rnoSoMjLpmR5NlIrUBcYlKpTZufcckwwo4BcUauXUWbMW0jAQlIQAISkEAQUNVl/ZcQfjK0FCrq3bt3yVZEG5/0s1SgcTqyNqAzjKNv8SYMDvaRe1GadO1PVGbyHXrJ8FpcNpCABCQgAQl0JaCq60ps6vYIu9evX1dnbXB0Ibkio4LE1WrHTWrQl6enp4wQibEthwqVGckWLIfjfXy37GszCUhAAhKQgARaElDVtQS1tWbEK1+9elWdfpW7jvqIvXJqbQzxxLCRr9oyDoukI3wcsWNiyuReEH6tLscaCUhAAhKQgAQ2JKCq2xDguN0jtwBhVDtN1V1XjL1Wn9YO0qOSI3EoM9yBa+86wX6ScPlmFuyJxN4eM9pFAhKQgAQkIIG1BLyFeC2ibTYgd7VhetxySKviyyEi9ork4jhdQ8dVjxiNMbmmLhqgEYt+tSQTIw6LpMMDhx9u1W0pPI3jfQyCnlvVbJUx1ktAAhKQgAQk0ImAqq4TrqkbI6Q4x0bqKGkKce1cyYKSAyxir3QpSr1Sl1U/uay4dorUPqk6aojDxvtkkY+1cg3NF8YQBR7jvRTJKgsSkIAEJCABCQQBI7C5/yXg6OI4GuIJFxq6qnRUDn9YJEawDNxssRjad10V4yDpuAYFrfZP3YfzcKUxUYHFSdNTPHxcMhySjhN4XjKcyFiQgAQkIAEJjEpAVTcq3iEHx/2Ghwx1xYdCio0mFxqCjPkQUulR++kZhPHRjpEJUe1YUpM0CBcdGi6OzUWXOAgY+jL8eT2Mqc5ujQQkIAEJSEACawmo6tYiyq4BAgsRdn19zak73HLJXRdaqiq/1i4AZYbDD0fd2pbFBsi1mCsd/sMAjtmFyPOS4SIryxKQgAQkIIEJCKjqJoA81hR4yxBPyDsmSD4zop9d5/vy5QtdeMdr147h2AsZR8iVqTEDtYc3sUcUuOvstpeABCQgAQlIoEhAVVekMcsyKorg6cXFBdZTXhVCbVhbnMy7e/duQ5vaR6EgcdG9ePHCS4YDERo3nKa1xEqV0ThkcemRPyUgAQlIQAJdCajquhLLtD3uMSw7ODjoYV9cZdKjb1KQb9++ZV78c/Hi2h42zL0LSg5du7+/j8szDjg2r4i85miMMqYLZWqau/hUAhKQgAQk0ExAVdfM5xY9xc/Xe7X379/nqB/h4E0G6T17Dh3xdKLPrq6u2hiD/uOVuOFYjTtocJdSwxHJNt1tIwEJSEACEqgloKqrxWJlNwJv3rwh47Vbn2W1JnEEz2UbfydePdTbr0zmv//Gq4ejlHSTUMOoPQ4mLguMq5GABCQggekIqOqmY+1MEoAAHk1ukCmmKpP1EtfTIOmMw/pHIgEJSEACvQmo6nqjy6tjZLBGNutkliUJ0sZHNZlVmU+EdEvnEZOpKWU4va4tPbIgAQlIQAISaEnAN4a1BJV7s8PDQ0xEMRDgq4qGZut7R/2IITIy02V1nI4zaqwonGGklxLfRPIWX2tGZchfKmstpzsNCI/eu3eP1RX9aiWStCT7GA6lKUrNij/x1RV/pjITsXekTaQaCxKQgAQkIIFOBPTVdcKVb2PUSYiPNgmYpWUgg7oKwRgBFUKhQfSUJhr1J6vgXFrkk6Lk+BnppVykRyIC9SiwVPnnzWdvb696q0gMQj1dfv78yTefajPWQktGQNIx0devX2nWJlsi0iOqKAJmv42ojmaNBCQgAQncQgL66paz6QgCxEeIg06rQuuskhoN4yCSQuugmRqaTfkIdcXbbJkRCJjHYTUUJ6sLWYYIo8w3wpenL1++pPHZ2Rmn3JKRiD3CyundtTjzGJMPI9CrKLkYB72IpAtRyyMUHt3TUJ0KQZJBemxEp4lsLAEJSEACCyagr245m4v4YDHoAyRL+1XRGK3Tw98Wh+qSj7D9jCO1RA8lVUSZQGcsinKcWkPqpRfdYnYosFCBYRKpqXxoXKTBmNGd9gksQyHpSPsttix17LTMeOta5Ex06mhjCUhAAhKQQCKgqksoZl9Af8Qa8D+1X0yIM3xR7btEy3BxpUm7dh+1fa3Hi8raU3TJkjjxVkUR+g/tm7JDomXI6NSdQo/3c9CLkVGTJY1YHNayBCQgAQlIoA0BVV0bSrNpE5fG4UaKiF7YjWhITqbiSkJMEElE7hSTCYptVpWZIkK9vWOOq0aevj5YgSgKVWWGQy7kYGSHYGF4+IbK/GUL4K+jbvqtd0YJSEACCyOgqlvUhp6enoabikNjaWE4lnZ3d3cqH9ItQ5N18u0xLOonuiAi8/TVpbW3KUSKQ/OlMCHg0MEMGHKWQrPnr83UtEEfoyPD+deyi80kIAEJSEACtQTMlqjFMtdKdAb6gMggwgsPULh/SoFCHqXTYDzqIcuQjHi2kI+IyLmSWm13SLfVz38/oWVo6N9VHUsEXjlRl1yAHXvbXAISkIAEJPA/Aqq6/+FYwA9UGi40PEB8yE5FwFHDZ6ilIRYjUol8HMRZNZRhG46TEDXcA5zUcMyFe28TVYfPD4bk225oud0lIAEJSEACQcAI7AL/EmrjsIOsEz2HWGSoZcReS0yOjo6oKWbFpgYRn42AddJ/mwgySOJJZYSqMg7CaWoLEpCABCQggZYEVHUtQc2pGUIhElSRDsRY24cUmxeJb+nZs2e0WWrstZrrGkBYOBFnNF/yzMVdJ8RPw22ZuLVETS9cnuxRSdIxCzaUKtPgFiQgAQlIQALNBFR1zXzm+pRYYeTDoki4qmND9w9qAxUSAhHNkX/sFYOrOweKamWxBiccS6MGZ2caIdYOz/Pz89QYN1tor8ePH0OGkfmE5KUNao/KVQoPSUcv2ldTWKjh8pSu+cjJKgsSkIAEJHDbCcT//VD4x89WCcQfItsxoBWcxE+OHyQLJ8Z6DM47FZKPqvcgPebt2gU70x8zq6ZMDUumkCDgb0O3MTKV6ZBcKOA0HdBYb4wQzfi+vr5ODaLAyCkUy97Rno7U0Jf2q1BTn4yp/acHL2BpotJPBo+OpXp/SkACEpDAbSNQVQ6/Dkj5n0QOfwfVvRnEKuRIHBdj/FAe7Yelb/rzoC8OqvZ9594S+YUo5NO8kFKzVWKueZBOT9OOdOplYwlIQAISWB6BqnIwBzaYLPYbNcb5LeJ6HNgimEhWLOIMl1LDnSY0oz3XbRAljDBixB+Tc2uxsAoLw+XGp1BRXyw1a9OlfiBrJSABCUhAAhsTUNVtjHAOA3BU6/DwMN5nz7kuPnHSDnlX0mooOZ4W14RzCCFYrLEsAQlIQAISkECGBFR1GW7KKCaF0w7FxvEvvlFvTMN3FEpT0jj58/Q/leD4UwISkIAEJJAnAVVdnvsyllV45pJzLsKsJc8cT+Pu4rEscFwJSEACEpCABMYhoKobh+scRiUsy2cOlmqjBCQgAQlIQALrCXhf3XpGtpCABCQgAQlIQAL5E1DV5b9HWigBCUhAAhKQgATWE1DVrWdkCwlIQAISkIAEJJA/AVVd/nukhRKQgAQkIAEJSGA9AbMl1jOassWnT59KSalTzu5c+RNY9XrZ/C3XQglIQAISGJuAqm5swt3G5/9s/9vuhszWEpCABCQgAQncEFDV5fKHwK2/uZiiHRKQgAQkIAEJzJCAqi6XTeNF8rmYoh0SkIAEJCABCcyQgNkSM9w0TZaABCQgAQlIQAIVAqq6ChIrJCABCUhAAhKQwAwJqOpmuGmaLAEJSEACEpCABCoEVHUVJFZIQAISkIAEJCCBGRJQ1c1w0zRZAhKQgAQkIAEJVAio6ipIrJCABCQgAQlIQAIzJKCqm+GmabIEJCABCUhAAhKoEFDVVZBYIQEJSEACEpCABGZIQFU3w03TZAlIQAISkIAEJFAhoKqrILFCAhKQgAQkIAEJzJCAqm6Gm6bJEpCABCQgAQlIoEJAVVdBYoUEJCABCUhAAhKYIYE7yeadnZ1UtiABCUhAAhKQgAQkMC8C+urmtV9aKwEJSEACEpCABOoJ3Hny5En9E2slIAEJSEACEpCABPImUBRy/wI0/yC625WbcAAAAABJRU5ErkJggg==)\n",
        "\n",
        "In code, modules are instances of the torch.nn.Module class (in PyTorch) or its subclasses. PyTorch models are generally built by combining various modules together."
      ],
      "metadata": {
        "id": "QaL2-xI2Tsn7"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "- `q_proj` (Query Projection): This layer projects the input into the query space.\n",
        "- `k_proj` (Key Projection): Projects the input into the key space.\n",
        "- `v_proj` (Value Projection): Projects the input into the value space.\n",
        "- `o_proj` (Output Projection): Combines the result of the attention mechanism back into the original feature size."
      ],
      "metadata": {
        "id": "kBkTMzAUahKQ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from peft import LoraConfig, PeftModel, prepare_model_for_kbit_training, get_peft_model\n",
        "model.gradient_checkpointing_enable()\n",
        "model = prepare_model_for_kbit_training(model)\n",
        "\n",
        "print(model)\n",
        "\n",
        "lora_config = LoraConfig(\n",
        "    r=64,\n",
        "    lora_alpha=32,\n",
        "    target_modules=modules,\n",
        "    lora_dropout=0.05,\n",
        "    bias=\"none\",\n",
        "    task_type=\"CAUSAL_LM\" # Causal Language Modeling (e.g., autoregressive models like GPT)\n",
        ")\n",
        "\n",
        "model = get_peft_model(model, lora_config)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "idbpGSBzX62q",
        "outputId": "32e4d3f1-07f2-450f-fc86-4cf3784b1e4e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "GemmaForCausalLM(\n",
            "  (model): GemmaModel(\n",
            "    (embed_tokens): Embedding(256000, 2048, padding_idx=0)\n",
            "    (layers): ModuleList(\n",
            "      (0-17): 18 x GemmaDecoderLayer(\n",
            "        (self_attn): GemmaSdpaAttention(\n",
            "          (q_proj): Linear4bit(in_features=2048, out_features=2048, bias=False)\n",
            "          (k_proj): Linear4bit(in_features=2048, out_features=256, bias=False)\n",
            "          (v_proj): Linear4bit(in_features=2048, out_features=256, bias=False)\n",
            "          (o_proj): Linear4bit(in_features=2048, out_features=2048, bias=False)\n",
            "          (rotary_emb): GemmaRotaryEmbedding()\n",
            "        )\n",
            "        (mlp): GemmaMLP(\n",
            "          (gate_proj): Linear4bit(in_features=2048, out_features=16384, bias=False)\n",
            "          (up_proj): Linear4bit(in_features=2048, out_features=16384, bias=False)\n",
            "          (down_proj): Linear4bit(in_features=16384, out_features=2048, bias=False)\n",
            "          (act_fn): GELUActivation()\n",
            "        )\n",
            "        (input_layernorm): GemmaRMSNorm()\n",
            "        (post_attention_layernorm): GemmaRMSNorm()\n",
            "      )\n",
            "    )\n",
            "    (norm): GemmaRMSNorm()\n",
            "  )\n",
            "  (lm_head): Linear(in_features=2048, out_features=256000, bias=False)\n",
            ")\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "1. **`r`**:\n",
        "    - The `r` parameter defines the rank for the low-rank adaptation. Essentially, it sets how much compression is applied to the adapter layers. A lower rank means a smaller set of parameters to update, which reduces the computational cost, while a higher rank increases the model's capacity for fine-tuning.\n",
        "  \n",
        "2. **`lora_alpha`**:\n",
        "    - The lora_alpha parameter is a scaling factor for the LoRA layers. It controls how much the updates from the low-rank matrices (adapters) are scaled during training. A higher value (in this case, 32) ensures that the LoRA updates have a significant impact during training, while a lower value would reduce their effect. This hyperparameter can be tuned to balance between preserving the original model and adapting it to new tasks.\n",
        "\n",
        "3. **`target_module`**:\n",
        "    - This specifies the target modules in the language model that will have LoRA adapters applied. In this case, the modules listed (e.g., `q_proj`, `o_proj`, etc.) represent different projection layers within a transformer model, commonly used in large language models (LLMs). By focusing on these specific layers, LoRA limits the amount of fine-tuning to only a subset of the model's total parameters, making the process more efficient.\n",
        "\n",
        "4. **`lora_dropout`**:\n",
        "    - This parameter adds dropout to the LoRA layers. By setting lora_dropout=0.05, we introduce a small probability (5%) of randomly dropping units from the LoRA layers during training. This helps prevent overfitting, especially when fine-tuning with limited data, as it encourages the model to generalize better to unseen inputs.\n",
        "\n",
        "5. **`bias`**:\n",
        "    - The bias=\"none\" option indicates that no additional bias parameters are added to the LoRA adapters. This is often done to simplify the model and reduce the number of trainable parameters further, making the model even more parameter-efficient.\n",
        "\n",
        "3. **`task_type`**:\n",
        "    - The `task_type` argument specifies the type of task the model is being fine-tuned for. Here, \"CAUSAL_LM\" refers to Causal Language Modeling, which is commonly used for autoregressive language models like GPT. This task type involves predicting the next token in a sequence based on the preceding tokens.\n",
        "\n",
        "Note: To see the available options for target modules in LoRA configuration, you can refer to the PEFT library on GitHub. LoRA is typically applied to attention-related layers in transformer models, such as q_proj, k_proj, v_proj, and others, but the options may vary depending on the architecture. For models that are mapped in the PEFT library, you don’t need to manually specify target modules, as the library automatically handles it for known architectures.\n",
        "\n",
        "However, for custom models or architectures not covered by the default mappings, you may need to specify target modules manually. You can retrieve the list of modules by printing them from the model, filtering by layer types like nn.Linear, which are often the layers LoRA targets. Some configurations, such as QLoRA, target all linear layers to maximize performance.\n",
        "\n",
        "For more information, including how to check the transformer model mappings and additional configuration options, you can check the PEFT GitHub repository and the HuggingFace documentation:\n",
        "- https://github.com/huggingface/peft\n",
        "- https://huggingface.co/docs/peft/main/en/conceptual_guides/lora\n",
        "\n"
      ],
      "metadata": {
        "id": "13XMm6YRYmic"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Number of trainable parameters\n",
        "trainable, total = model.get_nb_trainable_parameters()\n",
        "print(f\"Trainable: {trainable} | total: {total} | Percentage: {trainable/total*100:.4f}%\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8RcEVIQBY1qZ",
        "outputId": "325fc981-8a95-4c87-d09c-fb1bd81a33c7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Trainable: 78446592 | total: 2584619008 | Percentage: 3.0351%\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "By freezing most of the model, we retain the general knowledge that the model has already learned during its pre-training on massive datasets. Fine-tuning only a small part of the model allows it to adapt to a new task (or domain) while keeping most of its original capabilities intact.\n",
        "\n"
      ],
      "metadata": {
        "id": "06CV2dUJf-kP"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Train Model"
      ],
      "metadata": {
        "id": "865HcsQcblnJ"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Push the trained model to HF"
      ],
      "metadata": {
        "id": "xXNbCeWJfKpH"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import transformers\n",
        "from trl import SFTTrainer\n",
        "\n",
        "tokenizer.pad_token = tokenizer.eos_token\n",
        "tokenizer.padding_side='right'\n",
        "torch.cuda.empty_cache()\n",
        "\n",
        "trainer = SFTTrainer(\n",
        "    model=model,\n",
        "    train_dataset=train_data,\n",
        "    eval_dataset=test_data,\n",
        "    dataset_text_field=\"prompt\",\n",
        "    peft_config=lora_config,\n",
        "    max_seq_length=2500,\n",
        "    args=transformers.TrainingArguments(\n",
        "        per_device_train_batch_size=1,\n",
        "        gradient_accumulation_steps=2,\n",
        "        num_train_epochs=2,\n",
        "        warmup_steps=0.03,\n",
        "        max_steps=10,\n",
        "        learning_rate=2e-4,\n",
        "        logging_steps=20,\n",
        "        #logging_steps=10 if in case di gumana\n",
        "        output_dir=\"outputs\",\n",
        "        optim=\"paged_adamw_8bit\",\n",
        "        save_strategy=\"epoch\",\n",
        "        report_to=\"none\"\n",
        "    ),\n",
        "    data_collator=transformers.DataCollatorForLanguageModeling(tokenizer, mlm=False),\n",
        ")\n",
        "\n",
        "model.config.use_cache = False\n",
        "trainer.train()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 367,
          "referenced_widgets": [
            "6ac8f2f7f4664a26bb7e07c1cde83eae",
            "212c6a5679744ff891650f601cd4e6c9",
            "82ceff906c2443e2b05c4b6b379a489c",
            "83775177ca1b4c06a2a57dc2c6b41827",
            "2de0ee6fe2a14eb89e4aa61b3992a0c9",
            "be734680b384438d9304c133ad53d47d",
            "cfe81d1d55154c41a40c1171ed9ed765",
            "7ecb6130e52744bd834ffee0342932ee",
            "60961672e2774b1e9f45a6318e5251e2",
            "4b5ab79df25d4126955dc8ba22091e5f",
            "b1bfa85380784177ab5faec7614f1b3b",
            "e87c74286f65461a9b21b92b3d8efd32",
            "b0dda4fb1e394b6daa2ae25bc9286860",
            "4c6106a368874816a6b251221a50ac28",
            "75c8656786d648b09844a1600a7e7e28",
            "6c5ef84c98034728a5c54fe283fb3cbb",
            "b129f9ed2f5445319d75694e7d3883df",
            "7197f23c39b9405d8e6086c21c25f02a",
            "4371a7716a1c481e886f0539ced5cd9e",
            "5c519c6d3863421a937dd6104228df26",
            "b797e3464da3466ca368f9d0858255ce",
            "8b8c8938d90d4262ba6b8fd752c032e9"
          ]
        },
        "id": "gVArtPIzY0d2",
        "outputId": "c08fb558-ce63-4c24-94b9-7b63e4383bdb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1150: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map:   0%|          | 0/45 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "6ac8f2f7f4664a26bb7e07c1cde83eae"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map:   0%|          | 0/5 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "e87c74286f65461a9b21b92b3d8efd32"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/trl/trainer/sft_trainer.py:290: UserWarning: You passed a tokenizer with `padding_side` not equal to `right` to the SFTTrainer. This might lead to some unexpected behaviour due to overflow issues when training a model in half-precision. You might consider adding `tokenizer.padding_side = 'right'` to your code.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/torch/_dynamo/eval_frame.py:600: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.\n",
            "  return fn(*args, **kwargs)\n",
            "/usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py:295: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead.\n",
            "  with torch.enable_grad(), device_autocast_ctx, torch.cpu.amp.autocast(**ctx.cpu_autocast_kwargs):  # type: ignore[attr-defined]\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='10' max='10' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [10/10 00:38, Epoch 0/1]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Step</th>\n",
              "      <th>Training Loss</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "  </tbody>\n",
              "</table><p>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TrainOutput(global_step=10, training_loss=3.0606761932373048, metrics={'train_runtime': 44.3455, 'train_samples_per_second': 0.451, 'train_steps_per_second': 0.226, 'total_flos': 76681399455744.0, 'train_loss': 3.0606761932373048, 'epoch': 0.44})"
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "new_model = \"gemma-2b-instruct-ft-medical-qa\"\n",
        "\n",
        "trainer.model.save_pretrained(new_model)\n",
        "\n",
        "base_model = AutoModelForCausalLM.from_pretrained(\n",
        "    model_id,\n",
        "    low_cpu_mem_usage=True,\n",
        "    return_dict=True,\n",
        "    torch_dtype=torch.float16,\n",
        "    device_map={\"\": 0},\n",
        ")\n",
        "merged_model= PeftModel.from_pretrained(base_model, new_model)\n",
        "merged_model= merged_model.merge_and_unload()\n",
        "\n",
        "# Save the merged model locally\n",
        "# save_adapter=True, save_config=True\n",
        "merged_model.save_pretrained(\"merged_model\",safe_serialization=True)\n",
        "tokenizer.save_pretrained(\"merged_model\")\n",
        "tokenizer.pad_token = tokenizer.eos_token\n",
        "tokenizer.padding_side = \"right\"\n",
        "\n",
        "# Push the model and tokenizer to the Hugging Face Model Hub\n",
        "merged_model.push_to_hub(new_model, use_temp_dir=False)\n",
        "tokenizer.push_to_hub(new_model, use_temp_dir=False)"
      ],
      "metadata": {
        "id": "UGyD-qTjfKbP",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 599,
          "referenced_widgets": [
            "9515e687b4874d0b8f42b6bc120511ef",
            "9d974183fe4e4d188abb537b765e3cc4",
            "527d48bf918741c89d3828edd7845709",
            "f37123994dfe42c49772bd839400b0a9",
            "972afde01bb9485bae8b90fcc2f81367",
            "25acb2292d0a42e0ae32a5a90de06068",
            "87a766de8e644f7d8a0069946726155e",
            "769d9e2f800443d4acc608fbba2a1281",
            "3357e57e0c984418a0d60f8ed81f4be0",
            "38690d18d1d74f3caf389b8cdb503d64",
            "310f5bd29c684bd18a6504fd64df6e9f"
          ]
        },
        "outputId": "34fe4eb9-1dad-49e3-ff24-c68ebb04e33a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1150: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "9515e687b4874d0b8f42b6bc120511ef"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "error",
          "ename": "OutOfMemoryError",
          "evalue": "CUDA out of memory. Tried to allocate 1000.00 MiB. GPU 0 has a total capacity of 14.75 GiB of which 603.06 MiB is free. Process 55406 has 13.91 GiB memory in use. Of the allocated memory 13.72 GiB is allocated by PyTorch, and 49.62 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation.  See documentation for Memory Management  (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mOutOfMemoryError\u001b[0m                          Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-34-95ba0ba98db1>\u001b[0m in \u001b[0;36m<cell line: 5>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave_pretrained\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnew_model\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m base_model = AutoModelForCausalLM.from_pretrained(\n\u001b[0m\u001b[1;32m      6\u001b[0m     \u001b[0mmodel_id\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m     \u001b[0mlow_cpu_mem_usage\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/models/auto/auto_factory.py\u001b[0m in \u001b[0;36mfrom_pretrained\u001b[0;34m(cls, pretrained_model_name_or_path, *model_args, **kwargs)\u001b[0m\n\u001b[1;32m    559\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_model_mapping\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkeys\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    560\u001b[0m             \u001b[0mmodel_class\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_get_model_class\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_model_mapping\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 561\u001b[0;31m             return model_class.from_pretrained(\n\u001b[0m\u001b[1;32m    562\u001b[0m                 \u001b[0mpretrained_model_name_or_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mmodel_args\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mhub_kwargs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    563\u001b[0m             )\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/modeling_utils.py\u001b[0m in \u001b[0;36mfrom_pretrained\u001b[0;34m(cls, pretrained_model_name_or_path, config, cache_dir, ignore_mismatched_sizes, force_download, local_files_only, token, revision, use_safetensors, *model_args, **kwargs)\u001b[0m\n\u001b[1;32m   3500\u001b[0m                 \u001b[0moffload_index\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3501\u001b[0m                 \u001b[0merror_msgs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3502\u001b[0;31m             \u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_load_pretrained_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3503\u001b[0m                 \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3504\u001b[0m                 \u001b[0mstate_dict\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/modeling_utils.py\u001b[0m in \u001b[0;36m_load_pretrained_model\u001b[0;34m(cls, model, state_dict, loaded_keys, resolved_archive_file, pretrained_model_name_or_path, ignore_mismatched_sizes, sharded_metadata, _fast_init, low_cpu_mem_usage, device_map, offload_folder, offload_state_dict, dtype, hf_quantizer, keep_in_fp32_modules)\u001b[0m\n\u001b[1;32m   3924\u001b[0m                                     \u001b[0mhf_quantizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcreate_quantized_param\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparam\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"cpu\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate_dict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3925\u001b[0m                     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3926\u001b[0;31m                         new_error_msgs, offload_index, state_dict_index = _load_state_dict_into_meta_model(\n\u001b[0m\u001b[1;32m   3927\u001b[0m                             \u001b[0mmodel_to_load\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3928\u001b[0m                             \u001b[0mstate_dict\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/modeling_utils.py\u001b[0m in \u001b[0;36m_load_state_dict_into_meta_model\u001b[0;34m(model, state_dict, loaded_state_dict_keys, start_prefix, expected_keys, device_map, offload_folder, offload_index, state_dict_folder, state_dict_index, dtype, hf_quantizer, is_safetensors, keep_in_fp32_modules, unexpected_keys)\u001b[0m\n\u001b[1;32m    803\u001b[0m         ):\n\u001b[1;32m    804\u001b[0m             \u001b[0;31m# For backward compatibility with older versions of `accelerate` and for non-quantized params\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 805\u001b[0;31m             \u001b[0mset_module_tensor_to_device\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparam_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparam_device\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mset_module_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    806\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    807\u001b[0m             \u001b[0mhf_quantizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcreate_quantized_param\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparam\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparam_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparam_device\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0munexpected_keys\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/accelerate/utils/modeling.py\u001b[0m in \u001b[0;36mset_module_tensor_to_device\u001b[0;34m(module, tensor_name, device, value, dtype, fp16_statistics, tied_params_map)\u001b[0m\n\u001b[1;32m    382\u001b[0m                     \u001b[0mmodule\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_parameters\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtensor_name\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparam_cls\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnew_value\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrequires_grad\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mold_value\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrequires_grad\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    383\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 384\u001b[0;31m             \u001b[0mnew_value\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    385\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    386\u001b[0m             \u001b[0mnew_value\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mOutOfMemoryError\u001b[0m: CUDA out of memory. Tried to allocate 1000.00 MiB. GPU 0 has a total capacity of 14.75 GiB of which 603.06 MiB is free. Process 55406 has 13.91 GiB memory in use. Of the allocated memory 13.72 GiB is allocated by PyTorch, and 49.62 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation.  See documentation for Memory Management  (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Evaluation Metrics"
      ],
      "metadata": {
        "id": "_tubadZyeZ4f"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Load/define base (non-finetuned) vs finetuned models for comparison\n",
        "\n",
        "import torch\n",
        "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
        "from math import exp\n",
        "import bitsandbytes as bnb  # Ensure bitsandbytes is installed if using quantization\n",
        "from peft import PeftModel  # If using PEFT models\n",
        "\n",
        "# Define your Hugging Face username and model IDs\n",
        "username = \"HafthorJulian\"\n",
        "finetuned_model_id = f\"{username}/gemma-2b-instruct-ft-medical-qa\"\n",
        "base_model_id = \"google/gemma-2b-it\"\n",
        "\n",
        "# Set the qunatization config\n",
        "bnb_config = BitsAndBytesConfig(\n",
        "    load_in_4bit=True,\n",
        "    bnb_4bit_use_double_quant=True,\n",
        "    bnb_4bit_quant_type=\"nf4\",\n",
        "    bnb_4bit_compute_dtype=torch.bfloat16\n",
        ")\n",
        "\n",
        "model_id = \"google/gemma-2b-it\"\n",
        "\n",
        "base_model = AutoModelForCausalLM.from_pretrained(model_id, quantization_config=bnb_config, device_map={\"\":0})\n",
        "base_tokenizer = AutoTokenizer.from_pretrained(model_id, add_eos_token=True)\n",
        "\n",
        "# # Load the Finetuned Model and Tokenizer from local directory\n",
        "# finetuned_model_path = \"merged_model\"  # Path where you saved the merged model\n",
        "# finetuned_model = AutoModelForCausalLM.from_pretrained(\n",
        "#     finetuned_model_path,\n",
        "#     torch_dtype=torch.float16,\n",
        "#     device_map=\"auto\"\n",
        "# )\n",
        "# finetuned_tokenizer = AutoTokenizer.from_pretrained(finetuned_model_path)\n",
        "\n",
        "# Load the Finetuned Model and Tokenizer from HF\n",
        "finetuned_tokenizer = AutoTokenizer.from_pretrained(model_id)\n",
        "finetuned_model = AutoModelForCausalLM.from_pretrained(\n",
        "                  model_id,\n",
        "                  torch_dtype=torch.float16,\n",
        "                  device_map=\"auto\"\n",
        ")\n",
        "\n",
        "# Ensure the model is in evaluation mode\n",
        "base_model.eval()\n",
        "finetuned_model.eval()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 604,
          "referenced_widgets": [
            "54ebcbece75646b181f95d21d9cd6c19",
            "0dc9ff1f3937441f957d73d9aa93d49f",
            "dbfb7dd75d7e4990b011dd2f655bb7c2",
            "3afa987700db41e7a570fced3c789f42",
            "e83cdbb744b14f00b112d82cbeb6b87d",
            "223592b76a8b4c78ab42f5578ffad70b",
            "ef9b3255842747269119fdd87448c4d0",
            "e46d9a945ec94fdca298d8bbda84a350",
            "6f8e3a86ab3a49cda308b197faea4504",
            "edf90949427d4e7da4b987ef6b7fd7ad",
            "44c343d856bd40a48d848f89d03b9396",
            "9f01f00f98674ad7b4a7e298d34d5fda",
            "613ce7bace0f4126b1c1808c670efe6f",
            "31ee7bcfebe14211824d51387bb8c0fb",
            "cabaae388e514feba2e780f7f025f462",
            "97aa38a813aa414baf612248c70b87d5",
            "73c40a53096342d3964b5c098cad8cb9",
            "1c54bbf30b164c4897b8175d9a5204f8",
            "aef0f473777b448dad4a3e586be50377",
            "030d263dc79f49a197a23a32dc8593ea",
            "df3e1ebd8fe044b1b360d4caaae1cc86",
            "6309eeea5a964b31a72faa7f762e573f"
          ]
        },
        "id": "A0rSiQIqqHIc",
        "outputId": "761ee621-eddf-4d89-d333-a000f96b75c2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1150: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "54ebcbece75646b181f95d21d9cd6c19"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "9f01f00f98674ad7b4a7e298d34d5fda"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:root:Some parameters are on the meta device device because they were offloaded to the cpu.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "GemmaForCausalLM(\n",
              "  (model): GemmaModel(\n",
              "    (embed_tokens): Embedding(256000, 2048, padding_idx=0)\n",
              "    (layers): ModuleList(\n",
              "      (0-17): 18 x GemmaDecoderLayer(\n",
              "        (self_attn): GemmaSdpaAttention(\n",
              "          (q_proj): Linear(in_features=2048, out_features=2048, bias=False)\n",
              "          (k_proj): Linear(in_features=2048, out_features=256, bias=False)\n",
              "          (v_proj): Linear(in_features=2048, out_features=256, bias=False)\n",
              "          (o_proj): Linear(in_features=2048, out_features=2048, bias=False)\n",
              "          (rotary_emb): GemmaRotaryEmbedding()\n",
              "        )\n",
              "        (mlp): GemmaMLP(\n",
              "          (gate_proj): Linear(in_features=2048, out_features=16384, bias=False)\n",
              "          (up_proj): Linear(in_features=2048, out_features=16384, bias=False)\n",
              "          (down_proj): Linear(in_features=16384, out_features=2048, bias=False)\n",
              "          (act_fn): GELUActivation()\n",
              "        )\n",
              "        (input_layernorm): GemmaRMSNorm()\n",
              "        (post_attention_layernorm): GemmaRMSNorm()\n",
              "      )\n",
              "    )\n",
              "    (norm): GemmaRMSNorm()\n",
              "  )\n",
              "  (lm_head): Linear(in_features=2048, out_features=256000, bias=False)\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Perplexity\n",
        "\n",
        "Perplexity is a measurement derived from the probability distribution that a language model assigns to a sequence of words (i.e., a sentence or a corpus). Specifically, it quantifies how well a probability model predicts a sample. In the realm of language modeling, perplexity serves as an indicator of how \"confused\" the model is when predicting the next word in a sequence.\n",
        "* Lower Perplexity: Indicates that the model is better at predicting the sample. The model is less \"perplexed\" by the data.\n",
        "* Higher Perplexity: Suggests that the model is less certain about its predictions, implying poorer performance."
      ],
      "metadata": {
        "id": "qgBewc6YeeLx"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "test_data"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Mfe_KkD0wt0P",
        "outputId": "e31af3f1-d629-40ea-f782-c13662013bb7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Dataset({\n",
              "    features: ['question', 'question_id', 'question_source', 'entity_pages', 'search_results', 'answer', 'prompt'],\n",
              "    num_rows: 5\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test_data_subset = test_data.shuffle(seed=42).select(range(10))  # Get 10 random rows"
      ],
      "metadata": {
        "id": "ULFRgOJY4X6b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 339
        },
        "outputId": "2df28583-0436-4474-8e37-3122db9468a5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "IndexError",
          "evalue": "Index 9 out of range for dataset of size 5.",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-35-5b373ca778fd>\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtest_data_subset\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtest_data\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mseed\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m42\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mselect\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# Get 10 random rows\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/datasets/arrow_dataset.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    556\u001b[0m         }\n\u001b[1;32m    557\u001b[0m         \u001b[0;31m# apply actual function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 558\u001b[0;31m         \u001b[0mout\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mUnion\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"Dataset\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"DatasetDict\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    559\u001b[0m         \u001b[0mdatasets\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mList\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"Dataset\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    560\u001b[0m         \u001b[0;31m# re-apply format to the output\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/datasets/fingerprint.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    480\u001b[0m             \u001b[0;31m# Call actual function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    481\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 482\u001b[0;31m             \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    483\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    484\u001b[0m             \u001b[0;31m# Update fingerprint of in-place transforms + update in-place history of transforms\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/datasets/arrow_dataset.py\u001b[0m in \u001b[0;36mselect\u001b[0;34m(self, indices, keep_in_memory, indices_cache_file_name, writer_batch_size, new_fingerprint)\u001b[0m\n\u001b[1;32m   3816\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0m_is_range_contiguous\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindices\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mindices\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3817\u001b[0m                 \u001b[0mstart\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlength\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mindices\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindices\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstop\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mindices\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3818\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_select_contiguous\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlength\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnew_fingerprint\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnew_fingerprint\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3819\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3820\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/datasets/arrow_dataset.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    556\u001b[0m         }\n\u001b[1;32m    557\u001b[0m         \u001b[0;31m# apply actual function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 558\u001b[0;31m         \u001b[0mout\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mUnion\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"Dataset\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"DatasetDict\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    559\u001b[0m         \u001b[0mdatasets\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mList\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"Dataset\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    560\u001b[0m         \u001b[0;31m# re-apply format to the output\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/datasets/fingerprint.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    480\u001b[0m             \u001b[0;31m# Call actual function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    481\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 482\u001b[0;31m             \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    483\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    484\u001b[0m             \u001b[0;31m# Update fingerprint of in-place transforms + update in-place history of transforms\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/datasets/arrow_dataset.py\u001b[0m in \u001b[0;36m_select_contiguous\u001b[0;34m(self, start, length, new_fingerprint)\u001b[0m\n\u001b[1;32m   3877\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3878\u001b[0m         \u001b[0m_check_valid_indices_value\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3879\u001b[0;31m         \u001b[0m_check_valid_indices_value\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstart\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mlength\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3880\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_indices\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mlength\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3881\u001b[0m             return Dataset(\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/datasets/arrow_dataset.py\u001b[0m in \u001b[0;36m_check_valid_indices_value\u001b[0;34m(index, size)\u001b[0m\n\u001b[1;32m    648\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_check_valid_indices_value\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    649\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mindex\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0;36m0\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mindex\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0msize\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mindex\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0msize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 650\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mIndexError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"Index {index} out of range for dataset of size {size}.\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    651\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    652\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mIndexError\u001b[0m: Index 9 out of range for dataset of size 5."
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
        "from math import exp\n",
        "\n",
        "def calculate_perplexity(model, tokenizer, dataset):\n",
        "    total_loss = 0\n",
        "    total_tokens = 0\n",
        "\n",
        "    with torch.no_grad():\n",
        "        for example in dataset:\n",
        "            inputs = tokenizer(example[\"prompt\"], return_tensors=\"pt\").to(\"cuda\")\n",
        "            labels = inputs.input_ids\n",
        "            outputs = model(**inputs, labels=labels)\n",
        "            loss = outputs.loss\n",
        "            total_loss += loss.item() * inputs.input_ids.size(1)\n",
        "            total_tokens += inputs.input_ids.size(1)\n",
        "\n",
        "    perplexity = exp(total_loss / total_tokens)\n",
        "    return perplexity\n",
        "\n",
        "base_perplexity = calculate_perplexity(base_model, base_tokenizer, test_data_subset)\n",
        "finetuned_perplexity = calculate_perplexity(finetuned_model, finetuned_tokenizer, test_data_subset)\n",
        "print(f\"Non-finetuned Perplexity: {base_perplexity}\")\n",
        "print(f\"Finetuned Perplexity: {finetuned_perplexity}\\n\")"
      ],
      "metadata": {
        "id": "mXJpDkHscRq2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Semantic Similarity\n",
        "Semantic Similarity evaluates how close the model's generated responses are to the expected answers in terms of meaning.\n",
        "\n",
        "Implementation Steps:\n",
        "* Choose a Semantic Similarity Metric: Common choices include Cosine Similarity, BERTScore, or Sentence Transformers embeddings.\n",
        "* Compute Similarity Scores: Compare the generated responses with the ground truth."
      ],
      "metadata": {
        "id": "KSDStyKeqeBh"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "from torch.nn.functional import cosine_similarity\n",
        "\n",
        "def calculate_semantic_similarity(model, tokenizer, dataset):\n",
        "    similarities = []\n",
        "    model.eval()\n",
        "\n",
        "    with torch.no_grad():\n",
        "        for example in dataset:\n",
        "            # Encode the input prompt\n",
        "            input_ids = tokenizer.encode(example[\"prompt\"], return_tensors=\"pt\").to(\"cuda\")\n",
        "            # Generate output from the model\n",
        "            output_ids = model.generate(input_ids, max_length=2500)\n",
        "            generated_text = tokenizer.decode(output_ids[0], skip_special_tokens=True)\n",
        "\n",
        "            # Tokenize generated and reference texts\n",
        "            gen_inputs = tokenizer(generated_text, return_tensors='pt').to(\"cuda\")\n",
        "            ref_inputs = tokenizer(example[\"output\"], return_tensors='pt').to(\"cuda\")\n",
        "\n",
        "            # Get embeddings from the last hidden state\n",
        "            gen_outputs = model(**gen_inputs, output_hidden_states=True, return_dict=True)\n",
        "            ref_outputs = model(**ref_inputs, output_hidden_states=True, return_dict=True)\n",
        "\n",
        "            # Average pooling of the embeddings\n",
        "            gen_embedding = gen_outputs.hidden_states[-1].mean(dim=1).squeeze()\n",
        "            ref_embedding = ref_outputs.hidden_states[-1].mean(dim=1).squeeze()\n",
        "\n",
        "            # Compute cosine similarity\n",
        "            cosine_score = cosine_similarity(gen_embedding, ref_embedding, dim=0).item()\n",
        "            similarities.append(cosine_score)\n",
        "\n",
        "    average_similarity = sum(similarities) / len(similarities)\n",
        "    return average_similarity\n",
        "\n",
        "base_semantic_similarity = calculate_semantic_similarity(base_model, base_tokenizer, test_data_subset)\n",
        "finetuned_semantic_similarity = calculate_semantic_similarity(finetuned_model, finetuned_tokenizer, test_data_subset)\n",
        "print(f\"Non-finetuned Semantic Similarity: {base_semantic_similarity:.4f}\")\n",
        "print(f\"Finetuned Semantic Similarity: {finetuned_semantic_similarity:.4f}\\n\")"
      ],
      "metadata": {
        "id": "ZUJvEliypUjQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### BLEU (Bilingual Evaluation Understudy)\n",
        "\n",
        "BLEU measures the n-gram overlap between the generated text and reference text. It's widely used in machine translation but can be applied to other text generation tasks."
      ],
      "metadata": {
        "id": "Bsp1h8brrCA-"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from datasets import load_metric\n",
        "\n",
        "bleu = load_metric(\"bleu\")\n",
        "\n",
        "def calculate_bleu(model, tokenizer, dataset):\n",
        "    references = []\n",
        "    predictions = []\n",
        "    model.eval()\n",
        "\n",
        "    with torch.no_grad():\n",
        "        for example in dataset:\n",
        "            input_ids = tokenizer.encode(example[\"prompt\"], return_tensors=\"pt\").to(\"cuda\")\n",
        "            output_ids = model.generate(input_ids, max_length=2500)\n",
        "            generated_text = tokenizer.decode(output_ids[0], skip_special_tokens=True)\n",
        "\n",
        "            references.append([example[\"output\"].split()])\n",
        "            predictions.append(generated_text.split())\n",
        "\n",
        "    bleu_score = bleu.compute(predictions=predictions, references=references)\n",
        "    return bleu_score\n",
        "\n",
        "base_bleu = calculate_bleu(base_model, base_tokenizer, test_data_subset)\n",
        "finetuned_bleu = calculate_bleu(finetuned_model, finetuned_tokenizer, test_data_subset)\n",
        "print(f\"Non-finetuned BLEU Score: {base_bleu['bleu']:.4f}\")\n",
        "print(f\"Finetuned BLEU Score: {finetuned_bleu['bleu']:.4f}\\n\")"
      ],
      "metadata": {
        "id": "hklUvirurBjF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### ROUGE (Recall-Oriented Understudy for Gisting Evaluation)\n",
        "\n",
        "ROUGE focuses on the overlap of n-grams, word sequences, and word pairs between the generated and reference texts, emphasizing recall."
      ],
      "metadata": {
        "id": "giYslB0Tvihi"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from datasets import load_metric\n",
        "\n",
        "rouge = load_metric(\"rouge\")\n",
        "\n",
        "def compute_rouge(model, tokenizer, dataset):\n",
        "    references = []\n",
        "    predictions = []\n",
        "    model.eval()\n",
        "\n",
        "    with torch.no_grad():\n",
        "        for example in dataset:\n",
        "            input_ids = tokenizer.encode(example[\"prompt\"], return_tensors=\"pt\").to(\"cuda\")\n",
        "            output_ids = model.generate(input_ids, max_length=2500)\n",
        "            generated_text = tokenizer.decode(output_ids[0], skip_special_tokens=True)\n",
        "\n",
        "            references.append(example[\"output\"])\n",
        "            predictions.append(generated_text)\n",
        "\n",
        "    rouge_score = rouge.compute(predictions=predictions, references=references)\n",
        "    return rouge_score\n",
        "\n",
        "base_rouge = compute_rouge(base_model, base_tokenizer, test_data)\n",
        "finetuned_rouge = compute_rouge(finetuned_model, finetuned_tokenizer, test_data)\n",
        "print(f\"Non-finetuned ROUGE Scores: {base_rouge}\")\n",
        "print(f\"Finetuned ROUGE Scores: {finetuned_rouge}\")"
      ],
      "metadata": {
        "id": "pl8vgTWurQTH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### METEOR (Metric for Evaluation of Translation with Explicit ORdering)\n",
        "\n",
        "METEOR evaluates translation by considering synonymy and stemming, making it more flexible than BLEU."
      ],
      "metadata": {
        "id": "dbYvR1G9vmoy"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from datasets import load_metric\n",
        "\n",
        "meteor = load_metric(\"meteor\")\n",
        "\n",
        "def compute_meteor(model, tokenizer, dataset):\n",
        "    references = []\n",
        "    predictions = []\n",
        "    model.eval()\n",
        "\n",
        "    with torch.no_grad():\n",
        "        for example in dataset:\n",
        "            input_ids = tokenizer.encode(example[\"prompt\"], return_tensors=\"pt\").to(\"cuda\")\n",
        "            output_ids = model.generate(input_ids, max_length=2500)\n",
        "            generated_text = tokenizer.decode(output_ids[0], skip_special_tokens=True)\n",
        "\n",
        "            references.append(example[\"output\"])\n",
        "            predictions.append(generated_text)\n",
        "\n",
        "    meteor_score = meteor.compute(predictions=predictions, references=references)\n",
        "    return meteor_score\n",
        "\n",
        "base_meteor = compute_meteor(base_model, base_tokenizer, test_data)\n",
        "finetuned_meteor = compute_meteor(finetuned_model, finetuned_tokenizer, test_data)\n",
        "print(f\"METEOR Score: {base_meteor}\")\n",
        "print(f\"Finetuned METEOR Score: {finetuned_meteor}\")"
      ],
      "metadata": {
        "id": "J6ROcKFYvq-w"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}