RAG_Pipelines.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "mount_file_id": "1OlcbeSGRv4tbUR_gK3hNTbL6c__GHxMV",
      "authorship_tag": "ABX9TyPFDldbbBSkELjzuk2jSkDW",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "dd9eb798833c4a60827b9bac80f69e7f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_3f476df085f54c2fabf632f1446673ca",
              "IPY_MODEL_7c426c4f48ca468c89633fe38ad0af70",
              "IPY_MODEL_66f7b06bf50c43d18407e314bb86659e"
            ],
            "layout": "IPY_MODEL_a05e7d765f944ac6b744cb4ca7402745"
          }
        },
        "3f476df085f54c2fabf632f1446673ca": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0a9ac58dfd5f48ca823c2d89c63c9d9e",
            "placeholder": "​",
            "style": "IPY_MODEL_855cbbeaebda453a910c68f5ad2fc8a9",
            "value": "modules.json: 100%"
          }
        },
        "7c426c4f48ca468c89633fe38ad0af70": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_281a3284211d4fb3b8d01859767d08c5",
            "max": 387,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e40929753ffe41f78f74124a83765781",
            "value": 387
          }
        },
        "66f7b06bf50c43d18407e314bb86659e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_737c7724ccc84b4faf321be01ac95c71",
            "placeholder": "​",
            "style": "IPY_MODEL_2875b306a4674167a36bf7299c707f2d",
            "value": " 387/387 [00:00&lt;00:00, 16.2kB/s]"
          }
        },
        "a05e7d765f944ac6b744cb4ca7402745": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0a9ac58dfd5f48ca823c2d89c63c9d9e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "855cbbeaebda453a910c68f5ad2fc8a9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "281a3284211d4fb3b8d01859767d08c5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e40929753ffe41f78f74124a83765781": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "737c7724ccc84b4faf321be01ac95c71": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2875b306a4674167a36bf7299c707f2d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "12fea48807ac46a586a6914b6e1fd870": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_16bcc3e294894118bb9d094c2814fdd7",
              "IPY_MODEL_f6757e025a5a401eaba88e4281ffe37f",
              "IPY_MODEL_9f797358db574cf89b03e02ee942c016"
            ],
            "layout": "IPY_MODEL_9bb884d31ea34dc08b187a42a9f67610"
          }
        },
        "16bcc3e294894118bb9d094c2814fdd7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7ece6ebccbbf402abb0e6b944cd457cf",
            "placeholder": "​",
            "style": "IPY_MODEL_f36b8df972974f628f097683256810dd",
            "value": "README.md: 100%"
          }
        },
        "f6757e025a5a401eaba88e4281ffe37f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7c4b6b87417d4907bdedc688f2567724",
            "max": 179345,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_795cab2e7f434df4a1104bbf42ea4816",
            "value": 179345
          }
        },
        "9f797358db574cf89b03e02ee942c016": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bb02c7de892c44f48cbb60d267c0af4f",
            "placeholder": "​",
            "style": "IPY_MODEL_5fd7ded25de8422ea07c31b63137de8c",
            "value": " 179k/179k [00:00&lt;00:00, 2.16MB/s]"
          }
        },
        "9bb884d31ea34dc08b187a42a9f67610": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7ece6ebccbbf402abb0e6b944cd457cf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f36b8df972974f628f097683256810dd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7c4b6b87417d4907bdedc688f2567724": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "795cab2e7f434df4a1104bbf42ea4816": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "bb02c7de892c44f48cbb60d267c0af4f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5fd7ded25de8422ea07c31b63137de8c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "da5d5e324c154459adae403a545ba049": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_d45b5ca974314c3092424b7876a09f43",
              "IPY_MODEL_0fdc14f7f7d34a6b8a282bc20def9fba",
              "IPY_MODEL_a6e499c155724b90ae23deca08d60ace"
            ],
            "layout": "IPY_MODEL_bf149179793d483da86674c488d6d1bf"
          }
        },
        "d45b5ca974314c3092424b7876a09f43": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f72c96cd468a490fa97f1d0f0b241d58",
            "placeholder": "​",
            "style": "IPY_MODEL_88419c3743cb4db3b1d41aa89a184052",
            "value": "sentence_bert_config.json: 100%"
          }
        },
        "0fdc14f7f7d34a6b8a282bc20def9fba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_85b9d90eb8324711865396b90508d84e",
            "max": 57,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_6b10174df19542098ee0993dc963ad8a",
            "value": 57
          }
        },
        "a6e499c155724b90ae23deca08d60ace": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d30c42b9e85b4be7a293e6d659962040",
            "placeholder": "​",
            "style": "IPY_MODEL_aa79d39f39fb43329996048fbd64dbc3",
            "value": " 57.0/57.0 [00:00&lt;00:00, 2.06kB/s]"
          }
        },
        "bf149179793d483da86674c488d6d1bf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f72c96cd468a490fa97f1d0f0b241d58": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "88419c3743cb4db3b1d41aa89a184052": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "85b9d90eb8324711865396b90508d84e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6b10174df19542098ee0993dc963ad8a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "d30c42b9e85b4be7a293e6d659962040": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "aa79d39f39fb43329996048fbd64dbc3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c0535342db854c04b1ea160e7bb698f2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_925a175ac7a642e28a9226adb513d6bb",
              "IPY_MODEL_faf81eff6a79489698fcf35e17b1d361",
              "IPY_MODEL_349b1dbd156a47e290956d37e02c55e1"
            ],
            "layout": "IPY_MODEL_0b64db73949b43c894acb5125904ac60"
          }
        },
        "925a175ac7a642e28a9226adb513d6bb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6bd8bf91b9004830a2500a9abc4d4989",
            "placeholder": "​",
            "style": "IPY_MODEL_21a75b76406140c18ac7d972836f8be5",
            "value": "config.json: 100%"
          }
        },
        "faf81eff6a79489698fcf35e17b1d361": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_98c57cfa76dd43f4ab74a2af63bde4d1",
            "max": 694,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_64d91eccb19344e3958326fb796866dc",
            "value": 694
          }
        },
        "349b1dbd156a47e290956d37e02c55e1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_75b665fbc55e480ba1bb63e76acd2dee",
            "placeholder": "​",
            "style": "IPY_MODEL_d32b0dc1e519426bb5fb4528f22ff0ca",
            "value": " 694/694 [00:00&lt;00:00, 27.1kB/s]"
          }
        },
        "0b64db73949b43c894acb5125904ac60": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6bd8bf91b9004830a2500a9abc4d4989": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "21a75b76406140c18ac7d972836f8be5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "98c57cfa76dd43f4ab74a2af63bde4d1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "64d91eccb19344e3958326fb796866dc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "75b665fbc55e480ba1bb63e76acd2dee": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d32b0dc1e519426bb5fb4528f22ff0ca": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7c8e22ea91dd4574b20108c7dcb8d0d3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_5220a146513649ef9077a782688085c7",
              "IPY_MODEL_ec6e082736164bbeafe0ba690503ef52",
              "IPY_MODEL_7265fbd7933343c3a049e8952ace4379"
            ],
            "layout": "IPY_MODEL_44bb4f96758e4834881157aac10f9d1b"
          }
        },
        "5220a146513649ef9077a782688085c7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fe8b4d8cc6bf475c9b7c71d0296c8a86",
            "placeholder": "​",
            "style": "IPY_MODEL_79e69af2935c407f8cad0947d5bb5b7c",
            "value": "model.safetensors: 100%"
          }
        },
        "ec6e082736164bbeafe0ba690503ef52": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_01cda12e9f5648f490c55fb8d98d5e23",
            "max": 1112201288,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_15cbe85268be4b0c84a3baf3a034bff8",
            "value": 1112201288
          }
        },
        "7265fbd7933343c3a049e8952ace4379": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5aa64173bf024f7bb235a83a45a9709a",
            "placeholder": "​",
            "style": "IPY_MODEL_2fd7f1856833436b8b92327d05e7c917",
            "value": " 1.11G/1.11G [00:13&lt;00:00, 106MB/s]"
          }
        },
        "44bb4f96758e4834881157aac10f9d1b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fe8b4d8cc6bf475c9b7c71d0296c8a86": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "79e69af2935c407f8cad0947d5bb5b7c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "01cda12e9f5648f490c55fb8d98d5e23": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "15cbe85268be4b0c84a3baf3a034bff8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5aa64173bf024f7bb235a83a45a9709a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2fd7f1856833436b8b92327d05e7c917": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8cc091291e954490af82da443bba1936": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_aa4fd59e7bf64a708cbdcdb6518f4150",
              "IPY_MODEL_9c57763e9a38459eb517ab1e55e5c87d",
              "IPY_MODEL_ac5694ea9d5a4ef98923597ff47c8289"
            ],
            "layout": "IPY_MODEL_3c17064a87594dc2bac8d92ac2e4fbad"
          }
        },
        "aa4fd59e7bf64a708cbdcdb6518f4150": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_acbc63488c134ffa84e89f824b487c4c",
            "placeholder": "​",
            "style": "IPY_MODEL_0a303792cfa74419b2306872b6ba1ba2",
            "value": "tokenizer_config.json: 100%"
          }
        },
        "9c57763e9a38459eb517ab1e55e5c87d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bce72b19051e4ee587e5bfd91942f01e",
            "max": 418,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_5bafc8b65462486198838f4d1a257e06",
            "value": 418
          }
        },
        "ac5694ea9d5a4ef98923597ff47c8289": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cbc369e92bbf4f599d351a4e100fc333",
            "placeholder": "​",
            "style": "IPY_MODEL_a15805cb81ff4136aefc54479e160418",
            "value": " 418/418 [00:00&lt;00:00, 18.3kB/s]"
          }
        },
        "3c17064a87594dc2bac8d92ac2e4fbad": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "acbc63488c134ffa84e89f824b487c4c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0a303792cfa74419b2306872b6ba1ba2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bce72b19051e4ee587e5bfd91942f01e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5bafc8b65462486198838f4d1a257e06": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "cbc369e92bbf4f599d351a4e100fc333": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a15805cb81ff4136aefc54479e160418": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "407033b72f8c4a55969df05719095593": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_06dbfab0db0943fe8993ae07d1d1ff68",
              "IPY_MODEL_83abda0ddbe044e0b489ad277c885eaf",
              "IPY_MODEL_ac83575f372443b89bf23bd51e94c967"
            ],
            "layout": "IPY_MODEL_1959d9e87b6b4ce680e93a70c173cd46"
          }
        },
        "06dbfab0db0943fe8993ae07d1d1ff68": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_eae15deb028e46ffafc17dbe73144921",
            "placeholder": "​",
            "style": "IPY_MODEL_245bd71d73544a659eeaf40db0b52b2f",
            "value": "sentencepiece.bpe.model: 100%"
          }
        },
        "83abda0ddbe044e0b489ad277c885eaf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_54febfd872324020ba8cc07c0ceee490",
            "max": 5069051,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_cf7ee90018594a90a834bf0d62a225ea",
            "value": 5069051
          }
        },
        "ac83575f372443b89bf23bd51e94c967": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_524e2651bbeb4bc0849cb06444acbc47",
            "placeholder": "​",
            "style": "IPY_MODEL_c7c1da8cd0b64c5898b59f0fbb6a2c42",
            "value": " 5.07M/5.07M [00:00&lt;00:00, 16.9MB/s]"
          }
        },
        "1959d9e87b6b4ce680e93a70c173cd46": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "eae15deb028e46ffafc17dbe73144921": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "245bd71d73544a659eeaf40db0b52b2f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "54febfd872324020ba8cc07c0ceee490": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cf7ee90018594a90a834bf0d62a225ea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "524e2651bbeb4bc0849cb06444acbc47": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c7c1da8cd0b64c5898b59f0fbb6a2c42": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4bba4de607a149cab1f18de39faa45a6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_ff7f9ed18f2f4314afdaaf243a0f71c5",
              "IPY_MODEL_f663b96df4a24860890edc49a6457117",
              "IPY_MODEL_4e45832a271d4f49bf5aa1664d2726e9"
            ],
            "layout": "IPY_MODEL_f2705fa38649474abf77853143937596"
          }
        },
        "ff7f9ed18f2f4314afdaaf243a0f71c5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_77e6c4241cdf4da8a5e26d8ebdd9315b",
            "placeholder": "​",
            "style": "IPY_MODEL_76fdc72a319c468f971e62332eca4414",
            "value": "tokenizer.json: 100%"
          }
        },
        "f663b96df4a24860890edc49a6457117": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5187098e371843f091bf56323bd1db7a",
            "max": 17082660,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_708241eee2744d118bc55ec0f0a16051",
            "value": 17082660
          }
        },
        "4e45832a271d4f49bf5aa1664d2726e9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f18fe33963ea4843b32399a5d02f320d",
            "placeholder": "​",
            "style": "IPY_MODEL_623ab2035aa94727ad2b729dc9f1b3ac",
            "value": " 17.1M/17.1M [00:00&lt;00:00, 47.7MB/s]"
          }
        },
        "f2705fa38649474abf77853143937596": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "77e6c4241cdf4da8a5e26d8ebdd9315b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "76fdc72a319c468f971e62332eca4414": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5187098e371843f091bf56323bd1db7a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "708241eee2744d118bc55ec0f0a16051": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f18fe33963ea4843b32399a5d02f320d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "623ab2035aa94727ad2b729dc9f1b3ac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f669df67a908403cba6ae463fe8ff5b2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_87ac1c30bc804495b74b24ad1009983d",
              "IPY_MODEL_ab964e1f9a43422d8b1a4ece3cdb4393",
              "IPY_MODEL_250fbbdf68c04c958abcba234e54028b"
            ],
            "layout": "IPY_MODEL_28e403dce878472da6ddcb8883eef743"
          }
        },
        "87ac1c30bc804495b74b24ad1009983d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e024aa5407434c11986a67bb0a0bc2fe",
            "placeholder": "​",
            "style": "IPY_MODEL_1856d0dac54d43439cb9ecb598ca722b",
            "value": "special_tokens_map.json: 100%"
          }
        },
        "ab964e1f9a43422d8b1a4ece3cdb4393": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fb3eaa51f6c441fba1370c42a9a7a1b4",
            "max": 280,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_6aa88bbe8c864365b62255c1fb5b7b72",
            "value": 280
          }
        },
        "250fbbdf68c04c958abcba234e54028b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a344c4fab8ab461db379c2b46ac1c088",
            "placeholder": "​",
            "style": "IPY_MODEL_6c2906e9fcf240ae871b5122d6f1c37d",
            "value": " 280/280 [00:00&lt;00:00, 3.69kB/s]"
          }
        },
        "28e403dce878472da6ddcb8883eef743": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e024aa5407434c11986a67bb0a0bc2fe": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1856d0dac54d43439cb9ecb598ca722b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "fb3eaa51f6c441fba1370c42a9a7a1b4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6aa88bbe8c864365b62255c1fb5b7b72": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a344c4fab8ab461db379c2b46ac1c088": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6c2906e9fcf240ae871b5122d6f1c37d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7dd2e6d7c1474e3498518a488d5a21d7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_63c7a696c50e48229a1813f45a9b4b3c",
              "IPY_MODEL_04a57f4bd07c46f8968653fe94bc93d1",
              "IPY_MODEL_9062c36e474e4d3295d79b25cbd22920"
            ],
            "layout": "IPY_MODEL_ecfe5244bcb74f7c9cceefed9947778d"
          }
        },
        "63c7a696c50e48229a1813f45a9b4b3c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5e86185e492141c8ba2d011516adf7c7",
            "placeholder": "​",
            "style": "IPY_MODEL_f02ff8812b8a4c6f8cf3d6e36c7b39b1",
            "value": "1_Pooling/config.json: 100%"
          }
        },
        "04a57f4bd07c46f8968653fe94bc93d1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0444a11e9c3142dbb6720027cb37c2a3",
            "max": 200,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_d5f219ab3c7a4a799b1bc9cfd55080ec",
            "value": 200
          }
        },
        "9062c36e474e4d3295d79b25cbd22920": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ece4cc31c712415f8a745064d281d50a",
            "placeholder": "​",
            "style": "IPY_MODEL_7477ddcae27a43229d9fe5cced04518a",
            "value": " 200/200 [00:00&lt;00:00, 7.32kB/s]"
          }
        },
        "ecfe5244bcb74f7c9cceefed9947778d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5e86185e492141c8ba2d011516adf7c7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f02ff8812b8a4c6f8cf3d6e36c7b39b1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0444a11e9c3142dbb6720027cb37c2a3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d5f219ab3c7a4a799b1bc9cfd55080ec": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ece4cc31c712415f8a745064d281d50a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7477ddcae27a43229d9fe5cced04518a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b4dd8164c7fb43b09332c345740d3777": {
          "model_module": "@jupyter-widgets/output",
          "model_name": "OutputModel",
          "model_module_version": "1.0.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/output",
            "_model_module_version": "1.0.0",
            "_model_name": "OutputModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/output",
            "_view_module_version": "1.0.0",
            "_view_name": "OutputView",
            "layout": "IPY_MODEL_786120cb17e142dfa35105a91c213d69",
            "msg_id": "",
            "outputs": [
              {
                "output_type": "display_data",
                "data": {
                  "text/plain": "\u001b[35m  80%\u001b[0m \u001b[38;2;249;38;114m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[38;5;237m╺\u001b[0m\u001b[38;5;237m━━━━━━━━━━━━━━\u001b[0m \u001b[32m4/5 \u001b[0m [ \u001b[33m0:00:21\u001b[0m < \u001b[36m0:00:05\u001b[0m , \u001b[31m0 it/s\u001b[0m ]\n",
                  "text/html": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800080; text-decoration-color: #800080\">  80%</span> <span style=\"color: #f92672; text-decoration-color: #f92672\">━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━</span><span style=\"color: #3a3a3a; text-decoration-color: #3a3a3a\">╺━━━━━━━━━━━━━━</span> <span style=\"color: #008000; text-decoration-color: #008000\">4/5 </span> [ <span style=\"color: #808000; text-decoration-color: #808000\">0:00:21</span> &lt; <span style=\"color: #008080; text-decoration-color: #008080\">0:00:05</span> , <span style=\"color: #800000; text-decoration-color: #800000\">0 it/s</span> ]\n</pre>\n"
                },
                "metadata": {}
              }
            ]
          }
        },
        "786120cb17e142dfa35105a91c213d69": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/PinsaraPerera/Metadata_Enhancement_KeyBert/blob/main/RAG_Pipelines.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5iRH1wy93TPP",
        "outputId": "4479a03c-cc04-403c-e338-ba5ea768e18f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting keybert\n",
            "  Downloading keybert-0.8.4.tar.gz (29 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (4.38.1)\n",
            "Collecting langchain\n",
            "  Downloading langchain-0.1.11-py3-none-any.whl (807 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m807.5/807.5 kB\u001b[0m \u001b[31m9.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting tiktoken\n",
            "  Downloading tiktoken-0.6.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.8/1.8 MB\u001b[0m \u001b[31m15.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting sentence-transformers>=0.3.8 (from keybert)\n",
            "  Downloading sentence_transformers-2.5.1-py3-none-any.whl (156 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m156.5/156.5 kB\u001b[0m \u001b[31m17.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: scikit-learn>=0.22.2 in /usr/local/lib/python3.10/dist-packages (from keybert) (1.2.2)\n",
            "Requirement already satisfied: numpy>=1.18.5 in /usr/local/lib/python3.10/dist-packages (from keybert) (1.25.2)\n",
            "Requirement already satisfied: rich>=10.4.0 in /usr/local/lib/python3.10/dist-packages (from keybert) (13.7.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.13.1)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.19.3 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.20.3)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (23.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2023.12.25)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.31.0)\n",
            "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.15.2)\n",
            "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.4.2)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.66.2)\n",
            "Requirement already satisfied: SQLAlchemy<3,>=1.4 in /usr/local/lib/python3.10/dist-packages (from langchain) (2.0.27)\n",
            "Requirement already satisfied: aiohttp<4.0.0,>=3.8.3 in /usr/local/lib/python3.10/dist-packages (from langchain) (3.9.3)\n",
            "Requirement already satisfied: async-timeout<5.0.0,>=4.0.0 in /usr/local/lib/python3.10/dist-packages (from langchain) (4.0.3)\n",
            "Collecting dataclasses-json<0.7,>=0.5.7 (from langchain)\n",
            "  Downloading dataclasses_json-0.6.4-py3-none-any.whl (28 kB)\n",
            "Collecting jsonpatch<2.0,>=1.33 (from langchain)\n",
            "  Downloading jsonpatch-1.33-py2.py3-none-any.whl (12 kB)\n",
            "Collecting langchain-community<0.1,>=0.0.25 (from langchain)\n",
            "  Downloading langchain_community-0.0.25-py3-none-any.whl (1.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.8/1.8 MB\u001b[0m \u001b[31m19.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting langchain-core<0.2,>=0.1.29 (from langchain)\n",
            "  Downloading langchain_core-0.1.29-py3-none-any.whl (252 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m252.6/252.6 kB\u001b[0m \u001b[31m23.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting langchain-text-splitters<0.1,>=0.0.1 (from langchain)\n",
            "  Downloading langchain_text_splitters-0.0.1-py3-none-any.whl (21 kB)\n",
            "Collecting langsmith<0.2.0,>=0.1.17 (from langchain)\n",
            "  Downloading langsmith-0.1.22-py3-none-any.whl (66 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m66.6/66.6 kB\u001b[0m \u001b[31m5.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pydantic<3,>=1 in /usr/local/lib/python3.10/dist-packages (from langchain) (2.6.3)\n",
            "Requirement already satisfied: tenacity<9.0.0,>=8.1.0 in /usr/local/lib/python3.10/dist-packages (from langchain) (8.2.3)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.3.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (23.2.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.4.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (6.0.5)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.9.4)\n",
            "Collecting marshmallow<4.0.0,>=3.18.0 (from dataclasses-json<0.7,>=0.5.7->langchain)\n",
            "  Downloading marshmallow-3.21.1-py3-none-any.whl (49 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.4/49.4 kB\u001b[0m \u001b[31m4.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting typing-inspect<1,>=0.4.0 (from dataclasses-json<0.7,>=0.5.7->langchain)\n",
            "  Downloading typing_inspect-0.9.0-py3-none-any.whl (8.8 kB)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.19.3->transformers) (2023.6.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.19.3->transformers) (4.10.0)\n",
            "Collecting jsonpointer>=1.9 (from jsonpatch<2.0,>=1.33->langchain)\n",
            "  Downloading jsonpointer-2.4-py2.py3-none-any.whl (7.8 kB)\n",
            "Requirement already satisfied: anyio<5,>=3 in /usr/local/lib/python3.10/dist-packages (from langchain-core<0.2,>=0.1.29->langchain) (3.7.1)\n",
            "Collecting orjson<4.0.0,>=3.9.14 (from langsmith<0.2.0,>=0.1.17->langchain)\n",
            "  Downloading orjson-3.9.15-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (138 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m138.5/138.5 kB\u001b[0m \u001b[31m3.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1->langchain) (0.6.0)\n",
            "Requirement already satisfied: pydantic-core==2.16.3 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1->langchain) (2.16.3)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2024.2.2)\n",
            "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.10/dist-packages (from rich>=10.4.0->keybert) (3.0.0)\n",
            "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from rich>=10.4.0->keybert) (2.16.1)\n",
            "Requirement already satisfied: scipy>=1.3.2 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.22.2->keybert) (1.11.4)\n",
            "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.22.2->keybert) (1.3.2)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.22.2->keybert) (3.3.0)\n",
            "Requirement already satisfied: torch>=1.11.0 in /usr/local/lib/python3.10/dist-packages (from sentence-transformers>=0.3.8->keybert) (2.1.0+cu121)\n",
            "Requirement already satisfied: Pillow in /usr/local/lib/python3.10/dist-packages (from sentence-transformers>=0.3.8->keybert) (9.4.0)\n",
            "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/dist-packages (from SQLAlchemy<3,>=1.4->langchain) (3.0.3)\n",
            "Requirement already satisfied: sniffio>=1.1 in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3->langchain-core<0.2,>=0.1.29->langchain) (1.3.1)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3->langchain-core<0.2,>=0.1.29->langchain) (1.2.0)\n",
            "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py>=2.2.0->rich>=10.4.0->keybert) (0.1.2)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence-transformers>=0.3.8->keybert) (1.12)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence-transformers>=0.3.8->keybert) (3.2.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence-transformers>=0.3.8->keybert) (3.1.3)\n",
            "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence-transformers>=0.3.8->keybert) (2.1.0)\n",
            "Collecting mypy-extensions>=0.3.0 (from typing-inspect<1,>=0.4.0->dataclasses-json<0.7,>=0.5.7->langchain)\n",
            "  Downloading mypy_extensions-1.0.0-py3-none-any.whl (4.7 kB)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.11.0->sentence-transformers>=0.3.8->keybert) (2.1.5)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.11.0->sentence-transformers>=0.3.8->keybert) (1.3.0)\n",
            "Building wheels for collected packages: keybert\n",
            "  Building wheel for keybert (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for keybert: filename=keybert-0.8.4-py3-none-any.whl size=39199 sha256=ac2e9a40268e976acd32850773d3bd767907d204e81b5295b51da7991b0ae752\n",
            "  Stored in directory: /root/.cache/pip/wheels/97/ef/4c/6588bd7072b0cc04225b40e639b991e49ebd4e21fb81f0acee\n",
            "Successfully built keybert\n",
            "Installing collected packages: orjson, mypy-extensions, marshmallow, jsonpointer, typing-inspect, tiktoken, jsonpatch, langsmith, dataclasses-json, langchain-core, sentence-transformers, langchain-text-splitters, langchain-community, langchain, keybert\n",
            "Successfully installed dataclasses-json-0.6.4 jsonpatch-1.33 jsonpointer-2.4 keybert-0.8.4 langchain-0.1.11 langchain-community-0.0.25 langchain-core-0.1.29 langchain-text-splitters-0.0.1 langsmith-0.1.22 marshmallow-3.21.1 mypy-extensions-1.0.0 orjson-3.9.15 sentence-transformers-2.5.1 tiktoken-0.6.0 typing-inspect-0.9.0\n"
          ]
        }
      ],
      "source": [
        "!pip install keybert transformers langchain tiktoken"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from tqdm.rich import trange, tqdm\n",
        "from rich import console\n",
        "from rich.markdown import Markdown\n",
        "from rich.text import Text\n",
        "\n",
        "import warnings\n",
        "warnings.filterwarnings(\"ignore\")\n",
        "\n",
        "import datetime\n",
        "from rich.console import Console\n",
        "\n",
        "console = Console(width=110)\n",
        "\n",
        "from transformers import pipeline\n",
        "import os\n",
        "\n",
        "from keybert import KeyBERT"
      ],
      "metadata": {
        "id": "A5KF_voa3qqF"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "kw_model = KeyBERT(model = 'intfloat/multilingual-e5-base')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 337,
          "referenced_widgets": [
            "dd9eb798833c4a60827b9bac80f69e7f",
            "3f476df085f54c2fabf632f1446673ca",
            "7c426c4f48ca468c89633fe38ad0af70",
            "66f7b06bf50c43d18407e314bb86659e",
            "a05e7d765f944ac6b744cb4ca7402745",
            "0a9ac58dfd5f48ca823c2d89c63c9d9e",
            "855cbbeaebda453a910c68f5ad2fc8a9",
            "281a3284211d4fb3b8d01859767d08c5",
            "e40929753ffe41f78f74124a83765781",
            "737c7724ccc84b4faf321be01ac95c71",
            "2875b306a4674167a36bf7299c707f2d",
            "12fea48807ac46a586a6914b6e1fd870",
            "16bcc3e294894118bb9d094c2814fdd7",
            "f6757e025a5a401eaba88e4281ffe37f",
            "9f797358db574cf89b03e02ee942c016",
            "9bb884d31ea34dc08b187a42a9f67610",
            "7ece6ebccbbf402abb0e6b944cd457cf",
            "f36b8df972974f628f097683256810dd",
            "7c4b6b87417d4907bdedc688f2567724",
            "795cab2e7f434df4a1104bbf42ea4816",
            "bb02c7de892c44f48cbb60d267c0af4f",
            "5fd7ded25de8422ea07c31b63137de8c",
            "da5d5e324c154459adae403a545ba049",
            "d45b5ca974314c3092424b7876a09f43",
            "0fdc14f7f7d34a6b8a282bc20def9fba",
            "a6e499c155724b90ae23deca08d60ace",
            "bf149179793d483da86674c488d6d1bf",
            "f72c96cd468a490fa97f1d0f0b241d58",
            "88419c3743cb4db3b1d41aa89a184052",
            "85b9d90eb8324711865396b90508d84e",
            "6b10174df19542098ee0993dc963ad8a",
            "d30c42b9e85b4be7a293e6d659962040",
            "aa79d39f39fb43329996048fbd64dbc3",
            "c0535342db854c04b1ea160e7bb698f2",
            "925a175ac7a642e28a9226adb513d6bb",
            "faf81eff6a79489698fcf35e17b1d361",
            "349b1dbd156a47e290956d37e02c55e1",
            "0b64db73949b43c894acb5125904ac60",
            "6bd8bf91b9004830a2500a9abc4d4989",
            "21a75b76406140c18ac7d972836f8be5",
            "98c57cfa76dd43f4ab74a2af63bde4d1",
            "64d91eccb19344e3958326fb796866dc",
            "75b665fbc55e480ba1bb63e76acd2dee",
            "d32b0dc1e519426bb5fb4528f22ff0ca",
            "7c8e22ea91dd4574b20108c7dcb8d0d3",
            "5220a146513649ef9077a782688085c7",
            "ec6e082736164bbeafe0ba690503ef52",
            "7265fbd7933343c3a049e8952ace4379",
            "44bb4f96758e4834881157aac10f9d1b",
            "fe8b4d8cc6bf475c9b7c71d0296c8a86",
            "79e69af2935c407f8cad0947d5bb5b7c",
            "01cda12e9f5648f490c55fb8d98d5e23",
            "15cbe85268be4b0c84a3baf3a034bff8",
            "5aa64173bf024f7bb235a83a45a9709a",
            "2fd7f1856833436b8b92327d05e7c917",
            "8cc091291e954490af82da443bba1936",
            "aa4fd59e7bf64a708cbdcdb6518f4150",
            "9c57763e9a38459eb517ab1e55e5c87d",
            "ac5694ea9d5a4ef98923597ff47c8289",
            "3c17064a87594dc2bac8d92ac2e4fbad",
            "acbc63488c134ffa84e89f824b487c4c",
            "0a303792cfa74419b2306872b6ba1ba2",
            "bce72b19051e4ee587e5bfd91942f01e",
            "5bafc8b65462486198838f4d1a257e06",
            "cbc369e92bbf4f599d351a4e100fc333",
            "a15805cb81ff4136aefc54479e160418",
            "407033b72f8c4a55969df05719095593",
            "06dbfab0db0943fe8993ae07d1d1ff68",
            "83abda0ddbe044e0b489ad277c885eaf",
            "ac83575f372443b89bf23bd51e94c967",
            "1959d9e87b6b4ce680e93a70c173cd46",
            "eae15deb028e46ffafc17dbe73144921",
            "245bd71d73544a659eeaf40db0b52b2f",
            "54febfd872324020ba8cc07c0ceee490",
            "cf7ee90018594a90a834bf0d62a225ea",
            "524e2651bbeb4bc0849cb06444acbc47",
            "c7c1da8cd0b64c5898b59f0fbb6a2c42",
            "4bba4de607a149cab1f18de39faa45a6",
            "ff7f9ed18f2f4314afdaaf243a0f71c5",
            "f663b96df4a24860890edc49a6457117",
            "4e45832a271d4f49bf5aa1664d2726e9",
            "f2705fa38649474abf77853143937596",
            "77e6c4241cdf4da8a5e26d8ebdd9315b",
            "76fdc72a319c468f971e62332eca4414",
            "5187098e371843f091bf56323bd1db7a",
            "708241eee2744d118bc55ec0f0a16051",
            "f18fe33963ea4843b32399a5d02f320d",
            "623ab2035aa94727ad2b729dc9f1b3ac",
            "f669df67a908403cba6ae463fe8ff5b2",
            "87ac1c30bc804495b74b24ad1009983d",
            "ab964e1f9a43422d8b1a4ece3cdb4393",
            "250fbbdf68c04c958abcba234e54028b",
            "28e403dce878472da6ddcb8883eef743",
            "e024aa5407434c11986a67bb0a0bc2fe",
            "1856d0dac54d43439cb9ecb598ca722b",
            "fb3eaa51f6c441fba1370c42a9a7a1b4",
            "6aa88bbe8c864365b62255c1fb5b7b72",
            "a344c4fab8ab461db379c2b46ac1c088",
            "6c2906e9fcf240ae871b5122d6f1c37d",
            "7dd2e6d7c1474e3498518a488d5a21d7",
            "63c7a696c50e48229a1813f45a9b4b3c",
            "04a57f4bd07c46f8968653fe94bc93d1",
            "9062c36e474e4d3295d79b25cbd22920",
            "ecfe5244bcb74f7c9cceefed9947778d",
            "5e86185e492141c8ba2d011516adf7c7",
            "f02ff8812b8a4c6f8cf3d6e36c7b39b1",
            "0444a11e9c3142dbb6720027cb37c2a3",
            "d5f219ab3c7a4a799b1bc9cfd55080ec",
            "ece4cc31c712415f8a745064d281d50a",
            "7477ddcae27a43229d9fe5cced04518a"
          ]
        },
        "id": "4rBHVedL4xkN",
        "outputId": "750d82d2-e1ef-4d6c-dd61-bc820e5915d6"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "modules.json:   0%|          | 0.00/387 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "dd9eb798833c4a60827b9bac80f69e7f"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "README.md:   0%|          | 0.00/179k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "12fea48807ac46a586a6914b6e1fd870"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "sentence_bert_config.json:   0%|          | 0.00/57.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "da5d5e324c154459adae403a545ba049"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "config.json:   0%|          | 0.00/694 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c0535342db854c04b1ea160e7bb698f2"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model.safetensors:   0%|          | 0.00/1.11G [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "7c8e22ea91dd4574b20108c7dcb8d0d3"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer_config.json:   0%|          | 0.00/418 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "8cc091291e954490af82da443bba1936"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "sentencepiece.bpe.model:   0%|          | 0.00/5.07M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "407033b72f8c4a55969df05719095593"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer.json:   0%|          | 0.00/17.1M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "4bba4de607a149cab1f18de39faa45a6"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "special_tokens_map.json:   0%|          | 0.00/280 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "f669df67a908403cba6ae463fe8ff5b2"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "1_Pooling/config.json:   0%|          | 0.00/200 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "7dd2e6d7c1474e3498518a488d5a21d7"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "logfile = 'keybertLog.txt'"
      ],
      "metadata": {
        "id": "4jJaYFmo5Vd8"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def writeHistory(text):\n",
        "  with open(logfile, 'a', encoding='utf-8') as f:\n",
        "    f.write(text)\n",
        "    f.write('\\n')\n",
        "\n",
        "  f.close()"
      ],
      "metadata": {
        "id": "zW6SLUKD5dK9"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "Text = \"\"\"\n",
        "         Supervised learning is the machine learning task of learning a function that\n",
        "         maps an input to an output based on example input-output pairs. It infers a\n",
        "         function from labeled training data consisting of a set of training examples.\n",
        "         In supervised learning, each example is a pair consisting of an input object\n",
        "         (typically a vector) and a desired output value (also called the supervisory signal).\n",
        "         A supervised learning algorithm analyzes the training data and produces an inferred function,\n",
        "         which can be used for mapping new examples. An optimal scenario will allow for the\n",
        "         algorithm to correctly determine the class labels for unseen instances. This requires\n",
        "         the learning algorithm to generalize from the training data to unseen situations in a\n",
        "         'reasonable' way (see inductive bias).\n",
        "      \"\"\"\n",
        "\n",
        "phrases = kw_model.extract_keywords(Text, keyphrase_ngram_range=(1, 2), stop_words='english',\n",
        "                              use_mmr=True, diversity=0.7, highlight=True)\n",
        "\n",
        "print(phrases)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 146
        },
        "id": "hoeItdDK_y6N",
        "outputId": "cd39f72e-a8d2-4e0a-819d-cf48e288e3a4"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[30;48;2;255;255;0mSupervised learning\u001b[0m the machine learning task of learning function that maps an input to an \u001b[30;48;2;255;255;0moutput\u001b[0m based on example\n",
              "input \u001b[30;48;2;255;255;0moutput\u001b[0m pairs It infers function from labeled training data consisting of set of training examples In \n",
              "\u001b[30;48;2;255;255;0msupervised learning\u001b[0m example is \u001b[30;48;2;255;255;0mpair consisting\u001b[0m an input object typically vector and desired \u001b[30;48;2;255;255;0moutput\u001b[0m value also \n",
              "called the supervisory signal \u001b[30;48;2;255;255;0msupervised learning\u001b[0m analyzes the training data and produces an inferred function \n",
              "which can be used for mapping new examples An optimal scenario will allow for the algorithm to correctly determine \n",
              "the class labels for unseen instances This requires the learning algorithm to generalize from the training data to \n",
              "unseen situations in reasonable way see inductive bias\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">Supervised learning</span> the machine learning task of learning function that maps an input to an <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">output</span> based on example\n",
              "input <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">output</span> pairs It infers function from labeled training data consisting of set of training examples In \n",
              "<span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">supervised learning</span> example is <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">pair consisting</span> an input object typically vector and desired <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">output</span> value also \n",
              "called the supervisory signal <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">supervised learning</span> analyzes the training data and produces an inferred function \n",
              "which can be used for mapping new examples An optimal scenario will allow for the algorithm to correctly determine \n",
              "the class labels for unseen instances This requires the learning algorithm to generalize from the training data to \n",
              "unseen situations in reasonable way see inductive bias\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[('supervised learning', 0.8802), ('output', 0.7979), ('function maps', 0.7952), ('examples optimal', 0.7942), ('pair consisting', 0.762)]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import datetime\n",
        "import random\n",
        "\n",
        "def extract_keys(text, n_gram, diversity):\n",
        "  phrases = kw_model.extract_keywords(text, keyphrase_ngram_range=(1, n_gram), stop_words='english',\n",
        "                              use_mmr=True, diversity=diversity, highlight=True) # use_mmr allow to use spaces within the phrases\n",
        "\n",
        "  tags = []\n",
        "  for keyword in phrases:\n",
        "    tags.append(str(keyword[0])) # extract the word ex: ('supervised learning', 0.8802)\n",
        "\n",
        "  timeStamped = datetime.datetime.now()\n",
        "\n",
        "  # create the logs\n",
        "\n",
        "  logged_text = f'LOGGED ON:{str(timeStamped)}\\nMETADATA: {str(tags)}\\nSETTINGS: keyphrase_ngram_range: (1,{str(n_gram)}) Diversity: {str(diversity)}'\n",
        "  writeHistory(logged_text)\n",
        "\n",
        "  return tags\n"
      ],
      "metadata": {
        "id": "8zBbNMNg6C8k"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "text = \"\"\"\n",
        "\n",
        "Title: A Review on Background, Technology, Limitations, and Opportunities of Large Vision Models\n",
        "publish on Paperswithcode\n",
        "Authors: Yixin Liu, Kai Zhang, Yuan Li, Zhiling Yan, Chujie Gao, Ruoxi Chen, Zhengqing Yuan, Yue Huang, Hanchi Sun, Jianfeng Gao, Lifang He, Lichao Sun\n",
        "URL: https://arxiv.org/pdf/2402.17177v2.pdf\n",
        "Sora is a text-to-video generative AI model, released by OpenAI in February 2024. The model is\n",
        "trained to generate videos of realistic or imaginative scenes from text instructions and show potential\n",
        "in simulating the physical world. Based on public technical reports and reverse engineering, this paper\n",
        "presents a comprehensive review of the model’s background, related technologies, applications, remain-\n",
        "ing challenges, and future directions of text-to-video AI models. We first trace Sora’s development\n",
        "and investigate the underlying technologies used to build this “world simulator”. Then, we describe in\n",
        "detail the applications and potential impact of Sora in multiple industries ranging from film-making\n",
        "and education to marketing. We discuss the main challenges and limitations that need to be addressed to\n",
        "widely deploy Sora, such as ensuring safe and unbiased video generation. Lastly, we discuss the future\n",
        "development of Sora and video generation models in general, and how advancements in the field could\n",
        "enable new ways of human-AI interaction, boosting productivity and creativity of video generation.\n",
        "\n",
        "\"\"\""
      ],
      "metadata": {
        "id": "Eq79bFA0BT3G"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "extract_keywords = extract_keys(text, 1, 0.35)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 225
        },
        "id": "zpW_OmJsDWnc",
        "outputId": "a69a8824-4ba1-43de-f081-9e9f9564d330"
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Title Review on Background Technology Limitations and Opportunities of Large Vision Models publish on \n",
              "\u001b[30;48;2;255;255;0mPaperswithcode\u001b[0m Authors Yixin Liu Kai Zhang Yuan Li Zhiling Yan Chujie Gao Ruoxi Chen Zhengqing Yuan Yue Huang \n",
              "Hanchi Sun Jianfeng Gao Lifang He Lichao Sun URL https arxiv org pdf 2402 17177v2 pdf \u001b[30;48;2;255;255;0mSora\u001b[0m is text to video \n",
              "\u001b[30;48;2;255;255;0mgenerative\u001b[0m AI model released by \u001b[30;48;2;255;255;0mOpenAI\u001b[0m in February 2024 The model is trained to generate \u001b[30;48;2;255;255;0mvideos\u001b[0m of realistic or \n",
              "imaginative scenes from text instructions and show potential in simulating the physical world Based on public \n",
              "technical reports and reverse engineering this paper presents comprehensive review of the model background related \n",
              "technologies applications remain ing challenges and future directions of text to video AI models We first trace \n",
              "\u001b[30;48;2;255;255;0mSora\u001b[0m development and investigate the underlying technologies used to build this world simulator Then we describe in\n",
              "detail the applications and potential impact of \u001b[30;48;2;255;255;0mSora\u001b[0m in multiple industries ranging from film making and education \n",
              "to marketing We discuss the main challenges and limitations that need to be addressed to widely deploy \u001b[30;48;2;255;255;0mSora\u001b[0m such as\n",
              "ensuring safe and unbiased video generation Lastly we discuss the future development of \u001b[30;48;2;255;255;0mSora\u001b[0m and video generation \n",
              "models in general and how advancements in the field could enable new ways of human AI interaction boosting \n",
              "productivity and creativity of video generation\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Title Review on Background Technology Limitations and Opportunities of Large Vision Models publish on \n",
              "<span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">Paperswithcode</span> Authors Yixin Liu Kai Zhang Yuan Li Zhiling Yan Chujie Gao Ruoxi Chen Zhengqing Yuan Yue Huang \n",
              "Hanchi Sun Jianfeng Gao Lifang He Lichao Sun URL https arxiv org pdf 2402 17177v2 pdf <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">Sora</span> is text to video \n",
              "<span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">generative</span> AI model released by <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">OpenAI</span> in February 2024 The model is trained to generate <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">videos</span> of realistic or \n",
              "imaginative scenes from text instructions and show potential in simulating the physical world Based on public \n",
              "technical reports and reverse engineering this paper presents comprehensive review of the model background related \n",
              "technologies applications remain ing challenges and future directions of text to video AI models We first trace \n",
              "<span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">Sora</span> development and investigate the underlying technologies used to build this world simulator Then we describe in\n",
              "detail the applications and potential impact of <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">Sora</span> in multiple industries ranging from film making and education \n",
              "to marketing We discuss the main challenges and limitations that need to be addressed to widely deploy <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">Sora</span> such as\n",
              "ensuring safe and unbiased video generation Lastly we discuss the future development of <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">Sora</span> and video generation \n",
              "models in general and how advancements in the field could enable new ways of human AI interaction boosting \n",
              "productivity and creativity of video generation\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "console.print(f\"[bold]Keywords: {extract_keywords}\") # tags generator"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        },
        "id": "Af-njlldDlnb",
        "outputId": "5992c4b1-e652-4fc0-eabb-50501bfcce8e"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1mKeywords: \u001b[0m\u001b[1m[\u001b[0m\u001b[1;32m'sora'\u001b[0m\u001b[1m, \u001b[0m\u001b[1;32m'videos'\u001b[0m\u001b[1m, \u001b[0m\u001b[1;32m'openai'\u001b[0m\u001b[1m, \u001b[0m\u001b[1;32m'generative'\u001b[0m\u001b[1m, \u001b[0m\u001b[1;32m'paperswithcode'\u001b[0m\u001b[1m]\u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">Keywords: [</span><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">'sora'</span><span style=\"font-weight: bold\">, </span><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">'videos'</span><span style=\"font-weight: bold\">, </span><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">'openai'</span><span style=\"font-weight: bold\">, </span><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">'generative'</span><span style=\"font-weight: bold\">, </span><span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">'paperswithcode'</span><span style=\"font-weight: bold\">]</span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def generate_tags(keywords):\n",
        "  for kw in keywords:\n",
        "    console.print(f\"[bold]#{kw}\\n\")"
      ],
      "metadata": {
        "id": "2JGS6Yk1D0Ul"
      },
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "generate_tags(extract_keywords)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 177
        },
        "id": "TtNCQWs2EYK-",
        "outputId": "ae838643-48f2-4685-d834-0b13690a4672"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1m#sora\u001b[0m\n",
              "\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">#sora</span>\n",
              "\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1m#videos\u001b[0m\n",
              "\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">#videos</span>\n",
              "\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1m#openai\u001b[0m\n",
              "\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">#openai</span>\n",
              "\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1m#generative\u001b[0m\n",
              "\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">#generative</span>\n",
              "\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1m#paperswithcode\u001b[0m\n",
              "\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">#paperswithcode</span>\n",
              "\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "filename = 'sora.txt'\n",
        "with open(filename, encoding='utf-8') as f:\n",
        "  entire_text = f.read()\n",
        "f.close()\n",
        "\n",
        "console.print(\"Entire text has been [bold]saved\")\n",
        "\n",
        "title = 'Sora Diffusion Transformers'\n",
        "filename = 'sora.txt'\n",
        "author = 'Yixin Liu, Kai Zhang, Yuan Li, Zhiling Yan, Chujie Gao, Ruoxi Chen, Zhengqing Yuan, Yue Huang, Hanchi Sun, Jianfeng Gao, Lifang He, Lichao Sun'\n",
        "url = 'https://arxiv.org/pdf/2402.17177v2.pdf'\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        },
        "id": "6U4UxRi2TtpH",
        "outputId": "7ce91bff-5c0c-4fb2-ebc9-420d9d51cf65"
      },
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Entire text has been \u001b[1msaved\u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Entire text has been <span style=\"font-weight: bold\">saved</span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.document_loaders import TextLoader\n",
        "from langchain.text_splitter import TokenTextSplitter"
      ],
      "metadata": {
        "id": "MOAVBj0eEcUG"
      },
      "execution_count": 57,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "text_splitter = TokenTextSplitter(chunk_size=350, chunk_overlap=10) # divided into 330 token chunks"
      ],
      "metadata": {
        "id": "QLvvvL1mJDU8"
      },
      "execution_count": 58,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "splitted_text = text_splitter.split_text(entire_text)"
      ],
      "metadata": {
        "id": "ZgBrUwU9JPZ9"
      },
      "execution_count": 59,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "console.print(splitted_text)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "FMS5EY_OXu7X",
        "outputId": "5a077604-718e-4801-ca22-19684c720767"
      },
      "execution_count": 60,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1m[\u001b[0m\n",
              "    \u001b[32m'Image Diffusion Transformer. Traditional diffusion models \u001b[0m\u001b[32m[\u001b[0m\u001b[32m51, 52, 53\u001b[0m\u001b[32m]\u001b[0m\u001b[32m mainly leverage \u001b[0m\n",
              "\u001b[32mconvolutional\\nU-Nets that include downsampling and upsampling blocks for the denoising network backbone. \u001b[0m\n",
              "\u001b[32mHowever,\\nrecent studies show that the U-Net architecture is not crucial to the good performance of the \u001b[0m\n",
              "\u001b[32mdiffusion model.\\nBy incorporating a more flexible transformer architecture, the transformer-based diffusion \u001b[0m\n",
              "\u001b[32mmodels can use\\nmore training data and larger model parameters. Along this line, DiT \u001b[0m\u001b[32m[\u001b[0m\u001b[32m4\u001b[0m\u001b[32m]\u001b[0m\u001b[32m and U-ViT \u001b[0m\u001b[32m[\u001b[0m\u001b[32m54\u001b[0m\u001b[32m]\u001b[0m\u001b[32m are \u001b[0m\n",
              "\u001b[32mamong the first\\nworks to employ vision transformers for latent diffusion models. As in ViT, DiT employs a \u001b[0m\n",
              "\u001b[32mmulti-head self-\\nattention layer and a pointwise feed-forward network interlaced with some layer norm and \u001b[0m\n",
              "\u001b[32mscaling layers.\\nMoreover, as shown in Figure 11, DiT incorporates conditioning via adaptive layer norm \u001b[0m\n",
              "\u001b[32m(\u001b[0m\u001b[32mAdaLN\u001b[0m\u001b[32m)\u001b[0m\u001b[32m with an\\nadditional MLP layer for zero-initializing, which initializes each residual block as an \u001b[0m\n",
              "\u001b[32midentity function and\\nthus greatly stabilizes the training process. The scalability and flexibility of DiT is\u001b[0m\n",
              "\u001b[32mempirically validated.\\nDiT becomes the new backbone for diffusion models. In U-ViT, as shown in Figure 11, \u001b[0m\n",
              "\u001b[32mthey treat all inputs,\\nincluding time, condition, and noisy image patches, as tokens and propose long skip \u001b[0m\n",
              "\u001b[32mconnections between the\\nshallow and deep transformer layers. The results suggest that the downsampling and \u001b[0m\n",
              "\u001b[32mupsampling operators\\nin CNN-based U-Net are not always necessary, and U-ViT achieves record-breaking FID \u001b[0m\n",
              "\u001b[32mscores in image\\nand text-to-image generation.\\nLike Masked AutoEncoder \u001b[0m\u001b[32m(\u001b[0m\u001b[32mMAE\u001b[0m\u001b[32m)\u001b[0m\u001b[32m'\u001b[0m,\n",
              "    \u001b[32m'Like Masked AutoEncoder \u001b[0m\u001b[32m(\u001b[0m\u001b[32mMAE\u001b[0m\u001b[32m)\u001b[0m\u001b[32m \u001b[0m\u001b[32m[\u001b[0m\u001b[32m33\u001b[0m\u001b[32m]\u001b[0m\u001b[32m, Masked Diffusion Transformer \u001b[0m\u001b[32m(\u001b[0m\u001b[32mMDT\u001b[0m\u001b[32m)\u001b[0m\u001b[32m \u001b[0m\u001b[32m[\u001b[0m\u001b[32m55\u001b[0m\u001b[32m]\u001b[0m\u001b[32m incorporates mask\\nlatent \u001b[0m\n",
              "\u001b[32mmodeling into the diffusion process to explicitly enhance contextual relation learning among object\\nsemantic \u001b[0m\n",
              "\u001b[32mparts in image synthesis. Specifically, as shown in Figure 12, MDT uses a side-interpolated for an\\nadditional\u001b[0m\n",
              "\u001b[32mmasked token reconstruction task during training to boost the training efficiency and learn pow-\\nerful \u001b[0m\n",
              "\u001b[32mcontext-aware positional embedding for inference. Compared to DiT \u001b[0m\u001b[32m[\u001b[0m\u001b[32m4\u001b[0m\u001b[32m]\u001b[0m\u001b[32m, MDT achieves better perfor-\\nmance and \u001b[0m\n",
              "\u001b[32mfaster learning speed. Instead of using AdaLN \u001b[0m\u001b[32m(\u001b[0m\u001b[32mi.e., shifting and scaling\u001b[0m\u001b[32m)\u001b[0m\u001b[32m for time-conditioning\\nmodeling, \u001b[0m\n",
              "\u001b[32mHatamizadeh et al. \u001b[0m\u001b[32m[\u001b[0m\u001b[32m56\u001b[0m\u001b[32m]\u001b[0m\u001b[32m introduce Diffusion Vision Transformers \u001b[0m\u001b[32m(\u001b[0m\u001b[32mDiffiT\u001b[0m\u001b[32m)\u001b[0m\u001b[32m, which uses a time-\\ndependent \u001b[0m\n",
              "\u001b[32mself-attention \u001b[0m\u001b[32m(\u001b[0m\u001b[32mTMSA\u001b[0m\u001b[32m)\u001b[0m\u001b[32m module to model dynamic denoising behavior over sampling time steps.\\nBesides, DiffiT \u001b[0m\n",
              "\u001b[32muses two hybrid hierarchical architectures for efficient denoising in the pixel space and the\\nlatent space, \u001b[0m\n",
              "\u001b[32mrespectively, and achieves new state-of-the-art results across various generation tasks. Overall,\\nthese \u001b[0m\n",
              "\u001b[32mstudies show promising results in employing vision transformers for image latent diffusion, paving the\\nway \u001b[0m\n",
              "\u001b[32mfor future studies for other modalities.\\nincluding time, condition, and noisy image patches, as tokens and \u001b[0m\n",
              "\u001b[32mpropose long skip connections between the\\nshallow and deep transformer layers. The results suggest that the \u001b[0m\n",
              "\u001b[32mdownsampling and upsampling operators\\nin CNN-based U-Net are not always necessary, and U-ViT achieves \u001b[0m\n",
              "\u001b[32mrecord-breaking FID scores in image\\nand text-to-image generation.\\nLike'\u001b[0m,\n",
              "    \u001b[32m'and text-to-image generation.\\nLike Masked AutoEncoder \u001b[0m\u001b[32m(\u001b[0m\u001b[32mMAE\u001b[0m\u001b[32m)\u001b[0m\u001b[32m \u001b[0m\u001b[32m[\u001b[0m\u001b[32m33\u001b[0m\u001b[32m]\u001b[0m\u001b[32m, Masked Diffusion Transformer \u001b[0m\u001b[32m(\u001b[0m\u001b[32mMDT\u001b[0m\u001b[32m)\u001b[0m\u001b[32m \u001b[0m\n",
              "\u001b[32m[\u001b[0m\u001b[32m55\u001b[0m\u001b[32m]\u001b[0m\u001b[32m incorporates mask\\nlatent modeling into the diffusion process to explicitly enhance contextual relation \u001b[0m\n",
              "\u001b[32mlearning among object\\nsemantic parts in image synthesis. Specifically, as shown in Figure 12, MDT uses a \u001b[0m\n",
              "\u001b[32mside-interpolated for an\\nadditional masked token reconstruction task during training to boost the training \u001b[0m\n",
              "\u001b[32mefficiency and learn pow-\\nerful context-aware positional embedding for inference. Compared to DiT \u001b[0m\u001b[32m[\u001b[0m\u001b[32m4\u001b[0m\u001b[32m]\u001b[0m\u001b[32m, MDT \u001b[0m\n",
              "\u001b[32machieves better perfor-\\nmance and faster learning speed. Instead of using AdaLN \u001b[0m\u001b[32m(\u001b[0m\u001b[32mi.e., shifting and scaling\u001b[0m\u001b[32m)\u001b[0m\u001b[32m \u001b[0m\n",
              "\u001b[32mfor time-conditioning\\nmodeling, Hatamizadeh et al. \u001b[0m\u001b[32m[\u001b[0m\u001b[32m56\u001b[0m\u001b[32m]\u001b[0m\u001b[32m introduce Diffusion Vision Transformers \u001b[0m\u001b[32m(\u001b[0m\u001b[32mDiffiT\u001b[0m\u001b[32m)\u001b[0m\u001b[32m, \u001b[0m\n",
              "\u001b[32mwhich uses a time-\\ndependent self-attention \u001b[0m\u001b[32m(\u001b[0m\u001b[32mTMSA\u001b[0m\u001b[32m)\u001b[0m\u001b[32m module to model dynamic denoising behavior over sampling \u001b[0m\n",
              "\u001b[32mtime steps.\\nBesides, DiffiT uses two hybrid hierarchical architectures for efficient denoising in the pixel \u001b[0m\n",
              "\u001b[32mspace and the\\nlatent space, respectively, and achieves new state-of-the-art results across various generation\u001b[0m\n",
              "\u001b[32mtasks. Overall,\\nthese studies show promising results in employing vision transformers for image latent \u001b[0m\n",
              "\u001b[32mdiffusion, paving the\\nway for future studies for other modalities.\\nFigure 12: The overall framework of \u001b[0m\n",
              "\u001b[32mMasked Diffusion Transformer \u001b[0m\u001b[32m(\u001b[0m\u001b[32mMDT\u001b[0m\u001b[32m)\u001b[0m\u001b[32m. A solid/dotted line indicates\\nthe training/inference process for each \u001b[0m\n",
              "\u001b[32mtime step. Masking and side-interpolater are only used during training\\nand are removed during \u001b[0m\n",
              "\u001b[32minference.\\nVideo Diffusion Transformer. Building upon the foundational works in'\u001b[0m,\n",
              "    \u001b[32m'usion Transformer. Building upon the foundational works in text-to-image \u001b[0m\u001b[32m(\u001b[0m\u001b[32mT2I\u001b[0m\u001b[32m)\u001b[0m\u001b[32m diffusion mod-\\nels, \u001b[0m\n",
              "\u001b[32mrecent research has been focused on realizing the potential of diffusion transformers for text-to-video\\n\u001b[0m\u001b[32m(\u001b[0m\u001b[32mT2V\u001b[0m\u001b[32m)\u001b[0m\n",
              "\u001b[32mgeneration tasks. Due to the temporal nature of videos, key challenges for applying DiTs in the video\\ndomain \u001b[0m\n",
              "\u001b[32mare: i\u001b[0m\u001b[32m)\u001b[0m\u001b[32m how to compress the video spatially and temporally to a latent space for efficient denoising;\\nii\u001b[0m\u001b[32m)\u001b[0m\u001b[32m how\u001b[0m\n",
              "\u001b[32mto convert the compressed latent to patches and feed them to the transformer; and iii\u001b[0m\u001b[32m)\u001b[0m\u001b[32m how to \u001b[0m\n",
              "\u001b[32mhandle\\nlong-range temporal and spatial dependencies and ensure content consistency. Please refer to Section \u001b[0m\n",
              "\u001b[32m3.2.3\\nfor the first challenge. In this Section, we focus our discussion on transformer-based denoising \u001b[0m\n",
              "\u001b[32mnetwork ar-\\nchitectures designed to operate in the spatially and temporally compressed latent space. We give \u001b[0m\n",
              "\u001b[32ma detailed\\nreview of the two important works \u001b[0m\u001b[32m(\u001b[0m\u001b[32mImagen Video \u001b[0m\u001b[32m[\u001b[0m\u001b[32m29\u001b[0m\u001b[32m]\u001b[0m\u001b[32m and Video LDM \u001b[0m\u001b[32m[\u001b[0m\u001b[32m36\u001b[0m\u001b[32m]\u001b[0m\u001b[32m)\u001b[0m\u001b[32m described in the \u001b[0m\n",
              "\u001b[32mreference list\\nof the OpenAI Sora technique report.\\nImagen Video \u001b[0m\u001b[32m[\u001b[0m\u001b[32m29\u001b[0m\u001b[32m]\u001b[0m\u001b[32m, a text-to-video generation system \u001b[0m\n",
              "\u001b[32mdeveloped by Google Research, utilizes a cascade\\nof diffusion models, which consists of 7 sub-models that \u001b[0m\n",
              "\u001b[32mperform text-conditional video generation, spatial\\nsuper-resolution, and temporal super-resolution, to \u001b[0m\n",
              "\u001b[32mtransform textual prompts into high-definition videos.\\nAs shown in Figure 13, firstly, a frozen T5 text \u001b[0m\n",
              "\u001b[32mencoder generates contextual embeddings from the input text prompt. These embeddings are critical for aligning\u001b[0m\n",
              "\u001b[32mthe generated video with the text prompt and are\\ninjected into all models in the cascade, in addition to \u001b[0m\n",
              "\u001b[32mthe'\u001b[0m,\n",
              "    \u001b[32m' all models in the cascade, in addition to the base model. Subsequently, the embedding is fed to\\nthe \u001b[0m\n",
              "\u001b[32mbase model for low-resolution video generation, which is then refined by cascaded diffusion models \u001b[0m\n",
              "\u001b[32mto\\nincrease the resolution. The base video and super-resolution models use a 3D U-Net architecture in a \u001b[0m\n",
              "\u001b[32mspace-\\ntime separable fashion. This architecture weaves temporal attention and convolution layers with \u001b[0m\n",
              "\u001b[32mspatial\\ncounterparts to efficiently capture inter-frame dependencies. It employs v-prediction \u001b[0m\n",
              "\u001b[32mparameterization for\\nnumerical stability and conditioning augmentation to facilitate parallel training across\u001b[0m\n",
              "\u001b[32mmodels. The process\\ninvolves joint training on both images and videos, treating each image as a frame to \u001b[0m\n",
              "\u001b[32mleverage larger datasets,\\nand using classifier-free guidance \u001b[0m\u001b[32m[\u001b[0m\u001b[32m57\u001b[0m\u001b[32m]\u001b[0m\u001b[32m to enhance prompt fidelity. Progressive \u001b[0m\n",
              "\u001b[32mdistillation \u001b[0m\u001b[32m[\u001b[0m\u001b[32m58\u001b[0m\u001b[32m]\u001b[0m\u001b[32m is applied to\\nstreamline the sampling process, significantly reducing the computational \u001b[0m\n",
              "\u001b[32mload while maintaining perceptual\\nquality. Combining these methods and techniques allows Imagen Video to \u001b[0m\n",
              "\u001b[32mgenerate videos with not only\\nhigh fidelity but also remarkable controllability, as demonstrated by its \u001b[0m\n",
              "\u001b[32mability to produce diverse videos,\\ntext animations, and content in various artistic styles'\u001b[0m\n",
              "\u001b[1m]\u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">[</span>\n",
              "    <span style=\"color: #008000; text-decoration-color: #008000\">'Image Diffusion Transformer. Traditional diffusion models [51, 52, 53] mainly leverage </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">convolutional\\nU-Nets that include downsampling and upsampling blocks for the denoising network backbone. </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">However,\\nrecent studies show that the U-Net architecture is not crucial to the good performance of the </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">diffusion model.\\nBy incorporating a more flexible transformer architecture, the transformer-based diffusion </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">models can use\\nmore training data and larger model parameters. Along this line, DiT [4] and U-ViT [54] are </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">among the first\\nworks to employ vision transformers for latent diffusion models. As in ViT, DiT employs a </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">multi-head self-\\nattention layer and a pointwise feed-forward network interlaced with some layer norm and </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">scaling layers.\\nMoreover, as shown in Figure 11, DiT incorporates conditioning via adaptive layer norm </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">(AdaLN) with an\\nadditional MLP layer for zero-initializing, which initializes each residual block as an </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">identity function and\\nthus greatly stabilizes the training process. The scalability and flexibility of DiT is</span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">empirically validated.\\nDiT becomes the new backbone for diffusion models. In U-ViT, as shown in Figure 11, </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">they treat all inputs,\\nincluding time, condition, and noisy image patches, as tokens and propose long skip </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">connections between the\\nshallow and deep transformer layers. The results suggest that the downsampling and </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">upsampling operators\\nin CNN-based U-Net are not always necessary, and U-ViT achieves record-breaking FID </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">scores in image\\nand text-to-image generation.\\nLike Masked AutoEncoder (MAE)'</span>,\n",
              "    <span style=\"color: #008000; text-decoration-color: #008000\">'Like Masked AutoEncoder (MAE) [33], Masked Diffusion Transformer (MDT) [55] incorporates mask\\nlatent </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">modeling into the diffusion process to explicitly enhance contextual relation learning among object\\nsemantic </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">parts in image synthesis. Specifically, as shown in Figure 12, MDT uses a side-interpolated for an\\nadditional</span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">masked token reconstruction task during training to boost the training efficiency and learn pow-\\nerful </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">context-aware positional embedding for inference. Compared to DiT [4], MDT achieves better perfor-\\nmance and </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">faster learning speed. Instead of using AdaLN (i.e., shifting and scaling) for time-conditioning\\nmodeling, </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">Hatamizadeh et al. [56] introduce Diffusion Vision Transformers (DiffiT), which uses a time-\\ndependent </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">self-attention (TMSA) module to model dynamic denoising behavior over sampling time steps.\\nBesides, DiffiT </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">uses two hybrid hierarchical architectures for efficient denoising in the pixel space and the\\nlatent space, </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">respectively, and achieves new state-of-the-art results across various generation tasks. Overall,\\nthese </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">studies show promising results in employing vision transformers for image latent diffusion, paving the\\nway </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">for future studies for other modalities.\\nincluding time, condition, and noisy image patches, as tokens and </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">propose long skip connections between the\\nshallow and deep transformer layers. The results suggest that the </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">downsampling and upsampling operators\\nin CNN-based U-Net are not always necessary, and U-ViT achieves </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">record-breaking FID scores in image\\nand text-to-image generation.\\nLike'</span>,\n",
              "    <span style=\"color: #008000; text-decoration-color: #008000\">'and text-to-image generation.\\nLike Masked AutoEncoder (MAE) [33], Masked Diffusion Transformer (MDT) </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">[55] incorporates mask\\nlatent modeling into the diffusion process to explicitly enhance contextual relation </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">learning among object\\nsemantic parts in image synthesis. Specifically, as shown in Figure 12, MDT uses a </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">side-interpolated for an\\nadditional masked token reconstruction task during training to boost the training </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">efficiency and learn pow-\\nerful context-aware positional embedding for inference. Compared to DiT [4], MDT </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">achieves better perfor-\\nmance and faster learning speed. Instead of using AdaLN (i.e., shifting and scaling) </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">for time-conditioning\\nmodeling, Hatamizadeh et al. [56] introduce Diffusion Vision Transformers (DiffiT), </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">which uses a time-\\ndependent self-attention (TMSA) module to model dynamic denoising behavior over sampling </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">time steps.\\nBesides, DiffiT uses two hybrid hierarchical architectures for efficient denoising in the pixel </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">space and the\\nlatent space, respectively, and achieves new state-of-the-art results across various generation</span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">tasks. Overall,\\nthese studies show promising results in employing vision transformers for image latent </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">diffusion, paving the\\nway for future studies for other modalities.\\nFigure 12: The overall framework of </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">Masked Diffusion Transformer (MDT). A solid/dotted line indicates\\nthe training/inference process for each </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">time step. Masking and side-interpolater are only used during training\\nand are removed during </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">inference.\\nVideo Diffusion Transformer. Building upon the foundational works in'</span>,\n",
              "    <span style=\"color: #008000; text-decoration-color: #008000\">'usion Transformer. Building upon the foundational works in text-to-image (T2I) diffusion mod-\\nels, </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">recent research has been focused on realizing the potential of diffusion transformers for text-to-video\\n(T2V)</span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">generation tasks. Due to the temporal nature of videos, key challenges for applying DiTs in the video\\ndomain </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">are: i) how to compress the video spatially and temporally to a latent space for efficient denoising;\\nii) how</span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">to convert the compressed latent to patches and feed them to the transformer; and iii) how to </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">handle\\nlong-range temporal and spatial dependencies and ensure content consistency. Please refer to Section </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">3.2.3\\nfor the first challenge. In this Section, we focus our discussion on transformer-based denoising </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">network ar-\\nchitectures designed to operate in the spatially and temporally compressed latent space. We give </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">a detailed\\nreview of the two important works (Imagen Video [29] and Video LDM [36]) described in the </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">reference list\\nof the OpenAI Sora technique report.\\nImagen Video [29], a text-to-video generation system </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">developed by Google Research, utilizes a cascade\\nof diffusion models, which consists of 7 sub-models that </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">perform text-conditional video generation, spatial\\nsuper-resolution, and temporal super-resolution, to </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">transform textual prompts into high-definition videos.\\nAs shown in Figure 13, firstly, a frozen T5 text </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">encoder generates contextual embeddings from the input text prompt. These embeddings are critical for aligning</span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">the generated video with the text prompt and are\\ninjected into all models in the cascade, in addition to </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">the'</span>,\n",
              "    <span style=\"color: #008000; text-decoration-color: #008000\">' all models in the cascade, in addition to the base model. Subsequently, the embedding is fed to\\nthe </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">base model for low-resolution video generation, which is then refined by cascaded diffusion models </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">to\\nincrease the resolution. The base video and super-resolution models use a 3D U-Net architecture in a </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">space-\\ntime separable fashion. This architecture weaves temporal attention and convolution layers with </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">spatial\\ncounterparts to efficiently capture inter-frame dependencies. It employs v-prediction </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">parameterization for\\nnumerical stability and conditioning augmentation to facilitate parallel training across</span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">models. The process\\ninvolves joint training on both images and videos, treating each image as a frame to </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">leverage larger datasets,\\nand using classifier-free guidance [57] to enhance prompt fidelity. Progressive </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">distillation [58] is applied to\\nstreamline the sampling process, significantly reducing the computational </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">load while maintaining perceptual\\nquality. Combining these methods and techniques allows Imagen Video to </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">generate videos with not only\\nhigh fidelity but also remarkable controllability, as demonstrated by its </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">ability to produce diverse videos,\\ntext animations, and content in various artistic styles'</span>\n",
              "<span style=\"font-weight: bold\">]</span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "console.print(len(splitted_text))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        },
        "id": "YNGj8Ca6XwkO",
        "outputId": "82ac49a1-fef5-4841-f9c1-f83c6434dbcb"
      },
      "execution_count": 61,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1;36m5\u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "console.print(splitted_text[1])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 305
        },
        "id": "umGhmwalYBqH",
        "outputId": "382f0fb1-6165-4298-93f0-63a15c009514"
      },
      "execution_count": 62,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Like Masked AutoEncoder \u001b[1m(\u001b[0mMAE\u001b[1m)\u001b[0m \u001b[1m[\u001b[0m\u001b[1;36m33\u001b[0m\u001b[1m]\u001b[0m, Masked Diffusion Transformer \u001b[1m(\u001b[0mMDT\u001b[1m)\u001b[0m \u001b[1m[\u001b[0m\u001b[1;36m55\u001b[0m\u001b[1m]\u001b[0m incorporates mask\n",
              "latent modeling into the diffusion process to explicitly enhance contextual relation learning among object\n",
              "semantic parts in image synthesis. Specifically, as shown in Figure \u001b[1;36m12\u001b[0m, MDT uses a side-interpolated for an\n",
              "additional masked token reconstruction task during training to boost the training efficiency and learn pow-\n",
              "erful context-aware positional embedding for inference. Compared to DiT \u001b[1m[\u001b[0m\u001b[1;36m4\u001b[0m\u001b[1m]\u001b[0m, MDT achieves better perfor-\n",
              "mance and faster learning speed. Instead of using AdaLN \u001b[1m(\u001b[0mi.e., shifting and scaling\u001b[1m)\u001b[0m for time-conditioning\n",
              "modeling, Hatamizadeh et al. \u001b[1m[\u001b[0m\u001b[1;36m56\u001b[0m\u001b[1m]\u001b[0m introduce Diffusion Vision Transformers \u001b[1m(\u001b[0mDiffiT\u001b[1m)\u001b[0m, which uses a time-\n",
              "dependent self-attention \u001b[1m(\u001b[0mTMSA\u001b[1m)\u001b[0m module to model dynamic denoising behavior over sampling time steps.\n",
              "Besides, DiffiT uses two hybrid hierarchical architectures for efficient denoising in the pixel space and the\n",
              "latent space, respectively, and achieves new state-of-the-art results across various generation tasks. \n",
              "Overall,\n",
              "these studies show promising results in employing vision transformers for image latent diffusion, paving the\n",
              "way for future studies for other modalities.\n",
              "including time, condition, and noisy image patches, as tokens and propose long skip connections between the\n",
              "shallow and deep transformer layers. The results suggest that the downsampling and upsampling operators\n",
              "in CNN-based U-Net are not always necessary, and U-ViT achieves record-breaking FID scores in image\n",
              "and text-to-image generation.\n",
              "Like\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Like Masked AutoEncoder <span style=\"font-weight: bold\">(</span>MAE<span style=\"font-weight: bold\">)</span> <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">33</span><span style=\"font-weight: bold\">]</span>, Masked Diffusion Transformer <span style=\"font-weight: bold\">(</span>MDT<span style=\"font-weight: bold\">)</span> <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">55</span><span style=\"font-weight: bold\">]</span> incorporates mask\n",
              "latent modeling into the diffusion process to explicitly enhance contextual relation learning among object\n",
              "semantic parts in image synthesis. Specifically, as shown in Figure <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">12</span>, MDT uses a side-interpolated for an\n",
              "additional masked token reconstruction task during training to boost the training efficiency and learn pow-\n",
              "erful context-aware positional embedding for inference. Compared to DiT <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span><span style=\"font-weight: bold\">]</span>, MDT achieves better perfor-\n",
              "mance and faster learning speed. Instead of using AdaLN <span style=\"font-weight: bold\">(</span>i.e., shifting and scaling<span style=\"font-weight: bold\">)</span> for time-conditioning\n",
              "modeling, Hatamizadeh et al. <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">56</span><span style=\"font-weight: bold\">]</span> introduce Diffusion Vision Transformers <span style=\"font-weight: bold\">(</span>DiffiT<span style=\"font-weight: bold\">)</span>, which uses a time-\n",
              "dependent self-attention <span style=\"font-weight: bold\">(</span>TMSA<span style=\"font-weight: bold\">)</span> module to model dynamic denoising behavior over sampling time steps.\n",
              "Besides, DiffiT uses two hybrid hierarchical architectures for efficient denoising in the pixel space and the\n",
              "latent space, respectively, and achieves new state-of-the-art results across various generation tasks. \n",
              "Overall,\n",
              "these studies show promising results in employing vision transformers for image latent diffusion, paving the\n",
              "way for future studies for other modalities.\n",
              "including time, condition, and noisy image patches, as tokens and propose long skip connections between the\n",
              "shallow and deep transformer layers. The results suggest that the downsampling and upsampling operators\n",
              "in CNN-based U-Net are not always necessary, and U-ViT achieves record-breaking FID scores in image\n",
              "and text-to-image generation.\n",
              "Like\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "keys = []\n",
        "for i in trange(0, len(splitted_text)):\n",
        "  text = splitted_text[i]\n",
        "  keys.append({\n",
        "      'document': filename,\n",
        "      'title': title,\n",
        "      'author': author,\n",
        "      'url': url,\n",
        "      'doc': text,\n",
        "      'keywords': extract_keys(text, 1, 0.34)\n",
        "  })"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "b4dd8164c7fb43b09332c345740d3777",
            "786120cb17e142dfa35105a91c213d69"
          ]
        },
        "id": "a9iIYJazYa5o",
        "outputId": "6f2fecb1-bffa-42ca-c97f-f8e9b85700be"
      },
      "execution_count": 70,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Output()"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "b4dd8164c7fb43b09332c345740d3777"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Image \u001b[30;48;2;255;255;0mDiffusion\u001b[0m Transformer Traditional \u001b[30;48;2;255;255;0mdiffusion\u001b[0m models 51 52 53 mainly leverage \u001b[30;48;2;255;255;0mconvolutional\u001b[0m Nets that include \n",
              "\u001b[30;48;2;255;255;0mdownsampling\u001b[0m and upsampling blocks for the denoising network backbone However recent studies show that the \u001b[30;48;2;255;255;0mNet\u001b[0m \n",
              "architecture is not crucial to the good performance of the \u001b[30;48;2;255;255;0mdiffusion\u001b[0m model By incorporating more flexible \n",
              "transformer architecture the transformer based \u001b[30;48;2;255;255;0mdiffusion\u001b[0m models can use more training data and larger model \n",
              "parameters Along this line DiT and ViT 54 are among the first works to employ vision transformers for latent \n",
              "\u001b[30;48;2;255;255;0mdiffusion\u001b[0m models As in ViT DiT employs multi head self attention layer and pointwise feed forward network \n",
              "interlaced with some layer norm and scaling layers Moreover as shown in Figure 11 DiT incorporates conditioning via\n",
              "adaptive layer norm AdaLN with an additional MLP layer for zero initializing which initializes each residual block \n",
              "as an identity function and thus greatly stabilizes the training process The scalability and flexibility of DiT is \n",
              "empirically validated DiT becomes the new backbone for \u001b[30;48;2;255;255;0mdiffusion\u001b[0m models In ViT as shown in Figure 11 they treat all\n",
              "inputs including time condition and noisy image patches as tokens and propose long skip connections between the \n",
              "shallow and deep transformer layers The results suggest that the \u001b[30;48;2;255;255;0mdownsampling\u001b[0m and upsampling operators in CNN based\n",
              "\u001b[30;48;2;255;255;0mNet\u001b[0m are not always necessary and ViT achieves record breaking FID scores in image and text to image generation Like\n",
              "Masked \u001b[30;48;2;255;255;0mAutoEncoder\u001b[0m MAE\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Image <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">Diffusion</span> Transformer Traditional <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">diffusion</span> models 51 52 53 mainly leverage <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">convolutional</span> Nets that include \n",
              "<span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">downsampling</span> and upsampling blocks for the denoising network backbone However recent studies show that the <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">Net</span> \n",
              "architecture is not crucial to the good performance of the <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">diffusion</span> model By incorporating more flexible \n",
              "transformer architecture the transformer based <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">diffusion</span> models can use more training data and larger model \n",
              "parameters Along this line DiT and ViT 54 are among the first works to employ vision transformers for latent \n",
              "<span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">diffusion</span> models As in ViT DiT employs multi head self attention layer and pointwise feed forward network \n",
              "interlaced with some layer norm and scaling layers Moreover as shown in Figure 11 DiT incorporates conditioning via\n",
              "adaptive layer norm AdaLN with an additional MLP layer for zero initializing which initializes each residual block \n",
              "as an identity function and thus greatly stabilizes the training process The scalability and flexibility of DiT is \n",
              "empirically validated DiT becomes the new backbone for <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">diffusion</span> models In ViT as shown in Figure 11 they treat all\n",
              "inputs including time condition and noisy image patches as tokens and propose long skip connections between the \n",
              "shallow and deep transformer layers The results suggest that the <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">downsampling</span> and upsampling operators in CNN based\n",
              "<span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">Net</span> are not always necessary and ViT achieves record breaking FID scores in image and text to image generation Like\n",
              "Masked <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">AutoEncoder</span> MAE\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Like Masked \u001b[30;48;2;255;255;0mAutoEncoder\u001b[0m MAE 33 Masked \u001b[30;48;2;255;255;0mDiffusion\u001b[0m Transformer \u001b[30;48;2;255;255;0mMDT\u001b[0m 55 incorporates mask latent modeling into the \n",
              "\u001b[30;48;2;255;255;0mdiffusion\u001b[0m process to explicitly enhance \u001b[30;48;2;255;255;0mcontextual\u001b[0m relation learning among object semantic parts in image synthesis\n",
              "Specifically as shown in Figure 12 \u001b[30;48;2;255;255;0mMDT\u001b[0m uses side interpolated for an additional masked token reconstruction task \n",
              "during training to boost the training efficiency and learn pow erful context aware positional embedding for \n",
              "inference Compared to DiT \u001b[30;48;2;255;255;0mMDT\u001b[0m achieves better perfor mance and faster learning speed Instead of using AdaLN \n",
              "shifting and scaling for time conditioning modeling Hatamizadeh et al 56 introduce \u001b[30;48;2;255;255;0mDiffusion\u001b[0m Vision Transformers \n",
              "DiffiT which uses time dependent self attention TMSA module to model dynamic denoising behavior over sampling time \n",
              "steps Besides DiffiT uses two hybrid hierarchical architectures for efficient denoising in the pixel space and the \n",
              "latent space respectively and achieves new state of the art results across various generation tasks Overall these \n",
              "studies show promising results in employing vision transformers for image latent \u001b[30;48;2;255;255;0mdiffusion\u001b[0m paving the way for \n",
              "future studies for other modalities including time condition and noisy image patches as tokens and propose long \n",
              "skip connections between the shallow and deep transformer layers The results suggest that the \u001b[30;48;2;255;255;0mdownsampling\u001b[0m and \n",
              "upsampling operators in CNN based Net are not always necessary and ViT achieves record breaking FID scores in image\n",
              "and text to image generation Like\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Like Masked <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">AutoEncoder</span> MAE 33 Masked <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">Diffusion</span> Transformer <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">MDT</span> 55 incorporates mask latent modeling into the \n",
              "<span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">diffusion</span> process to explicitly enhance <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">contextual</span> relation learning among object semantic parts in image synthesis\n",
              "Specifically as shown in Figure 12 <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">MDT</span> uses side interpolated for an additional masked token reconstruction task \n",
              "during training to boost the training efficiency and learn pow erful context aware positional embedding for \n",
              "inference Compared to DiT <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">MDT</span> achieves better perfor mance and faster learning speed Instead of using AdaLN \n",
              "shifting and scaling for time conditioning modeling Hatamizadeh et al 56 introduce <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">Diffusion</span> Vision Transformers \n",
              "DiffiT which uses time dependent self attention TMSA module to model dynamic denoising behavior over sampling time \n",
              "steps Besides DiffiT uses two hybrid hierarchical architectures for efficient denoising in the pixel space and the \n",
              "latent space respectively and achieves new state of the art results across various generation tasks Overall these \n",
              "studies show promising results in employing vision transformers for image latent <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">diffusion</span> paving the way for \n",
              "future studies for other modalities including time condition and noisy image patches as tokens and propose long \n",
              "skip connections between the shallow and deep transformer layers The results suggest that the <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">downsampling</span> and \n",
              "upsampling operators in CNN based Net are not always necessary and ViT achieves record breaking FID scores in image\n",
              "and text to image generation Like\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "and text to image generation Like Masked \u001b[30;48;2;255;255;0mAutoEncoder\u001b[0m MAE 33 Masked \u001b[30;48;2;255;255;0mDiffusion\u001b[0m Transformer \u001b[30;48;2;255;255;0mMDT\u001b[0m 55 incorporates mask \n",
              "latent modeling into the \u001b[30;48;2;255;255;0mdiffusion\u001b[0m process to explicitly enhance \u001b[30;48;2;255;255;0mcontextual\u001b[0m relation learning among object semantic\n",
              "parts in image synthesis Specifically as shown in Figure 12 \u001b[30;48;2;255;255;0mMDT\u001b[0m uses side interpolated for an additional masked \n",
              "token reconstruction task during training to boost the training efficiency and learn pow erful context aware \n",
              "positional embedding for inference Compared to DiT \u001b[30;48;2;255;255;0mMDT\u001b[0m achieves better perfor mance and faster learning speed \n",
              "Instead of using AdaLN shifting and scaling for time conditioning modeling Hatamizadeh et al 56 introduce \u001b[30;48;2;255;255;0mDiffusion\u001b[0m\n",
              "Vision Transformers \u001b[30;48;2;255;255;0mDiffiT\u001b[0m which uses time dependent self attention TMSA module to model dynamic denoising behavior\n",
              "over sampling time steps Besides \u001b[30;48;2;255;255;0mDiffiT\u001b[0m uses two hybrid hierarchical architectures for efficient denoising in the \n",
              "pixel space and the latent space respectively and achieves new state of the art results across various generation \n",
              "tasks Overall these studies show promising results in employing vision transformers for image latent \u001b[30;48;2;255;255;0mdiffusion\u001b[0m \n",
              "paving the way for future studies for other modalities Figure 12 The overall framework of Masked \u001b[30;48;2;255;255;0mDiffusion\u001b[0m \n",
              "Transformer \u001b[30;48;2;255;255;0mMDT\u001b[0m solid dotted line indicates the training inference process for each time step Masking and side \n",
              "interpolater are only used during training and are removed during inference Video \u001b[30;48;2;255;255;0mDiffusion\u001b[0m Transformer Building \n",
              "upon the foundational works in\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">and text to image generation Like Masked <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">AutoEncoder</span> MAE 33 Masked <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">Diffusion</span> Transformer <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">MDT</span> 55 incorporates mask \n",
              "latent modeling into the <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">diffusion</span> process to explicitly enhance <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">contextual</span> relation learning among object semantic\n",
              "parts in image synthesis Specifically as shown in Figure 12 <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">MDT</span> uses side interpolated for an additional masked \n",
              "token reconstruction task during training to boost the training efficiency and learn pow erful context aware \n",
              "positional embedding for inference Compared to DiT <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">MDT</span> achieves better perfor mance and faster learning speed \n",
              "Instead of using AdaLN shifting and scaling for time conditioning modeling Hatamizadeh et al 56 introduce <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">Diffusion</span>\n",
              "Vision Transformers <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">DiffiT</span> which uses time dependent self attention TMSA module to model dynamic denoising behavior\n",
              "over sampling time steps Besides <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">DiffiT</span> uses two hybrid hierarchical architectures for efficient denoising in the \n",
              "pixel space and the latent space respectively and achieves new state of the art results across various generation \n",
              "tasks Overall these studies show promising results in employing vision transformers for image latent <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">diffusion</span> \n",
              "paving the way for future studies for other modalities Figure 12 The overall framework of Masked <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">Diffusion</span> \n",
              "Transformer <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">MDT</span> solid dotted line indicates the training inference process for each time step Masking and side \n",
              "interpolater are only used during training and are removed during inference Video <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">Diffusion</span> Transformer Building \n",
              "upon the foundational works in\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "usion \u001b[30;48;2;255;255;0mTransformer\u001b[0m Building upon the foundational works in text to image \u001b[30;48;2;255;255;0mT2I\u001b[0m diffusion mod els recent research has \n",
              "been focused on realizing the potential of diffusion transformers for text to video T2V generation tasks Due to the\n",
              "temporal nature of \u001b[30;48;2;255;255;0mvideos\u001b[0m key challenges for applying DiTs in the video domain are how to compress the video \n",
              "spatially and temporally to latent space for efficient denoising ii how to convert the compressed latent to patches\n",
              "and feed them to the \u001b[30;48;2;255;255;0mtransformer\u001b[0m and iii how to handle long range temporal and spatial dependencies and ensure \n",
              "content consistency Please refer to Section for the first challenge In this Section we focus our discussion on \n",
              "\u001b[30;48;2;255;255;0mtransformer\u001b[0m based denoising network ar chitectures designed to operate in the spatially and temporally compressed \n",
              "latent space We give detailed review of the two important works Imagen Video 29 and Video LDM 36 described in the \n",
              "reference list of the OpenAI Sora technique report Imagen Video 29 text to video generation system developed by \n",
              "Google Research utilizes cascade of diffusion models which consists of sub models that perform text conditional \n",
              "video generation spatial super resolution and temporal super resolution to transform textual prompts into high \n",
              "definition \u001b[30;48;2;255;255;0mvideos\u001b[0m As shown in Figure 13 firstly frozen T5 text encoder generates \u001b[30;48;2;255;255;0mcontextual\u001b[0m \u001b[30;48;2;255;255;0membeddings\u001b[0m from the \n",
              "input text prompt These \u001b[30;48;2;255;255;0membeddings\u001b[0m are critical for aligning the generated video with the text prompt and are \n",
              "injected into all models in the cascade in addition to the\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">usion <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">Transformer</span> Building upon the foundational works in text to image <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">T2I</span> diffusion mod els recent research has \n",
              "been focused on realizing the potential of diffusion transformers for text to video T2V generation tasks Due to the\n",
              "temporal nature of <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">videos</span> key challenges for applying DiTs in the video domain are how to compress the video \n",
              "spatially and temporally to latent space for efficient denoising ii how to convert the compressed latent to patches\n",
              "and feed them to the <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">transformer</span> and iii how to handle long range temporal and spatial dependencies and ensure \n",
              "content consistency Please refer to Section for the first challenge In this Section we focus our discussion on \n",
              "<span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">transformer</span> based denoising network ar chitectures designed to operate in the spatially and temporally compressed \n",
              "latent space We give detailed review of the two important works Imagen Video 29 and Video LDM 36 described in the \n",
              "reference list of the OpenAI Sora technique report Imagen Video 29 text to video generation system developed by \n",
              "Google Research utilizes cascade of diffusion models which consists of sub models that perform text conditional \n",
              "video generation spatial super resolution and temporal super resolution to transform textual prompts into high \n",
              "definition <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">videos</span> As shown in Figure 13 firstly frozen T5 text encoder generates <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">contextual</span> <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">embeddings</span> from the \n",
              "input text prompt These <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">embeddings</span> are critical for aligning the generated video with the text prompt and are \n",
              "injected into all models in the cascade in addition to the\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "all models in the cascade in addition to the base model Subsequently the \u001b[30;48;2;255;255;0membedding\u001b[0m is fed to the base model for low\n",
              "resolution video generation which is then refined by cascaded diffusion models to increase the resolution The base \n",
              "video and super resolution models use \u001b[30;48;2;255;255;0m3D\u001b[0m Net architecture in space time separable fashion This architecture weaves \n",
              "temporal attention and \u001b[30;48;2;255;255;0mconvolution\u001b[0m layers with spatial counterparts to efficiently capture inter frame dependencies\n",
              "It employs prediction parameterization for numerical stability and conditioning augmentation to facilitate \u001b[30;48;2;255;255;0mparallel\u001b[0m\n",
              "training across models The process involves joint training on both images and \u001b[30;48;2;255;255;0mvideos\u001b[0m treating each image as frame \n",
              "to leverage larger datasets and using classifier free guidance 57 to enhance prompt fidelity Progressive \n",
              "distillation 58 is applied to streamline the sampling process significantly reducing the computational load while \n",
              "maintaining perceptual quality Combining these methods and techniques allows Imagen Video to generate \u001b[30;48;2;255;255;0mvideos\u001b[0m with \n",
              "not only high fidelity but also remarkable controllability as demonstrated by its ability to produce diverse \u001b[30;48;2;255;255;0mvideos\u001b[0m\n",
              "text animations and content in various artistic styles\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">all models in the cascade in addition to the base model Subsequently the <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">embedding</span> is fed to the base model for low\n",
              "resolution video generation which is then refined by cascaded diffusion models to increase the resolution The base \n",
              "video and super resolution models use <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">3D</span> Net architecture in space time separable fashion This architecture weaves \n",
              "temporal attention and <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">convolution</span> layers with spatial counterparts to efficiently capture inter frame dependencies\n",
              "It employs prediction parameterization for numerical stability and conditioning augmentation to facilitate <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">parallel</span>\n",
              "training across models The process involves joint training on both images and <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">videos</span> treating each image as frame \n",
              "to leverage larger datasets and using classifier free guidance 57 to enhance prompt fidelity Progressive \n",
              "distillation 58 is applied to streamline the sampling process significantly reducing the computational load while \n",
              "maintaining perceptual quality Combining these methods and techniques allows Imagen Video to generate <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">videos</span> with \n",
              "not only high fidelity but also remarkable controllability as demonstrated by its ability to produce diverse <span style=\"color: #000000; text-decoration-color: #000000; background-color: #ffff00\">videos</span>\n",
              "text animations and content in various artistic styles\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "console.print(keys[0])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 385
        },
        "id": "gVRVPIQFZuCY",
        "outputId": "bf697a70-ba9a-4bee-a93e-6a81131c521e"
      },
      "execution_count": 71,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1m{\u001b[0m\n",
              "    \u001b[32m'document'\u001b[0m: \u001b[32m'sora.txt'\u001b[0m,\n",
              "    \u001b[32m'title'\u001b[0m: \u001b[32m'Sora Diffusion Transformers'\u001b[0m,\n",
              "    \u001b[32m'author'\u001b[0m: \u001b[32m'Yixin Liu, Kai Zhang, Yuan Li, Zhiling Yan, Chujie Gao, Ruoxi Chen, Zhengqing Yuan, Yue Huang, \u001b[0m\n",
              "\u001b[32mHanchi Sun, Jianfeng Gao, Lifang He, Lichao Sun'\u001b[0m,\n",
              "    \u001b[32m'url'\u001b[0m: \u001b[32m'https://arxiv.org/pdf/2402.17177v2.pdf'\u001b[0m,\n",
              "    \u001b[32m'doc'\u001b[0m: \u001b[32m'Image Diffusion Transformer. Traditional diffusion models \u001b[0m\u001b[32m[\u001b[0m\u001b[32m51, 52, 53\u001b[0m\u001b[32m]\u001b[0m\u001b[32m mainly leverage \u001b[0m\n",
              "\u001b[32mconvolutional\\nU-Nets that include downsampling and upsampling blocks for the denoising network backbone. \u001b[0m\n",
              "\u001b[32mHowever,\\nrecent studies show that the U-Net architecture is not crucial to the good performance of the \u001b[0m\n",
              "\u001b[32mdiffusion model.\\nBy incorporating a more flexible transformer architecture, the transformer-based diffusion \u001b[0m\n",
              "\u001b[32mmodels can use\\nmore training data and larger model parameters. Along this line, DiT \u001b[0m\u001b[32m[\u001b[0m\u001b[32m4\u001b[0m\u001b[32m]\u001b[0m\u001b[32m and U-ViT \u001b[0m\u001b[32m[\u001b[0m\u001b[32m54\u001b[0m\u001b[32m]\u001b[0m\u001b[32m are \u001b[0m\n",
              "\u001b[32mamong the first\\nworks to employ vision transformers for latent diffusion models. As in ViT, DiT employs a \u001b[0m\n",
              "\u001b[32mmulti-head self-\\nattention layer and a pointwise feed-forward network interlaced with some layer norm and \u001b[0m\n",
              "\u001b[32mscaling layers.\\nMoreover, as shown in Figure 11, DiT incorporates conditioning via adaptive layer norm \u001b[0m\n",
              "\u001b[32m(\u001b[0m\u001b[32mAdaLN\u001b[0m\u001b[32m)\u001b[0m\u001b[32m with an\\nadditional MLP layer for zero-initializing, which initializes each residual block as an \u001b[0m\n",
              "\u001b[32midentity function and\\nthus greatly stabilizes the training process. The scalability and flexibility of DiT is\u001b[0m\n",
              "\u001b[32mempirically validated.\\nDiT becomes the new backbone for diffusion models. In U-ViT, as shown in Figure 11, \u001b[0m\n",
              "\u001b[32mthey treat all inputs,\\nincluding time, condition, and noisy image patches, as tokens and propose long skip \u001b[0m\n",
              "\u001b[32mconnections between the\\nshallow and deep transformer layers. The results suggest that the downsampling and \u001b[0m\n",
              "\u001b[32mupsampling operators\\nin CNN-based U-Net are not always necessary, and U-ViT achieves record-breaking FID \u001b[0m\n",
              "\u001b[32mscores in image\\nand text-to-image generation.\\nLike Masked AutoEncoder \u001b[0m\u001b[32m(\u001b[0m\u001b[32mMAE\u001b[0m\u001b[32m)\u001b[0m\u001b[32m'\u001b[0m,\n",
              "    \u001b[32m'keywords'\u001b[0m: \u001b[1m[\u001b[0m\u001b[32m'convolutional'\u001b[0m, \u001b[32m'autoencoder'\u001b[0m, \u001b[32m'diffusion'\u001b[0m, \u001b[32m'downsampling'\u001b[0m, \u001b[32m'net'\u001b[0m\u001b[1m]\u001b[0m\n",
              "\u001b[1m}\u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">{</span>\n",
              "    <span style=\"color: #008000; text-decoration-color: #008000\">'document'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'sora.txt'</span>,\n",
              "    <span style=\"color: #008000; text-decoration-color: #008000\">'title'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'Sora Diffusion Transformers'</span>,\n",
              "    <span style=\"color: #008000; text-decoration-color: #008000\">'author'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'Yixin Liu, Kai Zhang, Yuan Li, Zhiling Yan, Chujie Gao, Ruoxi Chen, Zhengqing Yuan, Yue Huang, </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">Hanchi Sun, Jianfeng Gao, Lifang He, Lichao Sun'</span>,\n",
              "    <span style=\"color: #008000; text-decoration-color: #008000\">'url'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://arxiv.org/pdf/2402.17177v2.pdf'</span>,\n",
              "    <span style=\"color: #008000; text-decoration-color: #008000\">'doc'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'Image Diffusion Transformer. Traditional diffusion models [51, 52, 53] mainly leverage </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">convolutional\\nU-Nets that include downsampling and upsampling blocks for the denoising network backbone. </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">However,\\nrecent studies show that the U-Net architecture is not crucial to the good performance of the </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">diffusion model.\\nBy incorporating a more flexible transformer architecture, the transformer-based diffusion </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">models can use\\nmore training data and larger model parameters. Along this line, DiT [4] and U-ViT [54] are </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">among the first\\nworks to employ vision transformers for latent diffusion models. As in ViT, DiT employs a </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">multi-head self-\\nattention layer and a pointwise feed-forward network interlaced with some layer norm and </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">scaling layers.\\nMoreover, as shown in Figure 11, DiT incorporates conditioning via adaptive layer norm </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">(AdaLN) with an\\nadditional MLP layer for zero-initializing, which initializes each residual block as an </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">identity function and\\nthus greatly stabilizes the training process. The scalability and flexibility of DiT is</span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">empirically validated.\\nDiT becomes the new backbone for diffusion models. In U-ViT, as shown in Figure 11, </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">they treat all inputs,\\nincluding time, condition, and noisy image patches, as tokens and propose long skip </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">connections between the\\nshallow and deep transformer layers. The results suggest that the downsampling and </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">upsampling operators\\nin CNN-based U-Net are not always necessary, and U-ViT achieves record-breaking FID </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">scores in image\\nand text-to-image generation.\\nLike Masked AutoEncoder (MAE)'</span>,\n",
              "    <span style=\"color: #008000; text-decoration-color: #008000\">'keywords'</span>: <span style=\"font-weight: bold\">[</span><span style=\"color: #008000; text-decoration-color: #008000\">'convolutional'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'autoencoder'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'diffusion'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'downsampling'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'net'</span><span style=\"font-weight: bold\">]</span>\n",
              "<span style=\"font-weight: bold\">}</span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.schema.document import Document\n",
        "\n",
        "Documents = []\n",
        "for i in range(0, len(keys)):\n",
        "  Documents.append(Document(page_content=keys[i]['doc'], metadata= {\n",
        "      'source': keys[i]['document'],\n",
        "      'type': 'chunk',\n",
        "      'author': keys[i]['author'],\n",
        "      'url': keys[i]['url'],\n",
        "      'keywords': keys[i]['keywords']\n",
        "  }))"
      ],
      "metadata": {
        "id": "6Znxxr9RZynv"
      },
      "execution_count": 72,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "console.print(Documents[1])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 417
        },
        "id": "wKOZHefraAvY",
        "outputId": "9598e2bb-4501-4194-c6b6-e4fb51a6f797"
      },
      "execution_count": 73,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1;35mDocument\u001b[0m\u001b[1m(\u001b[0m\n",
              "    \u001b[33mpage_content\u001b[0m=\u001b[32m'Like Masked AutoEncoder \u001b[0m\u001b[32m(\u001b[0m\u001b[32mMAE\u001b[0m\u001b[32m)\u001b[0m\u001b[32m \u001b[0m\u001b[32m[\u001b[0m\u001b[32m33\u001b[0m\u001b[32m]\u001b[0m\u001b[32m, Masked Diffusion Transformer \u001b[0m\u001b[32m(\u001b[0m\u001b[32mMDT\u001b[0m\u001b[32m)\u001b[0m\u001b[32m \u001b[0m\u001b[32m[\u001b[0m\u001b[32m55\u001b[0m\u001b[32m]\u001b[0m\u001b[32m incorporates \u001b[0m\n",
              "\u001b[32mmask\\nlatent modeling into the diffusion process to explicitly enhance contextual relation learning among \u001b[0m\n",
              "\u001b[32mobject\\nsemantic parts in image synthesis. Specifically, as shown in Figure 12, MDT uses a side-interpolated \u001b[0m\n",
              "\u001b[32mfor an\\nadditional masked token reconstruction task during training to boost the training efficiency and learn\u001b[0m\n",
              "\u001b[32mpow-\\nerful context-aware positional embedding for inference. Compared to DiT \u001b[0m\u001b[32m[\u001b[0m\u001b[32m4\u001b[0m\u001b[32m]\u001b[0m\u001b[32m, MDT achieves better \u001b[0m\n",
              "\u001b[32mperfor-\\nmance and faster learning speed. Instead of using AdaLN \u001b[0m\u001b[32m(\u001b[0m\u001b[32mi.e., shifting and scaling\u001b[0m\u001b[32m)\u001b[0m\u001b[32m for \u001b[0m\n",
              "\u001b[32mtime-conditioning\\nmodeling, Hatamizadeh et al. \u001b[0m\u001b[32m[\u001b[0m\u001b[32m56\u001b[0m\u001b[32m]\u001b[0m\u001b[32m introduce Diffusion Vision Transformers \u001b[0m\u001b[32m(\u001b[0m\u001b[32mDiffiT\u001b[0m\u001b[32m)\u001b[0m\u001b[32m, which \u001b[0m\n",
              "\u001b[32muses a time-\\ndependent self-attention \u001b[0m\u001b[32m(\u001b[0m\u001b[32mTMSA\u001b[0m\u001b[32m)\u001b[0m\u001b[32m module to model dynamic denoising behavior over sampling time \u001b[0m\n",
              "\u001b[32msteps.\\nBesides, DiffiT uses two hybrid hierarchical architectures for efficient denoising in the pixel space \u001b[0m\n",
              "\u001b[32mand the\\nlatent space, respectively, and achieves new state-of-the-art results across various generation \u001b[0m\n",
              "\u001b[32mtasks. Overall,\\nthese studies show promising results in employing vision transformers for image latent \u001b[0m\n",
              "\u001b[32mdiffusion, paving the\\nway for future studies for other modalities.\\nincluding time, condition, and noisy \u001b[0m\n",
              "\u001b[32mimage patches, as tokens and propose long skip connections between the\\nshallow and deep transformer layers. \u001b[0m\n",
              "\u001b[32mThe results suggest that the downsampling and upsampling operators\\nin CNN-based U-Net are not always \u001b[0m\n",
              "\u001b[32mnecessary, and U-ViT achieves record-breaking FID scores in image\\nand text-to-image generation.\\nLike'\u001b[0m,\n",
              "    \u001b[33mmetadata\u001b[0m=\u001b[1m{\u001b[0m\n",
              "        \u001b[32m'source'\u001b[0m: \u001b[32m'sora.txt'\u001b[0m,\n",
              "        \u001b[32m'type'\u001b[0m: \u001b[32m'chunk'\u001b[0m,\n",
              "        \u001b[32m'author'\u001b[0m: \u001b[32m'Yixin Liu, Kai Zhang, Yuan Li, Zhiling Yan, Chujie Gao, Ruoxi Chen, Zhengqing Yuan, Yue \u001b[0m\n",
              "\u001b[32mHuang, Hanchi Sun, Jianfeng Gao, Lifang He, Lichao Sun'\u001b[0m,\n",
              "        \u001b[32m'url'\u001b[0m: \u001b[32m'https://arxiv.org/pdf/2402.17177v2.pdf'\u001b[0m,\n",
              "        \u001b[32m'keywords'\u001b[0m: \u001b[1m[\u001b[0m\u001b[32m'mdt'\u001b[0m, \u001b[32m'autoencoder'\u001b[0m, \u001b[32m'downsampling'\u001b[0m, \u001b[32m'diffusion'\u001b[0m, \u001b[32m'contextual'\u001b[0m\u001b[1m]\u001b[0m\n",
              "    \u001b[1m}\u001b[0m\n",
              "\u001b[1m)\u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Document</span><span style=\"font-weight: bold\">(</span>\n",
              "    <span style=\"color: #808000; text-decoration-color: #808000\">page_content</span>=<span style=\"color: #008000; text-decoration-color: #008000\">'Like Masked AutoEncoder (MAE) [33], Masked Diffusion Transformer (MDT) [55] incorporates </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">mask\\nlatent modeling into the diffusion process to explicitly enhance contextual relation learning among </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">object\\nsemantic parts in image synthesis. Specifically, as shown in Figure 12, MDT uses a side-interpolated </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">for an\\nadditional masked token reconstruction task during training to boost the training efficiency and learn</span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">pow-\\nerful context-aware positional embedding for inference. Compared to DiT [4], MDT achieves better </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">perfor-\\nmance and faster learning speed. Instead of using AdaLN (i.e., shifting and scaling) for </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">time-conditioning\\nmodeling, Hatamizadeh et al. [56] introduce Diffusion Vision Transformers (DiffiT), which </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">uses a time-\\ndependent self-attention (TMSA) module to model dynamic denoising behavior over sampling time </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">steps.\\nBesides, DiffiT uses two hybrid hierarchical architectures for efficient denoising in the pixel space </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">and the\\nlatent space, respectively, and achieves new state-of-the-art results across various generation </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">tasks. Overall,\\nthese studies show promising results in employing vision transformers for image latent </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">diffusion, paving the\\nway for future studies for other modalities.\\nincluding time, condition, and noisy </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">image patches, as tokens and propose long skip connections between the\\nshallow and deep transformer layers. </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">The results suggest that the downsampling and upsampling operators\\nin CNN-based U-Net are not always </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">necessary, and U-ViT achieves record-breaking FID scores in image\\nand text-to-image generation.\\nLike'</span>,\n",
              "    <span style=\"color: #808000; text-decoration-color: #808000\">metadata</span>=<span style=\"font-weight: bold\">{</span>\n",
              "        <span style=\"color: #008000; text-decoration-color: #008000\">'source'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'sora.txt'</span>,\n",
              "        <span style=\"color: #008000; text-decoration-color: #008000\">'type'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'chunk'</span>,\n",
              "        <span style=\"color: #008000; text-decoration-color: #008000\">'author'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'Yixin Liu, Kai Zhang, Yuan Li, Zhiling Yan, Chujie Gao, Ruoxi Chen, Zhengqing Yuan, Yue </span>\n",
              "<span style=\"color: #008000; text-decoration-color: #008000\">Huang, Hanchi Sun, Jianfeng Gao, Lifang He, Lichao Sun'</span>,\n",
              "        <span style=\"color: #008000; text-decoration-color: #008000\">'url'</span>: <span style=\"color: #008000; text-decoration-color: #008000\">'https://arxiv.org/pdf/2402.17177v2.pdf'</span>,\n",
              "        <span style=\"color: #008000; text-decoration-color: #008000\">'keywords'</span>: <span style=\"font-weight: bold\">[</span><span style=\"color: #008000; text-decoration-color: #008000\">'mdt'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'autoencoder'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'downsampling'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'diffusion'</span>, <span style=\"color: #008000; text-decoration-color: #008000\">'contextual'</span><span style=\"font-weight: bold\">]</span>\n",
              "    <span style=\"font-weight: bold\">}</span>\n",
              "<span style=\"font-weight: bold\">)</span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "kNWUGlhQwkiD"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}