Similar_Article.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "machine_shape": "hm",
      "gpuType": "T4",
      "authorship_tag": "ABX9TyPLw316RXwOKkp+W5vcMtLH",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "790f21fd485e416eadfad7e26a58dda8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_0992530c78a4432480932f7b3b4531a9",
              "IPY_MODEL_60d737be68ff4c26aa1f322751074f68",
              "IPY_MODEL_ed4e8e87f9be47a3a2dd9cf23aee245f"
            ],
            "layout": "IPY_MODEL_63ed7fdc7c2146059acffd5167b7c1d1"
          }
        },
        "0992530c78a4432480932f7b3b4531a9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ee0666dfb5124a7d828c41de1ac4d8ca",
            "placeholder": "​",
            "style": "IPY_MODEL_c4c9922e9aae493d966db37a5fe363bf",
            "value": "Downloading (…)solve/main/vocab.txt: 100%"
          }
        },
        "60d737be68ff4c26aa1f322751074f68": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_97f671a883fe4c4da13764cbfe1ca343",
            "max": 231508,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_2daf299bd7c24729bf4fc471dac28cb4",
            "value": 231508
          }
        },
        "ed4e8e87f9be47a3a2dd9cf23aee245f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6a7a9bca61094ea596351d1418b568a1",
            "placeholder": "​",
            "style": "IPY_MODEL_114d0deaaae74ac8a8db599d02da77c7",
            "value": " 232k/232k [00:00&lt;00:00, 594kB/s]"
          }
        },
        "63ed7fdc7c2146059acffd5167b7c1d1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ee0666dfb5124a7d828c41de1ac4d8ca": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c4c9922e9aae493d966db37a5fe363bf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "97f671a883fe4c4da13764cbfe1ca343": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2daf299bd7c24729bf4fc471dac28cb4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "6a7a9bca61094ea596351d1418b568a1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "114d0deaaae74ac8a8db599d02da77c7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2fb2f92909444a678a4bb4e48e321bb7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_cb1eea8a518e4b8495a0219200a5def9",
              "IPY_MODEL_a7a754bb09844ca7a4401849d529ff97",
              "IPY_MODEL_3a499783554449e1bbcec99184ff70ed"
            ],
            "layout": "IPY_MODEL_a96c1d77427b4465881be2a633ca8609"
          }
        },
        "cb1eea8a518e4b8495a0219200a5def9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_dff3d3fdbbac4376ad54fe49d37b8f6b",
            "placeholder": "​",
            "style": "IPY_MODEL_8f271bda1c994eceb78e4ea74e858136",
            "value": "Downloading (…)okenizer_config.json: 100%"
          }
        },
        "a7a754bb09844ca7a4401849d529ff97": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8c7e0ef42092460ebecae8ed94f9fb9c",
            "max": 28,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_892740163b654c109be8a3e09d255777",
            "value": 28
          }
        },
        "3a499783554449e1bbcec99184ff70ed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a3d9c8fb28014531b3974edb8fbe4cd7",
            "placeholder": "​",
            "style": "IPY_MODEL_36aef634e2ae4392a46fa8c94f7a7fab",
            "value": " 28.0/28.0 [00:00&lt;00:00, 1.97kB/s]"
          }
        },
        "a96c1d77427b4465881be2a633ca8609": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dff3d3fdbbac4376ad54fe49d37b8f6b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8f271bda1c994eceb78e4ea74e858136": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8c7e0ef42092460ebecae8ed94f9fb9c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "892740163b654c109be8a3e09d255777": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a3d9c8fb28014531b3974edb8fbe4cd7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "36aef634e2ae4392a46fa8c94f7a7fab": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "fbc827e3b75f4bc48c55bee15bee0380": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_24267fed290a4443a682e12d7f694120",
              "IPY_MODEL_fc49bee0f46c4ac98f631f05ca021303",
              "IPY_MODEL_bd0e8b700e3f4ab0b0643c5fab0d87ef"
            ],
            "layout": "IPY_MODEL_9dd41b2d167149acb6f6d8b1d69507f5"
          }
        },
        "24267fed290a4443a682e12d7f694120": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f7e9d0500fee41c788eaa964c5b6b0b6",
            "placeholder": "​",
            "style": "IPY_MODEL_79e6070a34f1466581dadd9279414a62",
            "value": "Downloading (…)lve/main/config.json: 100%"
          }
        },
        "fc49bee0f46c4ac98f631f05ca021303": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bd53a57436594f7f8ca1a86d64878caf",
            "max": 570,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_704aedbdd6ae40f4bf97bacba405cf58",
            "value": 570
          }
        },
        "bd0e8b700e3f4ab0b0643c5fab0d87ef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_42412c15b60241c49042ccfd81bcf581",
            "placeholder": "​",
            "style": "IPY_MODEL_26a201f68f54441ca36c287a06ad7f8a",
            "value": " 570/570 [00:00&lt;00:00, 31.6kB/s]"
          }
        },
        "9dd41b2d167149acb6f6d8b1d69507f5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f7e9d0500fee41c788eaa964c5b6b0b6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "79e6070a34f1466581dadd9279414a62": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bd53a57436594f7f8ca1a86d64878caf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "704aedbdd6ae40f4bf97bacba405cf58": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "42412c15b60241c49042ccfd81bcf581": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "26a201f68f54441ca36c287a06ad7f8a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/mohripan/Machine-Learning/blob/main/Similar_Article.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 90
        },
        "id": "rrEj3hll25Ta",
        "outputId": "e08e0738-4b6d-430b-f056-ee176308046c"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-9e14af99-dee6-40d8-8252-c283739e18b1\" name=\"files[]\" multiple disabled\n",
              "        style=\"border:none\" />\n",
              "     <output id=\"result-9e14af99-dee6-40d8-8252-c283739e18b1\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script>// Copyright 2017 Google LLC\n",
              "//\n",
              "// Licensed under the Apache License, Version 2.0 (the \"License\");\n",
              "// you may not use this file except in compliance with the License.\n",
              "// You may obtain a copy of the License at\n",
              "//\n",
              "//      http://www.apache.org/licenses/LICENSE-2.0\n",
              "//\n",
              "// Unless required by applicable law or agreed to in writing, software\n",
              "// distributed under the License is distributed on an \"AS IS\" BASIS,\n",
              "// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
              "// See the License for the specific language governing permissions and\n",
              "// limitations under the License.\n",
              "\n",
              "/**\n",
              " * @fileoverview Helpers for google.colab Python module.\n",
              " */\n",
              "(function(scope) {\n",
              "function span(text, styleAttributes = {}) {\n",
              "  const element = document.createElement('span');\n",
              "  element.textContent = text;\n",
              "  for (const key of Object.keys(styleAttributes)) {\n",
              "    element.style[key] = styleAttributes[key];\n",
              "  }\n",
              "  return element;\n",
              "}\n",
              "\n",
              "// Max number of bytes which will be uploaded at a time.\n",
              "const MAX_PAYLOAD_SIZE = 100 * 1024;\n",
              "\n",
              "function _uploadFiles(inputId, outputId) {\n",
              "  const steps = uploadFilesStep(inputId, outputId);\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  // Cache steps on the outputElement to make it available for the next call\n",
              "  // to uploadFilesContinue from Python.\n",
              "  outputElement.steps = steps;\n",
              "\n",
              "  return _uploadFilesContinue(outputId);\n",
              "}\n",
              "\n",
              "// This is roughly an async generator (not supported in the browser yet),\n",
              "// where there are multiple asynchronous steps and the Python side is going\n",
              "// to poll for completion of each step.\n",
              "// This uses a Promise to block the python side on completion of each step,\n",
              "// then passes the result of the previous step as the input to the next step.\n",
              "function _uploadFilesContinue(outputId) {\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  const steps = outputElement.steps;\n",
              "\n",
              "  const next = steps.next(outputElement.lastPromiseValue);\n",
              "  return Promise.resolve(next.value.promise).then((value) => {\n",
              "    // Cache the last promise value to make it available to the next\n",
              "    // step of the generator.\n",
              "    outputElement.lastPromiseValue = value;\n",
              "    return next.value.response;\n",
              "  });\n",
              "}\n",
              "\n",
              "/**\n",
              " * Generator function which is called between each async step of the upload\n",
              " * process.\n",
              " * @param {string} inputId Element ID of the input file picker element.\n",
              " * @param {string} outputId Element ID of the output display.\n",
              " * @return {!Iterable<!Object>} Iterable of next steps.\n",
              " */\n",
              "function* uploadFilesStep(inputId, outputId) {\n",
              "  const inputElement = document.getElementById(inputId);\n",
              "  inputElement.disabled = false;\n",
              "\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  outputElement.innerHTML = '';\n",
              "\n",
              "  const pickedPromise = new Promise((resolve) => {\n",
              "    inputElement.addEventListener('change', (e) => {\n",
              "      resolve(e.target.files);\n",
              "    });\n",
              "  });\n",
              "\n",
              "  const cancel = document.createElement('button');\n",
              "  inputElement.parentElement.appendChild(cancel);\n",
              "  cancel.textContent = 'Cancel upload';\n",
              "  const cancelPromise = new Promise((resolve) => {\n",
              "    cancel.onclick = () => {\n",
              "      resolve(null);\n",
              "    };\n",
              "  });\n",
              "\n",
              "  // Wait for the user to pick the files.\n",
              "  const files = yield {\n",
              "    promise: Promise.race([pickedPromise, cancelPromise]),\n",
              "    response: {\n",
              "      action: 'starting',\n",
              "    }\n",
              "  };\n",
              "\n",
              "  cancel.remove();\n",
              "\n",
              "  // Disable the input element since further picks are not allowed.\n",
              "  inputElement.disabled = true;\n",
              "\n",
              "  if (!files) {\n",
              "    return {\n",
              "      response: {\n",
              "        action: 'complete',\n",
              "      }\n",
              "    };\n",
              "  }\n",
              "\n",
              "  for (const file of files) {\n",
              "    const li = document.createElement('li');\n",
              "    li.append(span(file.name, {fontWeight: 'bold'}));\n",
              "    li.append(span(\n",
              "        `(${file.type || 'n/a'}) - ${file.size} bytes, ` +\n",
              "        `last modified: ${\n",
              "            file.lastModifiedDate ? file.lastModifiedDate.toLocaleDateString() :\n",
              "                                    'n/a'} - `));\n",
              "    const percent = span('0% done');\n",
              "    li.appendChild(percent);\n",
              "\n",
              "    outputElement.appendChild(li);\n",
              "\n",
              "    const fileDataPromise = new Promise((resolve) => {\n",
              "      const reader = new FileReader();\n",
              "      reader.onload = (e) => {\n",
              "        resolve(e.target.result);\n",
              "      };\n",
              "      reader.readAsArrayBuffer(file);\n",
              "    });\n",
              "    // Wait for the data to be ready.\n",
              "    let fileData = yield {\n",
              "      promise: fileDataPromise,\n",
              "      response: {\n",
              "        action: 'continue',\n",
              "      }\n",
              "    };\n",
              "\n",
              "    // Use a chunked sending to avoid message size limits. See b/62115660.\n",
              "    let position = 0;\n",
              "    do {\n",
              "      const length = Math.min(fileData.byteLength - position, MAX_PAYLOAD_SIZE);\n",
              "      const chunk = new Uint8Array(fileData, position, length);\n",
              "      position += length;\n",
              "\n",
              "      const base64 = btoa(String.fromCharCode.apply(null, chunk));\n",
              "      yield {\n",
              "        response: {\n",
              "          action: 'append',\n",
              "          file: file.name,\n",
              "          data: base64,\n",
              "        },\n",
              "      };\n",
              "\n",
              "      let percentDone = fileData.byteLength === 0 ?\n",
              "          100 :\n",
              "          Math.round((position / fileData.byteLength) * 100);\n",
              "      percent.textContent = `${percentDone}% done`;\n",
              "\n",
              "    } while (position < fileData.byteLength);\n",
              "  }\n",
              "\n",
              "  // All done.\n",
              "  yield {\n",
              "    response: {\n",
              "      action: 'complete',\n",
              "    }\n",
              "  };\n",
              "}\n",
              "\n",
              "scope.google = scope.google || {};\n",
              "scope.google.colab = scope.google.colab || {};\n",
              "scope.google.colab._files = {\n",
              "  _uploadFiles,\n",
              "  _uploadFilesContinue,\n",
              "};\n",
              "})(self);\n",
              "</script> "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Saving data.csv to data.csv\n",
            "move data.csv to upload/data.csv\n"
          ]
        }
      ],
      "source": [
        "import os\n",
        "from google.colab import files\n",
        "import shutil\n",
        "\n",
        "upload_folder = 'upload'\n",
        "result_folder = 'results'\n",
        "\n",
        "if os.path.isdir(upload_folder):\n",
        "    shutil.rmtree(upload_folder)\n",
        "if os.path.isdir(result_folder):\n",
        "    shutil.rmtree(result_folder)\n",
        "os.mkdir(upload_folder)\n",
        "os.mkdir(result_folder)\n",
        "\n",
        "# upload images\n",
        "uploaded = files.upload()\n",
        "for filename in uploaded.keys():\n",
        "  dst_path = os.path.join(upload_folder, filename)\n",
        "  print(f'move {filename} to {dst_path}')\n",
        "  shutil.move(filename, dst_path)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install transformers"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "I5-EIYcuZ0D4",
        "outputId": "730e735e-f5de-4a66-b273-3719dc9fd4d0"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting transformers\n",
            "  Downloading transformers-4.30.2-py3-none-any.whl (7.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.2/7.2 MB\u001b[0m \u001b[31m53.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.12.2)\n",
            "Collecting huggingface-hub<1.0,>=0.14.1 (from transformers)\n",
            "  Downloading huggingface_hub-0.15.1-py3-none-any.whl (236 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m236.8/236.8 kB\u001b[0m \u001b[31m11.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.22.4)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (23.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2022.10.31)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.27.1)\n",
            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1 (from transformers)\n",
            "  Downloading tokenizers-0.13.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m97.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting safetensors>=0.3.1 (from transformers)\n",
            "  Downloading safetensors-0.3.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m77.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.65.0)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers) (2023.6.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers) (4.6.3)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (1.26.16)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2023.5.7)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.12)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.4)\n",
            "Installing collected packages: tokenizers, safetensors, huggingface-hub, transformers\n",
            "Successfully installed huggingface-hub-0.15.1 safetensors-0.3.1 tokenizers-0.13.3 transformers-4.30.2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import BertTokenizer\n",
        "import torch\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "df = pd.read_csv('upload/data.csv')\n",
        "\n",
        "# Load the BERT tokenizer.\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)\n",
        "\n",
        "# Create a function to tokenize a set of texts\n",
        "def preprocessing_for_bert(data):\n",
        "    \"\"\"Perform required preprocessing steps for pretrained BERT.\n",
        "    @param    data (np.array): Array of texts to be processed.\n",
        "    @return   input_ids (torch.Tensor): Tensor of token ids to be fed to a model.\n",
        "    @return   attention_masks (torch.Tensor): Tensor of indices specifying which tokens should be attended to by the model.\n",
        "    \"\"\"\n",
        "    # Create empty lists to store outputs\n",
        "    input_ids = []\n",
        "    attention_masks = []\n",
        "\n",
        "    # For every sentence...\n",
        "    for sent in data:\n",
        "        # `encode_plus` will:\n",
        "        #    (1) Tokenize the sentence\n",
        "        #    (2) Add the `[CLS]` and `[SEP]` token to the start and end\n",
        "        #    (3) Map tokens to their IDs\n",
        "        #    (4) Pad or truncate the sentence to `max_length`\n",
        "        #    (5) Create attention masks for [PAD] tokens\n",
        "        encoded_sent = tokenizer.encode_plus(\n",
        "            text=sent,  # Preprocess sentence\n",
        "            add_special_tokens=True,        # Add `[CLS]` and `[SEP]`\n",
        "            max_length=512,                  # Max length to truncate/pad\n",
        "            pad_to_max_length=True,         # Pad sentence to max length\n",
        "            #return_tensors='pt',           # Return PyTorch tensor\n",
        "            return_attention_mask=True,      # Return attention mask\n",
        "            truncation=True\n",
        "            )\n",
        "\n",
        "        # Add the outputs to the lists\n",
        "        input_ids.append(encoded_sent.get('input_ids'))\n",
        "        attention_masks.append(encoded_sent.get('attention_mask'))\n",
        "\n",
        "    # Convert lists to tensors\n",
        "    input_ids = torch.tensor(input_ids)\n",
        "    attention_masks = torch.tensor(attention_masks)\n",
        "\n",
        "    return input_ids, attention_masks\n",
        "\n",
        "\n",
        "# Get the lists of sentences.\n",
        "sentences = df['0'].values\n",
        "\n",
        "# Run the function `preprocessing_for_bert` on the sentences\n",
        "input_ids, attention_masks = preprocessing_for_bert(sentences)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 168,
          "referenced_widgets": [
            "790f21fd485e416eadfad7e26a58dda8",
            "0992530c78a4432480932f7b3b4531a9",
            "60d737be68ff4c26aa1f322751074f68",
            "ed4e8e87f9be47a3a2dd9cf23aee245f",
            "63ed7fdc7c2146059acffd5167b7c1d1",
            "ee0666dfb5124a7d828c41de1ac4d8ca",
            "c4c9922e9aae493d966db37a5fe363bf",
            "97f671a883fe4c4da13764cbfe1ca343",
            "2daf299bd7c24729bf4fc471dac28cb4",
            "6a7a9bca61094ea596351d1418b568a1",
            "114d0deaaae74ac8a8db599d02da77c7",
            "2fb2f92909444a678a4bb4e48e321bb7",
            "cb1eea8a518e4b8495a0219200a5def9",
            "a7a754bb09844ca7a4401849d529ff97",
            "3a499783554449e1bbcec99184ff70ed",
            "a96c1d77427b4465881be2a633ca8609",
            "dff3d3fdbbac4376ad54fe49d37b8f6b",
            "8f271bda1c994eceb78e4ea74e858136",
            "8c7e0ef42092460ebecae8ed94f9fb9c",
            "892740163b654c109be8a3e09d255777",
            "a3d9c8fb28014531b3974edb8fbe4cd7",
            "36aef634e2ae4392a46fa8c94f7a7fab",
            "fbc827e3b75f4bc48c55bee15bee0380",
            "24267fed290a4443a682e12d7f694120",
            "fc49bee0f46c4ac98f631f05ca021303",
            "bd0e8b700e3f4ab0b0643c5fab0d87ef",
            "9dd41b2d167149acb6f6d8b1d69507f5",
            "f7e9d0500fee41c788eaa964c5b6b0b6",
            "79e6070a34f1466581dadd9279414a62",
            "bd53a57436594f7f8ca1a86d64878caf",
            "704aedbdd6ae40f4bf97bacba405cf58",
            "42412c15b60241c49042ccfd81bcf581",
            "26a201f68f54441ca36c287a06ad7f8a"
          ]
        },
        "id": "r8hxFCUEV-CI",
        "outputId": "75a07599-6235-4cfe-c980-899533fa17e6"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)solve/main/vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "790f21fd485e416eadfad7e26a58dda8"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)okenizer_config.json:   0%|          | 0.00/28.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "2fb2f92909444a678a4bb4e48e321bb7"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)lve/main/config.json:   0%|          | 0.00/570 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "fbc827e3b75f4bc48c55bee15bee0380"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py:2377: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  warnings.warn(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import BertModel, BertTokenizer\n",
        "import torch\n",
        "import gc\n",
        "from torch.utils.data import DataLoader, SequentialSampler, TensorDataset\n",
        "\n",
        "# Load the BERT model\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
        "bert_model = BertModel.from_pretrained('bert-base-uncased')\n",
        "\n",
        "# Set batch size (you may need to adjust this depending on your available memory)\n",
        "batch_size = 16  # Adjust based on your system's capability\n",
        "\n",
        "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
        "bert_model = bert_model.to(device)  # Move the model to the device\n",
        "\n",
        "# Tokenize and encode the text\n",
        "def tokenize_and_encode(text):\n",
        "    encoded_text = tokenizer.encode_plus(\n",
        "        text,\n",
        "        max_length=512,\n",
        "        add_special_tokens=True,  # Add ['CLS'] and ['SEP']\n",
        "        return_token_type_ids=False,\n",
        "        padding='max_length',\n",
        "        return_attention_mask=True,\n",
        "        return_tensors='pt',  # Return PyTorch tensors\n",
        "        truncation=True\n",
        "    )\n",
        "    return encoded_text[\"input_ids\"], encoded_text[\"attention_mask\"]\n",
        "\n",
        "# Embed the text\n",
        "def bert_embeddings(input_ids, attention_masks):\n",
        "    with torch.no_grad():  # Deactivate gradients for the following block\n",
        "        input_ids = input_ids.to(device)\n",
        "        attention_masks = attention_masks.to(device)\n",
        "        outputs = bert_model(input_ids, attention_mask=attention_masks)\n",
        "        cls_embeddings = outputs[0][:, 0, :]  # Get the [CLS] embeddings\n",
        "    return cls_embeddings.cpu().numpy()  # Move embeddings to CPU and convert to numpy\n",
        "\n",
        "# Split the text into chunks and get embeddings for each\n",
        "def get_document_embeddings(text):\n",
        "    text_chunks = [text[i:i+512] for i in range(0, len(text), 512)]\n",
        "    document_embeddings = []\n",
        "    for chunk in text_chunks:\n",
        "        input_ids, attention_masks = tokenize_and_encode(chunk)\n",
        "        chunk_embeddings = bert_embeddings(input_ids, attention_masks)\n",
        "        document_embeddings.append(chunk_embeddings)\n",
        "    return np.concatenate(document_embeddings, axis=0)\n",
        "\n",
        "# Get the document embeddings for the entire dataset\n",
        "df['embeddings'] = df['0'].apply(get_document_embeddings)\n",
        "\n",
        "# Flatten the list of embeddings into a 2D array\n",
        "embeddings = np.concatenate(df['embeddings'].values, axis=0)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NZwmETBkZya4",
        "outputId": "b0c3b038-7cce-4099-e185-bd7accd526c5"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertModel: ['cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.bias', 'cls.seq_relationship.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.bias']\n",
            "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install corextopic"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "900vXtudjTMv",
        "outputId": "5ca3a7e7-c440-45fd-cdbf-64083115c508"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: corextopic in /usr/local/lib/python3.10/dist-packages (1.1)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install cPickle"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "g_CivaJSo8Rf",
        "outputId": "c27ffc11-d8ff-4172-be7d-2cd0dcb45f84"
      },
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[31mERROR: Could not find a version that satisfies the requirement cPickle (from versions: none)\u001b[0m\u001b[31m\n",
            "\u001b[0m\u001b[31mERROR: No matching distribution found for cPickle\u001b[0m\u001b[31m\n",
            "\u001b[0m"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from corextopic import corextopic as ct\n",
        "import pickle\n",
        "\n",
        "words = [str(i) for i in range(768)]\n",
        "\n",
        "# Initialize CorEx\n",
        "topic_model = ct.Corex(n_hidden=20, words=words, max_iter=200, verbose=False, seed=42)\n",
        "\n",
        "# Train CorEx\n",
        "topic_model.fit(embeddings, words=words)\n",
        "\n",
        "# You can save your trained model if you want\n",
        "topic_model.save('corex_model')\n",
        "\n",
        "pickle.dump(topic_model, open('corex_model.pkl', 'wb'))\n",
        "\n",
        "# Get the topics\n",
        "topics = topic_model.get_topics()\n",
        "for n,topic in enumerate(topics):\n",
        "    topic_words,_,_ = zip(*topic)\n",
        "    print('{}: '.format(n) + ', '.join(topic_words))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dLEe8bWxbXBP",
        "outputId": "77f53620-9759-44bb-c218-4f50c8fba83b"
      },
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "WARNING: Some words never appear (or always appear)\n",
            "0: 88, 692, 127, 105, 319, 734, 165, 198, 762, 605\n",
            "1: 728, 10, 591, 100, 428, 286, 527, 122, 23, 316\n",
            "2: 136, 218, 164, 6, 650, 72, 517, 456, 685, 107\n",
            "3: 236, 582, 18, 162, 531, 589, 225, 571, 274, 261\n",
            "4: 550, 731, 625, 177, 594, 467, 147, 233, 244, 189\n",
            "5: 87, 14, 193, 471, 518, 552, 435, 479, 302, 503\n",
            "6: 337, 245, 112, 214, 95, 732, 546, 282, 151, 506\n",
            "7: 158, 672, 365, 161, 355, 206, 680, 155, 249, 1\n",
            "8: 195, 358, 36, 675, 146, 132, 389, 138, 614, 211\n",
            "9: 547, 648, 656, 76, 444, 117, 463, 447, 741, 134\n",
            "10: 0, 266, 469, 700, 294, 39, 252, 398, 584, 541\n",
            "11: 446, 92, 455, 20, 543, 209, 58, 478, 481, 665\n",
            "12: 179, 239, 310, 44, 283, 486, 160, 240, 392, 482\n",
            "13: 50, 226, 12, 445, 86, 426, 565, 596, 278, 213\n",
            "14: 259, 351, 13, 748, 704, 706, 521, 55, 196, 312\n",
            "15: 323, 125, 128, 157, 170, 67, 654, 745, 551, 390\n",
            "16: 148, 234, 102, 321, 169, 70, 513, 599, 79, 554\n",
            "17: 489, 427, 756, 59, 632, 210, 676, 371, 520, 8\n",
            "18: 498, 199, 192, 621, 242, 519, 145, 332, 254, 452\n",
            "19: 705, 568, 270, 442, 695, 301, 558, 567, 299, 615\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Load the model if it's not already in memory\n",
        "from corextopic import corextopic as ct\n",
        "# topic_model = pickle.load(open('corex_model.pkl', 'rb'))\n",
        "\n",
        "# Define the document\n",
        "document = df['0'][0]\n",
        "\n",
        "print(document)\n",
        "\n",
        "# Get the embeddings for the document\n",
        "document_embeddings = get_document_embeddings(document)\n",
        "\n",
        "print()\n",
        "print(document_embeddings)\n",
        "print()\n",
        "\n",
        "# Transform with CorEx\n",
        "topic_distribution = topic_model.transform(document_embeddings)\n",
        "\n",
        "# Get the top topic for the document\n",
        "top_topic = np.argmax(topic_distribution[0])\n",
        "print(\"The top topic for the document is:\", top_topic)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mm3pl9n0jRWs",
        "outputId": "314500a3-4220-4cbf-d58d-6f682733a742"
      },
      "execution_count": 72,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " 7 Listen Share Word embedding is one of the most important techniques in natural language processing(NLP), where words are mapped to vectors of real numbers. Word embedding is capable of capturing the meaning of a word in a document, semantic and syntactic similarity, relation with other words. It also has been widely used for recommender systems and text classification. This tutorial will show a brief introduction of genism word2vec model with an example of generating word embedding for the vehicle make model. Word2vec is one of the most popular technique to learn word embeddings using a two-layer neural network. Its input is a text corpus and its output is a set of vectors. Word embedding via word2vec can make natural language computer-readable, then further implementation of mathematical operations on words can be used to detect their similarities. A well-trained set of word vectors will place similar words close to each other in that space. For instance, the words women, men, and human might cluster in one corner, while yellow, red and blue cluster together in another. There are two main training algorithms for word2vec, one is the continuous bag of words(CBOW), another is called skip-gram. The major difference between these two methods is that CBOW is using context to predict a target word while skip-gram is using a word to predict a target context. Generally, the skip-gram method can have a better performance compared with CBOW method, for it can capture two semantics for a single word. For instance, it will have two vector representations for Apple, one for the company and another for the fruit. For more details about the word2vec algorithm, please check here. Gensim is an open source python library for natural language processing and it was developed and is maintained by the Czech natural language processing researcher Radim Řehůřek. Gensim library will enable us to develop word embeddings by training our own word2vec models on a custom corpus either with CBOW of skip-grams algorithms. At first, we need to install the genism package. Gensim runs on Linux, Windows and Mac OS X, and should run on any other platform that supports Python 2.7+ and NumPy. Gensim depends on the following software: There are two ways for installation. We could run the following code in our terminal to install genism package. Or, alternatively for Conda environments: In this tutorial, I will show how to generate word embedding with genism using a concrete example. The dataset I used for this tutorial is from Kaggle Dataset. This vehicle dataset includes features such as make, model, year, engine, and other properties of the car. We will use these features to generate the word embeddings for each make model and then compare the similarities between different make model. The full python tutorial can be found here. Since the purpose of this tutorial is to learn how to generate word embeddings using genism library, we will not do the EDA and feature selection for the word2vec model for the sake of simplicity. Genism word2vec requires that a format of ‘list of lists’ for training where every document is contained in a list and every list contains lists of tokens of that document. At first, we need to generate a format of ‘list of lists’ for training the make model word embedding. To be more specific, each make model is contained in a list and every list contains lists of features of that make model. To achieve this, we need to do the following things : a. Create a new column for Make Model b. Generate a format of ‘ list of lists’ for each Make Model with the following features: Engine Fuel Type, Transmission Type, Driven_Wheels, Market Category, Vehicle Size, Vehicle Style. We can train the genism word2vec model with our own custom corpus as following: Let’s try to understand the hyperparameters of this model. size: The number of dimensions of the embeddings and the default is 100. window: The maximum distance between a target word and words around the target word. The default window is 5. min_count: The minimum count of words to consider when training the model; words with occurrence less than this count will be ignored. The default for min_count is 5. workers: The number of partitions during training and the default workers is 3. sg: The training algorithm, either CBOW(0) or skip gram(1). The default training algorithm is CBOW. After training the word2vec model, we can obtain the word embedding directly from the training model as following. Now we could even use Word2vec to compute the similarity between two Make Models in the vocabulary by invoking the model.similarity( ) and passing in the relevant words. For instance, model.similarity(‘Porsche 718 Cayman’, ‘Nissan Van’) This will give us the Euclidian similarity between Porsche 718 Cayman and Nissan Van. From the above examples, we can tell that Porsche 718 Cayman is more similar to Mercedes-Benz SLK-Class than Nissan Van. We also can use the built-in function model.most_similar() to get a set of the most similar make models for a given make model based on the Euclidean distance. However, Euclidian similarity cannot work well for the high-dimensional word vectors. This is because Euclidian similarity will increase as the number of dimensions increases, even if the word embedding stands for different meanings. Alternatively, we can use cosine similarity to measure the similarity between two vectors. Mathematically, it measures the cosine of the angle between two vectors projected in a multi-dimensional space. The cosine similarity captures the angle of the word vectors and not the magnitude. Under cosine similarity, no similarity is expressed as a 90-degree angle while the total similarity of 1 is at a 0-degree angle. The following function shows how can we generate the most similar make model based on cosine similarity. It’s hard to visualize the word embedding directly, for they usually have more than 3 dimensions. T-SNE is a useful tool to visualize high-dimensional data by dimension reduction while keeping relative pairwise distance between points. It can be said that T-SNE looking for a new data representation where the neighborhood relations are preserved. The following code shows how to plot the word embedding with T-SNE plot. This T-SNE plot shows the top 10 similar vehicles to the Porsche 718 Cayman in two-dimensional space. I am a master student in Data Science at the University of San Francisco. I am passionate about using Machine Learning to solve business challenges. You can also find me through Linkedin. \n",
            "\n",
            "[[-0.19346423 -0.3378449   0.26839834 ... -0.39094856  0.135079\n",
            "   0.53015673]\n",
            " [-0.37906685 -0.21589787  0.02734006 ... -0.30819687  0.20046076\n",
            "   1.0484372 ]\n",
            " [-0.44988152 -0.44624826  0.02568227 ... -0.21243116 -0.00944902\n",
            "   0.93836033]\n",
            " ...\n",
            " [-0.4376922  -0.16307181 -0.6989803  ...  0.05319526 -0.32193893\n",
            "   0.8699194 ]\n",
            " [-0.37499982 -0.01559641  0.35701433 ... -0.23336625  0.09810454\n",
            "   0.81751686]\n",
            " [-0.17860521 -0.23243466  0.12000003 ... -0.31988612  0.39327028\n",
            "   0.71790856]]\n",
            "\n",
            "The top topic for the document is: 0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.metrics.pairwise import cosine_similarity\n",
        "from scipy import sparse\n",
        "\n",
        "# Assuming embeddings is a 2D numpy array where each row is the embedding for a specific document\n",
        "sparse_embeddings = sparse.csr_matrix(embeddings)\n",
        "\n",
        "# Calculate cosine similarity for all pairs\n",
        "cosine_similarities = cosine_similarity(sparse_embeddings)\n",
        "\n",
        "# Get the top 5 articles most similar to the first one\n",
        "first_article_similarities = cosine_similarities[0]\n",
        "top_5_similar_articles = np.argsort(first_article_similarities)[-6:-1]  # -6:-1 instead of -5: to exclude the first article itself"
      ],
      "metadata": {
        "id": "KQLscLbnk0UE"
      },
      "execution_count": 75,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "top_5_similar_articles"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mD_RtsTQu590",
        "outputId": "1c9c607a-d82b-4458-d021-20bc2af7c493"
      },
      "execution_count": 76,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([ 1260, 10894, 12337,  1711, 13065])"
            ]
          },
          "metadata": {},
          "execution_count": 76
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.metrics.pairwise import cosine_similarity\n",
        "from scipy import sparse\n",
        "\n",
        "# Average the embeddings for each article\n",
        "df['average_embedding'] = df['embeddings'].apply(lambda x: np.mean(x, axis=0))\n",
        "\n",
        "# Get a matrix of all average embeddings\n",
        "average_embeddings = np.stack(df['average_embedding'].values)\n",
        "\n",
        "# Calculate cosine similarity for all pairs\n",
        "sparse_average_embeddings = sparse.csr_matrix(average_embeddings)\n",
        "cosine_similarities = cosine_similarity(sparse_average_embeddings)\n",
        "\n",
        "# Get the top 5 articles most similar to the first one\n",
        "first_article_similarities = cosine_similarities[4]\n",
        "top_5_similar_articles = np.argsort(first_article_similarities)[-6:-1]  # -6:-1 instead of -5: to exclude the first article itself"
      ],
      "metadata": {
        "id": "cufLDKtovAAT"
      },
      "execution_count": 97,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "top_5_similar_articles"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-M17df3zvafT",
        "outputId": "85b0d75f-7064-4c1f-be34-4835c74138fe"
      },
      "execution_count": 98,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([1049,  685,  392, 1296, 1070])"
            ]
          },
          "metadata": {},
          "execution_count": 98
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df['0'][1049]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "Uwn1052DxCLq",
        "outputId": "775caa7d-3092-4ee1-8160-382be6c7c7bf"
      },
      "execution_count": 100,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "' 15 Listen Share Deep Learning has taken over the majority of fields in solving complex problems, and the geospatial field is no exception. The title of the article interests you and hence, I hope that you are familiar with satellite datasets; for now, Landsat 5 TM. Little knowledge of how Machine Learning (ML) algorithms work, will help you grasp this hands-on tutorial quickly. For those who are unfamiliar with ML concept, in a nutshell, it is establishing the relationship between a few characteristics (features or Xs) of an entity with its other property (value or label or Y) — we provide plenty of examples (labelled data) to the model so that it learns from it and then predicts values/ labels for the new data (unlabelled data). That is enough of theory brush-up for machine learning! The general problem with satellite data: Two or more feature classes (e.g. built-up/ barren/ quarry) in the satellite data can have similar spectral values, which has made the classification a challenging task in the past couple of decades. The conventional supervised and unsupervised methods fail to be the perfect classifier due to the aforementioned issue, although they robustly perform the classification. But, there are always related issues. Let us understand this with the example below: In the above figure, if you were to use a vertical line as a classifier and move it only along the x-axis in such a way that it classifies all the images to its right as houses, the answer might not be straight forward. This is because the distribution of data is in such a way that it is impossible to separate them with just one vertical line. However, this doesn’t mean that the houses can’t be classified at all! Let us say you use the red line, as shown in the figure above, to separate the two features. In this instance, the majority of the houses were identified by the classifier but, a house was still left out, and a tree got misclassified as a house. To make sure that not even a single house is left behind, you might use the blue line. In that case, the classifier will cover all the house; this is called a high recall. However, not all the classified images are truly houses, this is called a low precision. Similarly, if we use the green line, all the images classified as houses are houses; therefore, the classifier possesses high precision. The recall will be lesser in this case because three houses were still left out. In the majority of cases, this trade-off between precision and recall holds. The house and tree problem demonstrated above is analogous to the built-up, quarry and barren land case. The classification priorities for satellite data can vary with the purpose. For example, if you want to make sure that all the built-up cells are classified as built-up, leaving none behind, and you care less about pixels of other classes with similar signatures being classified as built-up, then a model with a high recall is required. On the contrary, if the priority is to classify pure built-up pixels only without including any of the other class pixels, and you are okay to let go of mixed built-up pixels, then a high precision classifier is required. A generic model will use the red line in the case of the house and the tree to maintain the balance between precision and recall. Data used in the current scope Here, we will treat six bands (band 2 — band 7) of Landsat 5 TM as features and try to predict the binary built-up class. A multispectral Landsat 5 data acquired in the year 2011 for Bangalore and its corresponding binary built-up layer will be used for training and testing. Finally, another multispectral Landsat 5 data acquired in the year 2011 for Hyderabad will be used for new predictions. To know more about how to create training data for the model, you can check out this video. Since we are using labelled data to train the model, this is a supervised ML approach. We will be using Google’s Tensorflow library in Python to build a Neural Network (NN). The following other libraries will be required, please make sure you install them in advance (check out this video for installation instructions): Without further delay, let us get started with coding. Place all the three files in a directory — assign the path and input file names in the script, and read the GeoTIFF files. The raster module of the pyrsgis package reads the GeoTIFF’s geolocation information and the digital number (DN) values as a NumPy array separately. For details on this, please refer to the pyrsgis page. Let us print the size of the data that we have read. Output: As evident from the output, the number of rows and columns in the Bangalore images is the same, and the number of layers in the multispectral images are the same. The model will learn to decide whether a pixel is built-up or not based on the respective DN values across all the bands, and therefore, both the multispectral images should have the same number of features (bands) stacked in the same order. We will now change the shape of the arrays to a two-dimensional array, which is expected by the majority of ML algorithms, where each row represents a pixel. The convert module of the pyrsgis package will do that for us. Output: In the seventh line of the code snippet above, we extract all the pixels with the value one. This is a fail-safe to avoid issues due to NoData pixels that often has extreme high and low values. Now, we will split the data for training and validation. This is done to make sure that the model has not seen the test data and it performs equally well on new data. Otherwise, the model will overfit and perform well only on training data. Output: The test_size (0.4) in the code snippet above signifies that the training-testing proportion is 60/40. Many ML algorithms including NNs expect normalised data. This means that the histogram is stretched and scaled between a certain range (here, 0 to 1). We will normalise our features to suffice this requirement. Normalisation can be achieved by subtracting the minimum value and dividing by range. Since the Landsat data is an 8-bit data, the minimum and maximum values are 0 and 255 (2⁸ = 256 values). Note that it is always a good practice to calculate the minimum and maximum values from the data for normalisation. To avoid complexity, we will stick to the default rage of the 8-bit data here. Another additional pre-processing step is to reshape the features from two-dimensions to three-dimensions, such that each row represents an individual pixel. Output: Now that everything is in place, let us build the model using keras. To start with, we will use the sequential model, to add the layers one after the other. There is one input layer with the number of nodes equal to nBands. One hidden layer with 14 nodes and ‘relu’ as the activation function is used. The final layer contains two nodes for the binary built-up class with ‘softmax’ activation function, which is suitable for categorical output. You can find more activation functions here. As mentioned in line 10, we compile the model with ‘adam’ optimiser. (There are several others that you can check.) The loss type that we will be using, for now, is the categorical-sparse-crossentropy. You can check details here. The metric for model performance evaluation is ‘accuracy’. Finally, we run the model on xTrain and yTrain with two epochs (or iterations). Fitting the model will take some time depending on your data size and computational power. The following can be seen after the model compilation: Let us predict the values for the test data that we have kept separately, and perform various accuracy checks. The softmax function generates separate columns for each class type probability values. We extract only for class one (built-up), as mentioned in the sixth line in the code snippet above. The models for geospatial-related analysis become tricky to evaluate because unlike other general ML problems, it would not be fair to rely on a generalised summed up error; the spatial location is the key to the winning model. Therefore, the confusion matrix, precision and recall can reflect a clearer picture of how well the model performs. As seen in the confusion matrix above, there are thousands of built-up pixels classified as non-built-up and vice versa, but the proportion to the total data size is less. The precision and recall as obtained on the test data are more than 0.8. You can always spend some time and perform a few iterations to find the optimum number of hidden layers, the number of nodes in each hidden layer, and the number of epochs to get accuracy. Some commonly used remote sensing indices such as the NDBI or NDWI can also be used as features, as and when required. Once the desired accuracy is reached, use the model to predict for the new data and export the GeoTIFF. A similar model with minor tweaks can be applied for similar applications. Note that we are exporting the GeoTIFF with the predicted probability values, and not its thresholded binary version. We can always threshold the float type layer in a GIS environment later, as shown in the image below. The accuracy of the model has been evaluated already with precision and recall — you can also do the traditional checks (e.g. kappa coefficient) on the new predicted raster. Apart from the aforementioned challenges of satellite data classification, other intuitive limitations include the inability of the model to predict on data acquired in different seasons and over different regions, due to variation in the spectral signatures. The model that we used in the present article is a very basic architecture of the NN, some of the complex models including Convolution Neural Networks (CNN) have been proven by researchers to produce better results. To get started with CNN for satellite data classification, you can check out this post “Is CNN equally shiny on mid-resolution satellite data?”. The major advantage of these methods is the scalability once the model is trained. Please find the data used and the full script here. '"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 100
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "topic_words,_,_ = zip(*topic_model.get_topics(topic=top_topic))\n",
        "print('Top topic words: ', ', '.join(topic_words))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VZs_czI4p-Q5",
        "outputId": "8d147344-c8fc-4d29-8be9-a080e3f9be37"
      },
      "execution_count": 69,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Top topic words:  88, 692, 127, 105, 319, 734, 165, 198, 762, 605\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install faiss-gpu"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KUSHM8Uoswib",
        "outputId": "93c136b4-7e91-4b2a-91b2-9bf45ed9411d"
      },
      "execution_count": 105,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting faiss-gpu\n",
            "  Downloading faiss_gpu-1.7.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (85.5 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m85.5/85.5 MB\u001b[0m \u001b[31m12.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: faiss-gpu\n",
            "Successfully installed faiss-gpu-1.7.2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import faiss\n",
        "\n",
        "# Get a matrix of all average embeddings\n",
        "average_embeddings = np.stack(df['average_embedding'].values).astype('float32')  # Faiss requires float32 type\n",
        "\n",
        "# Build the index\n",
        "dimension = average_embeddings.shape[1]  # dimension of the vectors\n",
        "index = faiss.IndexFlatL2(dimension)\n",
        "index.add(average_embeddings)\n",
        "\n",
        "# Now you can use the index for searching the most similar vectors\n",
        "def search(index, vector, k=5):\n",
        "    # vector should be a numpy array of shape (1, dimension)\n",
        "    distances, indices = index.search(vector, k)\n",
        "    return distances, indices\n",
        "\n",
        "# Find the 5 most similar articles to the first one\n",
        "distances, indices = search(index, average_embeddings[0].reshape(1, -1), k=10)\n",
        "print(indices)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9y5d9mVt5bkC",
        "outputId": "76c6363e-69b6-4095-b090-83124a746089"
      },
      "execution_count": 114,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[[   0 1313  667  400  538  539    1  658 1049  653]]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df['0'][1313]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 140
        },
        "id": "mZBS7j9P5e3x",
        "outputId": "23ae26db-9278-4673-9095-745f3a15d19c"
      },
      "execution_count": 113,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "' 4 Listen Share A chatbot is an artificial intelligence software that can simulate a conversation (or a chat) with a user in natural language through messaging applications, websites, mobile apps or through the telephone, etc. These chatbots can be used in various industries for different purposes. We are going to build a simple chatbot using nltk library( Natural Language Toolkit). It is a leading platform for building Python programs to work with human language data. importing necessary libraries. import the dataset into the pandas data frame. The above data contains 1592 data points and two columns context which can be inferred as the query and text response is the response for that query. we can see that there are null values in the dataset if you open the dataset in excel and observe we can find that in our dataset the data is in different clusters i.e, the same type of questions in one place and then followed by next similar kind of questions. Null values are present for the same type of questions whose response can be almost similar and in that similar group of questions, the response is given to the first and the rest filled with null.so what we can do is use ffill() which returns the value of previous response in place of null values as below. So let’s get into our first step i.e, text normalization where we convert the data into lower case and then remove special characters and then perform lemmatization. Let us create a function that converts given text to lower case and removes special characters and numbers. We can see that our text is clean. Word tokenizing is the process of converting the normal text strings into a list of tokens. The pos_tag function returns the parts of speech of each token so that the lemmatizer function detects the parts of speech of token and then it converts the token to its root word as below We shall now create a function that performs all the steps mentioned above Let’s check our function and apply it to the dataset. We can see that our function worked well and thus we applied the same to our data. Our next step is word embedding, it is representation for text where words that have the same meaning have a similar representation. We have two models for this process bag of words (bow) and tf-idf ( Term Frequency-Inverse Document Frequency). The bag-of-words is a representation of text that describes the occurrence of words within a document. Consider if our dictionary contains the words {Playing, is, love}, and we want to vectorize the text “Playing football is love”, we would have the following vector: (1, 0, 1, 1). Consider the row 0 of the dataset the bow marks all the words present in the text as 1 and rest as 0. Stop words are extremely common words that would appear to be of little value in matching a user’s need and hence they are excluded from the vocabulary entirely. Below are the predefined stop words. Let us consider an example and try getting a response to the query. From above We can see that we have taken a question ‘Will you help me and tell me about yourself more’ and then perform text normalization and then applying the bow to the question. Now to get the related response we shall find the cosine similarity between the question and the lemmatized text we have. Cosine similarity is a measure of similarity between two vectors. It returns a value that is computed by taking the dot product and dividing that by the product of their norms between two vectors. Cosine Similarity (a, b) = Dot product(a, b) / ||a|| * ||b|| We can see that at index 194 we have the highest similarity text for the query we considered. Let us print the text at that position and see whether it is related or not. We can see that our model worked pretty well :) tf is Term Frequency, scoring of the frequency of the word in the current document and idf is Inverse Document Frequency, scoring of how rare the word is across documents. Here document represents a single text i.e, say row 0 or row1, etc, where documents refer to all the rows in the dataset. The above are the values obtained using tf-idf. Now using cosine similarity lets us find the response that we get with tf-idf. At index 4 we got higher similarity text that relates to our question. Let us see the response to our question. Using tf-idf we got a different response which is also a pretty good response to the question. Now let’s build a function that returns the response to the query using tf-idf. It is very simple we just have to combine all the topics we saw earlier in this article. Let’s see some output responses for a different queries. That’s cool know, our model responded pretty well to all the queries. We can also build a bot using the bow in the same way, all we have to do is to follow three steps normalization of text, word embedding, and similarity. The model we built doesn’t have any artificial intelligence, but still, it responded pretty well. For complete code, you can check out my GitHub. Thanks for reading. '"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 113
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "lNhayWxY6AmX"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}