langchain_llamacpp.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/yasheshshroff/NLPworkshop/blob/main/langchain_llamacpp.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5OU4nwVR8v9f",
        "outputId": "99b626c2-f7ea-48a8-e22b-00925d28c41e"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting llama-cpp-python\n",
            "  Downloading llama_cpp_python-0.2.11.tar.gz (3.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.6/3.6 MB\u001b[0m \u001b[31m15.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Installing backend dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: typing-extensions>=4.5.0 in /usr/local/lib/python3.10/dist-packages (from llama-cpp-python) (4.5.0)\n",
            "Requirement already satisfied: numpy>=1.20.0 in /usr/local/lib/python3.10/dist-packages (from llama-cpp-python) (1.23.5)\n",
            "Collecting diskcache>=5.6.1 (from llama-cpp-python)\n",
            "  Downloading diskcache-5.6.3-py3-none-any.whl (45 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m45.5/45.5 kB\u001b[0m \u001b[31m5.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hBuilding wheels for collected packages: llama-cpp-python\n",
            "  Building wheel for llama-cpp-python (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for llama-cpp-python: filename=llama_cpp_python-0.2.11-cp310-cp310-manylinux_2_35_x86_64.whl size=6423399 sha256=aca0780f07248ed75360a0d08e5141ce9caf80762428f883c5b77f8527a4ce01\n",
            "  Stored in directory: /root/.cache/pip/wheels/dc/42/77/a3ab0d02700427ea364de5797786c0272779dce795f62c3bc2\n",
            "Successfully built llama-cpp-python\n",
            "Installing collected packages: diskcache, llama-cpp-python\n",
            "Successfully installed diskcache-5.6.3 llama-cpp-python-0.2.11\n"
          ]
        }
      ],
      "source": [
        "!CMAKE_ARGS=\"-DLLAMA_CUBLAS=on\" FORCE_CMAKE=1 pip install llama-cpp-python"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install langchain pypdf sentence-transformers chromadb"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "idqqGDgw9PHY",
        "outputId": "4e71418c-0e28-4b04-9ca8-ef388e601f6d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: langchain in /usr/local/lib/python3.10/dist-packages (0.0.309)\n",
            "Requirement already satisfied: pypdf in /usr/local/lib/python3.10/dist-packages (3.16.2)\n",
            "Requirement already satisfied: sentence-transformers in /usr/local/lib/python3.10/dist-packages (2.2.2)\n",
            "Collecting chromadb\n",
            "  Downloading chromadb-0.4.13-py3-none-any.whl (437 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m437.8/437.8 kB\u001b[0m \u001b[31m7.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: PyYAML>=5.3 in /usr/local/lib/python3.10/dist-packages (from langchain) (6.0.1)\n",
            "Requirement already satisfied: SQLAlchemy<3,>=1.4 in /usr/local/lib/python3.10/dist-packages (from langchain) (2.0.21)\n",
            "Requirement already satisfied: aiohttp<4.0.0,>=3.8.3 in /usr/local/lib/python3.10/dist-packages (from langchain) (3.8.5)\n",
            "Requirement already satisfied: anyio<4.0 in /usr/local/lib/python3.10/dist-packages (from langchain) (3.7.1)\n",
            "Requirement already satisfied: async-timeout<5.0.0,>=4.0.0 in /usr/local/lib/python3.10/dist-packages (from langchain) (4.0.3)\n",
            "Requirement already satisfied: dataclasses-json<0.7,>=0.5.7 in /usr/local/lib/python3.10/dist-packages (from langchain) (0.6.1)\n",
            "Requirement already satisfied: jsonpatch<2.0,>=1.33 in /usr/local/lib/python3.10/dist-packages (from langchain) (1.33)\n",
            "Requirement already satisfied: langsmith<0.1.0,>=0.0.40 in /usr/local/lib/python3.10/dist-packages (from langchain) (0.0.42)\n",
            "Requirement already satisfied: numpy<2,>=1 in /usr/local/lib/python3.10/dist-packages (from langchain) (1.23.5)\n",
            "Requirement already satisfied: pydantic<3,>=1 in /usr/local/lib/python3.10/dist-packages (from langchain) (1.10.13)\n",
            "Requirement already satisfied: requests<3,>=2 in /usr/local/lib/python3.10/dist-packages (from langchain) (2.31.0)\n",
            "Requirement already satisfied: tenacity<9.0.0,>=8.1.0 in /usr/local/lib/python3.10/dist-packages (from langchain) (8.2.3)\n",
            "Requirement already satisfied: transformers<5.0.0,>=4.6.0 in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (4.34.0)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (4.66.1)\n",
            "Requirement already satisfied: torch>=1.6.0 in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (2.0.1+cu118)\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (0.15.2+cu118)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (1.2.2)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (1.11.3)\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (3.8.1)\n",
            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (0.1.99)\n",
            "Requirement already satisfied: huggingface-hub>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from sentence-transformers) (0.16.4)\n",
            "Collecting chroma-hnswlib==0.7.3 (from chromadb)\n",
            "  Downloading chroma_hnswlib-0.7.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.4/2.4 MB\u001b[0m \u001b[31m17.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting fastapi>=0.95.2 (from chromadb)\n",
            "  Downloading fastapi-0.103.2-py3-none-any.whl (66 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m66.3/66.3 kB\u001b[0m \u001b[31m5.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting uvicorn[standard]>=0.18.3 (from chromadb)\n",
            "  Downloading uvicorn-0.23.2-py3-none-any.whl (59 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m59.5/59.5 kB\u001b[0m \u001b[31m7.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting posthog>=2.4.0 (from chromadb)\n",
            "  Downloading posthog-3.0.2-py2.py3-none-any.whl (37 kB)\n",
            "Requirement already satisfied: typing-extensions>=4.5.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (4.5.0)\n",
            "Collecting pulsar-client>=3.1.0 (from chromadb)\n",
            "  Downloading pulsar_client-3.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (5.4 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.4/5.4 MB\u001b[0m \u001b[31m35.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting onnxruntime>=1.14.1 (from chromadb)\n",
            "  Downloading onnxruntime-1.16.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (6.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.2/6.2 MB\u001b[0m \u001b[31m61.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tokenizers>=0.13.2 in /usr/local/lib/python3.10/dist-packages (from chromadb) (0.14.0)\n",
            "Collecting pypika>=0.48.9 (from chromadb)\n",
            "  Downloading PyPika-0.48.9.tar.gz (67 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m67.3/67.3 kB\u001b[0m \u001b[31m9.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting overrides>=7.3.1 (from chromadb)\n",
            "  Downloading overrides-7.4.0-py3-none-any.whl (17 kB)\n",
            "Requirement already satisfied: importlib-resources in /usr/local/lib/python3.10/dist-packages (from chromadb) (6.1.0)\n",
            "Collecting bcrypt>=4.0.1 (from chromadb)\n",
            "  Downloading bcrypt-4.0.1-cp36-abi3-manylinux_2_28_x86_64.whl (593 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m593.7/593.7 kB\u001b[0m \u001b[31m48.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: typer>=0.9.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (0.9.0)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (23.1.0)\n",
            "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (3.3.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (6.0.4)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.9.2)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.4.0)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.3.1)\n",
            "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.10/dist-packages (from anyio<4.0->langchain) (3.4)\n",
            "Requirement already satisfied: sniffio>=1.1 in /usr/local/lib/python3.10/dist-packages (from anyio<4.0->langchain) (1.3.0)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<4.0->langchain) (1.1.3)\n",
            "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /usr/local/lib/python3.10/dist-packages (from dataclasses-json<0.7,>=0.5.7->langchain) (3.20.1)\n",
            "Requirement already satisfied: typing-inspect<1,>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from dataclasses-json<0.7,>=0.5.7->langchain) (0.9.0)\n",
            "Collecting starlette<0.28.0,>=0.27.0 (from fastapi>=0.95.2->chromadb)\n",
            "  Downloading starlette-0.27.0-py3-none-any.whl (66 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m67.0/67.0 kB\u001b[0m \u001b[31m8.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.4.0->sentence-transformers) (3.12.4)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.4.0->sentence-transformers) (2023.6.0)\n",
            "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.4.0->sentence-transformers) (23.2)\n",
            "Requirement already satisfied: jsonpointer>=1.9 in /usr/local/lib/python3.10/dist-packages (from jsonpatch<2.0,>=1.33->langchain) (2.4)\n",
            "Collecting coloredlogs (from onnxruntime>=1.14.1->chromadb)\n",
            "  Downloading coloredlogs-15.0.1-py2.py3-none-any.whl (46 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.0/46.0 kB\u001b[0m \u001b[31m6.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: flatbuffers in /usr/local/lib/python3.10/dist-packages (from onnxruntime>=1.14.1->chromadb) (23.5.26)\n",
            "Requirement already satisfied: protobuf in /usr/local/lib/python3.10/dist-packages (from onnxruntime>=1.14.1->chromadb) (3.20.3)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from onnxruntime>=1.14.1->chromadb) (1.12)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from posthog>=2.4.0->chromadb) (1.16.0)\n",
            "Collecting monotonic>=1.5 (from posthog>=2.4.0->chromadb)\n",
            "  Downloading monotonic-1.6-py2.py3-none-any.whl (8.2 kB)\n",
            "Collecting backoff>=1.10.0 (from posthog>=2.4.0->chromadb)\n",
            "  Downloading backoff-2.2.1-py3-none-any.whl (15 kB)\n",
            "Requirement already satisfied: python-dateutil>2.1 in /usr/local/lib/python3.10/dist-packages (from posthog>=2.4.0->chromadb) (2.8.2)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from pulsar-client>=3.1.0->chromadb) (2023.7.22)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain) (2.0.6)\n",
            "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/dist-packages (from SQLAlchemy<3,>=1.4->langchain) (3.0.0)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->sentence-transformers) (3.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->sentence-transformers) (3.1.2)\n",
            "Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.6.0->sentence-transformers) (2.0.0)\n",
            "Requirement already satisfied: cmake in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.6.0->sentence-transformers) (3.27.6)\n",
            "Requirement already satisfied: lit in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.6.0->sentence-transformers) (17.0.2)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers) (2023.6.3)\n",
            "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers) (0.3.3)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.10/dist-packages (from typer>=0.9.0->chromadb) (8.1.7)\n",
            "Collecting h11>=0.8 (from uvicorn[standard]>=0.18.3->chromadb)\n",
            "  Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m7.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting httptools>=0.5.0 (from uvicorn[standard]>=0.18.3->chromadb)\n",
            "  Downloading httptools-0.6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (428 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m428.8/428.8 kB\u001b[0m \u001b[31m40.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting python-dotenv>=0.13 (from uvicorn[standard]>=0.18.3->chromadb)\n",
            "  Downloading python_dotenv-1.0.0-py3-none-any.whl (19 kB)\n",
            "Collecting uvloop!=0.15.0,!=0.15.1,>=0.14.0 (from uvicorn[standard]>=0.18.3->chromadb)\n",
            "  Downloading uvloop-0.17.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (4.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.1/4.1 MB\u001b[0m \u001b[31m75.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting watchfiles>=0.13 (from uvicorn[standard]>=0.18.3->chromadb)\n",
            "  Downloading watchfiles-0.20.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m65.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting websockets>=10.4 (from uvicorn[standard]>=0.18.3->chromadb)\n",
            "  Downloading websockets-11.0.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (129 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m129.9/129.9 kB\u001b[0m \u001b[31m15.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk->sentence-transformers) (1.3.2)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->sentence-transformers) (3.2.0)\n",
            "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /usr/local/lib/python3.10/dist-packages (from torchvision->sentence-transformers) (9.4.0)\n",
            "Requirement already satisfied: mypy-extensions>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from typing-inspect<1,>=0.4.0->dataclasses-json<0.7,>=0.5.7->langchain) (1.0.0)\n",
            "Collecting humanfriendly>=9.1 (from coloredlogs->onnxruntime>=1.14.1->chromadb)\n",
            "  Downloading humanfriendly-10.0-py2.py3-none-any.whl (86 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.8/86.8 kB\u001b[0m \u001b[31m11.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.6.0->sentence-transformers) (2.1.3)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->onnxruntime>=1.14.1->chromadb) (1.3.0)\n",
            "Building wheels for collected packages: pypika\n",
            "  Building wheel for pypika (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pypika: filename=PyPika-0.48.9-py2.py3-none-any.whl size=53723 sha256=f925e3c9d3e54ca1381f287c07224d1b1e9a97b91565665be2a6fd896eff3483\n",
            "  Stored in directory: /root/.cache/pip/wheels/e1/26/51/d0bffb3d2fd82256676d7ad3003faea3bd6dddc9577af665f4\n",
            "Successfully built pypika\n",
            "Installing collected packages: pypika, monotonic, websockets, uvloop, python-dotenv, pulsar-client, overrides, humanfriendly, httptools, h11, chroma-hnswlib, bcrypt, backoff, watchfiles, uvicorn, starlette, posthog, coloredlogs, onnxruntime, fastapi, chromadb\n",
            "Successfully installed backoff-2.2.1 bcrypt-4.0.1 chroma-hnswlib-0.7.3 chromadb-0.4.13 coloredlogs-15.0.1 fastapi-0.103.2 h11-0.14.0 httptools-0.6.0 humanfriendly-10.0 monotonic-1.6 onnxruntime-1.16.0 overrides-7.4.0 posthog-3.0.2 pulsar-client-3.3.0 pypika-0.48.9 python-dotenv-1.0.0 starlette-0.27.0 uvicorn-0.23.2 uvloop-0.17.0 watchfiles-0.20.0 websockets-11.0.3\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from langchain.llms import LlamaCpp\n",
        "from langchain.prompts import PromptTemplate\n",
        "from langchain.chains import LLMChain\n",
        "from langchain.callbacks.manager import CallbackManager\n",
        "from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler"
      ],
      "metadata": {
        "id": "GgdoNsmG93kM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Callbacks support token-wise streaming\n",
        "callback_manager = CallbackManager([StreamingStdOutCallbackHandler()])"
      ],
      "metadata": {
        "id": "aghHSMCi99WE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oqk79lP9-G0z",
        "outputId": "74dec133-9ab0-4ad1-aebf-c73b3bcc75e9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "llm = LlamaCpp(\n",
        "    model_path=\"drive/MyDrive/ggml-model-q4_0.gguf\",\n",
        "    temperature=0.5,\n",
        "    max_tokens=2000,\n",
        "    top_p=1,\n",
        "    n_ctx=4000,\n",
        "    callback_manager=callback_manager,\n",
        "    verbose=True, # Verbose is required to pass to the callback manager\n",
        ")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Xpny6fZ2-BCS",
        "outputId": "6f7a1f5c-a693-4e0a-ca85-e7f237779ba0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "AVX = 1 | AVX2 = 1 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | FMA = 1 | NEON = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# the simplest way to ask Llama some general questions\n",
        "question = \"who wrote the book Innovator's dilemma?\"\n",
        "answer = llm(question)"
      ],
      "metadata": {
        "id": "QQSYNuWn-V50",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "693f01ab-e105-4795-e572-683713cd3c96"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "Clayton Christensen is the author of \"The Innovator's Dilemma,\" which was first published in 1997. The book explores why successful companies often struggle to adapt to disruptive technologies and new business models, leading to their eventual decline."
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# a more flexible way to ask Llama general questions using LangChain's PromptTemplate and LLMChain\n",
        "prompt = PromptTemplate.from_template(\n",
        "    \"who wrote {book}?\"\n",
        ")\n",
        "chain = LLMChain(llm=llm, prompt=prompt)\n",
        "answer = chain.run(\"innovator's dilemma\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Pqfa5UqEE86T",
        "outputId": "611de701-1d37-402e-9fc1-7e167e04f298"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Llama.generate: prefix-match hit\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "The Innovator's Dilemma is a book written by Clayton Christensen, a Harvard Business School professor and expert on disruptive innovation. The book was first published in 1997 and has since become a widely recognized and influential work on the topic of innovation and business strategy.\n",
            "\n",
            "In the book, Christensen argues that successful companies often struggle to adapt to disruptive technologies or business models because they are too focused on sustaining their existing businesses. This can lead to a \"dilemma\" in which the company is unable to successfully adopt new innovations and ultimately becomes disrupted by them.\n",
            "\n",
            "The Innovator's Dilemma has been widely praised for its insights into the challenges of managing innovation and the importance of understanding the needs of customers. It has also been applied to a wide range of industries, from technology to healthcare to finance."
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# let's see how Llama2 hallucinates, because it doesn't have the knowledge about Llama2 while the model was trained,\n",
        "# but by default it behaves like a know-it-all expert who can't afford to say I don't know\n",
        "prompt = PromptTemplate.from_template(\n",
        "    \"What is {what}?\"\n",
        ")\n",
        "chain = LLMChain(llm=llm, prompt=prompt)\n",
        "answer = chain.run(\"llama2\")"
      ],
      "metadata": {
        "id": "3hefEAcnIBMR",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7fbb9b61-ada8-41b0-813f-1ed64c50fdb3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Llama.generate: prefix-match hit\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "Llama2 is a software tool for aligning multiple sequencing reads to a reference genome. It is designed to be fast and accurate, and can handle a wide range of input formats. Llama2 is particularly useful for aligning RNA-seq and ChIP-seq data, but it can also be used for other types of sequencing data.\n",
            "\n",
            "Llama2 uses a probabilistic approach to assign reads to the reference genome, which allows it to handle gaps and indels in the data. It also includes a number of features for visualizing and interpreting the alignment results, such as a graphical user interface and tools for summarizing the alignment statistics.\n",
            "\n",
            "Llama2 is widely used in the scientific community and has been cited in many research papers. It is open-source software, which means that it is free to use and that the source code is available for modification and redistribution."
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# to fix the LLM's hallucination, one way is to use RAG, to augment it with more recent or custom data that holds the info for it to answer correctly\n",
        "# first load the Llama2 paper via the LangChain's PDF loader\n",
        "from langchain.document_loaders import PyPDFLoader\n",
        "loader = PyPDFLoader(\"https://arxiv.org/pdf/2307.09288.pdf\")\n",
        "documents = loader.load()"
      ],
      "metadata": {
        "id": "g1oUGb1qIoJR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# quick check on the loaded document for the correct pages etc\n",
        "print(len(documents), documents[0].page_content[0:300])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ELOm96cOIy5T",
        "outputId": "c422230e-79ab-47f7-f58b-341b29d7a583"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "77 Llama 2 : Open Foundation and Fine-Tuned Chat Models\n",
            "Hugo Touvron∗Louis Martin†Kevin Stone†\n",
            "Peter Albert Amjad Almahairi Yasmine Babaei Nikolay Bashlykov Soumya Batra\n",
            "Prajjwal Bhargava Shruti Bhosale Dan Bikel Lukas Blecher Cristian Canton Ferrer Moya Chen\n",
            "Guillem Cucurull David Esiobu Jude Fernande\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# there're more 30 vector stores (DBs) supported by LangChain. Chroma is light-weight and in memory so it's easy to get started with\n",
        "# other vector stores can be used to store large amount of data - see https://python.langchain.com/docs/integrations/vectorstores\n",
        "from langchain.vectorstores import Chroma\n",
        "\n",
        "# embeddings are numerical representations of the question and answer text\n",
        "from langchain.embeddings import HuggingFaceEmbeddings\n",
        "\n",
        "# use a common text splitter to split text into chunks\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter"
      ],
      "metadata": {
        "id": "ug8ZuCZfI6Bx"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# split the loaded documents into chunks\n",
        "text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=20)\n",
        "all_splits = text_splitter.split_documents(documents)\n",
        "\n",
        "# create the vector db to store all the split chunks as embeddings\n",
        "embeddings = HuggingFaceEmbeddings()\n",
        "vectordb = Chroma.from_documents(\n",
        "    documents=all_splits,\n",
        "    embedding=embeddings,\n",
        ")"
      ],
      "metadata": {
        "id": "ESUxojvXLNUI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# use another LangChain's chain, RetrievalQA, to associate Llama with the loaded documents stored in the vector db\n",
        "from langchain.chains import RetrievalQA\n",
        "\n",
        "qa_chain = RetrievalQA.from_chain_type(\n",
        "    llm,\n",
        "    retriever=vectordb.as_retriever()\n",
        ")\n",
        "\n",
        "# for each question, LangChain performs a semantic similarity search of it in the vector db, then passes the search results as the context\n",
        "# the Llama to answer question about the data stored in the verctor db\n",
        "question = \"What is llama2?\"\n",
        "result = qa_chain({\"query\": question})\n",
        "# it takes close to 2 minutes to return the result (but using other vector store than Chroma such as FAISS can take longer), because\n",
        "# Llama2 is running on a local Mac. To get much faster results, you can use a cloud service with GPU used for inference - see HelloLlamaCloud\n",
        "# for a demo."
      ],
      "metadata": {
        "id": "bCoYN6KnNSs5",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a7fcadca-1e23-4264-a524-32e8d1ffe24f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " Llama 2 is a new language model developed by Meta AI that has not been covered in all scenarios and may produce inaccurate or objectionable responses to user prompts. It is released openly to encourage responsible AI innovation and collaboration, but developers must comply with the terms of the provided license and acceptable use policy before deploying any applications of the model. The model has been fine-tuned for dialogue use cases and has achieved improvements in truthfulness and toxicity compared to the pretrained version."
          ]
        }
      ]
    }
  ]
}