preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "# import dataset1 (Github repository by Kunal Kumar)\n",
    "questions = []\n",
    "answers = []\n",
    "with open('final1.txt', 'r') as file:\n",
    "    lines = file.readlines()\n",
    "\n",
    "    for i in range(0, len(lines), 2):\n",
    "        question = lines[i].strip()  \n",
    "        if i + 1 < len(lines):  \n",
    "            answer = lines[i + 1].strip()\n",
    "        else:\n",
    "            answer = ''  \n",
    "\n",
    "        questions.append(question)\n",
    "        answers.append(answer)\n",
    "\n",
    "df1 = pd.DataFrame({'Question': questions, 'Answer': answers})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Question</th>\n",
       "      <th>Answer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>How does randomised algorithm work</td>\n",
       "      <td>The algorithm typically uses uniformly random ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>what do you mean by bestfirst search</td>\n",
       "      <td>bestfirst search is a search algorithm which e...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>how do you explain a daemon</td>\n",
       "      <td>daemon  disk and execution monitor is a proces...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>What is phonetic algorithm</td>\n",
       "      <td>A phonetic algorithm is an algorithm for index...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>what do you mean by uniform costsearch</td>\n",
       "      <td>a tree search that finds the lowestcost route ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2750</th>\n",
       "      <td>define smp</td>\n",
       "      <td>smp stands for symmetric multiprocessing it is...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2751</th>\n",
       "      <td>what do you mean by the resident set and worki...</td>\n",
       "      <td>resident set is that portion of the process im...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2752</th>\n",
       "      <td>how do you explain the main purpose of an oper...</td>\n",
       "      <td>there are two main purposes of an operating sy...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2753</th>\n",
       "      <td>what do you mean by executive in windows nt</td>\n",
       "      <td>in windows nt executive refers to the operatin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2754</th>\n",
       "      <td>what do you mean by a kernel</td>\n",
       "      <td>kernel is the part of os which handles all det...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2755 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               Question  \\\n",
       "0                    How does randomised algorithm work   \n",
       "1                  what do you mean by bestfirst search   \n",
       "2                           how do you explain a daemon   \n",
       "3                            What is phonetic algorithm   \n",
       "4                what do you mean by uniform costsearch   \n",
       "...                                                 ...   \n",
       "2750                                         define smp   \n",
       "2751  what do you mean by the resident set and worki...   \n",
       "2752  how do you explain the main purpose of an oper...   \n",
       "2753        what do you mean by executive in windows nt   \n",
       "2754                       what do you mean by a kernel   \n",
       "\n",
       "                                                 Answer  \n",
       "0     The algorithm typically uses uniformly random ...  \n",
       "1     bestfirst search is a search algorithm which e...  \n",
       "2     daemon  disk and execution monitor is a proces...  \n",
       "3     A phonetic algorithm is an algorithm for index...  \n",
       "4     a tree search that finds the lowestcost route ...  \n",
       "...                                                 ...  \n",
       "2750  smp stands for symmetric multiprocessing it is...  \n",
       "2751  resident set is that portion of the process im...  \n",
       "2752  there are two main purposes of an operating sy...  \n",
       "2753  in windows nt executive refers to the operatin...  \n",
       "2754  kernel is the part of os which handles all det...  \n",
       "\n",
       "[2755 rows x 2 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import dataset2 (Hugging Face - data-science-qa): \n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "df2 = pd.read_csv(\"hf://datasets/team-bay/data-science-qa/DataScienceDataset.csv\")\n",
    "df2 = df2[['question', 'answer']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question</th>\n",
       "      <th>answer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>what is Supervised Learning?</td>\n",
       "      <td>Supervised learning is a machine learning para...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Explain Unsupervised Learning</td>\n",
       "      <td>Unsupervised learning is a machine learning pa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>What is Regression?</td>\n",
       "      <td>Regression is a statistical method used for mo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Can you explain Classification?</td>\n",
       "      <td>Classification is a task in supervised learnin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>What is Clustering?</td>\n",
       "      <td>Clustering is an unsupervised learning task wh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>468</th>\n",
       "      <td>please explain Spline Interpolation</td>\n",
       "      <td>Spline Interpolation is a method used to inter...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>469</th>\n",
       "      <td>what is Spatial Autocorrelation</td>\n",
       "      <td>Spatial Autocorrelation is a measure of the de...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>470</th>\n",
       "      <td>can you please explain Graphical Models</td>\n",
       "      <td>Graphical Models are a framework for represent...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>471</th>\n",
       "      <td>Can you explain Variational Inference</td>\n",
       "      <td>Variational Inference is a method used to appr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>472</th>\n",
       "      <td>what is Topological Data Analysis</td>\n",
       "      <td>Topological Data Analysis is an approach to an...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>473 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    question  \\\n",
       "0               what is Supervised Learning?   \n",
       "1              Explain Unsupervised Learning   \n",
       "2                        What is Regression?   \n",
       "3            Can you explain Classification?   \n",
       "4                        What is Clustering?   \n",
       "..                                       ...   \n",
       "468      please explain Spline Interpolation   \n",
       "469          what is Spatial Autocorrelation   \n",
       "470  can you please explain Graphical Models   \n",
       "471    Can you explain Variational Inference   \n",
       "472        what is Topological Data Analysis   \n",
       "\n",
       "                                                answer  \n",
       "0    Supervised learning is a machine learning para...  \n",
       "1    Unsupervised learning is a machine learning pa...  \n",
       "2    Regression is a statistical method used for mo...  \n",
       "3    Classification is a task in supervised learnin...  \n",
       "4    Clustering is an unsupervised learning task wh...  \n",
       "..                                                 ...  \n",
       "468  Spline Interpolation is a method used to inter...  \n",
       "469  Spatial Autocorrelation is a measure of the de...  \n",
       "470  Graphical Models are a framework for represent...  \n",
       "471  Variational Inference is a method used to appr...  \n",
       "472  Topological Data Analysis is an approach to an...  \n",
       "\n",
       "[473 rows x 2 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import dataset3 (Kaggle - Software Engineering Interview Questions Dataset)\n",
    "\n",
    "df3 = pd.read_csv('SoftwareQuestions.csv', encoding='ISO-8859-1')\n",
    "\n",
    "df3 = df3[['Question', 'Answer']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Question</th>\n",
       "      <th>Answer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>What is the difference between compilation and...</td>\n",
       "      <td>Compilation translates source code into machin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Explain the concept of polymorphism.</td>\n",
       "      <td>Polymorphism allows objects of different class...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Define encapsulation and give an example.</td>\n",
       "      <td>Encapsulation bundles data and methods in a cl...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>What is an abstract class, and how is it diffe...</td>\n",
       "      <td>An abstract class can't be instantiated and ca...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Describe the principles of Object-Oriented Pro...</td>\n",
       "      <td>OOP principles include encapsulation, inherita...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>195</th>\n",
       "      <td>Design a cloud-based virtual desktop infrastru...</td>\n",
       "      <td>Address virtualization technologies, resource ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>196</th>\n",
       "      <td>Create a real-time sports analytics system usi...</td>\n",
       "      <td>Utilize streaming data processing, machine lea...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>197</th>\n",
       "      <td>Implement a quantum algorithm for solving a we...</td>\n",
       "      <td>Discuss quantum computing principles, qubit ma...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>198</th>\n",
       "      <td>Design a secure mobile payment system for deve...</td>\n",
       "      <td>Focus on security protocols, offline capabilit...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199</th>\n",
       "      <td>Build a scalable image processing pipeline for...</td>\n",
       "      <td>Implement distributed processing, efficient st...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>200 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              Question  \\\n",
       "0    What is the difference between compilation and...   \n",
       "1                 Explain the concept of polymorphism.   \n",
       "2            Define encapsulation and give an example.   \n",
       "3    What is an abstract class, and how is it diffe...   \n",
       "4    Describe the principles of Object-Oriented Pro...   \n",
       "..                                                 ...   \n",
       "195  Design a cloud-based virtual desktop infrastru...   \n",
       "196  Create a real-time sports analytics system usi...   \n",
       "197  Implement a quantum algorithm for solving a we...   \n",
       "198  Design a secure mobile payment system for deve...   \n",
       "199  Build a scalable image processing pipeline for...   \n",
       "\n",
       "                                                Answer  \n",
       "0    Compilation translates source code into machin...  \n",
       "1    Polymorphism allows objects of different class...  \n",
       "2    Encapsulation bundles data and methods in a cl...  \n",
       "3    An abstract class can't be instantiated and ca...  \n",
       "4    OOP principles include encapsulation, inherita...  \n",
       "..                                                 ...  \n",
       "195  Address virtualization technologies, resource ...  \n",
       "196  Utilize streaming data processing, machine lea...  \n",
       "197  Discuss quantum computing principles, qubit ma...  \n",
       "198  Focus on security protocols, offline capabilit...  \n",
       "199  Implement distributed processing, efficient st...  \n",
       "\n",
       "[200 rows x 2 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import dataset4 (Kaggle - Computer Science Theory QA Dataset)\n",
    "\n",
    "import json\n",
    "import pandas as pd\n",
    "\n",
    "with open('intents.json', 'r') as file:\n",
    "    data = json.load(file)\n",
    "\n",
    "questions = []\n",
    "responses = []\n",
    "\n",
    "for intent in data['intents']:\n",
    "    patterns = intent['patterns']\n",
    "    response_list = intent['responses']\n",
    "    \n",
    "    for pattern in patterns:\n",
    "        questions.append(pattern)\n",
    "        responses.append(response_list[0])\n",
    "\n",
    "df4 = pd.DataFrame({'Question': questions, 'Answer': responses})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Question</th>\n",
       "      <th>Answer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Explain data abstraction.</td>\n",
       "      <td>Data abstraction is a technique used in comput...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>What is data abstraction?</td>\n",
       "      <td>Data abstraction is a technique used in comput...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Define data abstraction.</td>\n",
       "      <td>Data abstraction is a technique used in comput...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>What is a syntax error</td>\n",
       "      <td>A syntax error is an error in the structure of...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Explain syntax error</td>\n",
       "      <td>A syntax error is an error in the structure of...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>342</th>\n",
       "      <td>Explain Bias-Variance Tradeoff.</td>\n",
       "      <td>Bias-Variance Tradeoff is a concept in machine...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>343</th>\n",
       "      <td>What is Stochastic Gradient Descent (SGD) in m...</td>\n",
       "      <td>Stochastic Gradient Descent (SGD) is an optimi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>344</th>\n",
       "      <td>Explain Stochastic Gradient Descent.</td>\n",
       "      <td>Stochastic Gradient Descent (SGD) is an optimi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>345</th>\n",
       "      <td>What is the Backpropagation Algorithm in machi...</td>\n",
       "      <td>The Backpropagation Algorithm is a widely used...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>346</th>\n",
       "      <td>Explain Backpropagation Algorithm.</td>\n",
       "      <td>The Backpropagation Algorithm is a widely used...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>347 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              Question  \\\n",
       "0                            Explain data abstraction.   \n",
       "1                            What is data abstraction?   \n",
       "2                             Define data abstraction.   \n",
       "3                               What is a syntax error   \n",
       "4                                 Explain syntax error   \n",
       "..                                                 ...   \n",
       "342                    Explain Bias-Variance Tradeoff.   \n",
       "343  What is Stochastic Gradient Descent (SGD) in m...   \n",
       "344               Explain Stochastic Gradient Descent.   \n",
       "345  What is the Backpropagation Algorithm in machi...   \n",
       "346                 Explain Backpropagation Algorithm.   \n",
       "\n",
       "                                                Answer  \n",
       "0    Data abstraction is a technique used in comput...  \n",
       "1    Data abstraction is a technique used in comput...  \n",
       "2    Data abstraction is a technique used in comput...  \n",
       "3    A syntax error is an error in the structure of...  \n",
       "4    A syntax error is an error in the structure of...  \n",
       "..                                                 ...  \n",
       "342  Bias-Variance Tradeoff is a concept in machine...  \n",
       "343  Stochastic Gradient Descent (SGD) is an optimi...  \n",
       "344  Stochastic Gradient Descent (SGD) is an optimi...  \n",
       "345  The Backpropagation Algorithm is a widely used...  \n",
       "346  The Backpropagation Algorithm is a widely used...  \n",
       "\n",
       "[347 rows x 2 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Question</th>\n",
       "      <th>Answer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>How does randomised algorithm work</td>\n",
       "      <td>The algorithm typically uses uniformly random ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>what do you mean by bestfirst search</td>\n",
       "      <td>bestfirst search is a search algorithm which e...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>how do you explain a daemon</td>\n",
       "      <td>daemon  disk and execution monitor is a proces...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>What is phonetic algorithm</td>\n",
       "      <td>A phonetic algorithm is an algorithm for index...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>what do you mean by uniform costsearch</td>\n",
       "      <td>a tree search that finds the lowestcost route ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3770</th>\n",
       "      <td>Explain Bias-Variance Tradeoff.</td>\n",
       "      <td>Bias-Variance Tradeoff is a concept in machine...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3771</th>\n",
       "      <td>What is Stochastic Gradient Descent (SGD) in m...</td>\n",
       "      <td>Stochastic Gradient Descent (SGD) is an optimi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3772</th>\n",
       "      <td>Explain Stochastic Gradient Descent.</td>\n",
       "      <td>Stochastic Gradient Descent (SGD) is an optimi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3773</th>\n",
       "      <td>What is the Backpropagation Algorithm in machi...</td>\n",
       "      <td>The Backpropagation Algorithm is a widely used...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3774</th>\n",
       "      <td>Explain Backpropagation Algorithm.</td>\n",
       "      <td>The Backpropagation Algorithm is a widely used...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3775 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               Question  \\\n",
       "0                    How does randomised algorithm work   \n",
       "1                  what do you mean by bestfirst search   \n",
       "2                           how do you explain a daemon   \n",
       "3                            What is phonetic algorithm   \n",
       "4                what do you mean by uniform costsearch   \n",
       "...                                                 ...   \n",
       "3770                    Explain Bias-Variance Tradeoff.   \n",
       "3771  What is Stochastic Gradient Descent (SGD) in m...   \n",
       "3772               Explain Stochastic Gradient Descent.   \n",
       "3773  What is the Backpropagation Algorithm in machi...   \n",
       "3774                 Explain Backpropagation Algorithm.   \n",
       "\n",
       "                                                 Answer  \n",
       "0     The algorithm typically uses uniformly random ...  \n",
       "1     bestfirst search is a search algorithm which e...  \n",
       "2     daemon  disk and execution monitor is a proces...  \n",
       "3     A phonetic algorithm is an algorithm for index...  \n",
       "4     a tree search that finds the lowestcost route ...  \n",
       "...                                                 ...  \n",
       "3770  Bias-Variance Tradeoff is a concept in machine...  \n",
       "3771  Stochastic Gradient Descent (SGD) is an optimi...  \n",
       "3772  Stochastic Gradient Descent (SGD) is an optimi...  \n",
       "3773  The Backpropagation Algorithm is a widely used...  \n",
       "3774  The Backpropagation Algorithm is a widely used...  \n",
       "\n",
       "[3775 rows x 2 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "questions_combined = pd.concat(\n",
    "    [df1['Question'], df2['question'], df3['Question'], df4['Question']],\n",
    "    axis=0\n",
    ").reset_index(drop=True)\n",
    "\n",
    "answers_combined = pd.concat(\n",
    "    [df1['Answer'], df2['answer'], df3['Answer'], df4['Answer']],\n",
    "    axis=0\n",
    ").reset_index(drop=True)\n",
    "\n",
    "combined_df = pd.DataFrame({\n",
    "    'Question': questions_combined,\n",
    "    'Answer': answers_combined\n",
    "})\n",
    "\n",
    "combined_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Question    0\n",
       "Answer      0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check for NULL values\n",
    "combined_df.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/97/1ldlnt8n5fqbgwqrjjql278c0000gn/T/ipykernel_84315/673652426.py:8: MarkupResemblesLocatorWarning: The input looks more like a filename than markup. You may want to open this file and pass the filehandle into Beautiful Soup.\n",
      "  text = BeautifulSoup(text, \"html.parser\").get_text()\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Question</th>\n",
       "      <th>Answer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>how does randomised algorithm work</td>\n",
       "      <td>the algorithm typically uses uniformly random ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>what do you mean by bestfirst search</td>\n",
       "      <td>bestfirst search is a search algorithm which e...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>how do you explain a daemon</td>\n",
       "      <td>daemon disk and execution monitor is a process...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>what is phonetic algorithm</td>\n",
       "      <td>a phonetic algorithm is an algorithm for index...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>what do you mean by uniform costsearch</td>\n",
       "      <td>a tree search that finds the lowestcost route ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3770</th>\n",
       "      <td>explain biasvariance tradeoff</td>\n",
       "      <td>biasvariance tradeoff is a concept in machine ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3771</th>\n",
       "      <td>what is stochastic gradient descent sgd in mac...</td>\n",
       "      <td>stochastic gradient descent sgd is an optimiza...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3772</th>\n",
       "      <td>explain stochastic gradient descent</td>\n",
       "      <td>stochastic gradient descent sgd is an optimiza...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3773</th>\n",
       "      <td>what is the backpropagation algorithm in machi...</td>\n",
       "      <td>the backpropagation algorithm is a widely used...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3774</th>\n",
       "      <td>explain backpropagation algorithm</td>\n",
       "      <td>the backpropagation algorithm is a widely used...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3775 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               Question  \\\n",
       "0                    how does randomised algorithm work   \n",
       "1                  what do you mean by bestfirst search   \n",
       "2                           how do you explain a daemon   \n",
       "3                            what is phonetic algorithm   \n",
       "4                what do you mean by uniform costsearch   \n",
       "...                                                 ...   \n",
       "3770                      explain biasvariance tradeoff   \n",
       "3771  what is stochastic gradient descent sgd in mac...   \n",
       "3772                explain stochastic gradient descent   \n",
       "3773  what is the backpropagation algorithm in machi...   \n",
       "3774                  explain backpropagation algorithm   \n",
       "\n",
       "                                                 Answer  \n",
       "0     the algorithm typically uses uniformly random ...  \n",
       "1     bestfirst search is a search algorithm which e...  \n",
       "2     daemon disk and execution monitor is a process...  \n",
       "3     a phonetic algorithm is an algorithm for index...  \n",
       "4     a tree search that finds the lowestcost route ...  \n",
       "...                                                 ...  \n",
       "3770  biasvariance tradeoff is a concept in machine ...  \n",
       "3771  stochastic gradient descent sgd is an optimiza...  \n",
       "3772  stochastic gradient descent sgd is an optimiza...  \n",
       "3773  the backpropagation algorithm is a widely used...  \n",
       "3774  the backpropagation algorithm is a widely used...  \n",
       "\n",
       "[3775 rows x 2 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import re\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "# Function to clean text\n",
    "def clean_text(text):\n",
    "    # Remove HTML tags\n",
    "    text = BeautifulSoup(text, \"html.parser\").get_text()\n",
    "    # Remove special characters and unnecessary punctuation\n",
    "    text = re.sub(r'[^a-zA-Z0-9\\s]', '', text)\n",
    "    # Remove extra whitespace\n",
    "    text = re.sub(r'\\s+', ' ', text).strip()\n",
    "    return text\n",
    "\n",
    "# Apply the cleaning function to the DataFrame\n",
    "combined_df['Question'] = combined_df['Question'].apply(clean_text).str.lower()\n",
    "combined_df['Answer'] = combined_df['Answer'].apply(clean_text).str.lower()\n",
    "combined_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save the DataFrame to a CSV file\n",
    "combined_df.to_csv('combined_data.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}