examples/Cheetah/00-Cheetah-example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os \n",
    "import ast\n",
    "import pickle\n",
    "import pathlib\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from TELF.applications import Cheetah\n",
    "from TELF.pre_processing import Vulture"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "DATA_PATH = os.path.join('..', '..', 'data')\n",
    "DATA_FILE = 'sample.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 940 entries, 0 to 939\n",
      "Data columns (total 8 columns):\n",
      " #   Column          Non-Null Count  Dtype \n",
      "---  ------          --------------  ----- \n",
      " 0   eid             940 non-null    object\n",
      " 1   title           940 non-null    object\n",
      " 2   year            940 non-null    int64 \n",
      " 3   abstract        940 non-null    object\n",
      " 4   authors         940 non-null    object\n",
      " 5   author_ids      940 non-null    object\n",
      " 6   references      843 non-null    object\n",
      " 7   clean_abstract  940 non-null    object\n",
      "dtypes: int64(1), object(7)\n",
      "memory usage: 58.9+ KB\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv(os.path.join(DATA_PATH, DATA_FILE))\n",
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>eid</th>\n",
       "      <th>title</th>\n",
       "      <th>year</th>\n",
       "      <th>abstract</th>\n",
       "      <th>authors</th>\n",
       "      <th>author_ids</th>\n",
       "      <th>references</th>\n",
       "      <th>clean_abstract</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3cbdf82a-6781-11ee-b983-4ab2673ea3f0</td>\n",
       "      <td>Paper Title</td>\n",
       "      <td>2016</td>\n",
       "      <td>Supervisory Control and Data Acquisition (SCAD...</td>\n",
       "      <td>Name;Name;Name;Name;Name</td>\n",
       "      <td>3df61b32-6781-11ee-b983-4ab2673ea3f0;3df61c18-...</td>\n",
       "      <td>3cbe2bec-6781-11ee-b983-4ab2673ea3f0;3cbe6d64-...</td>\n",
       "      <td>supervisory control acquisition system often s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3cbdf960-6781-11ee-b983-4ab2673ea3f0</td>\n",
       "      <td>Paper Title</td>\n",
       "      <td>2002</td>\n",
       "      <td>Supervisory Control and Data Acquisition (SCAD...</td>\n",
       "      <td>Name;Name</td>\n",
       "      <td>3df61cc2-6781-11ee-b983-4ab2673ea3f0;3df61cea-...</td>\n",
       "      <td>3cbe45d2-6781-11ee-b983-4ab2673ea3f0;3cbe0338-...</td>\n",
       "      <td>supervisory control acquisition system often s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3cbdf992-6781-11ee-b983-4ab2673ea3f0</td>\n",
       "      <td>Paper Title</td>\n",
       "      <td>1995</td>\n",
       "      <td>Malware is one of the most dangerous and costl...</td>\n",
       "      <td>Name;Name;Name;Name</td>\n",
       "      <td>3df61d1c-6781-11ee-b983-4ab2673ea3f0;3df61d3a-...</td>\n",
       "      <td>3cbe5c8e-6781-11ee-b983-4ab2673ea3f0;3cbe1bac-...</td>\n",
       "      <td>malware dangerous costly cyber threat organiza...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3cbdf9ba-6781-11ee-b983-4ab2673ea3f0</td>\n",
       "      <td>Paper Title</td>\n",
       "      <td>2003</td>\n",
       "      <td>Malware is one of the most dangerous and costl...</td>\n",
       "      <td>Name;Name;Name;Name;Name;Name;Name;Name;Name;Name</td>\n",
       "      <td>3df61dbc-6781-11ee-b983-4ab2673ea3f0;3df61dda-...</td>\n",
       "      <td>3cbe0040-6781-11ee-b983-4ab2673ea3f0;3cbe08e2-...</td>\n",
       "      <td>malware dangerous costly cyber threat national...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3cbdf9e2-6781-11ee-b983-4ab2673ea3f0</td>\n",
       "      <td>Paper Title</td>\n",
       "      <td>1997</td>\n",
       "      <td>We propose an efficient, distributed, out-of-m...</td>\n",
       "      <td>Name;Name;Name;Name;Name;Name;Name;Name;Name;Name</td>\n",
       "      <td>3df61f24-6781-11ee-b983-4ab2673ea3f0;3df61f56-...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>propose efficient distribute memory implementa...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    eid        title  year  \\\n",
       "0  3cbdf82a-6781-11ee-b983-4ab2673ea3f0  Paper Title  2016   \n",
       "1  3cbdf960-6781-11ee-b983-4ab2673ea3f0  Paper Title  2002   \n",
       "2  3cbdf992-6781-11ee-b983-4ab2673ea3f0  Paper Title  1995   \n",
       "3  3cbdf9ba-6781-11ee-b983-4ab2673ea3f0  Paper Title  2003   \n",
       "4  3cbdf9e2-6781-11ee-b983-4ab2673ea3f0  Paper Title  1997   \n",
       "\n",
       "                                            abstract  \\\n",
       "0  Supervisory Control and Data Acquisition (SCAD...   \n",
       "1  Supervisory Control and Data Acquisition (SCAD...   \n",
       "2  Malware is one of the most dangerous and costl...   \n",
       "3  Malware is one of the most dangerous and costl...   \n",
       "4  We propose an efficient, distributed, out-of-m...   \n",
       "\n",
       "                                             authors  \\\n",
       "0                           Name;Name;Name;Name;Name   \n",
       "1                                          Name;Name   \n",
       "2                                Name;Name;Name;Name   \n",
       "3  Name;Name;Name;Name;Name;Name;Name;Name;Name;Name   \n",
       "4  Name;Name;Name;Name;Name;Name;Name;Name;Name;Name   \n",
       "\n",
       "                                          author_ids  \\\n",
       "0  3df61b32-6781-11ee-b983-4ab2673ea3f0;3df61c18-...   \n",
       "1  3df61cc2-6781-11ee-b983-4ab2673ea3f0;3df61cea-...   \n",
       "2  3df61d1c-6781-11ee-b983-4ab2673ea3f0;3df61d3a-...   \n",
       "3  3df61dbc-6781-11ee-b983-4ab2673ea3f0;3df61dda-...   \n",
       "4  3df61f24-6781-11ee-b983-4ab2673ea3f0;3df61f56-...   \n",
       "\n",
       "                                          references  \\\n",
       "0  3cbe2bec-6781-11ee-b983-4ab2673ea3f0;3cbe6d64-...   \n",
       "1  3cbe45d2-6781-11ee-b983-4ab2673ea3f0;3cbe0338-...   \n",
       "2  3cbe5c8e-6781-11ee-b983-4ab2673ea3f0;3cbe1bac-...   \n",
       "3  3cbe0040-6781-11ee-b983-4ab2673ea3f0;3cbe08e2-...   \n",
       "4                                                NaN   \n",
       "\n",
       "                                      clean_abstract  \n",
       "0  supervisory control acquisition system often s...  \n",
       "1  supervisory control acquisition system often s...  \n",
       "2  malware dangerous costly cyber threat organiza...  \n",
       "3  malware dangerous costly cyber threat national...  \n",
       "4  propose efficient distribute memory implementa...  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create Clean Text Columns for Title & Abstract prior to Using Cheetah"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "vulture = Vulture(n_jobs  = 1, \n",
    "                  verbose = 0,  # Disable == 0, Verbose >= 1\n",
    "                 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 940 entries, 0 to 939\n",
      "Data columns (total 9 columns):\n",
      " #   Column          Non-Null Count  Dtype \n",
      "---  ------          --------------  ----- \n",
      " 0   eid             940 non-null    object\n",
      " 1   title           940 non-null    object\n",
      " 2   year            940 non-null    int64 \n",
      " 3   abstract        940 non-null    object\n",
      " 4   authors         940 non-null    object\n",
      " 5   author_ids      940 non-null    object\n",
      " 6   references      843 non-null    object\n",
      " 7   clean_abstract  940 non-null    object\n",
      " 8   clean_title     940 non-null    object\n",
      "dtypes: int64(1), object(8)\n",
      "memory usage: 66.2+ KB\n"
     ]
    }
   ],
   "source": [
    "df = vulture.clean_dataframe(df, concat_cleaned_cols=False, append_to_original_df=True, columns=[\"title\", \"abstract\"])\n",
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# if remove_english=True, this will remove non-english documents\n",
    "df.dropna(subset=['clean_title', 'clean_abstract'], inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Use Cheetah"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Examine the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# setup the cheetah columns. the keys are cheetah categories and the values are corresponding column names\n",
    "# not all columns need to be present, missing columns will not be indexed\n",
    "cheetah_columns = {\n",
    "    'title': 'clean_title', \n",
    "    'abstract': 'clean_abstract',\n",
    "    'year': 'year',\n",
    "    'author_ids': 'author_ids',\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Note:**\n",
    "- The key is the name for this column known to Cheetah. The value is the corresponding column in the DataFrame. For example, for a column called clean_abstract, an entry into columns would like **'abstract': 'clean_abstract'**.\n",
    "- If any of the above columns are missing, Cheetah will not attempt to index the missing column. If the user attempts to perform search on an information for a missing column, error will be raised.\n",
    "- Although any of the above columns could be missing, they must follow the correct format if they do exist. *See below for the correct formats.*"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Note that under author_ids column, each ID is seperated with \";\" symbol**:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'3df61b32-6781-11ee-b983-4ab2673ea3f0;3df61c18-6781-11ee-b983-4ab2673ea3f0;3df61c4a-6781-11ee-b983-4ab2673ea3f0;3df61c72-6781-11ee-b983-4ab2673ea3f0;3df61c9a-6781-11ee-b983-4ab2673ea3f0'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.iloc[0][cheetah_columns[\"author_ids\"]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Add some NaNs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Cheetah should be able to handle missing data. If some papers have NaN entries for certain columns, the associated documents should not be indexed for those values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def insert_random_nans(df, cols, perc=5, seed=42):\n",
    "    \"\"\"\n",
    "    Introduce NaN values randomly into specified columns of a DataFrame.\n",
    "    \n",
    "    Parameters:\n",
    "    -----------\n",
    "    df: pd.Dataframe\n",
    "        Target DataFrame\n",
    "    cols: List \n",
    "        A list of columns in which NaNs will be introduced\n",
    "    perc: int\n",
    "        Percentage of the total dataframe length to be replaced with NaNs in each column\n",
    "    seed: int\n",
    "        Random seed for reproducibility\n",
    "    \n",
    "    Returns:\n",
    "    --------\n",
    "    pd.DataFrame\n",
    "        Modified dataframe with random NaNs\n",
    "    \"\"\"\n",
    "    rng = np.random.default_rng(seed) \n",
    "    num_nans = int(len(df) * perc / 100)\n",
    "    for col in cols:\n",
    "        nan_indices = rng.choice(df.index, size=num_nans, replace=False)\n",
    "        df.loc[nan_indices, col] = np.nan\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 940 entries, 0 to 939\n",
      "Data columns (total 9 columns):\n",
      " #   Column          Non-Null Count  Dtype  \n",
      "---  ------          --------------  -----  \n",
      " 0   eid             940 non-null    object \n",
      " 1   title           893 non-null    object \n",
      " 2   year            893 non-null    float64\n",
      " 3   abstract        893 non-null    object \n",
      " 4   authors         940 non-null    object \n",
      " 5   author_ids      893 non-null    object \n",
      " 6   references      843 non-null    object \n",
      " 7   clean_abstract  940 non-null    object \n",
      " 8   clean_title     940 non-null    object \n",
      "dtypes: float64(1), object(8)\n",
      "memory usage: 66.2+ KB\n"
     ]
    }
   ],
   "source": [
    "df = insert_random_nans(df, cols=['author_ids', 'year', 'abstract', 'title'], perc=5, seed=42)\n",
    "df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Searching"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create or load indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting existing index.\n",
      "Indexing abstract\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 940/940 [00:00<00:00, 49669.88it/s]\n",
      "100%|██████████| 526/526 [00:00<00:00, 372543.72it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Indexing title\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 940/940 [00:00<00:00, 2223714.47it/s]\n",
      "100%|██████████| 1/1 [00:00<00:00, 26379.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Indexing years\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 940/940 [00:00<00:00, 2127709.53it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Indexing author IDs\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 940/940 [00:00<00:00, 193970.57it/s]\n",
      "100%|██████████| 6730/6730 [00:00<00:00, 2512028.65it/s]\n"
     ]
    }
   ],
   "source": [
    "cheetah = Cheetah(verbose=True)\n",
    "index_file = os.path.join(DATA_PATH, 'sample_index.p')\n",
    "\n",
    "cheetah.index(df, \n",
    "              columns=cheetah_columns, \n",
    "              index_file=index_file,\n",
    "              reindex=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Set criteria and search"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Search for single word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Found 113 papers in 0.0016 seconds\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>eid</th>\n",
       "      <th>title</th>\n",
       "      <th>year</th>\n",
       "      <th>abstract</th>\n",
       "      <th>authors</th>\n",
       "      <th>author_ids</th>\n",
       "      <th>references</th>\n",
       "      <th>clean_abstract</th>\n",
       "      <th>clean_title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3cbdf82a-6781-11ee-b983-4ab2673ea3f0</td>\n",
       "      <td>Paper Title</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>Supervisory Control and Data Acquisition (SCAD...</td>\n",
       "      <td>Name;Name;Name;Name;Name</td>\n",
       "      <td>3df61b32-6781-11ee-b983-4ab2673ea3f0;3df61c18-...</td>\n",
       "      <td>3cbe2bec-6781-11ee-b983-4ab2673ea3f0;3cbe6d64-...</td>\n",
       "      <td>supervisory control acquisition scada serve ne...</td>\n",
       "      <td>title</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3cbdf960-6781-11ee-b983-4ab2673ea3f0</td>\n",
       "      <td>Paper Title</td>\n",
       "      <td>2002.0</td>\n",
       "      <td>Supervisory Control and Data Acquisition (SCAD...</td>\n",
       "      <td>Name;Name</td>\n",
       "      <td>3df61cc2-6781-11ee-b983-4ab2673ea3f0;3df61cea-...</td>\n",
       "      <td>3cbe45d2-6781-11ee-b983-4ab2673ea3f0;3cbe0338-...</td>\n",
       "      <td>supervisory control acquisition scada serve ne...</td>\n",
       "      <td>title</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>770</th>\n",
       "      <td>3cbe6666-6781-11ee-b983-4ab2673ea3f0</td>\n",
       "      <td>Paper Title</td>\n",
       "      <td>2018.0</td>\n",
       "      <td>Supervisory Control and Data Acquisition (SCAD...</td>\n",
       "      <td>Name;Name</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3cbe5bb2-6781-11ee-b983-4ab2673ea3f0;3cbe7232-...</td>\n",
       "      <td>supervisory control acquisition scada serve ne...</td>\n",
       "      <td>title</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>518</th>\n",
       "      <td>3cbe42bc-6781-11ee-b983-4ab2673ea3f0</td>\n",
       "      <td>Paper Title</td>\n",
       "      <td>1997.0</td>\n",
       "      <td>Supervisory Control and Data Acquisition (SCAD...</td>\n",
       "      <td>Name;Name;Name;Name;Name;Name</td>\n",
       "      <td>3df891dc-6781-11ee-b983-4ab2673ea3f0;3df89204-...</td>\n",
       "      <td>3cbe504a-6781-11ee-b983-4ab2673ea3f0;3cbe7ac0-...</td>\n",
       "      <td>supervisory control acquisition scada serve ne...</td>\n",
       "      <td>title</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>774</th>\n",
       "      <td>3cbe66f2-6781-11ee-b983-4ab2673ea3f0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2009.0</td>\n",
       "      <td>Supervisory Control and Data Acquisition (SCAD...</td>\n",
       "      <td>Name;Name</td>\n",
       "      <td>3df7a10a-6781-11ee-b983-4ab2673ea3f0;3df98cc2-...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>supervisory control acquisition scada serve ne...</td>\n",
       "      <td>title</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      eid        title    year  \\\n",
       "0    3cbdf82a-6781-11ee-b983-4ab2673ea3f0  Paper Title  2016.0   \n",
       "1    3cbdf960-6781-11ee-b983-4ab2673ea3f0  Paper Title  2002.0   \n",
       "770  3cbe6666-6781-11ee-b983-4ab2673ea3f0  Paper Title  2018.0   \n",
       "518  3cbe42bc-6781-11ee-b983-4ab2673ea3f0  Paper Title  1997.0   \n",
       "774  3cbe66f2-6781-11ee-b983-4ab2673ea3f0          NaN  2009.0   \n",
       "\n",
       "                                              abstract  \\\n",
       "0    Supervisory Control and Data Acquisition (SCAD...   \n",
       "1    Supervisory Control and Data Acquisition (SCAD...   \n",
       "770  Supervisory Control and Data Acquisition (SCAD...   \n",
       "518  Supervisory Control and Data Acquisition (SCAD...   \n",
       "774  Supervisory Control and Data Acquisition (SCAD...   \n",
       "\n",
       "                           authors  \\\n",
       "0         Name;Name;Name;Name;Name   \n",
       "1                        Name;Name   \n",
       "770                      Name;Name   \n",
       "518  Name;Name;Name;Name;Name;Name   \n",
       "774                      Name;Name   \n",
       "\n",
       "                                            author_ids  \\\n",
       "0    3df61b32-6781-11ee-b983-4ab2673ea3f0;3df61c18-...   \n",
       "1    3df61cc2-6781-11ee-b983-4ab2673ea3f0;3df61cea-...   \n",
       "770                                                NaN   \n",
       "518  3df891dc-6781-11ee-b983-4ab2673ea3f0;3df89204-...   \n",
       "774  3df7a10a-6781-11ee-b983-4ab2673ea3f0;3df98cc2-...   \n",
       "\n",
       "                                            references  \\\n",
       "0    3cbe2bec-6781-11ee-b983-4ab2673ea3f0;3cbe6d64-...   \n",
       "1    3cbe45d2-6781-11ee-b983-4ab2673ea3f0;3cbe0338-...   \n",
       "770  3cbe5bb2-6781-11ee-b983-4ab2673ea3f0;3cbe7232-...   \n",
       "518  3cbe504a-6781-11ee-b983-4ab2673ea3f0;3cbe7ac0-...   \n",
       "774                                                NaN   \n",
       "\n",
       "                                        clean_abstract clean_title  \n",
       "0    supervisory control acquisition scada serve ne...       title  \n",
       "1    supervisory control acquisition scada serve ne...       title  \n",
       "770  supervisory control acquisition scada serve ne...       title  \n",
       "518  supervisory control acquisition scada serve ne...       title  \n",
       "774  supervisory control acquisition scada serve ne...       title  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "search_parameters = {\n",
    "    \"query\": \"scada\",\n",
    "    \"in_title\":True,         # if true searches for query title\n",
    "    \"in_abstract\":True,      # if true searches for query in abstract\n",
    "    \"do_results_table\": True # if true, generate DataFrame for explainability\n",
    "}\n",
    "\n",
    "search_results, search_criteria = cheetah.search(**search_parameters)\n",
    "search_results.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filter_type</th>\n",
       "      <th>filter_value</th>\n",
       "      <th>num_papers</th>\n",
       "      <th>included_ids</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>query</td>\n",
       "      <td>scada</td>\n",
       "      <td>113</td>\n",
       "      <td>0;1;770;518;774;9;521;11;779;780;526;273;786;2...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  filter_type filter_value  num_papers  \\\n",
       "0       query        scada         113   \n",
       "\n",
       "                                        included_ids  \n",
       "0  0;1;770;518;774;9;521;11;779;780;526;273;786;2...  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "search_criteria"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Search for single phrase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Found 208 papers in 0.0034 seconds\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>eid</th>\n",
       "      <th>title</th>\n",
       "      <th>year</th>\n",
       "      <th>abstract</th>\n",
       "      <th>authors</th>\n",
       "      <th>author_ids</th>\n",
       "      <th>references</th>\n",
       "      <th>clean_abstract</th>\n",
       "      <th>clean_title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3cbdf992-6781-11ee-b983-4ab2673ea3f0</td>\n",
       "      <td>Paper Title</td>\n",
       "      <td>1995.0</td>\n",
       "      <td>Malware is one of the most dangerous and costl...</td>\n",
       "      <td>Name;Name;Name;Name</td>\n",
       "      <td>3df61d1c-6781-11ee-b983-4ab2673ea3f0;3df61d3a-...</td>\n",
       "      <td>3cbe5c8e-6781-11ee-b983-4ab2673ea3f0;3cbe1bac-...</td>\n",
       "      <td>malware dangerous costly cyber threats organiz...</td>\n",
       "      <td>title</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>515</th>\n",
       "      <td>3cbe424e-6781-11ee-b983-4ab2673ea3f0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Malware is one of the most dangerous and costl...</td>\n",
       "      <td>Name;Name</td>\n",
       "      <td>3df89074-6781-11ee-b983-4ab2673ea3f0;3df8909c-...</td>\n",
       "      <td>3cbe0612-6781-11ee-b983-4ab2673ea3f0</td>\n",
       "      <td>malware dangerous costly cyber threats nationa...</td>\n",
       "      <td>title</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3cbdf9ba-6781-11ee-b983-4ab2673ea3f0</td>\n",
       "      <td>Paper Title</td>\n",
       "      <td>2003.0</td>\n",
       "      <td>Malware is one of the most dangerous and costl...</td>\n",
       "      <td>Name;Name;Name;Name;Name;Name;Name;Name;Name;Name</td>\n",
       "      <td>3df61dbc-6781-11ee-b983-4ab2673ea3f0;3df61dda-...</td>\n",
       "      <td>3cbe0040-6781-11ee-b983-4ab2673ea3f0;3cbe08e2-...</td>\n",
       "      <td>malware dangerous costly cyber threats nationa...</td>\n",
       "      <td>title</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>3cbdfa32-6781-11ee-b983-4ab2673ea3f0</td>\n",
       "      <td>Paper Title</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Malware is one of the most dangerous and costl...</td>\n",
       "      <td>Name;Name;Name;Name;Name</td>\n",
       "      <td>3df62348-6781-11ee-b983-4ab2673ea3f0;3df62370-...</td>\n",
       "      <td>3cbe37ea-6781-11ee-b983-4ab2673ea3f0;3cbe789a-...</td>\n",
       "      <td>malware dangerous costly cyber threats nationa...</td>\n",
       "      <td>title</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>520</th>\n",
       "      <td>3cbe4302-6781-11ee-b983-4ab2673ea3f0</td>\n",
       "      <td>Paper Title</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Malware is one of the most dangerous and costl...</td>\n",
       "      <td>Name;Name</td>\n",
       "      <td>3df833a4-6781-11ee-b983-4ab2673ea3f0;3df89326-...</td>\n",
       "      <td>3cbe4bcc-6781-11ee-b983-4ab2673ea3f0;3cbe7b74-...</td>\n",
       "      <td>malware dangerous costly cyber threats nationa...</td>\n",
       "      <td>title</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      eid        title    year  \\\n",
       "2    3cbdf992-6781-11ee-b983-4ab2673ea3f0  Paper Title  1995.0   \n",
       "515  3cbe424e-6781-11ee-b983-4ab2673ea3f0          NaN     NaN   \n",
       "3    3cbdf9ba-6781-11ee-b983-4ab2673ea3f0  Paper Title  2003.0   \n",
       "6    3cbdfa32-6781-11ee-b983-4ab2673ea3f0  Paper Title     NaN   \n",
       "520  3cbe4302-6781-11ee-b983-4ab2673ea3f0  Paper Title     NaN   \n",
       "\n",
       "                                              abstract  \\\n",
       "2    Malware is one of the most dangerous and costl...   \n",
       "515  Malware is one of the most dangerous and costl...   \n",
       "3    Malware is one of the most dangerous and costl...   \n",
       "6    Malware is one of the most dangerous and costl...   \n",
       "520  Malware is one of the most dangerous and costl...   \n",
       "\n",
       "                                               authors  \\\n",
       "2                                  Name;Name;Name;Name   \n",
       "515                                          Name;Name   \n",
       "3    Name;Name;Name;Name;Name;Name;Name;Name;Name;Name   \n",
       "6                             Name;Name;Name;Name;Name   \n",
       "520                                          Name;Name   \n",
       "\n",
       "                                            author_ids  \\\n",
       "2    3df61d1c-6781-11ee-b983-4ab2673ea3f0;3df61d3a-...   \n",
       "515  3df89074-6781-11ee-b983-4ab2673ea3f0;3df8909c-...   \n",
       "3    3df61dbc-6781-11ee-b983-4ab2673ea3f0;3df61dda-...   \n",
       "6    3df62348-6781-11ee-b983-4ab2673ea3f0;3df62370-...   \n",
       "520  3df833a4-6781-11ee-b983-4ab2673ea3f0;3df89326-...   \n",
       "\n",
       "                                            references  \\\n",
       "2    3cbe5c8e-6781-11ee-b983-4ab2673ea3f0;3cbe1bac-...   \n",
       "515               3cbe0612-6781-11ee-b983-4ab2673ea3f0   \n",
       "3    3cbe0040-6781-11ee-b983-4ab2673ea3f0;3cbe08e2-...   \n",
       "6    3cbe37ea-6781-11ee-b983-4ab2673ea3f0;3cbe789a-...   \n",
       "520  3cbe4bcc-6781-11ee-b983-4ab2673ea3f0;3cbe7b74-...   \n",
       "\n",
       "                                        clean_abstract clean_title  \n",
       "2    malware dangerous costly cyber threats organiz...       title  \n",
       "515  malware dangerous costly cyber threats nationa...       title  \n",
       "3    malware dangerous costly cyber threats nationa...       title  \n",
       "6    malware dangerous costly cyber threats nationa...       title  \n",
       "520  malware dangerous costly cyber threats nationa...       title  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# search in a small window (0 words between the words in the query)\n",
    "search_parameters = {\n",
    "    \"query\": \"cyber threats\",\n",
    "    \"in_title\": True,        # if true, searches for query title\n",
    "    \"in_abstract\": True,     # if true, searches for query in abstract\n",
    "    \"ngram_window_size\": 2,  # search substrings of this size\n",
    "    \"ngram_ordered\": False,  # if true, preserves order of tokens in query\n",
    "    \"do_results_table\": True # if true, generate DataFrame for explainability\n",
    "}\n",
    "\n",
    "search_results, search_criteria = cheetah.search(**search_parameters)\n",
    "search_results.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filter_type</th>\n",
       "      <th>filter_value</th>\n",
       "      <th>num_papers</th>\n",
       "      <th>included_ids</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>query</td>\n",
       "      <td>cyber threats</td>\n",
       "      <td>208</td>\n",
       "      <td>2;515;3;6;520;12;13;530;20;532;543;33;548;38;5...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  filter_type   filter_value  num_papers  \\\n",
       "0       query  cyber threats         208   \n",
       "\n",
       "                                        included_ids  \n",
       "0  2;515;3;6;520;12;13;530;20;532;543;33;548;38;5...  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "search_criteria"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Intersect search for multiple words/phrases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Found 208 papers in 0.0035 seconds\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>eid</th>\n",
       "      <th>title</th>\n",
       "      <th>year</th>\n",
       "      <th>abstract</th>\n",
       "      <th>authors</th>\n",
       "      <th>author_ids</th>\n",
       "      <th>references</th>\n",
       "      <th>clean_abstract</th>\n",
       "      <th>clean_title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3cbdf992-6781-11ee-b983-4ab2673ea3f0</td>\n",
       "      <td>Paper Title</td>\n",
       "      <td>1995.0</td>\n",
       "      <td>Malware is one of the most dangerous and costl...</td>\n",
       "      <td>Name;Name;Name;Name</td>\n",
       "      <td>3df61d1c-6781-11ee-b983-4ab2673ea3f0;3df61d3a-...</td>\n",
       "      <td>3cbe5c8e-6781-11ee-b983-4ab2673ea3f0;3cbe1bac-...</td>\n",
       "      <td>malware dangerous costly cyber threats organiz...</td>\n",
       "      <td>title</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3cbdf9ba-6781-11ee-b983-4ab2673ea3f0</td>\n",
       "      <td>Paper Title</td>\n",
       "      <td>2003.0</td>\n",
       "      <td>Malware is one of the most dangerous and costl...</td>\n",
       "      <td>Name;Name;Name;Name;Name;Name;Name;Name;Name;Name</td>\n",
       "      <td>3df61dbc-6781-11ee-b983-4ab2673ea3f0;3df61dda-...</td>\n",
       "      <td>3cbe0040-6781-11ee-b983-4ab2673ea3f0;3cbe08e2-...</td>\n",
       "      <td>malware dangerous costly cyber threats nationa...</td>\n",
       "      <td>title</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>515</th>\n",
       "      <td>3cbe424e-6781-11ee-b983-4ab2673ea3f0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Malware is one of the most dangerous and costl...</td>\n",
       "      <td>Name;Name</td>\n",
       "      <td>3df89074-6781-11ee-b983-4ab2673ea3f0;3df8909c-...</td>\n",
       "      <td>3cbe0612-6781-11ee-b983-4ab2673ea3f0</td>\n",
       "      <td>malware dangerous costly cyber threats nationa...</td>\n",
       "      <td>title</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>3cbdfa32-6781-11ee-b983-4ab2673ea3f0</td>\n",
       "      <td>Paper Title</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Malware is one of the most dangerous and costl...</td>\n",
       "      <td>Name;Name;Name;Name;Name</td>\n",
       "      <td>3df62348-6781-11ee-b983-4ab2673ea3f0;3df62370-...</td>\n",
       "      <td>3cbe37ea-6781-11ee-b983-4ab2673ea3f0;3cbe789a-...</td>\n",
       "      <td>malware dangerous costly cyber threats nationa...</td>\n",
       "      <td>title</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>520</th>\n",
       "      <td>3cbe4302-6781-11ee-b983-4ab2673ea3f0</td>\n",
       "      <td>Paper Title</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Malware is one of the most dangerous and costl...</td>\n",
       "      <td>Name;Name</td>\n",
       "      <td>3df833a4-6781-11ee-b983-4ab2673ea3f0;3df89326-...</td>\n",
       "      <td>3cbe4bcc-6781-11ee-b983-4ab2673ea3f0;3cbe7b74-...</td>\n",
       "      <td>malware dangerous costly cyber threats nationa...</td>\n",
       "      <td>title</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      eid        title    year  \\\n",
       "2    3cbdf992-6781-11ee-b983-4ab2673ea3f0  Paper Title  1995.0   \n",
       "3    3cbdf9ba-6781-11ee-b983-4ab2673ea3f0  Paper Title  2003.0   \n",
       "515  3cbe424e-6781-11ee-b983-4ab2673ea3f0          NaN     NaN   \n",
       "6    3cbdfa32-6781-11ee-b983-4ab2673ea3f0  Paper Title     NaN   \n",
       "520  3cbe4302-6781-11ee-b983-4ab2673ea3f0  Paper Title     NaN   \n",
       "\n",
       "                                              abstract  \\\n",
       "2    Malware is one of the most dangerous and costl...   \n",
       "3    Malware is one of the most dangerous and costl...   \n",
       "515  Malware is one of the most dangerous and costl...   \n",
       "6    Malware is one of the most dangerous and costl...   \n",
       "520  Malware is one of the most dangerous and costl...   \n",
       "\n",
       "                                               authors  \\\n",
       "2                                  Name;Name;Name;Name   \n",
       "3    Name;Name;Name;Name;Name;Name;Name;Name;Name;Name   \n",
       "515                                          Name;Name   \n",
       "6                             Name;Name;Name;Name;Name   \n",
       "520                                          Name;Name   \n",
       "\n",
       "                                            author_ids  \\\n",
       "2    3df61d1c-6781-11ee-b983-4ab2673ea3f0;3df61d3a-...   \n",
       "3    3df61dbc-6781-11ee-b983-4ab2673ea3f0;3df61dda-...   \n",
       "515  3df89074-6781-11ee-b983-4ab2673ea3f0;3df8909c-...   \n",
       "6    3df62348-6781-11ee-b983-4ab2673ea3f0;3df62370-...   \n",
       "520  3df833a4-6781-11ee-b983-4ab2673ea3f0;3df89326-...   \n",
       "\n",
       "                                            references  \\\n",
       "2    3cbe5c8e-6781-11ee-b983-4ab2673ea3f0;3cbe1bac-...   \n",
       "3    3cbe0040-6781-11ee-b983-4ab2673ea3f0;3cbe08e2-...   \n",
       "515               3cbe0612-6781-11ee-b983-4ab2673ea3f0   \n",
       "6    3cbe37ea-6781-11ee-b983-4ab2673ea3f0;3cbe789a-...   \n",
       "520  3cbe4bcc-6781-11ee-b983-4ab2673ea3f0;3cbe7b74-...   \n",
       "\n",
       "                                        clean_abstract clean_title  \n",
       "2    malware dangerous costly cyber threats organiz...       title  \n",
       "3    malware dangerous costly cyber threats nationa...       title  \n",
       "515  malware dangerous costly cyber threats nationa...       title  \n",
       "6    malware dangerous costly cyber threats nationa...       title  \n",
       "520  malware dangerous costly cyber threats nationa...       title  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# search in a small window (max 8 words between the words in the query)\n",
    "search_parameters = {\n",
    "    \"query\": [\"cyber threats\", \"malware\"],\n",
    "    \"and_search\": True,      # if true, intersect search results for multiple queries\n",
    "    \"in_title\": True,        # if true, searches for query title\n",
    "    \"in_abstract\": True,     # if true, searches for query in abstract\n",
    "    \"ngram_window_size\": 5,  # search substrings of this size\n",
    "    \"ngram_ordered\": True,   # if true, preserves order of tokens in query\n",
    "    \"do_results_table\": True # if true, generate DataFrame for explainability\n",
    "}\n",
    "\n",
    "search_results, search_criteria = cheetah.search(**search_parameters)\n",
    "search_results.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filter_type</th>\n",
       "      <th>filter_value</th>\n",
       "      <th>num_papers</th>\n",
       "      <th>included_ids</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>query</td>\n",
       "      <td>cyber threats</td>\n",
       "      <td>208</td>\n",
       "      <td>2;515;3;6;520;12;13;530;20;532;543;33;548;38;5...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>query</td>\n",
       "      <td>malware</td>\n",
       "      <td>306</td>\n",
       "      <td>2;3;6;12;13;17;20;30;33;34;38;41;43;47;51;53;5...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  filter_type   filter_value  num_papers  \\\n",
       "0       query  cyber threats         208   \n",
       "1       query        malware         306   \n",
       "\n",
       "                                        included_ids  \n",
       "0  2;515;3;6;520;12;13;530;20;532;543;33;548;38;5...  \n",
       "1  2;3;6;12;13;17;20;30;33;34;38;41;43;47;51;53;5...  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "search_criteria"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Union search for multiple words/phrases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Found 306 papers in 0.0034 seconds\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>eid</th>\n",
       "      <th>title</th>\n",
       "      <th>year</th>\n",
       "      <th>abstract</th>\n",
       "      <th>authors</th>\n",
       "      <th>author_ids</th>\n",
       "      <th>references</th>\n",
       "      <th>clean_abstract</th>\n",
       "      <th>clean_title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3cbdf992-6781-11ee-b983-4ab2673ea3f0</td>\n",
       "      <td>Paper Title</td>\n",
       "      <td>1995.0</td>\n",
       "      <td>Malware is one of the most dangerous and costl...</td>\n",
       "      <td>Name;Name;Name;Name</td>\n",
       "      <td>3df61d1c-6781-11ee-b983-4ab2673ea3f0;3df61d3a-...</td>\n",
       "      <td>3cbe5c8e-6781-11ee-b983-4ab2673ea3f0;3cbe1bac-...</td>\n",
       "      <td>malware dangerous costly cyber threats organiz...</td>\n",
       "      <td>title</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3cbdf9ba-6781-11ee-b983-4ab2673ea3f0</td>\n",
       "      <td>Paper Title</td>\n",
       "      <td>2003.0</td>\n",
       "      <td>Malware is one of the most dangerous and costl...</td>\n",
       "      <td>Name;Name;Name;Name;Name;Name;Name;Name;Name;Name</td>\n",
       "      <td>3df61dbc-6781-11ee-b983-4ab2673ea3f0;3df61dda-...</td>\n",
       "      <td>3cbe0040-6781-11ee-b983-4ab2673ea3f0;3cbe08e2-...</td>\n",
       "      <td>malware dangerous costly cyber threats nationa...</td>\n",
       "      <td>title</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>3cbdfa32-6781-11ee-b983-4ab2673ea3f0</td>\n",
       "      <td>Paper Title</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Malware is one of the most dangerous and costl...</td>\n",
       "      <td>Name;Name;Name;Name;Name</td>\n",
       "      <td>3df62348-6781-11ee-b983-4ab2673ea3f0;3df62370-...</td>\n",
       "      <td>3cbe37ea-6781-11ee-b983-4ab2673ea3f0;3cbe789a-...</td>\n",
       "      <td>malware dangerous costly cyber threats nationa...</td>\n",
       "      <td>title</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>3cbdfb18-6781-11ee-b983-4ab2673ea3f0</td>\n",
       "      <td>Paper Title</td>\n",
       "      <td>2000.0</td>\n",
       "      <td>Malware is one of the most dangerous and costl...</td>\n",
       "      <td>Name</td>\n",
       "      <td>3df62f82-6781-11ee-b983-4ab2673ea3f0</td>\n",
       "      <td>3cbe0a68-6781-11ee-b983-4ab2673ea3f0;3cbe2138-...</td>\n",
       "      <td>malware dangerous costly cyber threats organiz...</td>\n",
       "      <td>title</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>3cbdfb36-6781-11ee-b983-4ab2673ea3f0</td>\n",
       "      <td>Paper Title</td>\n",
       "      <td>2003.0</td>\n",
       "      <td>Malware is one of the most dangerous and costl...</td>\n",
       "      <td>Name;Name;Name;Name;Name;Name</td>\n",
       "      <td>3df62faa-6781-11ee-b983-4ab2673ea3f0;3df62fd2-...</td>\n",
       "      <td>3cbe07a2-6781-11ee-b983-4ab2673ea3f0;3cbe3cae-...</td>\n",
       "      <td>malware dangerous costly cyber threats nationa...</td>\n",
       "      <td>title</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     eid        title    year  \\\n",
       "2   3cbdf992-6781-11ee-b983-4ab2673ea3f0  Paper Title  1995.0   \n",
       "3   3cbdf9ba-6781-11ee-b983-4ab2673ea3f0  Paper Title  2003.0   \n",
       "6   3cbdfa32-6781-11ee-b983-4ab2673ea3f0  Paper Title     NaN   \n",
       "12  3cbdfb18-6781-11ee-b983-4ab2673ea3f0  Paper Title  2000.0   \n",
       "13  3cbdfb36-6781-11ee-b983-4ab2673ea3f0  Paper Title  2003.0   \n",
       "\n",
       "                                             abstract  \\\n",
       "2   Malware is one of the most dangerous and costl...   \n",
       "3   Malware is one of the most dangerous and costl...   \n",
       "6   Malware is one of the most dangerous and costl...   \n",
       "12  Malware is one of the most dangerous and costl...   \n",
       "13  Malware is one of the most dangerous and costl...   \n",
       "\n",
       "                                              authors  \\\n",
       "2                                 Name;Name;Name;Name   \n",
       "3   Name;Name;Name;Name;Name;Name;Name;Name;Name;Name   \n",
       "6                            Name;Name;Name;Name;Name   \n",
       "12                                               Name   \n",
       "13                      Name;Name;Name;Name;Name;Name   \n",
       "\n",
       "                                           author_ids  \\\n",
       "2   3df61d1c-6781-11ee-b983-4ab2673ea3f0;3df61d3a-...   \n",
       "3   3df61dbc-6781-11ee-b983-4ab2673ea3f0;3df61dda-...   \n",
       "6   3df62348-6781-11ee-b983-4ab2673ea3f0;3df62370-...   \n",
       "12               3df62f82-6781-11ee-b983-4ab2673ea3f0   \n",
       "13  3df62faa-6781-11ee-b983-4ab2673ea3f0;3df62fd2-...   \n",
       "\n",
       "                                           references  \\\n",
       "2   3cbe5c8e-6781-11ee-b983-4ab2673ea3f0;3cbe1bac-...   \n",
       "3   3cbe0040-6781-11ee-b983-4ab2673ea3f0;3cbe08e2-...   \n",
       "6   3cbe37ea-6781-11ee-b983-4ab2673ea3f0;3cbe789a-...   \n",
       "12  3cbe0a68-6781-11ee-b983-4ab2673ea3f0;3cbe2138-...   \n",
       "13  3cbe07a2-6781-11ee-b983-4ab2673ea3f0;3cbe3cae-...   \n",
       "\n",
       "                                       clean_abstract clean_title  \n",
       "2   malware dangerous costly cyber threats organiz...       title  \n",
       "3   malware dangerous costly cyber threats nationa...       title  \n",
       "6   malware dangerous costly cyber threats nationa...       title  \n",
       "12  malware dangerous costly cyber threats organiz...       title  \n",
       "13  malware dangerous costly cyber threats nationa...       title  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# search in a small window (max 8 words between the words in the query)\n",
    "search_parameters = {\n",
    "    \"query\": [\"cyber threats\", \"malware\"],\n",
    "    \"and_search\": False,     # if true, intersect search results for multiple queries\n",
    "    \"in_title\": True,        # if true, searches for query title\n",
    "    \"in_abstract\": True,     # if true, searches for query in abstract\n",
    "    \"ngram_window_size\": 5,  # search substrings of this size\n",
    "    \"ngram_ordered\": True,   # if true, preserves order of tokens in query\n",
    "    \"do_results_table\": True # if true, generate DataFrame for explainability\n",
    "}\n",
    "\n",
    "search_results, search_criteria = cheetah.search(**search_parameters)\n",
    "search_results.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filter_type</th>\n",
       "      <th>filter_value</th>\n",
       "      <th>num_papers</th>\n",
       "      <th>included_ids</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>query</td>\n",
       "      <td>cyber threats</td>\n",
       "      <td>208</td>\n",
       "      <td>2;515;3;6;520;12;13;530;20;532;543;33;548;38;5...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>query</td>\n",
       "      <td>malware</td>\n",
       "      <td>306</td>\n",
       "      <td>2;3;6;12;13;17;20;30;33;34;38;41;43;47;51;53;5...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  filter_type   filter_value  num_papers  \\\n",
       "0       query  cyber threats         208   \n",
       "1       query        malware         306   \n",
       "\n",
       "                                        included_ids  \n",
       "0  2;515;3;6;520;12;13;530;20;532;543;33;548;38;5...  \n",
       "1  2;3;6;12;13;17;20;30;33;34;38;41;43;47;51;53;5...  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "search_criteria"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Use Negative Prompts in Search"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Dictionaries in the search query serve as negative prompts. This means that a given term will be looked up but results must exclude the word(s) in the negative prompt. The negative prompt should only contain unigrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Found 113 papers in 0.0016 seconds\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>eid</th>\n",
       "      <th>title</th>\n",
       "      <th>year</th>\n",
       "      <th>abstract</th>\n",
       "      <th>authors</th>\n",
       "      <th>author_ids</th>\n",
       "      <th>references</th>\n",
       "      <th>clean_abstract</th>\n",
       "      <th>clean_title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3cbdf82a-6781-11ee-b983-4ab2673ea3f0</td>\n",
       "      <td>Paper Title</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>Supervisory Control and Data Acquisition (SCAD...</td>\n",
       "      <td>Name;Name;Name;Name;Name</td>\n",
       "      <td>3df61b32-6781-11ee-b983-4ab2673ea3f0;3df61c18-...</td>\n",
       "      <td>3cbe2bec-6781-11ee-b983-4ab2673ea3f0;3cbe6d64-...</td>\n",
       "      <td>supervisory control acquisition scada serve ne...</td>\n",
       "      <td>title</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3cbdf960-6781-11ee-b983-4ab2673ea3f0</td>\n",
       "      <td>Paper Title</td>\n",
       "      <td>2002.0</td>\n",
       "      <td>Supervisory Control and Data Acquisition (SCAD...</td>\n",
       "      <td>Name;Name</td>\n",
       "      <td>3df61cc2-6781-11ee-b983-4ab2673ea3f0;3df61cea-...</td>\n",
       "      <td>3cbe45d2-6781-11ee-b983-4ab2673ea3f0;3cbe0338-...</td>\n",
       "      <td>supervisory control acquisition scada serve ne...</td>\n",
       "      <td>title</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>770</th>\n",
       "      <td>3cbe6666-6781-11ee-b983-4ab2673ea3f0</td>\n",
       "      <td>Paper Title</td>\n",
       "      <td>2018.0</td>\n",
       "      <td>Supervisory Control and Data Acquisition (SCAD...</td>\n",
       "      <td>Name;Name</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3cbe5bb2-6781-11ee-b983-4ab2673ea3f0;3cbe7232-...</td>\n",
       "      <td>supervisory control acquisition scada serve ne...</td>\n",
       "      <td>title</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>518</th>\n",
       "      <td>3cbe42bc-6781-11ee-b983-4ab2673ea3f0</td>\n",
       "      <td>Paper Title</td>\n",
       "      <td>1997.0</td>\n",
       "      <td>Supervisory Control and Data Acquisition (SCAD...</td>\n",
       "      <td>Name;Name;Name;Name;Name;Name</td>\n",
       "      <td>3df891dc-6781-11ee-b983-4ab2673ea3f0;3df89204-...</td>\n",
       "      <td>3cbe504a-6781-11ee-b983-4ab2673ea3f0;3cbe7ac0-...</td>\n",
       "      <td>supervisory control acquisition scada serve ne...</td>\n",
       "      <td>title</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>774</th>\n",
       "      <td>3cbe66f2-6781-11ee-b983-4ab2673ea3f0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2009.0</td>\n",
       "      <td>Supervisory Control and Data Acquisition (SCAD...</td>\n",
       "      <td>Name;Name</td>\n",
       "      <td>3df7a10a-6781-11ee-b983-4ab2673ea3f0;3df98cc2-...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>supervisory control acquisition scada serve ne...</td>\n",
       "      <td>title</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      eid        title    year  \\\n",
       "0    3cbdf82a-6781-11ee-b983-4ab2673ea3f0  Paper Title  2016.0   \n",
       "1    3cbdf960-6781-11ee-b983-4ab2673ea3f0  Paper Title  2002.0   \n",
       "770  3cbe6666-6781-11ee-b983-4ab2673ea3f0  Paper Title  2018.0   \n",
       "518  3cbe42bc-6781-11ee-b983-4ab2673ea3f0  Paper Title  1997.0   \n",
       "774  3cbe66f2-6781-11ee-b983-4ab2673ea3f0          NaN  2009.0   \n",
       "\n",
       "                                              abstract  \\\n",
       "0    Supervisory Control and Data Acquisition (SCAD...   \n",
       "1    Supervisory Control and Data Acquisition (SCAD...   \n",
       "770  Supervisory Control and Data Acquisition (SCAD...   \n",
       "518  Supervisory Control and Data Acquisition (SCAD...   \n",
       "774  Supervisory Control and Data Acquisition (SCAD...   \n",
       "\n",
       "                           authors  \\\n",
       "0         Name;Name;Name;Name;Name   \n",
       "1                        Name;Name   \n",
       "770                      Name;Name   \n",
       "518  Name;Name;Name;Name;Name;Name   \n",
       "774                      Name;Name   \n",
       "\n",
       "                                            author_ids  \\\n",
       "0    3df61b32-6781-11ee-b983-4ab2673ea3f0;3df61c18-...   \n",
       "1    3df61cc2-6781-11ee-b983-4ab2673ea3f0;3df61cea-...   \n",
       "770                                                NaN   \n",
       "518  3df891dc-6781-11ee-b983-4ab2673ea3f0;3df89204-...   \n",
       "774  3df7a10a-6781-11ee-b983-4ab2673ea3f0;3df98cc2-...   \n",
       "\n",
       "                                            references  \\\n",
       "0    3cbe2bec-6781-11ee-b983-4ab2673ea3f0;3cbe6d64-...   \n",
       "1    3cbe45d2-6781-11ee-b983-4ab2673ea3f0;3cbe0338-...   \n",
       "770  3cbe5bb2-6781-11ee-b983-4ab2673ea3f0;3cbe7232-...   \n",
       "518  3cbe504a-6781-11ee-b983-4ab2673ea3f0;3cbe7ac0-...   \n",
       "774                                                NaN   \n",
       "\n",
       "                                        clean_abstract clean_title  \n",
       "0    supervisory control acquisition scada serve ne...       title  \n",
       "1    supervisory control acquisition scada serve ne...       title  \n",
       "770  supervisory control acquisition scada serve ne...       title  \n",
       "518  supervisory control acquisition scada serve ne...       title  \n",
       "774  supervisory control acquisition scada serve ne...       title  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "search_parameters = {\n",
    "    \"query\": {\"scada\": \"malware\"},\n",
    "    \"in_title\":True,         # if true searches for query title\n",
    "    \"in_abstract\":True,     # if true searches for query in abstract\n",
    "    \"do_results_table\": True # if true, generate DataFrame for explainability\n",
    "}\n",
    "\n",
    "search_results, search_criteria = cheetah.search(**search_parameters)\n",
    "search_results.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filter_type</th>\n",
       "      <th>filter_value</th>\n",
       "      <th>num_papers</th>\n",
       "      <th>included_ids</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>query</td>\n",
       "      <td>scada</td>\n",
       "      <td>113</td>\n",
       "      <td>0;1;770;518;774;9;521;11;779;780;526;273;786;2...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  filter_type filter_value  num_papers  \\\n",
       "0       query        scada         113   \n",
       "\n",
       "                                        included_ids  \n",
       "0  0;1;770;518;774;9;521;11;779;780;526;273;786;2...  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "search_criteria"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Found 113 papers in 0.0018 seconds\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>eid</th>\n",
       "      <th>title</th>\n",
       "      <th>year</th>\n",
       "      <th>abstract</th>\n",
       "      <th>authors</th>\n",
       "      <th>author_ids</th>\n",
       "      <th>references</th>\n",
       "      <th>clean_abstract</th>\n",
       "      <th>clean_title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3cbdf82a-6781-11ee-b983-4ab2673ea3f0</td>\n",
       "      <td>Paper Title</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>Supervisory Control and Data Acquisition (SCAD...</td>\n",
       "      <td>Name;Name;Name;Name;Name</td>\n",
       "      <td>3df61b32-6781-11ee-b983-4ab2673ea3f0;3df61c18-...</td>\n",
       "      <td>3cbe2bec-6781-11ee-b983-4ab2673ea3f0;3cbe6d64-...</td>\n",
       "      <td>supervisory control acquisition scada serve ne...</td>\n",
       "      <td>title</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3cbdf960-6781-11ee-b983-4ab2673ea3f0</td>\n",
       "      <td>Paper Title</td>\n",
       "      <td>2002.0</td>\n",
       "      <td>Supervisory Control and Data Acquisition (SCAD...</td>\n",
       "      <td>Name;Name</td>\n",
       "      <td>3df61cc2-6781-11ee-b983-4ab2673ea3f0;3df61cea-...</td>\n",
       "      <td>3cbe45d2-6781-11ee-b983-4ab2673ea3f0;3cbe0338-...</td>\n",
       "      <td>supervisory control acquisition scada serve ne...</td>\n",
       "      <td>title</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>770</th>\n",
       "      <td>3cbe6666-6781-11ee-b983-4ab2673ea3f0</td>\n",
       "      <td>Paper Title</td>\n",
       "      <td>2018.0</td>\n",
       "      <td>Supervisory Control and Data Acquisition (SCAD...</td>\n",
       "      <td>Name;Name</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3cbe5bb2-6781-11ee-b983-4ab2673ea3f0;3cbe7232-...</td>\n",
       "      <td>supervisory control acquisition scada serve ne...</td>\n",
       "      <td>title</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>518</th>\n",
       "      <td>3cbe42bc-6781-11ee-b983-4ab2673ea3f0</td>\n",
       "      <td>Paper Title</td>\n",
       "      <td>1997.0</td>\n",
       "      <td>Supervisory Control and Data Acquisition (SCAD...</td>\n",
       "      <td>Name;Name;Name;Name;Name;Name</td>\n",
       "      <td>3df891dc-6781-11ee-b983-4ab2673ea3f0;3df89204-...</td>\n",
       "      <td>3cbe504a-6781-11ee-b983-4ab2673ea3f0;3cbe7ac0-...</td>\n",
       "      <td>supervisory control acquisition scada serve ne...</td>\n",
       "      <td>title</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>774</th>\n",
       "      <td>3cbe66f2-6781-11ee-b983-4ab2673ea3f0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2009.0</td>\n",
       "      <td>Supervisory Control and Data Acquisition (SCAD...</td>\n",
       "      <td>Name;Name</td>\n",
       "      <td>3df7a10a-6781-11ee-b983-4ab2673ea3f0;3df98cc2-...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>supervisory control acquisition scada serve ne...</td>\n",
       "      <td>title</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      eid        title    year  \\\n",
       "0    3cbdf82a-6781-11ee-b983-4ab2673ea3f0  Paper Title  2016.0   \n",
       "1    3cbdf960-6781-11ee-b983-4ab2673ea3f0  Paper Title  2002.0   \n",
       "770  3cbe6666-6781-11ee-b983-4ab2673ea3f0  Paper Title  2018.0   \n",
       "518  3cbe42bc-6781-11ee-b983-4ab2673ea3f0  Paper Title  1997.0   \n",
       "774  3cbe66f2-6781-11ee-b983-4ab2673ea3f0          NaN  2009.0   \n",
       "\n",
       "                                              abstract  \\\n",
       "0    Supervisory Control and Data Acquisition (SCAD...   \n",
       "1    Supervisory Control and Data Acquisition (SCAD...   \n",
       "770  Supervisory Control and Data Acquisition (SCAD...   \n",
       "518  Supervisory Control and Data Acquisition (SCAD...   \n",
       "774  Supervisory Control and Data Acquisition (SCAD...   \n",
       "\n",
       "                           authors  \\\n",
       "0         Name;Name;Name;Name;Name   \n",
       "1                        Name;Name   \n",
       "770                      Name;Name   \n",
       "518  Name;Name;Name;Name;Name;Name   \n",
       "774                      Name;Name   \n",
       "\n",
       "                                            author_ids  \\\n",
       "0    3df61b32-6781-11ee-b983-4ab2673ea3f0;3df61c18-...   \n",
       "1    3df61cc2-6781-11ee-b983-4ab2673ea3f0;3df61cea-...   \n",
       "770                                                NaN   \n",
       "518  3df891dc-6781-11ee-b983-4ab2673ea3f0;3df89204-...   \n",
       "774  3df7a10a-6781-11ee-b983-4ab2673ea3f0;3df98cc2-...   \n",
       "\n",
       "                                            references  \\\n",
       "0    3cbe2bec-6781-11ee-b983-4ab2673ea3f0;3cbe6d64-...   \n",
       "1    3cbe45d2-6781-11ee-b983-4ab2673ea3f0;3cbe0338-...   \n",
       "770  3cbe5bb2-6781-11ee-b983-4ab2673ea3f0;3cbe7232-...   \n",
       "518  3cbe504a-6781-11ee-b983-4ab2673ea3f0;3cbe7ac0-...   \n",
       "774                                                NaN   \n",
       "\n",
       "                                        clean_abstract clean_title  \n",
       "0    supervisory control acquisition scada serve ne...       title  \n",
       "1    supervisory control acquisition scada serve ne...       title  \n",
       "770  supervisory control acquisition scada serve ne...       title  \n",
       "518  supervisory control acquisition scada serve ne...       title  \n",
       "774  supervisory control acquisition scada serve ne...       title  "
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "search_parameters = {\n",
    "    \"query\": {\"scada\": [\"malware\", \"threat\"]},\n",
    "    \"in_title\":True,         # if true searches for query title\n",
    "    \"in_abstract\":True,      # if true searches for query in abstract\n",
    "    \"do_results_table\": True # if true, generate DataFrame for explainability\n",
    "}\n",
    "\n",
    "search_results, search_criteria = cheetah.search(**search_parameters)\n",
    "search_results.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filter_type</th>\n",
       "      <th>filter_value</th>\n",
       "      <th>num_papers</th>\n",
       "      <th>included_ids</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>query</td>\n",
       "      <td>scada</td>\n",
       "      <td>113</td>\n",
       "      <td>0;1;770;518;774;9;521;11;779;780;526;273;786;2...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  filter_type filter_value  num_papers  \\\n",
       "0       query        scada         113   \n",
       "\n",
       "                                        included_ids  \n",
       "0  0;1;770;518;774;9;521;11;779;780;526;273;786;2...  "
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "search_criteria"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}