GPT_calls_NCBI_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c127d843",
   "metadata": {},
   "source": [
    "# Annotating NCBI test dataset with GPT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "43a694a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import openai\n",
    "import csv\n",
    "import json\n",
    "from nltk.tokenize import word_tokenize\n",
    "import pandas as pd\n",
    "import re\n",
    "\n",
    "from seqeval.metrics import classification_report\n",
    "from seqeval.scheme import IOB2\n",
    "\n",
    "import copy\n",
    "\n",
    "import time\n",
    "import ast"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "11e69f07",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 941 entries, 0 to 940\n",
      "Data columns (total 4 columns):\n",
      " #   Column          Non-Null Count  Dtype \n",
      "---  ------          --------------  ----- \n",
      " 0   id              941 non-null    int64 \n",
      " 1   tokens          941 non-null    object\n",
      " 2   original_tags   941 non-null    object\n",
      " 3   predicted_tags  0 non-null      object\n",
      "dtypes: int64(1), object(3)\n",
      "memory usage: 36.8+ KB\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>tokens</th>\n",
       "      <th>original_tags</th>\n",
       "      <th>predicted_tags</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>[Clustering, of, missense, mutations, in, the,...</td>\n",
       "      <td>[O, O, O, O, O, O, B-DIS, I-DIS, I-DIS, O, O, ...</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>[Ataxia, -, telangiectasia, (, A, -, T, ), is,...</td>\n",
       "      <td>[B-DIS, I-DIS, I-DIS, O, B-DIS, I-DIS, I-DIS, ...</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>[The, risk, of, cancer, ,, especially, lymphoi...</td>\n",
       "      <td>[O, O, O, B-DIS, O, O, B-DIS, I-DIS, O, O, O, ...</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>[By, analysing, tumour, DNA, from, patients, w...</td>\n",
       "      <td>[O, O, B-DIS, O, O, O, O, B-DIS, I-DIS, I-DIS,...</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>[In, marked, contrast, to, the, ATM, mutation,...</td>\n",
       "      <td>[O, O, O, O, O, O, O, O, O, B-DIS, I-DIS, I-DI...</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id                                             tokens  \\\n",
       "0   0  [Clustering, of, missense, mutations, in, the,...   \n",
       "1   1  [Ataxia, -, telangiectasia, (, A, -, T, ), is,...   \n",
       "2   2  [The, risk, of, cancer, ,, especially, lymphoi...   \n",
       "3   3  [By, analysing, tumour, DNA, from, patients, w...   \n",
       "4   4  [In, marked, contrast, to, the, ATM, mutation,...   \n",
       "\n",
       "                                       original_tags predicted_tags  \n",
       "0  [O, O, O, O, O, O, B-DIS, I-DIS, I-DIS, O, O, ...           None  \n",
       "1  [B-DIS, I-DIS, I-DIS, O, B-DIS, I-DIS, I-DIS, ...           None  \n",
       "2  [O, O, O, B-DIS, O, O, B-DIS, I-DIS, O, O, O, ...           None  \n",
       "3  [O, O, B-DIS, O, O, O, O, B-DIS, I-DIS, I-DIS,...           None  \n",
       "4  [O, O, O, O, O, O, O, O, O, B-DIS, I-DIS, I-DI...           None  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def convert_to_list(string):\n",
    "    # Convert a list that is in string format to list\n",
    "    try:\n",
    "        return ast.literal_eval(string)\n",
    "    except (SyntaxError, ValueError):\n",
    "        return None\n",
    "\n",
    "ncbi_dataframe = pd.read_csv(\"ncbi_test.csv\", index_col=0, header=0, converters={'tokens': convert_to_list, 'original_tags': convert_to_list})\n",
    "ncbi_dataframe[\"predicted_tags\"] = None\n",
    "ncbi_dataframe.info()\n",
    "ncbi_dataframe.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c3d859ec",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'The evidence of a significant proportion of loss - of - function mutations and a complete absence of the normal copy of ATM in the majority of mutated tumours establishes somatic inactivation of this gene in the pathogenesis of sporadic T - PLL and suggests that ATM acts as a tumour suppressor .'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\" \".join(ncbi_dataframe.loc[10][\"tokens\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "68370d2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "openai.api_type = \"\"\n",
    "openai.api_key = \"\"\n",
    "openai.api_base = \"\"\n",
    "openai.api_version = \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "89785d43",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt = \"In the text below, give a list of disease entities. Words need to be in exactly the same format as in input text. Format the output in JSON with only DISEASE key. INPUT: \"\n",
    "model_temperature = 0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3043aaa0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def getGPTResponses(start_id, end_id):\n",
    "    with open(\"ncbi_responses_test.csv\", \"a\") as response_file:\n",
    "        writer = csv.writer(response_file, delimiter=',')\n",
    "        for i in range(start_id, end_id+1):\n",
    "            print(i)\n",
    "            text = \" \".join(ncbi_dataframe.loc[i][\"tokens\"])\n",
    "            time.sleep(1)\n",
    "            try:\n",
    "                response = openai.ChatCompletion.create(deployment_id = \"name\",\n",
    "            model = \"gpt-35-turbo\",temperature=model_temperature,\n",
    "\n",
    "                                                    messages=[{\"role\": \"user\", \"content\": prompt + text}])\n",
    "                writer.writerow([i, response[\"choices\"][0][\"message\"][\"content\"]])\n",
    "            except openai.error.InvalidRequestError as e:\n",
    "                writer.writerow([i, None])\n",
    "            except:\n",
    "                response = openai.ChatCompletion.create(deployment_id = \"name\",\n",
    "            model = \"gpt-35-turbo\",temperature=model_temperature,\n",
    "\n",
    "                                                    messages=[{\"role\": \"user\", \"content\": prompt + text}])\n",
    "                writer.writerow([i,response[\"choices\"][0][\"message\"][\"content\"]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "06704477",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Only run when you want to get new responses\n",
    "#getGPTResponses(0, 940)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ae7eebbb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 941 entries, 0 to 940\n",
      "Data columns (total 2 columns):\n",
      " #   Column  Non-Null Count  Dtype \n",
      "---  ------  --------------  ----- \n",
      " 0   0       941 non-null    int64 \n",
      " 1   1       938 non-null    object\n",
      "dtypes: int64(1), object(1)\n",
      "memory usage: 14.8+ KB\n"
     ]
    }
   ],
   "source": [
    "gpt_responses = pd.read_csv(\"ncbi_responses_test.csv\", header=None)\n",
    "gpt_responses.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "2628824f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# provided by supervisor and modified by student\n",
    "def formatToIOB(data):\n",
    "    prev_tag = None\n",
    "    new_data = []\n",
    "    for tag in data:\n",
    "        if tag == 'O':\n",
    "            new_data.append(tag)\n",
    "            prev_tag = None\n",
    "        elif prev_tag == tag:\n",
    "            new_data.append('I-'+tag)\n",
    "        else:\n",
    "            new_data.append('B-'+tag)\n",
    "            prev_tag = tag\n",
    "    return new_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "c0a6109b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def getGPTTags():\n",
    "    # Extract ner tags from GPT responses\n",
    "    tags_gpt = []\n",
    "    for i in range(len(gpt_responses)): # Go throught every response\n",
    "        \n",
    "        current_idx = gpt_responses.loc[i][0]\n",
    "        if gpt_responses.loc[i][1] != None:\n",
    "            res = gpt_responses.loc[i][1]\n",
    "        else:\n",
    "            res = None\n",
    "        \n",
    "        try:# Sometimes GPT model can return non JSON format\n",
    "            data = json.loads(res)\n",
    "        except Exception as e:\n",
    "            data = {}\n",
    "            data[\"DISEASE\"] = []\n",
    "        \n",
    "        pred_tags = ['O']*len(ncbi_dataframe.loc[ncbi_dataframe[\"id\"]==current_idx][\"original_tags\"].to_list()[0])\n",
    "        tokens_data = ncbi_dataframe.loc[ncbi_dataframe[\"id\"]==current_idx][\"tokens\"].to_list()[0]\n",
    "        for i in range(len(tokens_data)):\n",
    "            tokens_data[i] = tokens_data[i].lower()\n",
    "        for key, value in data.items(): # for each key and value parse the result if it is suitable\n",
    "            if key.strip().lower() != \"disease\" and key.strip().lower() != \"diseases\":\n",
    "                print(\"wrong key\", key)\n",
    "            \n",
    "            if isinstance(value, str):\n",
    "                value = [value]\n",
    "            elif not isinstance(value, list):\n",
    "                value = []\n",
    "            \n",
    "            for unit in value:\n",
    "                unit_tokens = word_tokenize(unit)# mostly the result consists of many words\n",
    "                tag = 'DIS'\n",
    "                for token in unit_tokens:\n",
    "                    token = token.lower()\n",
    "                    if token in tokens_data: # if the token from GPT is in the actual text, then annotate it\n",
    "                        for idx in range(len(tokens_data)):\n",
    "                            if tokens_data[idx] == token:\n",
    "                                pred_tags[idx] = tag\n",
    "                    else:\n",
    "                        if \"-\" in token or \"/\" in token:# word_tokenize does not tokenize words with - and /, this is done manually\n",
    "                            temp_tokens = re.split(r'([-/])', token)\n",
    "                            idx = 0\n",
    "                            while idx < len(tokens_data):\n",
    "                                if tokens_data[idx] == temp_tokens[0]:\n",
    "                                    start_idx = idx\n",
    "                                    temp_idx = 0\n",
    "                                    while temp_idx < len(temp_tokens) and tokens_data[idx] == temp_tokens[temp_idx]:\n",
    "                                        idx += 1\n",
    "                                        temp_idx += 1\n",
    "                                    if temp_idx >= len(temp_tokens):\n",
    "                                        pred_tags[start_idx:start_idx+len(temp_tokens)] = [tag]*(len(temp_tokens))\n",
    "                                else:        \n",
    "                                    idx += 1\n",
    "                        else:\n",
    "                            print(\"word\",token,\"not in data\",tokens_data)\n",
    "            pred_tags = formatToIOB(pred_tags)\n",
    "            tags_gpt.append(pred_tags)\n",
    "    return tags_gpt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "dd18d6dd",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "word deficiency not in data ['c5', 'was', 'undetectable', 'in', 'her', 'serum', 'by', 'both', 'immunodiffusion', 'and', 'hemolytic', 'assays', '.']\n",
      "word deficiency not in data ['thus', ',', 'the', 'postulated', 'rate', 'mutant', 'gene', 'appears', 'to', 'code', 'for', 'the', 'expression', 'of', 'low', 'amounts', 'of', 'hex', 'a', '.']\n",
      "word angelman not in data ['in', 'two', 'familial', 'cases', 'and', 'one', 'sporadic', 'case', ',', 'mosaicism', 'for', 'ube3a', 'mutations', 'was', 'detected', 'in', 'the', 'mother', 'of', 'three', 'as', 'sons', ',', 'in', 'the', 'maternal', 'grandfather', 'of', 'two', 'as', 'first', 'cousins', ',', 'and', 'in', 'the', 'mother', 'of', 'an', 'as', 'daughter', '.']\n",
      "word syndrome not in data ['in', 'two', 'familial', 'cases', 'and', 'one', 'sporadic', 'case', ',', 'mosaicism', 'for', 'ube3a', 'mutations', 'was', 'detected', 'in', 'the', 'mother', 'of', 'three', 'as', 'sons', ',', 'in', 'the', 'maternal', 'grandfather', 'of', 'two', 'as', 'first', 'cousins', ',', 'and', 'in', 'the', 'mother', 'of', 'an', 'as', 'daughter', '.']\n",
      "word deficiency not in data ['pulse', '-', 'chase', 'analysis', 'using', 'proband', 'fibroblasts', 'revealed', 'that', 'the', 'w474c', '-', 'containing', 'alpha', '-', 'subunit', 'precursor', 'was', 'normally', 'synthesized', ',', 'but', 'not', 'phosphorylated', 'or', 'secreted', ',', 'and', 'the', 'mature', 'lysosomal', 'alpha', '-', 'subunit', 'was', 'not', 'detected', '.']\n",
      "word lorenzo not in data ['oleic', 'and', 'erucic', 'acids', '(', 'lorenzos', 'oil', ')', 'were', 'administered', 'to', 'patients', '1', 'and', '4', ',', 'but', 'sufficient', 'effectiveness', 'was', 'not', 'obtained', '.']\n",
      "word 's not in data ['oleic', 'and', 'erucic', 'acids', '(', 'lorenzos', 'oil', ')', 'were', 'administered', 'to', 'patients', '1', 'and', '4', ',', 'but', 'sufficient', 'effectiveness', 'was', 'not', 'obtained', '.']\n",
      "word deficiency not in data ['direct', 'sequencing', 'of', 'amplified', 'c9', 'cdna', 'and', 'dna', 'revealed', 'a', 'nonsense', 'substitution', '(', 'cga', '-', '-', '>', 'tga', ')', 'at', 'codon', '95', 'in', 'exon', '4', 'in', 'the', 'four', 'c9', '-', 'deficient', 'individuals', '.']\n",
      "word cancer not in data ['risk', 'of', 'developing', 'colorectal', ',', 'breast', 'and', 'other', 'cancers', 'were', 'compared', 'between', 'genotyped', 'i1307k', 'carriers', 'and', 'non', '-', 'carriers', 'and', 'their', 'first', '-', 'degree', 'relatives', '.']\n",
      "word cancer not in data ['risk', 'of', 'developing', 'colorectal', ',', 'breast', 'and', 'other', 'cancers', 'were', 'compared', 'between', 'genotyped', 'i1307k', 'carriers', 'and', 'non', '-', 'carriers', 'and', 'their', 'first', '-', 'degree', 'relatives', '.']\n"
     ]
    }
   ],
   "source": [
    "tags = getGPTTags()\n",
    "ncbi_dataframe[\"predicted_tags\"] = tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "3ad08b24",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>tokens</th>\n",
       "      <th>original_tags</th>\n",
       "      <th>predicted_tags</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>[clustering, of, missense, mutations, in, the,...</td>\n",
       "      <td>[O, O, O, O, O, O, B-DIS, I-DIS, I-DIS, O, O, ...</td>\n",
       "      <td>[O, O, O, O, O, O, B-DIS, I-DIS, I-DIS, O, O, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>[ataxia, -, telangiectasia, (, a, -, t, ), is,...</td>\n",
       "      <td>[B-DIS, I-DIS, I-DIS, O, B-DIS, I-DIS, I-DIS, ...</td>\n",
       "      <td>[B-DIS, I-DIS, I-DIS, O, O, O, O, O, O, O, O, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>[the, risk, of, cancer, ,, especially, lymphoi...</td>\n",
       "      <td>[O, O, O, B-DIS, O, O, B-DIS, I-DIS, O, O, O, ...</td>\n",
       "      <td>[O, O, O, B-DIS, O, O, B-DIS, I-DIS, O, O, O, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>[by, analysing, tumour, dna, from, patients, w...</td>\n",
       "      <td>[O, O, B-DIS, O, O, O, O, B-DIS, I-DIS, I-DIS,...</td>\n",
       "      <td>[O, O, O, O, O, O, O, B-DIS, I-DIS, I-DIS, I-D...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>[in, marked, contrast, to, the, atm, mutation,...</td>\n",
       "      <td>[O, O, O, O, O, O, O, O, O, B-DIS, I-DIS, I-DI...</td>\n",
       "      <td>[O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>936</th>\n",
       "      <td>936</td>\n",
       "      <td>[in, an, attempt, to, resolve, this, issue, ,,...</td>\n",
       "      <td>[O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...</td>\n",
       "      <td>[O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>937</th>\n",
       "      <td>937</td>\n",
       "      <td>[these, reagents, detect, a, 220, -, kd, prote...</td>\n",
       "      <td>[O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...</td>\n",
       "      <td>[O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>938</th>\n",
       "      <td>938</td>\n",
       "      <td>[immunohistochemical, staining, of, human, bre...</td>\n",
       "      <td>[O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...</td>\n",
       "      <td>[O, O, O, O, B-DIS, O, O, O, O, O, O, O, B-DIS...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>939</th>\n",
       "      <td>939</td>\n",
       "      <td>[conversely, ,, brca1, expression, was, reduce...</td>\n",
       "      <td>[O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...</td>\n",
       "      <td>[O, O, O, O, O, O, O, O, O, O, O, O, B-DIS, I-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>940</th>\n",
       "      <td>940</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>941 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      id                                             tokens  \\\n",
       "0      0  [clustering, of, missense, mutations, in, the,...   \n",
       "1      1  [ataxia, -, telangiectasia, (, a, -, t, ), is,...   \n",
       "2      2  [the, risk, of, cancer, ,, especially, lymphoi...   \n",
       "3      3  [by, analysing, tumour, dna, from, patients, w...   \n",
       "4      4  [in, marked, contrast, to, the, atm, mutation,...   \n",
       "..   ...                                                ...   \n",
       "936  936  [in, an, attempt, to, resolve, this, issue, ,,...   \n",
       "937  937  [these, reagents, detect, a, 220, -, kd, prote...   \n",
       "938  938  [immunohistochemical, staining, of, human, bre...   \n",
       "939  939  [conversely, ,, brca1, expression, was, reduce...   \n",
       "940  940                                                 []   \n",
       "\n",
       "                                         original_tags  \\\n",
       "0    [O, O, O, O, O, O, B-DIS, I-DIS, I-DIS, O, O, ...   \n",
       "1    [B-DIS, I-DIS, I-DIS, O, B-DIS, I-DIS, I-DIS, ...   \n",
       "2    [O, O, O, B-DIS, O, O, B-DIS, I-DIS, O, O, O, ...   \n",
       "3    [O, O, B-DIS, O, O, O, O, B-DIS, I-DIS, I-DIS,...   \n",
       "4    [O, O, O, O, O, O, O, O, O, B-DIS, I-DIS, I-DI...   \n",
       "..                                                 ...   \n",
       "936  [O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...   \n",
       "937  [O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...   \n",
       "938  [O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...   \n",
       "939  [O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...   \n",
       "940                                                 []   \n",
       "\n",
       "                                        predicted_tags  \n",
       "0    [O, O, O, O, O, O, B-DIS, I-DIS, I-DIS, O, O, ...  \n",
       "1    [B-DIS, I-DIS, I-DIS, O, O, O, O, O, O, O, O, ...  \n",
       "2    [O, O, O, B-DIS, O, O, B-DIS, I-DIS, O, O, O, ...  \n",
       "3    [O, O, O, O, O, O, O, B-DIS, I-DIS, I-DIS, I-D...  \n",
       "4    [O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...  \n",
       "..                                                 ...  \n",
       "936  [O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...  \n",
       "937  [O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...  \n",
       "938  [O, O, O, O, B-DIS, O, O, O, O, O, O, O, B-DIS...  \n",
       "939  [O, O, O, O, O, O, O, O, O, O, O, O, B-DIS, I-...  \n",
       "940                                                 []  \n",
       "\n",
       "[941 rows x 4 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ncbi_dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "8eb111ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "def getScoresIOB(tags_data, tags_gpt):\n",
    "    # Classification report with IOB format\n",
    "    y_true = tags_data\n",
    "    y_pred = tags_gpt\n",
    "    print(classification_report(y_true, y_pred, mode='strict', scheme=IOB2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "4723ed48",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "         DIS       0.40      0.58      0.47       960\n",
      "\n",
      "   micro avg       0.40      0.58      0.47       960\n",
      "   macro avg       0.40      0.58      0.47       960\n",
      "weighted avg       0.40      0.58      0.47       960\n",
      "\n"
     ]
    }
   ],
   "source": [
    "ncbi_dataframe.drop(940) # remove empty row\n",
    "original_tags_IOB = ncbi_dataframe[\"original_tags\"].tolist()\n",
    "predicted_tags_IOB = ncbi_dataframe[\"predicted_tags\"].tolist()\n",
    "getScoresIOB(original_tags_IOB, predicted_tags_IOB)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "a6118af4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "          IS       0.40      0.58      0.47       955\n",
      "\n",
      "   micro avg       0.40      0.58      0.47       955\n",
      "   macro avg       0.40      0.58      0.47       955\n",
      "weighted avg       0.40      0.58      0.47       955\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# To compare these results to the other dataframe used in thesis we need to have it without IOB format.\n",
    "def removeIOB(data):\n",
    "    for i in range(len(data)):\n",
    "        row = data[i]\n",
    "        for j in range(len(row)):\n",
    "            tag = row[j]\n",
    "            if \"DIS\" in tag:\n",
    "                data[i][j] = \"DIS\"\n",
    "    return data\n",
    "\n",
    "\n",
    "def getScores(tags_data, tags_gpt):\n",
    "    # Classification report without IOB format\n",
    "    y_true = tags_data\n",
    "    y_pred = tags_gpt\n",
    "    print(classification_report(y_true, y_pred))\n",
    "\n",
    "\n",
    "original_tags_no_IOB = removeIOB(copy.deepcopy(original_tags_IOB))\n",
    "\n",
    "predicted_tags_no_IOB = removeIOB(copy.deepcopy(predicted_tags_IOB))\n",
    "\n",
    "getScores(original_tags_no_IOB, predicted_tags_no_IOB)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3e2d7d68",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}