qualitativeComparePapers.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6a6bafc4",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from gensim.utils import simple_preprocess\n",
    "from gensim.models.word2vec import LineSentence\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from nltk.tokenize import word_tokenize\n",
    "from gensim.models import word2vec, FastText\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "import re\n",
    "import string\n",
    "import pandas as pd\n",
    "import gensim\n",
    "import PyPDF2 \n",
    "import nltk.data\n",
    "from nltk.stem.snowball import SnowballStemmer\n",
    "from gensim.parsing.preprocessing import remove_stopwords\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "25668415",
   "metadata": {},
   "source": [
    "This NLP code consideres two papers that are on the same topic and compares each of them to a list of key words from a research topic. This is indended to save time and get the most, qualitative, information."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f4fbd8ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "with open('/Users/Lumin/Desktop/NLP DataTools1/keywords_bats.txt', 'r') as f:\n",
    "    batReader = f.read()\n",
    "f.close()\n",
    "    \n",
    "readerChiropteraYang = PyPDF2.PdfReader('/Users/Lumin/Desktop/NLP DataTools1/Molecular_Phylogenetics_and_Taxonomic_Review_of_No.pdf')\n",
    "readerChiropteraVespert = PyPDF2.PdfReader('/Users/Lumin/Desktop/NLP DataTools1/Mehdizadehetal.2021.pdf')\n",
    "#cVerpert = readerChiropteraVespert.metadata\n",
    "#cYang = readerChiropteraYang.metadata\n",
    "len(readerChiropteraYang.pages) #14\n",
    "len(readerChiropteraVespert.pages) #12\n",
    "\n",
    "\n",
    "#fullYang\n",
    "fullYang = ''\n",
    "for p in range(len(readerChiropteraYang.pages)):\n",
    "    page = readerChiropteraYang.pages[p]\n",
    "    fullYang += page.extract_text()\n",
    "#fullVespert\n",
    "fullVespert = ''\n",
    "for p in range(len(readerChiropteraVespert.pages)):\n",
    "    page = readerChiropteraVespert.pages[p]\n",
    "    fullVespert += page.extract_text()\n",
    "    \n",
    "tokenizedBat = word_tokenize(batReader)\n",
    "tokenizedYang = word_tokenize(fullYang)\n",
    "tokenizedVespert = word_tokenize(fullVespert)\n",
    "\n",
    "plowBat = [word.lower() for word in tokenizedBat if word.isalpha()]\n",
    "\n",
    "plowYang=[word.lower() for word in tokenizedYang if word.isalpha()]\n",
    "\n",
    "plowVespert=[word.lower() for word in tokenizedVespert if word.isalpha()]\n",
    "\n",
    "stopWords = nltk.download('stopwords')\n",
    "stops = set(stopwords.words('english'))\n",
    "\n",
    "stoppedBat = []\n",
    "for w in plowBat:\n",
    "    if w not in stops:\n",
    "        stoppedBat.append(w)\n",
    "\n",
    "stoppedYang = []\n",
    "for w in plowYang:\n",
    "    if w not in stops:\n",
    "        stoppedYang.append(w)\n",
    "        \n",
    "stoppedVespert = []\n",
    "for w in plowVespert:\n",
    "    if w not in stops:\n",
    "        stoppedVespert.append(w)\n",
    "\n",
    "\n",
    "snowed = SnowballStemmer(language='english')\n",
    "\n",
    "stemmedBat = []\n",
    "for w in stoppedBat:\n",
    "    x = snowed.stem(w)\n",
    "    stemmedBat.append(x)\n",
    "\n",
    "stemmedYang = []\n",
    "for w in stoppedYang:\n",
    "    x = snowed.stem(w)\n",
    "    stemmedYang.append(x)\n",
    "\n",
    "stemmedVespert = []\n",
    "for w in stoppedVespert:\n",
    "    x = snowed.stem(w)\n",
    "    stemmedVespert.append(x)\n",
    "\n",
    "\n",
    "batYangSimSt =  len([i for i in stoppedBat if i in stoppedYang])\n",
    "batVespertSimSt = len([i for i in stoppedBat if i in stoppedVespert])\n",
    "#print(\"batYangSimStopped\", batYangSim, \"batVespertSimStopped\", batVespertSim)\n",
    "#for all the words in the research group key words, one article  has more related data than than the other on the same topic with:\n",
    "'''batYangSimStopped 77 batVespertSimStopped 48''' #not stemmed\n",
    "\n",
    "batYangSimStem =  [i for i in stoppedBat if i in stemmedYang]\n",
    "batVespertSimStem = [i for i in stoppedBat if i in stemmedVespert]\n",
    "#print(\"batYangSimStem\", batYangSimStem, \"batVespertSimStem\", batVespertSimStem)\n",
    "'''batYangSimStem 40 batVespertSimStem 28'''#stemmed\n",
    "\n",
    "if len(batYangSimStem) > len(batVespertSimStem):\n",
    "    print(\"ChiropteraYang...what you get\",'\\n', batYangSimStem, \"\\nNot included: \",'\\n', [set(batVespertSimStem) - set(batYangSimStem)])\n",
    "    \n",
    "else:\n",
    "    print(\"ChiropteraVespert...what you get: \\n\",batVespertSimStem, '\\nNot included: ','\\n',[set(batYangSimStem)-set(batVespertSimStem)])\n",
    "    \n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}