examples/Beaver/tensor-Participation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "4d57e6e4-657d-45e0-9d8e-ff050047f3e0",
   "metadata": {},
   "source": [
    "# Load Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7e3ccf65-3f76-4acd-a516-c4f6a9e16982",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "from collections import defaultdict\n",
    "\n",
    "from TELF.pre_processing import Beaver"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "af84282b-848e-4d3c-bd81-e2add24db2b1",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 940 entries, 0 to 939\n",
      "Data columns (total 9 columns):\n",
      " #   Column          Non-Null Count  Dtype \n",
      "---  ------          --------------  ----- \n",
      " 0   eid             940 non-null    object\n",
      " 1   title           940 non-null    object\n",
      " 2   countries       940 non-null    object\n",
      " 3   year            940 non-null    int64 \n",
      " 4   abstract        940 non-null    object\n",
      " 5   authors         940 non-null    object\n",
      " 6   author_ids      940 non-null    object\n",
      " 7   references      843 non-null    object\n",
      " 8   clean_abstract  940 non-null    object\n",
      "dtypes: int64(1), object(8)\n",
      "memory usage: 66.2+ KB\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv(os.path.join('..', '..', 'data', 'sample.csv'))\n",
    "df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "90e3f42e-612f-4b9b-9fda-a0271829a41d",
   "metadata": {},
   "source": [
    "### Check how many total author publications are in the dataset\n",
    "Tensor should contain this many non-zero values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "525c18b6-13c7-4720-b039-71b04d0b921b",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9917"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "author_pub_counts = defaultdict(int)\n",
    "for authors in df.author_ids.to_list():\n",
    "    for author in authors.split(';'):\n",
    "        author_pub_counts[author] += 1\n",
    "        \n",
    "sum(author_pub_counts.values())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b24aa37a-484e-48ea-b335-afc56f5d2f74",
   "metadata": {},
   "source": [
    "# Build Participation Tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "62b79669-7b50-49c6-8fbc-22c3f25ef7bd",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "beaver = Beaver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "dd074491-6fee-4ccf-b2be-16e02d1ab935",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "preparing dictionaries for communication\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1/1 [00:00<00:00, 94.27it/s]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 9917/9917.0 [00:00<00:00, 474768.15it/s]\n"
     ]
    }
   ],
   "source": [
    "settings = {\n",
    "    \"dataset\":df,\n",
    "    \"target_columns\":(\"author_ids\", \"eid\", \"year\"),\n",
    "    \"dimension_order\":[0,1,2],\n",
    "    \"split_authors_with\":\";\",\n",
    "    \"save_path\":None,\n",
    "    \"verbose\":True,\n",
    "    \"n_jobs\":1,\n",
    "    \"n_nodes\":1\n",
    "}\n",
    "\n",
    "X, author_ids, paper_ids, years = beaver.participation_tensor(**settings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "8ccfa04f-6786-4b40-9a2d-fc757c66abef",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table><tbody><tr><th style=\"text-align: left\">Format</th><td style=\"text-align: left\">coo</td></tr><tr><th style=\"text-align: left\">Data Type</th><td style=\"text-align: left\">float64</td></tr><tr><th style=\"text-align: left\">Shape</th><td style=\"text-align: left\">(7054, 940, 29)</td></tr><tr><th style=\"text-align: left\">nnz</th><td style=\"text-align: left\">9917</td></tr><tr><th style=\"text-align: left\">Density</th><td style=\"text-align: left\">5.15725975968636e-05</td></tr><tr><th style=\"text-align: left\">Read-only</th><td style=\"text-align: left\">True</td></tr><tr><th style=\"text-align: left\">Size</th><td style=\"text-align: left\">309.9K</td></tr><tr><th style=\"text-align: left\">Storage ratio</th><td style=\"text-align: left\">0.0</td></tr></tbody></table>"
      ],
      "text/plain": [
       "<COO: shape=(7054, 940, 29), dtype=float64, nnz=9917, fill_value=0.0>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "bcab5f20-693d-431a-a8df-b7993204f253",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def inz_paper_author(X, i, author_ids, paper_ids, dimension_order=[0,1,2]):\n",
    "    \"\"\" return the paper and author associated with the ith non-zero entry in X \"\"\"\n",
    "    \n",
    "    nz = X.nonzero()\n",
    "    \n",
    "    # get author and paper dimension index corresponding to X \n",
    "    author_index = dimension_order.index(0)\n",
    "    paper_index = dimension_order.index(1)\n",
    "    i_author = nz[author_index][i]\n",
    "    i_paper = nz[paper_index][i]\n",
    "    return ( author_ids[i_author], paper_ids[i_paper] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "0820562c-d7be-4f0f-9574-dd58675902d1",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('3cbdf82a-6781-11ee-b983-4ab2673ea3f0',\n",
       " '3df61b32-6781-11ee-b983-4ab2673ea3f0')"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "author_id, eid = inz_paper_author(X, 0, author_ids, paper_ids, dimension_order=settings['dimension_order'])\n",
    "eid, author_id"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "95d651f7-2d31-493a-b3e3-a7d2cc40c2c6",
   "metadata": {},
   "source": [
    "### Paper matching eid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "dbad937e-f7d9-4d18-8560-53eee57fdbcc",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>eid</th>\n",
       "      <th>title</th>\n",
       "      <th>countries</th>\n",
       "      <th>year</th>\n",
       "      <th>abstract</th>\n",
       "      <th>authors</th>\n",
       "      <th>author_ids</th>\n",
       "      <th>references</th>\n",
       "      <th>clean_abstract</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3cbdf82a-6781-11ee-b983-4ab2673ea3f0</td>\n",
       "      <td>Paper Title</td>\n",
       "      <td>New Zealand;United States</td>\n",
       "      <td>2016</td>\n",
       "      <td>Supervisory Control and Data Acquisition (SCAD...</td>\n",
       "      <td>Name;Name;Name;Name;Name</td>\n",
       "      <td>3df61b32-6781-11ee-b983-4ab2673ea3f0;3df61c18-...</td>\n",
       "      <td>3cbe2bec-6781-11ee-b983-4ab2673ea3f0;3cbe6d64-...</td>\n",
       "      <td>supervisory control acquisition system often s...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    eid        title  \\\n",
       "0  3cbdf82a-6781-11ee-b983-4ab2673ea3f0  Paper Title   \n",
       "\n",
       "                   countries  year  \\\n",
       "0  New Zealand;United States  2016   \n",
       "\n",
       "                                            abstract  \\\n",
       "0  Supervisory Control and Data Acquisition (SCAD...   \n",
       "\n",
       "                    authors  \\\n",
       "0  Name;Name;Name;Name;Name   \n",
       "\n",
       "                                          author_ids  \\\n",
       "0  3df61b32-6781-11ee-b983-4ab2673ea3f0;3df61c18-...   \n",
       "\n",
       "                                          references  \\\n",
       "0  3cbe2bec-6781-11ee-b983-4ab2673ea3f0;3cbe6d64-...   \n",
       "\n",
       "                                      clean_abstract  \n",
       "0  supervisory control acquisition system often s...  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.loc[(df.eid == eid)]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "027f9e80-5c6a-42b9-8d11-73f06ccbf07c",
   "metadata": {},
   "source": [
    "### Papers published by author_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "4215a768-f3db-4114-9d84-00ef3f791dbc",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>eid</th>\n",
       "      <th>title</th>\n",
       "      <th>countries</th>\n",
       "      <th>year</th>\n",
       "      <th>abstract</th>\n",
       "      <th>authors</th>\n",
       "      <th>author_ids</th>\n",
       "      <th>references</th>\n",
       "      <th>clean_abstract</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3cbdf82a-6781-11ee-b983-4ab2673ea3f0</td>\n",
       "      <td>Paper Title</td>\n",
       "      <td>New Zealand;United States</td>\n",
       "      <td>2016</td>\n",
       "      <td>Supervisory Control and Data Acquisition (SCAD...</td>\n",
       "      <td>Name;Name;Name;Name;Name</td>\n",
       "      <td>3df61b32-6781-11ee-b983-4ab2673ea3f0;3df61c18-...</td>\n",
       "      <td>3cbe2bec-6781-11ee-b983-4ab2673ea3f0;3cbe6d64-...</td>\n",
       "      <td>supervisory control acquisition system often s...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    eid        title  \\\n",
       "0  3cbdf82a-6781-11ee-b983-4ab2673ea3f0  Paper Title   \n",
       "\n",
       "                   countries  year  \\\n",
       "0  New Zealand;United States  2016   \n",
       "\n",
       "                                            abstract  \\\n",
       "0  Supervisory Control and Data Acquisition (SCAD...   \n",
       "\n",
       "                    authors  \\\n",
       "0  Name;Name;Name;Name;Name   \n",
       "\n",
       "                                          author_ids  \\\n",
       "0  3df61b32-6781-11ee-b983-4ab2673ea3f0;3df61c18-...   \n",
       "\n",
       "                                          references  \\\n",
       "0  3cbe2bec-6781-11ee-b983-4ab2673ea3f0;3cbe6d64-...   \n",
       "\n",
       "                                      clean_abstract  \n",
       "0  supervisory control acquisition system often s...  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.loc[(df.author_ids.str.contains(str(author_id)))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "b6df15c3-3196-4af1-9064-6c6fa150b4a3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# check if entry in X is valid\n",
    "assert len(df.loc[(df.eid == eid) & (df.author_ids.str.contains(str(author_id)))]) > 0, 'Invalid entry in X'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "21ff9bc2-86a7-4447-beb2-ed6e024c2606",
   "metadata": {},
   "source": [
    "### Check for all entries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "5781b1e4-79f3-4939-a05d-48a80a4f2b5f",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 9917/9917 [00:11<00:00, 889.15it/s] \n"
     ]
    }
   ],
   "source": [
    "dimension_order=settings['dimension_order']\n",
    "for i in tqdm(range(X.nnz)):\n",
    "    author_id, eid = inz_paper_author(X, i, author_ids, paper_ids, dimension_order)\n",
    "    assert len(df.loc[(df.eid == eid) & (df.author_ids.str.contains(str(author_id)))]) > 0, f'Invalid entry #{i} in X'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a7763543",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}