examples/Beaver/tensor-Citation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "4d57e6e4-657d-45e0-9d8e-ff050047f3e0",
   "metadata": {},
   "source": [
    "# Load Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7e3ccf65-3f76-4acd-a516-c4f6a9e16982",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "from collections import defaultdict\n",
    "\n",
    "from TELF.pre_processing import Beaver"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "af84282b-848e-4d3c-bd81-e2add24db2b1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 940 entries, 0 to 939\n",
      "Data columns (total 9 columns):\n",
      " #   Column          Non-Null Count  Dtype \n",
      "---  ------          --------------  ----- \n",
      " 0   eid             940 non-null    object\n",
      " 1   title           940 non-null    object\n",
      " 2   countries       940 non-null    object\n",
      " 3   year            940 non-null    int64 \n",
      " 4   abstract        940 non-null    object\n",
      " 5   authors         940 non-null    object\n",
      " 6   author_ids      940 non-null    object\n",
      " 7   references      843 non-null    object\n",
      " 8   clean_abstract  940 non-null    object\n",
      "dtypes: int64(1), object(8)\n",
      "memory usage: 66.2+ KB\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv(os.path.join('..', '..', 'data', 'sample.csv'))\n",
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "08d63541-5e83-4590-8874-9bc8d1349a1b",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 940 entries, 0 to 939\n",
      "Data columns (total 9 columns):\n",
      " #   Column          Non-Null Count  Dtype \n",
      "---  ------          --------------  ----- \n",
      " 0   eid             940 non-null    object\n",
      " 1   title           940 non-null    object\n",
      " 2   countries       940 non-null    object\n",
      " 3   year            940 non-null    int64 \n",
      " 4   abstract        940 non-null    object\n",
      " 5   authors         940 non-null    object\n",
      " 6   author_ids      940 non-null    object\n",
      " 7   references      940 non-null    object\n",
      " 8   clean_abstract  940 non-null    object\n",
      "dtypes: int64(1), object(8)\n",
      "memory usage: 66.2+ KB\n"
     ]
    }
   ],
   "source": [
    "# replace the missing citations with an empty string\n",
    "# these entries will be ignored by Beaver.citation_tensor()\n",
    "df.replace(np.nan, \"\", regex=True, inplace=True)\n",
    "df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b24aa37a-484e-48ea-b335-afc56f5d2f74",
   "metadata": {},
   "source": [
    "# Build Participation Tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "62b79669-7b50-49c6-8fbc-22c3f25ef7bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "beaver = Beaver()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "dd074491-6fee-4ccf-b2be-16e02d1ab935",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "preparing dictionaries for communication\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1/1 [00:00<00:00, 17.34it/s]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 51119/51119.0 [00:00<00:00, 604814.70it/s]\n"
     ]
    }
   ],
   "source": [
    "# using joblib_backend='multiprocessing' leads to \"NameError: name 'null' is not defined\" \n",
    "# in python 3.8 multiprocessing module when trying to spawn multiple processes for n_jobs > 1 \n",
    "\n",
    "settings = {\n",
    "    \"dataset\":df,\n",
    "    \"target_columns\":(\"author_ids\", \"eid\", \"references\", \"year\"),\n",
    "    \"dimension_order\":[0,1,2],\n",
    "    \"split_authors_with\":\";\",\n",
    "    \"split_references_with\":\";\",\n",
    "    \"save_path\":None,\n",
    "    \"verbose\":True,\n",
    "    \"n_jobs\":1,\n",
    "    \"n_nodes\":1,\n",
    "    #\"joblib_backend\": \"multiprocessing\", \n",
    "}\n",
    "\n",
    "X, author_ids, paper_ids, years = beaver.citation_tensor(**settings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c90a51c9-85c1-4009-95e9-670ff08b98ae",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table><tbody><tr><th style=\"text-align: left\">Format</th><td style=\"text-align: left\">coo</td></tr><tr><th style=\"text-align: left\">Data Type</th><td style=\"text-align: left\">float64</td></tr><tr><th style=\"text-align: left\">Shape</th><td style=\"text-align: left\">(7054, 940, 29)</td></tr><tr><th style=\"text-align: left\">nnz</th><td style=\"text-align: left\">51119</td></tr><tr><th style=\"text-align: left\">Density</th><td style=\"text-align: left\">0.0002658404372848715</td></tr><tr><th style=\"text-align: left\">Read-only</th><td style=\"text-align: left\">True</td></tr><tr><th style=\"text-align: left\">Size</th><td style=\"text-align: left\">1.6M</td></tr><tr><th style=\"text-align: left\">Storage ratio</th><td style=\"text-align: left\">0.0</td></tr></tbody></table>"
      ],
      "text/plain": [
       "<COO: shape=(7054, 940, 29), dtype=float64, nnz=51119, fill_value=0.0>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e573bf8e-ef90-45a3-af85-2a5e51bb6c5f",
   "metadata": {},
   "source": [
    "# Test Validity "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8e641708-5d38-4901-b826-03cd4b8eef4a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# match the values in settings\n",
    "author_ids_col        = settings['target_columns'][0]\n",
    "paper_id_col          = settings['target_columns'][1]\n",
    "references_col        = settings['target_columns'][2]\n",
    "time_col              = settings['target_columns'][3]\n",
    "\n",
    "author_ids_idx = settings['dimension_order'][0]\n",
    "paper_id_idx   = settings['dimension_order'][1]\n",
    "references_idx = settings['dimension_order'][2]\n",
    "\n",
    "split_authors_with    = settings['split_authors_with']\n",
    "split_references_with = settings['split_references_with']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6536fbf9-5071-44dd-93f7-babba9994d4b",
   "metadata": {},
   "source": [
    "### Test for valid non-zero entry positions in the tensor\n",
    "\n",
    "To test the this property of this tensor, we can create a map of all authors and the papers they cite using the DataFrame. <br>\n",
    "Then we do the same for the tensor. If the two maps match, then the tensor correctly has entries for all authors citing some paper at some time. <br>\n",
    "**NOTE:** This check will not validate the actual values (# of times the citation has been made)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "34f3f999-8290-462e-9c14-4eb374683e5d",
   "metadata": {},
   "source": [
    "#### Form map from DataFame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "5f30f7f7-70d8-4c34-b751-fc14a1b050f0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "\n",
    "def map_from_dataframe(df, author_ids='author_ids', paper_id='eid', references='references', time='year', split_authors_with=';', split_references_with=';'):\n",
    "    \n",
    "    # get all of the paper ids found in the dataset (to exclude cited paper ids outside of the dataset)\n",
    "    paper_ids = set(df[paper_id].unique())\n",
    "    \n",
    "    # this line is functionally equivalent to defining an empty 2d dict of sets\n",
    "    df_map = defaultdict(lambda: defaultdict(set))\n",
    "    \n",
    "    for author_ids_list, references_list, year in zip(df[author_ids].to_list(), df[references].to_list(), df[time].to_list()):\n",
    "        year = int(year)\n",
    "        for author_id in author_ids_list.split(split_authors_with):\n",
    "            for paper_id in references_list.split(split_references_with):\n",
    "                if paper_id not in paper_ids:\n",
    "                    continue\n",
    "                df_map[author_id][year].add(paper_id)\n",
    "    \n",
    "    # convert to dict of dicts on return \n",
    "    df_map = dict(df_map)\n",
    "    return {k: dict(v) for k,v in df_map.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "aefe8044-e5e1-4ea4-87ef-41965d0fc2f9",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'3df61b32-6781-11ee-b983-4ab2673ea3f0': {2016: {'3cbe02ac-6781-11ee-b983-4ab2673ea3f0',\n",
       "   '3cbe2674-6781-11ee-b983-4ab2673ea3f0',\n",
       "   '3cbe2bec-6781-11ee-b983-4ab2673ea3f0',\n",
       "   '3cbe2f98-6781-11ee-b983-4ab2673ea3f0',\n",
       "   '3cbe41c2-6781-11ee-b983-4ab2673ea3f0',\n",
       "   '3cbe5342-6781-11ee-b983-4ab2673ea3f0',\n",
       "   '3cbe6206-6781-11ee-b983-4ab2673ea3f0',\n",
       "   '3cbe6d64-6781-11ee-b983-4ab2673ea3f0',\n",
       "   '3cbe6dd2-6781-11ee-b983-4ab2673ea3f0',\n",
       "   '3cbe717e-6781-11ee-b983-4ab2673ea3f0'}}}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_map = map_from_dataframe(df, author_ids_col, paper_id_col, references_col, time_col, split_authors_with, split_references_with)\n",
    "dict(list(df_map.items())[:1])  # example of single entry in the map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ca708932-47b4-403a-83b4-a39614a18882",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6606"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_map)  # number of authors that have cited another paper in this dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0356ff97-a144-40ce-9aea-27472dba0e58",
   "metadata": {},
   "source": [
    "#### Form map from tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "3493a27d-d41e-4a8e-a239-adcf4e1e3472",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "\n",
    "def map_from_tensor(X, author_ids, paper_ids, time, author_index=0, paper_index=1, time_idx=2):\n",
    "    \n",
    "    # get the locations of non-zero entries in X\n",
    "    nz = X.nonzero()\n",
    "    \n",
    "    # this line is functionally equivalent to defining an empty 2d dict of sets\n",
    "    X_map = defaultdict(lambda: defaultdict(set))\n",
    "    \n",
    "    for i in range(len(nz[0])):  # for every non-zero value\n",
    "        \n",
    "        # get coordinates for maps from non-zero entry\n",
    "        i_author, i_paper, i_time = nz[author_index][i], nz[paper_index][i], nz[time_idx][i]\n",
    "    \n",
    "        # convert coordinates to data\n",
    "        author_id = author_ids[i_author]\n",
    "        paper_id = paper_ids[i_paper]\n",
    "        year = time[i_time]\n",
    "        \n",
    "        # add to dict\n",
    "        X_map[author_id][year].add(paper_id)\n",
    "    \n",
    "    # convert to dict of dicts on return \n",
    "    X_map = dict(X_map)\n",
    "    return {k: dict(v) for k,v in X_map.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "36ca2712-8b5c-4bff-917c-da0cf6f31f18",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'3df61b32-6781-11ee-b983-4ab2673ea3f0': {2016: {'3cbe02ac-6781-11ee-b983-4ab2673ea3f0',\n",
       "   '3cbe2674-6781-11ee-b983-4ab2673ea3f0',\n",
       "   '3cbe2bec-6781-11ee-b983-4ab2673ea3f0',\n",
       "   '3cbe2f98-6781-11ee-b983-4ab2673ea3f0',\n",
       "   '3cbe41c2-6781-11ee-b983-4ab2673ea3f0',\n",
       "   '3cbe5342-6781-11ee-b983-4ab2673ea3f0',\n",
       "   '3cbe6206-6781-11ee-b983-4ab2673ea3f0',\n",
       "   '3cbe6d64-6781-11ee-b983-4ab2673ea3f0',\n",
       "   '3cbe6dd2-6781-11ee-b983-4ab2673ea3f0',\n",
       "   '3cbe717e-6781-11ee-b983-4ab2673ea3f0'}}}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_map = map_from_tensor(X, author_ids, paper_ids, years, author_ids_idx, paper_id_idx, references_idx)\n",
    "dict(list(X_map.items())[:1])  # example of single entry in the map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "f60c54d0-d58c-4995-b343-f9c378aa4d38",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6606"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(X_map)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d566c190-8042-4674-bdae-1146a08acf55",
   "metadata": {},
   "source": [
    "#### Compare"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "58820705-4d8e-472f-871d-03233590eb7d",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_map == X_map"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ef23e9a6-f15c-47b9-9e89-92e40996bc54",
   "metadata": {},
   "source": [
    "### Test sum to check if counts are correct"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8171edb6-cde7-4ce4-add6-8e77f92a91c2",
   "metadata": {},
   "source": [
    "#### DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "1a5541b0-aade-467d-939c-774e60e2ada5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def sum_from_dataframe(df, author_ids='author_ids', paper_id='eid', references='references', time='year', split_authors_with=';', split_references_with=';'):\n",
    "    \n",
    "    # get all of the paper ids found in the dataset (to exclude cited paper ids outside of the dataset)\n",
    "    paper_ids = set(df[paper_id].unique())\n",
    "    \n",
    "    sum = 0\n",
    "    for author_ids_list, references_list, year in zip(df[author_ids].to_list(), df[references].to_list(), df[time].to_list()):\n",
    "        year = int(year)\n",
    "        for author_id in author_ids_list.split(split_authors_with):\n",
    "            for paper_id in references_list.split(split_references_with):\n",
    "                if paper_id in paper_ids:\n",
    "                    sum += 1\n",
    "    return sum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "b4e555d8-19c5-43c0-b063-d482f21d90f4",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "51148"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum_from_dataframe(df, author_ids_col, paper_id_col, references_col, time_col, split_authors_with, split_references_with)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "012b7f1d-3b3e-489a-bfcd-a03121052b85",
   "metadata": {},
   "source": [
    "#### tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "0e79e6d6-301b-4137-a9a8-62a3355efb01",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "51148.0"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "32e4c3d7",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}