notebooks/supervised/classification/facts/1.mergining-streams.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "99O9ukh3S6UR"
   },
   "source": [
    "# Facts: collecting datasets\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cellView": "form",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 74
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 6224,
     "status": "ok",
     "timestamp": 1593746113000,
     "user": {
      "displayName": "Lucas",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GgO-xqdSSwbB8xXos7Shqhpq7QJ06mYVUvdLDRcbQ=s64",
      "userId": "01670254553878112810"
     },
     "user_tz": 180
    },
    "id": "NKy65_S_4p0P",
    "outputId": "b5379d20-1e5e-41b4-c97f-d60a1ade5bde"
   },
   "outputs": [],
   "source": [
    "#@title\n",
    "\n",
    "!pip -q install shap\n",
    "\n",
    "import os\n",
    "from functools import reduce\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "import shap\n",
    "from sklearn.preprocessing import LabelEncoder, StandardScaler\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from mpl_toolkits.mplot3d import Axes3D\n",
    "import seaborn as sns\n",
    "from google.colab import drive\n",
    "\n",
    "sns.set(palette=sns.color_palette(\"hls\", 8))\n",
    "pd.set_option('display.max_columns', None)\n",
    "# pd.set_option('display.max_colwidth', None)\n",
    "\n",
    "from IPython.display import display_html\n",
    "def display_side_by_side(*args):\n",
    "    html_str=''\n",
    "    for df in args:\n",
    "        html_str+=df.to_html()\n",
    "    display_html(html_str.replace('table','table style=\"display:inline\"'),raw=True)\n",
    "\n",
    "drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 6220,
     "status": "ok",
     "timestamp": 1593746113002,
     "user": {
      "displayName": "Lucas",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GgO-xqdSSwbB8xXos7Shqhpq7QJ06mYVUvdLDRcbQ=s64",
      "userId": "01670254553878112810"
     },
     "user_tz": 180
    },
    "id": "c9iKdvc1TnZa"
   },
   "outputs": [],
   "source": [
    "SEED = 180120342\n",
    "OUTPUT_DATASET = '/content/drive/My Drive/datasets/facts.csv'\n",
    "\n",
    "rs = np.random.RandomState(SEED)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "kX0dQV3BU1Wb"
   },
   "source": [
    "#### 20 News Groups"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 0
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 6917,
     "status": "ok",
     "timestamp": 1593746113703,
     "user": {
      "displayName": "Lucas",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GgO-xqdSSwbB8xXos7Shqhpq7QJ06mYVUvdLDRcbQ=s64",
      "userId": "01670254553878112810"
     },
     "user_tz": 180
    },
    "id": "YzdVqSE3TcgN",
    "outputId": "3653b639-0564-46c5-9fce-f3f041707d9c"
   },
   "outputs": [],
   "source": [
    "from sklearn.datasets import fetch_20newsgroups\n",
    "\n",
    "x = fetch_20newsgroups(subset='train', random_state=rs)\n",
    "t = fetch_20newsgroups(subset='test', random_state=rs)\n",
    "labels = np.asarray(x.target_names)\n",
    "\n",
    "x = pd.DataFrame({'text': x.data, 'category': labels[x.target], 'stage': 'train'})\n",
    "xt = pd.DataFrame({'text': t.data, 'category': labels[t.target], 'stage': 'test'})\n",
    "d20newsgroups = x.append(xt).assign(target='true').assign(source='20newsgroups')\n",
    "\n",
    "d20newsgroups.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 0
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 6915,
     "status": "ok",
     "timestamp": 1593746113704,
     "user": {
      "displayName": "Lucas",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GgO-xqdSSwbB8xXos7Shqhpq7QJ06mYVUvdLDRcbQ=s64",
      "userId": "01670254553878112810"
     },
     "user_tz": 180
    },
    "id": "CUsIER-6fePZ",
    "outputId": "3a44b0d3-69a7-45df-97ed-c39b6e2fd043"
   },
   "outputs": [],
   "source": [
    "print('samples:', len(d20newsgroups))\n",
    "print(dict(zip(*np.unique(d20newsgroups.stage, return_counts=True))))\n",
    "d20newsgroups.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "PI9x6HVFVlKK"
   },
   "source": [
    "#### Fake and real news dataset\n",
    "\n",
    "[fake-and-real-news-dataset](https://www.kaggle.com/clmentbisaillon/fake-and-real-news-dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 6910,
     "status": "ok",
     "timestamp": 1593746113704,
     "user": {
      "displayName": "Lucas",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GgO-xqdSSwbB8xXos7Shqhpq7QJ06mYVUvdLDRcbQ=s64",
      "userId": "01670254553878112810"
     },
     "user_tz": 180
    },
    "id": "EDU2wwT94p0g"
   },
   "outputs": [],
   "source": [
    "DATASET = ('/content/drive/My Drive/Colab Notebooks/ml-notes/'\n",
    "           'datasets/572515_1037534_bundle_archive.zip')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 0
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 9399,
     "status": "ok",
     "timestamp": 1593746116198,
     "user": {
      "displayName": "Lucas",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GgO-xqdSSwbB8xXos7Shqhpq7QJ06mYVUvdLDRcbQ=s64",
      "userId": "01670254553878112810"
     },
     "user_tz": 180
    },
    "id": "-llQQgGuE5U3",
    "outputId": "02690c09-35fc-45c6-f2d4-d85a3ecd263c"
   },
   "outputs": [],
   "source": [
    "import zipfile\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "with zipfile.ZipFile(DATASET) as z:\n",
    "    z.extractall('./ds/')\n",
    "\n",
    "t, f = (pd.read_csv('./ds/True.csv'),\n",
    "        pd.read_csv('./ds/Fake.csv'))\n",
    "\n",
    "def preprocess(t, f):\n",
    "    t['target'] = 'true'\n",
    "    f['target'] = 'fake'\n",
    "\n",
    "    x = t.append(f)\n",
    "\n",
    "    ds = x.date.str.strip()\n",
    "    x['created_at'] = pd.to_datetime(ds, format='%B %d, %Y', errors='coerce')\n",
    "    x.loc[x.created_at.isnull(), 'created_at'] = pd.to_datetime(ds[x.created_at.isnull()], format='%b %d, %Y', errors='coerce')\n",
    "    x.loc[x.created_at.isnull(), 'created_at'] = pd.to_datetime(ds[x.created_at.isnull()], format='%d-%b-%y', errors='coerce')\n",
    "\n",
    "    return x.dropna().rename(columns={'subject': 'category'}).drop(columns=['date'])\n",
    "\n",
    "fake_and_real_news = preprocess(t, f).assign(source='fake_and_real_news')\n",
    "fake_and_real_news.sort_values('created_at')\n",
    "middle = fake_and_real_news.iloc[len(fake_and_real_news) // 2].created_at\n",
    "fake_and_real_news['stage'] = 'train'\n",
    "fake_and_real_news.loc[fake_and_real_news.created_at >= middle, 'stage'] = 'test'\n",
    "\n",
    "fake_and_real_news = fake_and_real_news.drop(columns=['created_at'])\n",
    "\n",
    "fake_and_real_news.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 0
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 9784,
     "status": "ok",
     "timestamp": 1593746116586,
     "user": {
      "displayName": "Lucas",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GgO-xqdSSwbB8xXos7Shqhpq7QJ06mYVUvdLDRcbQ=s64",
      "userId": "01670254553878112810"
     },
     "user_tz": 180
    },
    "id": "g4f9ly_zfoK_",
    "outputId": "6485ca43-4635-465c-f01d-1829ec010e78"
   },
   "outputs": [],
   "source": [
    "print('samples:', len(fake_and_real_news))\n",
    "print(dict(zip(*np.unique(fake_and_real_news.stage, return_counts=True))))\n",
    "fake_and_real_news.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "XE3nZKkGZZNk"
   },
   "source": [
    "#### Fake News\n",
    "\n",
    "https://www.kaggle.com/c/fake-news/data?select=train.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 9783,
     "status": "ok",
     "timestamp": 1593746116589,
     "user": {
      "displayName": "Lucas",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GgO-xqdSSwbB8xXos7Shqhpq7QJ06mYVUvdLDRcbQ=s64",
      "userId": "01670254553878112810"
     },
     "user_tz": 180
    },
    "id": "Xcxnm-nzZaeM"
   },
   "outputs": [],
   "source": [
    "DATASET_DIR = '/content/drive/My Drive/datasets/fake-news'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 12389,
     "status": "ok",
     "timestamp": 1593746119198,
     "user": {
      "displayName": "Lucas",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GgO-xqdSSwbB8xXos7Shqhpq7QJ06mYVUvdLDRcbQ=s64",
      "userId": "01670254553878112810"
     },
     "user_tz": 180
    },
    "id": "fZhP8_CaauJp"
   },
   "outputs": [],
   "source": [
    "with zipfile.ZipFile(DATASET_DIR + '/train.csv.zip') as z:\n",
    "    z.extractall('./dsfn/')\n",
    "\n",
    "with zipfile.ZipFile(DATASET_DIR + '/test.csv.zip') as z:\n",
    "    z.extractall('./dsfn/')\n",
    "\n",
    "x, t = (pd.read_csv('./dsfn/train.csv').assign(stage='train'),\n",
    "        pd.read_csv('./dsfn/test.csv').assign(stage='test'))\n",
    "\n",
    "s = pd.read_csv(DATASET_DIR + '/submit.csv')\n",
    "\n",
    "t = t.merge(s, how='left', on='id')\n",
    "\n",
    "fake_news_2 = x.append(t)\n",
    "fake_news_2.loc[fake_news_2.label == 1, 'label'] = 'unreliable'\n",
    "fake_news_2.loc[fake_news_2.label == 0, 'label'] = 'reliable'\n",
    "\n",
    "fake_news_2 = (fake_news_2\n",
    "               .rename(columns={'label': 'target'})\n",
    "               .drop(columns=['id'])\n",
    "               .assign(source='kg_fake_news_2'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 12388,
     "status": "ok",
     "timestamp": 1593746119200,
     "user": {
      "displayName": "Lucas",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GgO-xqdSSwbB8xXos7Shqhpq7QJ06mYVUvdLDRcbQ=s64",
      "userId": "01670254553878112810"
     },
     "user_tz": 180
    },
    "id": "_fqNbt6wdxYJ",
    "outputId": "207c0033-6b6f-45a7-e3a2-ca520e8cbbaf"
   },
   "outputs": [],
   "source": [
    "x.shape, t.shape, dict(zip(*np.unique(fake_news_2.stage, return_counts=True)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 98
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 12384,
     "status": "ok",
     "timestamp": 1593746119201,
     "user": {
      "displayName": "Lucas",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GgO-xqdSSwbB8xXos7Shqhpq7QJ06mYVUvdLDRcbQ=s64",
      "userId": "01670254553878112810"
     },
     "user_tz": 180
    },
    "id": "hLt11U0pa4H9",
    "outputId": "c200d5d9-e2c8-4a3f-f038-4546e64267b7"
   },
   "outputs": [],
   "source": [
    "fake_news_2.head(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "v3-tC1jLYLec"
   },
   "source": [
    "#### Saving"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 12381,
     "status": "ok",
     "timestamp": 1593746119203,
     "user": {
      "displayName": "Lucas",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GgO-xqdSSwbB8xXos7Shqhpq7QJ06mYVUvdLDRcbQ=s64",
      "userId": "01670254553878112810"
     },
     "user_tz": 180
    },
    "id": "PJgobDOIWhc8"
   },
   "outputs": [],
   "source": [
    "all_sets = [\n",
    "    d20newsgroups,\n",
    "    fake_and_real_news,\n",
    "    fake_news_2\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 12377,
     "status": "ok",
     "timestamp": 1593746119203,
     "user": {
      "displayName": "Lucas",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GgO-xqdSSwbB8xXos7Shqhpq7QJ06mYVUvdLDRcbQ=s64",
      "userId": "01670254553878112810"
     },
     "user_tz": 180
    },
    "id": "tkFO0Rmjkkdf"
   },
   "outputs": [],
   "source": [
    "all_columns = reduce(lambda a, b: a | b, [set(s.columns) for s in all_sets])\n",
    "\n",
    "for s in all_sets:\n",
    "    for c in all_columns - set(s.columns):\n",
    "        s[c] = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 81
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 12373,
     "status": "ok",
     "timestamp": 1593746119204,
     "user": {
      "displayName": "Lucas",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GgO-xqdSSwbB8xXos7Shqhpq7QJ06mYVUvdLDRcbQ=s64",
      "userId": "01670254553878112810"
     },
     "user_tz": 180
    },
    "id": "jcSErdYVYWvJ",
    "outputId": "d049362d-d0e9-4806-bf6a-786796a64615"
   },
   "outputs": [],
   "source": [
    "x = pd.concat(all_sets)\n",
    "x.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 376
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 12982,
     "status": "ok",
     "timestamp": 1593746119818,
     "user": {
      "displayName": "Lucas",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GgO-xqdSSwbB8xXos7Shqhpq7QJ06mYVUvdLDRcbQ=s64",
      "userId": "01670254553878112810"
     },
     "user_tz": 180
    },
    "id": "IUAGE2llebLe",
    "outputId": "f7a7c372-ed20-41c3-d3b6-0ae21e2e8970"
   },
   "outputs": [],
   "source": [
    "print('samples:', len(x))\n",
    "print('Null values:')\n",
    "print(x.isnull().mean().round(2))\n",
    "\n",
    "x.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 22528,
     "status": "ok",
     "timestamp": 1593746129369,
     "user": {
      "displayName": "Lucas",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GgO-xqdSSwbB8xXos7Shqhpq7QJ06mYVUvdLDRcbQ=s64",
      "userId": "01670254553878112810"
     },
     "user_tz": 180
    },
    "id": "A1aDiVo6YZFj"
   },
   "outputs": [],
   "source": [
    "x.to_csv(OUTPUT_DATASET, index=False)"
   ]
  }
 ],
 "metadata": {
  "jupytext": {
   "main_language": "python"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}