train.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow.keras as keras\n",
    "from tensorflow.python.keras.utils import to_categorical\n",
    "import numpy as np\n",
    "import os, sys\n",
    "from data_helper import read_data, sents2sequences\n",
    "from model import summary_model\n",
    "from model_helper import plot_attention_weights\n",
    "from logger import get_logger"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "project_path = os.getcwd()#'/home/swayam/Desktop/tf2_project/'\n",
    "if project_path not in sys.path:\n",
    "    sys.path.append(project_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_dir = os.getcwd()#'/home/swayam/Desktop/tf2_project/'\n",
    "logger = get_logger(\"model.train\",os.path.join(base_dir, 'logs'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 64\n",
    "hidden_size = 96\n",
    "ip_timesteps, op_timesteps = 20, 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_data(train_size, random_seed=100):\n",
    "#randomly shuffle train/test\n",
    "\n",
    "    ip_text = read_data(os.path.join(project_path, 'data', 'text.txt'))\n",
    "    op_text = read_data(os.path.join(project_path, 'data', 'summary.txt'))\n",
    "    logger.info('Length of text: {}'.format(len(ip_text)))\n",
    "\n",
    "    op_text = ['sos ' + sent[:-1] + 'eos .'  if sent.endswith('.') else 'sos ' + sent + ' eos .' for sent in op_text]\n",
    "\n",
    "    np.random.seed(random_seed)\n",
    "    inds = np.arange(len(ip_text))\n",
    "    np.random.shuffle(inds)\n",
    "\n",
    "    train_inds = inds[:train_size]\n",
    "    test_inds = inds[train_size:]\n",
    "    tr_ip_text = [ip_text[ti] for ti in train_inds]\n",
    "    tr_op_text = [op_text[ti] for ti in train_inds]\n",
    "\n",
    "    ts_ip_text = [ip_text[ti] for ti in test_inds]\n",
    "    ts_op_text = [op_text[ti] for ti in test_inds]\n",
    "\n",
    "    return tr_ip_text, tr_op_text, ts_ip_text, ts_op_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_data(ip_tokenizer, op_tokenizer, ip_text, op_text, in_timesteps, op_timesteps):\n",
    "#Preprocessing and generating sequence of word indices\n",
    "\n",
    "    ip_seq = sents2sequences(ip_tokenizer, ip_text, reverse=False, padding_type='pre', pad_length=ip_timesteps)\n",
    "    op_seq = sents2sequences(op_tokenizer, op_text, pad_length=op_timesteps)\n",
    "    logger.info('Vocabulary size (Input): {}'.format(np.max(ip_seq)+1))\n",
    "    logger.info('Vocabulary size (Output): {}'.format(np.max(op_seq)+1))\n",
    "    logger.debug('IP text shape: {}'.format(ip_seq.shape))\n",
    "    logger.debug('OP text shape: {}'.format(op_seq.shape))\n",
    "\n",
    "    return ip_seq, op_seq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(full_model, ip_seq, op_seq, batch_size, n_epochs=10):\n",
    "#Training the model\n",
    "\n",
    "    for ep in range(n_epochs):\n",
    "        losses = []\n",
    "        for bi in range(0, ip_seq.shape[0] - batch_size, batch_size):\n",
    "\n",
    "            ip_onehot_seq = to_categorical(ip_seq[bi:bi + batch_size, :], num_classes=ip_vsize)\n",
    "            op_onehot_seq = to_categorical(op_seq[bi:bi + batch_size, :], num_classes=op_vsize)\n",
    "\n",
    "            full_model.train_on_batch([ip_onehot_seq, op_onehot_seq[:, :-1, :]], op_onehot_seq[:, 1:, :])\n",
    "\n",
    "            l = full_model.evaluate([ip_onehot_seq, op_onehot_seq[:, :-1, :]], op_onehot_seq[:, 1:, :],\n",
    "                                    batch_size=batch_size, verbose=0)\n",
    "\n",
    "            losses.append(l)\n",
    "        if (ep + 1) % 1 == 0:\n",
    "            logger.info(\"Loss in epoch {}: {}\".format(ep + 1, np.mean(losses)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if __name__ == '__main__':\n",
    "\n",
    "    debug = True\n",
    "\n",
    "    train_size = 100000 if not debug else 8000\n",
    "    filename = ''\n",
    "\n",
    "    tr_ip_text, tr_op_text, ts_ip_text, ts_op_text = get_data(train_size=train_size)\n",
    "\n",
    "#Defining tokenizers\n",
    "    ip_tokenizer = keras.preprocessing.text.Tokenizer(oov_token='UNK')\n",
    "    ip_tokenizer.fit_on_texts(tr_ip_text)\n",
    "\n",
    "    op_tokenizer = keras.preprocessing.text.Tokenizer(oov_token='UNK')\n",
    "    op_tokenizer.fit_on_texts(tr_op_text)\n",
    "\n",
    "#Getting preprocessed data\n",
    "    ip_seq, op_seq = preprocess_data(ip_tokenizer, op_tokenizer, tr_ip_text, tr_op_text, ip_timesteps, op_timesteps)\n",
    "\n",
    "    ip_vsize = max(ip_tokenizer.index_word.keys()) + 1\n",
    "    op_vsize = max(op_tokenizer.index_word.keys()) + 1\n",
    "\n",
    "#Defining the full model\n",
    "    full_model, infer_enc_model, infer_dec_model = summary_model(hidden_size=hidden_size, batch_size=batch_size,\n",
    "        ip_timesteps=ip_timesteps, op_timesteps=op_timesteps,ip_vsize=ip_vsize, op_vsize=op_vsize)\n",
    "\n",
    "    n_epochs = 1000 if not debug else 100\n",
    "    train(full_model, ip_seq, op_seq, batch_size, n_epochs)\n",
    "\n",
    "#save model\n",
    "    full_model.save('summarizer.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}