Models_preparation.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Models preparation.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TLzJVjg-8qJW"
      },
      "source": [
        "This notebook should be run first once, completely. Its goal is to pre-compute the word2vec embeddings, PMI and PCI matrices, and N[W,c] count; steps that consume a high amount of time and memory.\r\n",
        "\r\n",
        "The computations can hardly be optimised further as we need the complete counts and accurate matrices to perform our analyses. However if needed, it is possible to limit the vocabulary size (and thus also the context words taken into account in the corpus), which limits importantly the implications but impacts the memory used quadratically."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2c9ZB13Bf_jz"
      },
      "source": [
        "import gensim\n",
        "from gensim import utils, matutils\n",
        "import gensim.downloader as api\n",
        "from gensim.models.word2vec import Word2Vec\n",
        "\n",
        "import time\n",
        "import logging\n",
        "from itertools import chain\n",
        "import logging\n",
        "import numpy as np\n",
        "import pickle\n",
        "import os\n",
        "\n",
        "import scipy\n",
        "from scipy import sparse\n",
        "\n",
        "text8 = api.load(\"text8\")\n",
        "\n",
        "def save_obj(obj, name ):\n",
        "    with open( name + '.pkl', 'wb') as f:\n",
        "        pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6n8OduYIf_j9"
      },
      "source": [
        "# word2vec model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EhibDRSSf_j-"
      },
      "source": [
        "# Define and saving a model\n",
        "model = Word2Vec(text8, negative=1, hs=0, sg=1, ns_exponent=1, vector_size=500, epochs=15, sample=0) \n",
        "Word2Vec.save(model, \"./Models/word2vec_clean.model\") \n",
        "\n",
        "# Loading a model\n",
        "# model = Word2Vec.load(\"./Models/word2vec_clean.model\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SdjBLOjRHaBt"
      },
      "source": [
        "# Check that the model has trained well. Is Queen there?\r\n",
        "\r\n",
        "model.wv.most_similar('king')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KtMKVNXE7_6l"
      },
      "source": [
        "# Vocabulary definition, inversed vocabulary etc...\r\n",
        "\r\n",
        "# For memory purposes, we recommend to limit the vocabulary for tests. \r\n",
        "# Replace by None if you wish to use the entirety of the vocabulary. \r\n",
        "vocab_limit = 2000\r\n",
        "\r\n",
        "vocabulary_keys = model.wv.index_to_key\r\n",
        "vocabulary = set(vocabulary_keys)\r\n",
        "vocabulary_list = np.array(list(vocabulary_keys[:vocab_limit]))\r\n",
        "len_vocabulary = len(list(vocabulary_keys[:vocab_limit]))\r\n",
        "\r\n",
        "invdict_vocabulary = dict.fromkeys(vocabulary_list)\r\n",
        "for i,wi in enumerate(vocabulary_list):\r\n",
        "    invdict_vocabulary[wi] = i"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tlDtMOSXf_j_"
      },
      "source": [
        "# Count dictionnaries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rucUQZWvf_kA"
      },
      "source": [
        "def count_bioccurences(corpus, vocabulary_list, context_size=5, print_size=200):\n",
        "    '''\n",
        "    Create a dict of dict, counting the occurences of context words for each word of the vocabulary, in a corpus.\n",
        "    '''\n",
        "    vocabulary = set(vocabulary_list)\n",
        "    count_dict_ij = dict.fromkeys(vocabulary_list)\n",
        "    for v in count_dict_ij:\n",
        "        count_dict_ij[v] = dict()\n",
        "    \n",
        "    for nd,doc in enumerate(corpus):\n",
        "        if nd%print_size==0:\n",
        "            print(nd)\n",
        "            \n",
        "        for i,wi in enumerate(doc):\n",
        "            for j in range(max(0,i-context_size),min(len(doc),i+context_size+1)):\n",
        "                cj = doc[j]\n",
        "                if wi in vocabulary and cj in vocabulary and j != i:\n",
        "                    if not cj in count_dict_ij[wi]:\n",
        "                        count_dict_ij[wi][cj] = 1\n",
        "                    else:\n",
        "                        count_dict_ij[wi][cj] += 1\n",
        "\n",
        "    return(count_dict_ij)\n",
        "\n",
        "\n",
        "def count_trioccurences(corpus, top_voc, vocabulary_list, context_size=5, print_size=200):\n",
        "    '''\n",
        "    Create a dict of dict of dict, counting the occurences of context pairs of each word of the vocabulary, in a corpus.\n",
        "    '''\n",
        "    vocabulary = set(vocabulary_list)\n",
        "    count_dict_cj_tri = dict.fromkeys(vocabulary_list)\n",
        "    \n",
        "    for nd,doc in enumerate(corpus):\n",
        "        if nd%print_size==0:\n",
        "            print(nd)\n",
        "            \n",
        "        for i,wi in enumerate(doc):\n",
        "            for j in range( max(0, i-context_size), min(len(doc), i+context_size+1) ):\n",
        "                cj = doc[j]\n",
        "                for k in range(j+1,min(len(doc),i+context_size+1)):\n",
        "                    ck = doc[k]\n",
        "                    if cj in top_voc and ck in top_voc and wi in vocabulary and j != i and k != i:\n",
        "                        if count_dict_cj_tri[wi] is None:\n",
        "                            count_dict_cj_tri[wi] = dict()\n",
        "                        if not cj in count_dict_cj_tri[wi]:\n",
        "                            count_dict_cj_tri[wi][cj] = dict()\n",
        "                        if not ck in count_dict_cj_tri[wi][cj]:\n",
        "                            count_dict_cj_tri[wi][cj][ck] = 1\n",
        "                        else:\n",
        "                            count_dict_cj_tri[wi][cj][ck] += 1\n",
        "    return(count_dict_cj_tri)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j9ZeoVBVf_kA"
      },
      "source": [
        "## Bioccurences count\n",
        "\n",
        "count_dict_ij = count_bioccurences(text8, vocabulary_list, context_size=5, print_size=200)\n",
        "pickle.dump(count_dict_ij, open(\"./Models/count_dict_ij.pkl\", \"wb\"))\n",
        "\n",
        "# count_dict_ij = pickle.load(open(\"./Models/count_dict_ij.pkl\", \"rb\"))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zYk1hqFEPDwW"
      },
      "source": [
        "### Trioccurence count\r\n",
        "\r\n",
        "# Top indexes vocabulary\r\n",
        "count_dict_i = dict.fromkeys(vocabulary_list)\r\n",
        "for w in count_dict_ij:\r\n",
        "    count_dict_i[w] = sum(count_dict_ij[w].values())\r\n",
        "n_max = 10000    \r\n",
        "sorted_count_dict = {k: v for k, v in sorted(count_dict_i.items(), key=lambda item: item[1])}\r\n",
        "sorted_vocabulary = list(reversed([[k, v] for (k, v) in sorted_count_dict.items()]))\r\n",
        "top_voc_list = [w[0] for w in sorted_vocabulary[:n_max]]\r\n",
        "top_voc = set(top_voc_list)\r\n",
        "invdict_top_voc = {w:i for i,w in enumerate(top_voc_list)}\r\n",
        "\r\n",
        "# Computation\r\n",
        "count_dict_cj_tri = count_trioccurences(text8, top_voc, vocabulary_list, context_size=5, print_size=200)\r\n",
        "pickle.dump(count_dict_cj_tri, open(\"./Models/count_dict_cj_tri.pkl\", \"wb\"))\r\n",
        "\r\n",
        "# count_dict_cj_tri = pickle.load(open(\"./Models/count_dict_cj_tri.pkl\", \"rb\"))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gNCmoteSf_kB"
      },
      "source": [
        "# Count matrices"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_TYdI6EMf_kB"
      },
      "source": [
        "# Heavily inspired by https://www.kaggle.com/kenshoresearch/kdwd-pmi-word-vectors\n",
        "def get_count_matrix(skipgrams, tok2indx):\n",
        "    '''\n",
        "    Build a sparse matrix of bioccurences from a dict of dict skipgrams.\n",
        "    '''\n",
        "    row_indxs = []                                                                       \n",
        "    col_indxs = []                                                                       \n",
        "    dat_values = []\n",
        "    \n",
        "    for i,wi in enumerate(skipgrams):\n",
        "        if i%500==0:\n",
        "            print(i)\n",
        "        for wj in skipgrams[wi]:\n",
        "            row_indxs.append(tok2indx[wi])\n",
        "            col_indxs.append(tok2indx[wj])\n",
        "            dat_values.append(skipgrams[wi][wj])\n",
        "            \n",
        "    print('building sparse bicount matrix')\n",
        "    return sparse.csr_matrix((dat_values, (row_indxs, col_indxs)))\n",
        "\n",
        "def get_count_matrix_tri(skipgrams, tok2indx, tok2indx_top):\n",
        "    '''\n",
        "    Build a sparse matrix of trioccurences from a dict of dict of dict skipgrams.\n",
        "    The matrix will be of dimension 2.\n",
        "    '''\n",
        "    row_indxs = []                                                                       \n",
        "    col_indxs = []\n",
        "    dat_values = []    \n",
        "    \n",
        "    for k,ck in enumerate(skipgrams):\n",
        "        if k%3000==0:\n",
        "            print(k)\n",
        "        if not skipgrams[ck] is None:\n",
        "            for wi in skipgrams[ck]:\n",
        "                for wj in skipgrams[ck][wi]:\n",
        "                    if tok2indx_top[wi] > tok2indx_top[wj] and (not wj in skipgrams[ck] or not wi in skipgrams[ck][wj]):\n",
        "                        row_indxs.append(10000*tok2indx_top[wj] + tok2indx_top[wi])\n",
        "                        col_indxs.append(tok2indx[ck])\n",
        "                        dat_values.append(skipgrams[ck][wi][wj])   \n",
        "                        \n",
        "                    if tok2indx_top[wi] <= tok2indx_top[wj]:\n",
        "                        s = 0\n",
        "                        if wi != wj and wj in skipgrams[ck]:\n",
        "                            if wi in skipgrams[ck][wj]:\n",
        "                                s = skipgrams[ck][wj][wi]\n",
        "                        row_indxs.append(10000*tok2indx_top[wi] + tok2indx_top[wj])\n",
        "                        col_indxs.append(tok2indx[ck])\n",
        "                        dat_values.append(skipgrams[ck][wi][wj] + s)\n",
        "                        \n",
        "    print('building sparse tricount matrix')\n",
        "    return(sparse.csr_matrix((dat_values, (row_indxs, col_indxs))))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Dar5GstEf_kC"
      },
      "source": [
        "count_matrix = get_count_matrix(count_dict_ij, invdict_vocabulary)\n",
        "\n",
        "scipy.sparse.save_npz('./Models/count_matrix.npz', count_matrix)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V_rZBdqCnF81"
      },
      "source": [
        "count_matrix_tri = get_count_matrix_tri(count_dict_cj_tri, invdict_vocabulary, invdict_top_voc)\r\n",
        "\r\n",
        "scipy.sparse.save_npz('./Models/count_matrix_tri.npz', count_matrix_tri)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qGuqEgWEf_kD"
      },
      "source": [
        "# PMI and PCI matrix"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EfIrzqXaf_kD"
      },
      "source": [
        "def get_pci_matrix(count_matrix_tri, proba=False):\n",
        "    '''\n",
        "    Build the sparse (proba) paraphrase error matrix from the sparse trioccurences matrix\n",
        "    '''\n",
        "    sum_over_contexts = np.array(count_matrix_tri.sum(axis=1)).flatten()\n",
        "    div_sum_over_contexts = np.clip(1/sum_over_contexts,0,1)\n",
        "    para = count_matrix_tri.T.multiply(div_sum_over_contexts).T\n",
        "    para_csr = para.tocsr()\n",
        "    if not proba:\n",
        "        para_csr.data = np.log(para_csr.data)\n",
        "    return(para_csr)\n",
        "\n",
        "def get_pmi_matrix(skipgrams, count_matrix, tok2indx, alpha=0.75, ppmi_bool=False, min_context=False, not_pmi=False):\n",
        "    \n",
        "    # for standard PPMI\n",
        "    DD = 0\n",
        "    for wi in skipgrams:\n",
        "        DD += sum(skipgrams[wi].values())\n",
        "\n",
        "    sum_over_contexts = np.array(count_matrix.sum(axis=1)).flatten()\n",
        "    sum_over_words = np.array(count_matrix.sum(axis=0)).flatten()\n",
        "        \n",
        "    # for context distribution smoothing (cds)\n",
        "    sum_over_words_alpha = sum_over_words**alpha\n",
        "    Pc_alpha_denom = np.sum(sum_over_words_alpha)\n",
        "        \n",
        "    row_indxs = []\n",
        "    col_indxs = []\n",
        "    ppmi_dat_values = []   # positive pointwise mutual information\n",
        "    \n",
        "    for i,wi in enumerate(skipgrams):\n",
        "        if i%5000==0:\n",
        "            print(i)\n",
        "        for wj in skipgrams[wi]:    \n",
        "            tok_word_indx, tok_context_indx, pound_wc = tok2indx[wi], tok2indx[wj], skipgrams[wi][wj]\n",
        "            pound_w = sum_over_contexts[tok_word_indx]\n",
        "            pound_c = sum_over_words[tok_context_indx]\n",
        "            pound_c_alpha = sum_over_words_alpha[tok_context_indx]\n",
        "            \n",
        "            # Doesn't actually work here, as pound_wc is >=1. The sparse structure keep at 0 all values where pound_wc =0\n",
        "            if min_context:\n",
        "                pound_wc = max(pound_wc,1)\n",
        "            Pwc = pound_wc / DD\n",
        "            Pw = pound_w / DD\n",
        "            Pc = pound_c / DD\n",
        "            Pc_alpha = pound_c_alpha / Pc_alpha_denom\n",
        "\n",
        "            if not_pmi:\n",
        "                Pc_alpha = 1\n",
        "            \n",
        "            pmi = np.log2(Pwc / (Pw * Pc_alpha))\n",
        "            \n",
        "            if ppmi_bool:\n",
        "                ppmi = max(pmi, 0)\n",
        "            else:\n",
        "                ppmi = pmi\n",
        "\n",
        "            row_indxs.append(tok_word_indx)\n",
        "            col_indxs.append(tok_context_indx)\n",
        "            ppmi_dat_values.append(ppmi)\n",
        "    \n",
        "    if ppmi_bool:\n",
        "        print('building ppmi matrix') \n",
        "    else:\n",
        "        print('building pmi matrix') \n",
        "    return sparse.csr_matrix((ppmi_dat_values, (row_indxs, col_indxs)))\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ly2W1iOuf_kE"
      },
      "source": [
        "pmi_matrix = get_pmi_matrix(count_dict_ij, count_matrix, invdict_vocabulary, alpha=1, ppmi_bool=False, min_context=False)\n",
        "\n",
        "scipy.sparse.save_npz('./Models/pmi_matrix_nominctxt.npz', pmi_matrix)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jm3AFo_SoP_W"
      },
      "source": [
        "para_matrix = get_pci_matrix(count_matrix_tri)\r\n",
        "\r\n",
        "scipy.sparse.save_npz('./Models/paraphrase_matrix.npz', para_matrix)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A5rXXM0Of_kF"
      },
      "source": [
        "# N[W,c] "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o72wuzimf_kF"
      },
      "source": [
        "## Preparation of the analogy pairs used in the main notebook to pre-compute N[W,c].\n",
        "\n",
        "# Loading of the BATS dataset\n",
        "\n",
        "directory = './BATS_3.0'\n",
        "names = []\n",
        "pairs_sets = []\n",
        "\n",
        "for d in os.listdir(directory):\n",
        "    if d != 'metadata.json':\n",
        "        for f in os.listdir(os.path.join(directory,str(d))):\n",
        "            names.append(str(f)[:-4])\n",
        "            pairs_sets.append(set())\n",
        "            with utils.open_file(os.path.join(directory,str(d),str(f))) as fin:\n",
        "                for line_no, line in enumerate(fin):\n",
        "                    line = utils.to_unicode(line)\n",
        "                    a, b = [word.lower() for word in line.split()]\n",
        "                    list_b = b.split('/')\n",
        "                    if list_b[0] != a: #Keeping only the first analogy pair\n",
        "                        pairs_sets[-1].add((a, list_b[0]))\n",
        "\n",
        "pairs_sets = [list(d) for d in pairs_sets]\n",
        "\n",
        "# Creation of the suitable W and W* pairs.\n",
        "\n",
        "def possible_analogies_and_voc(pairs_sets, count_dict_ij, invdict_top_voc):\n",
        "  possible_analogies = []\n",
        "  for k in range(len(pairs_sets)):\n",
        "      possible_analogies.append([])\n",
        "      for i in range(len(pairs_sets[k])):\n",
        "          p1_a, p1_ap = pairs_sets[k][i]\n",
        "          for j in range(i+1, len(pairs_sets[k])):\n",
        "              p2_a, p2_ap = pairs_sets[k][j]\n",
        "\n",
        "              if (p1_a in count_dict_ij and p2_ap in count_dict_ij[p1_a]) or (p2_ap in count_dict_ij and p1_a in count_dict_ij[p2_ap]):\n",
        "                  if (p2_a in count_dict_ij and p1_ap in count_dict_ij[p2_a]) or (p1_ap in count_dict_ij and p2_a in count_dict_ij[p1_ap]):\n",
        "                      if p1_a in invdict_top_voc and p1_ap in invdict_top_voc and p2_ap in invdict_top_voc and p2_a in invdict_top_voc:\n",
        "                          possible_analogies[-1].append([(p1_a, p2_ap), (p1_ap, p2_a)])\n",
        "                          \n",
        "  voc_possible_analogies = set()\n",
        "  for k in range(len(pairs_sets)):\n",
        "      for p in possible_analogies[k]:\n",
        "          t1, t2 = p[0], p[1]\n",
        "          p0,p1 = t1\n",
        "          n0,n1 = t2\n",
        "          voc_possible_analogies.add(p0)\n",
        "          voc_possible_analogies.add(p1)\n",
        "          voc_possible_analogies.add(n0)\n",
        "          voc_possible_analogies.add(n1)\n",
        "  voc_possible_analogies = list(voc_possible_analogies)\n",
        "\n",
        "  return(possible_analogies, voc_possible_analogies)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RuPdcqJTf_kF"
      },
      "source": [
        "possible_analogies, voc_possible_analogies = possible_analogies_and_voc(pairs_sets, count_dict_ij, invdict_top_voc)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SDPh4knk9oll"
      },
      "source": [
        "def get_n_w_c(voc_possible_analogies, vocabulary_list, top_voc_list, invdict_top_voc, count_matrix_tri):\r\n",
        "    n_w_c = dict()\r\n",
        "    for i, wi in enumerate(voc_possible_analogies):\r\n",
        "        n_wi_c = dict()\r\n",
        "        for wj in top_voc_list:\r\n",
        "\r\n",
        "            if invdict_top_voc[wi] > invdict_top_voc[wj]:\r\n",
        "                n_wiwj_c = count_matrix_tri[10000*invdict_top_voc[wj] + invdict_top_voc[wi]]\r\n",
        "            else:\r\n",
        "                n_wiwj_c = count_matrix_tri[10000*invdict_top_voc[wi] + invdict_top_voc[wj]]\r\n",
        "\r\n",
        "            for ik, ind in enumerate(n_wiwj_c.indices):\r\n",
        "                w_ind = vocabulary_list[ind]\r\n",
        "                if not w_ind in n_wi_c:\r\n",
        "                    n_wi_c[w_ind] = n_wiwj_c.data[ik]\r\n",
        "                else:\r\n",
        "                    n_wi_c[w_ind] += n_wiwj_c.data[ik]\r\n",
        "        n_w_c[wi] = n_wi_c\r\n",
        "        \r\n",
        "    return(n_w_c)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bfzKBgizk2Kf"
      },
      "source": [
        "n_w_c = get_n_w_c(voc_possible_analogies, vocabulary_list, top_voc_list, invdict_top_voc, count_matrix_tri)\r\n",
        "save_obj(n_w_c, \"./Models/n_w_c\")"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}