Sentiment-Analysis.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Copy of Copy of HW_1.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/JayLeoK/NLP-Sentiment-Analysis/blob/master/Sentiment-Analysis.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bymVftHXImRa",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import sys, argparse\n",
        "from scipy import sparse\n",
        "from sklearn import linear_model\n",
        "from collections import Counter\n",
        "import numpy as np\n",
        "import re\n",
        "from collections import Counter\n",
        "from collections import defaultdict"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KYRARhMtI7AR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "######################################################################\n",
        "## This defines the dumb features the model starts with.\n",
        "######################################################################\n",
        "\n",
        "\n",
        "def dumb_featurize(text):\n",
        "\tfeats = {}\n",
        "\twords = text.split(\" \")\n",
        "\n",
        "\tfor word in words:\n",
        "\t\tif word == \"love\" or word == \"like\" or word == \"best\":\n",
        "\t\t\tfeats[\"contains_positive_word\"] = 1\n",
        "\t\tif word == \"hate\" or word == \"dislike\" or word == \"worst\" or word == \"awful\":\n",
        "\t\t\tfeats[\"contains_negative_word\"] = 1\n",
        "\n",
        "\treturn feats\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CGiM8qQiJOBU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "######################################################################\n",
        "## This defines the sentiment classification class which\n",
        "## loads the data and sets up the model.\n",
        "######################################################################\n",
        "\n",
        "class SentimentClassifier:\n",
        "\n",
        "\tdef __init__(self, feature_method):\n",
        "\t\tself.feature_vocab = {}\n",
        "\t\tself.feature_method = feature_method\n",
        "\n",
        "\n",
        "\t# Read data from file\n",
        "\tdef load_data(self, filename):\n",
        "\t\tdata = []\n",
        "\t\twith open(filename, encoding=\"utf8\") as file:\n",
        "\t\t\tfor line in file:\n",
        "\t\t\t\tcols = line.split(\"\\t\")\n",
        "\t\t\t\tlabel = cols[0]\n",
        "\t\t\t\ttext = cols[1].rstrip()\n",
        "\n",
        "\t\t\t\tdata.append((label, text))\n",
        "\t\treturn data\n",
        "\n",
        "\t# Featurize entire dataset\n",
        "\tdef featurize(self, data):\n",
        "\t\tfeaturized_data = []\n",
        "\t\tfor label, text in data:\n",
        "\t\t\tfeats = self.feature_method(text)\n",
        "\t\t\tfeaturized_data.append((label, feats))\n",
        "\t\treturn featurized_data\n",
        "\n",
        "\t# Read dataset and returned featurized representation as sparse matrix + label array\n",
        "\tdef process(self, dataFile, training = False):\n",
        "\t\tdata = self.load_data(dataFile)\n",
        "\t\tdata = self.featurize(data)\n",
        "\n",
        "\t\tif training:\t\t\t\n",
        "\t\t\tfid = 0\n",
        "\t\t\tfeature_doc_count = Counter()\n",
        "\t\t\tfor label, feats in data:\n",
        "\t\t\t\tfor feat in feats:\n",
        "\t\t\t\t\tfeature_doc_count[feat]+= 1\n",
        "\n",
        "\t\t\tfor feat in feature_doc_count:\n",
        "\t\t\t\tif feature_doc_count[feat] >= MIN_FEATURE_COUNT[self.feature_method.__name__]:\n",
        "\t\t\t\t\tself.feature_vocab[feat] = fid\n",
        "\t\t\t\t\tfid += 1\n",
        "\n",
        "\t\tF = len(self.feature_vocab)\n",
        "\t\tD = len(data)\n",
        "\t\tX = sparse.dok_matrix((D, F))\n",
        "\t\tY = np.zeros(D)\n",
        "\t\tfor idx, (label, feats) in enumerate(data):\n",
        "\t\t\tfor feat in feats:\n",
        "\t\t\t\tif feat in self.feature_vocab:\n",
        "\t\t\t\t\tX[idx, self.feature_vocab[feat]] = feats[feat]\n",
        "\t\t\tY[idx] = 1 if label == \"pos\" else 0\n",
        "\n",
        "\t\treturn X, Y\n",
        "\n",
        "\tdef load_test(self, dataFile):\n",
        "\t\tdata = self.load_data(dataFile)\n",
        "\t\tdata = self.featurize(data)\n",
        "\n",
        "\t\tF = len(self.feature_vocab)\n",
        "\t\tD = len(data)\n",
        "\t\tX = sparse.dok_matrix((D, F))\n",
        "\t\tY = np.zeros(D, dtype = int)\n",
        "\t\tfor idx, (data_id, feats) in enumerate(data):\n",
        "\t\t\t# print (data_id)\n",
        "\t\t\tfor feat in feats:\n",
        "\t\t\t\tif feat in self.feature_vocab:\n",
        "\t\t\t\t\tX[idx, self.feature_vocab[feat]] = feats[feat]\n",
        "\t\t\tY[idx] = data_id\n",
        "\n",
        "\t\treturn X, Y\n",
        "\n",
        "\t# Train model and evaluate on held-out data\n",
        "\tdef evaluate(self, trainX, trainY, devX, devY):\n",
        "\t\t(D,F) = trainX.shape\n",
        "\t\tself.log_reg = linear_model.LogisticRegression(C = L2_REGULARIZATION_STRENGTH[self.feature_method.__name__])\t\n",
        "\t\tself.log_reg.fit(trainX, trainY)\n",
        "\t\ttraining_accuracy = self.log_reg.score(trainX, trainY)\n",
        "\t\tdevelopment_accuracy = self.log_reg.score(devX, devY)\n",
        "\t\tprint(\"Method: %s, Features: %s, Train accuracy: %.3f, Dev accuracy: %.3f\" % (self.feature_method.__name__, F, training_accuracy, development_accuracy))\n",
        "\t\t\n",
        "\n",
        "\t# Predict labels for new data\n",
        "\tdef predict(self, testX, idsX):\n",
        "\t\tpredX = self.log_reg.predict(testX)\n",
        "\n",
        "\t\tout = open(\"%s_%s\" % (self.feature_method.__name__, \"predictions.csv\"), \"w\", encoding=\"utf8\")\n",
        "\t\tout.write(\"Id,Expected\\n\")\n",
        "\t\tfor idx, data_id in enumerate(testX):\n",
        "\t\t\tout.write(\"%s,%s\\n\" % (idsX[idx], int(predX[idx])))\n",
        "\t\tout.close()\n",
        "\n",
        "\t# Write learned parameters to file\n",
        "\tdef printWeights(self):\n",
        "\t\tout = open(\"%s_%s\" % (self.feature_method.__name__, \"weights.txt\"), \"w\", encoding=\"utf8\")\n",
        "\t\treverseVocab = [None]*len(self.feature_vocab)\n",
        "\t\tfor feat in self.feature_vocab:\n",
        "\t\t\treverseVocab[self.feature_vocab[feat]] = feat\n",
        "\n",
        "\t\tout.write(\"%.5f\\t__BIAS__\\n\" % self.log_reg.intercept_)\n",
        "\t\tfor (weight, feat) in sorted(zip(self.log_reg.coef_[0], reverseVocab)):\n",
        "\t\t\tout.write(\"%.5f\\t%s\\n\" % (weight, feat))\n",
        "\t\tout.close()\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Jnqjxd6fKPiP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "######################################################################\n",
        "##change these parameters to prevent the model from overfitting \n",
        "##and achieve higher performance\n",
        "######################################################################\n",
        "\n",
        "# regularization strength to control overfitting (values closer to 0  = stronger regularization)\n",
        "L2_REGULARIZATION_STRENGTH = {\"dumb_featurize\": 1, \"fancy_featurize\": 0.15 }\n",
        "\n",
        "# must observe feature at least this many times in training data to include in model\n",
        "MIN_FEATURE_COUNT = {\"dumb_featurize\": 10,  \"fancy_featurize\": 7}"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jxKmEqI5JY71",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# fancy_featurization implementation\n",
        "def fancy_featurize(text):\n",
        "  features = {}\n",
        "  # adds bag of word representation to features\n",
        "\n",
        "  features.update(bag_of_words(text))\n",
        "  # Your code goes here\n",
        "  words = text.split(' ')\n",
        "  features.update({'number of words':len(words)})\n",
        "\n",
        "  features.update(VADER_ngram(words))\n",
        "\n",
        "  features.update(MPQA_counts(words))\n",
        "\n",
        "  #try has positive, has negative, and count positive, count negative\n",
        "  return features"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UDr4xZw1JbuZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Adds the bag of words representation of the text to feats\n",
        "def bag_of_words(text):\n",
        "  word_bag = {}\n",
        "\t# Your code goes here\n",
        "  used = set()\n",
        "  for word in text.split(' '):\n",
        "    if word in used:\n",
        "      word_bag[word]+=1\n",
        "    else:\n",
        "      word_bag[word]=1\n",
        "      used.add(word)\n",
        "  return word_bag"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0dLu5Y_eHUey",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def MPQA_counts(words):    #adds MPQA negative and positive counts\n",
        "  #mounted google drive before accessing the .tff file\n",
        "  features = {}\n",
        "  with open('/content/drive/My Drive/subjclueslen1-HLTEMNLP05.tff', 'r') as file:\n",
        "    pos_lexicon = set()\n",
        "    neg_lexicon = set()\n",
        "    for line in file:\n",
        "      if \"priorpolarity=positive\" in line:\n",
        "        word = line.split()[2].split(\"=\")[1]\n",
        "        pos_lexicon.add(word)\n",
        "      if \"priorpolarity=negative\" in line:\n",
        "        word = line.split()[2].split(\"=\")[1]\n",
        "        neg_lexicon.add(word)\n",
        "  \n",
        "  sentiments = []\n",
        "  for word in words:\n",
        "    if word in pos_lexicon:\n",
        "      sentiments.append(1)\n",
        "    elif word in neg_lexicon:\n",
        "      sentiments.append(-1)\n",
        "    else:\n",
        "      sentiments.append(0)\n",
        "    \n",
        "  hasPos,hasNeg = 0,0\n",
        "  posCount = sum([i>0 for i in sentiments])\n",
        "  negCount = sum([i<0 for i in sentiments])\n",
        "  if posCount>1: hasPos = 1\n",
        "  if negCount>1: hasNeg = 1\n",
        "  features['MPQA Has Positive']= hasPos\n",
        "  features['MPQA Has Negative']= hasNeg\n",
        "  features['MPQA count difference'] = posCount-negCount\n",
        "  features['MPQA Positive Count']= posCount\n",
        "  features['MPQA Negative Count']= negCount\n",
        "  features['MPQA pos/neg Ratio']= (posCount+1)/(negCount+1)\n",
        "  \n",
        "  return features\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VjRNN5JC2vDV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def count_threshold(value,pos,neg,threshold):\n",
        "  if value<(-threshold/2):\n",
        "    neg = neg+1\n",
        "  elif value>(threshold/2):\n",
        "    pos = pos+1\n",
        "  return pos,neg"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x4Gr_ITzHUN-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def VADER_ngram(words):   #implements bigram, trigram, and negation\n",
        "  with open('./vader_lexicon.txt') as file:\n",
        "    lexicon = {}\n",
        "    tokens = set()\n",
        "    for line in file:\n",
        "      items = line.split('\\t')\n",
        "      token,rating = items[0], float(items[1])\n",
        "      tokens.add(token)\n",
        "      lexicon[token]=rating\n",
        "\n",
        "  features = {}\n",
        "  sentiments = []\n",
        "  for i in range(0,len(words),3):\n",
        "    first = words[i]\n",
        "    if first in tokens:\n",
        "      sentiments.append(lexicon[first])\n",
        "    \n",
        "    negation = first == ('not' or 'n\\'t')\n",
        "    used = set()\n",
        "    if i+1<len(words):\n",
        "      second = words[i+1]\n",
        "      if not negation: \n",
        "        bigram = first +' '+ second\n",
        "        if second in tokens: sentiments.append(lexicon[second])\n",
        "        else: sentiments.append(0)\n",
        "      else:   #negation reverses effect on count, sentiment\n",
        "        bigram = first+' NEG_'+second\n",
        "        if second in tokens: sentiments.append(-lexicon[second]) #flips sign\n",
        "        else: sentiments.append(0)\n",
        "      if bigram in used:\n",
        "        features[bigram] += 1 \n",
        "      else:\n",
        "        features[bigram] = 1\n",
        "        used.add(bigram)\n",
        "\n",
        "    if i+2<len(words):\n",
        "      third = words[i+2]\n",
        "      if not negation: \n",
        "        trigram = bigram +' '+ third\n",
        "        if third in tokens: sentiments.append(lexicon[third])\n",
        "        else: sentiments.append(0)\n",
        "      else:   #negation reverses effect on count, sentiment\n",
        "        trigram = bigram+' NEG_'+third\n",
        "        if third in lexicon:sentiments.append(-lexicon[third]) #flips sign\n",
        "        else: sentiments.append(0)\n",
        "      if trigram in used:\n",
        "        features[trigram] += 1 \n",
        "      else:\n",
        "        features[trigram] = 1\n",
        "        used.add(trigram)\n",
        "  threshold = 2\n",
        "  posCount, negCount = sum([i>threshold/2 for i in sentiments]), sum([i<-threshold/2 for i in sentiments])\n",
        "  features['VADER positive count'] = posCount\n",
        "  features['VADER negative count'] = negCount\n",
        "  features['VADER count difference'] = posCount-negCount\n",
        "  features['VADER total sentiment'] = sum(sentiments)\n",
        "  features['VADER pos/neg ratio'] = (posCount+1)/(negCount+1)\n",
        "\n",
        "  return features"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2O7w65CyuP19",
        "colab_type": "code",
        "outputId": "865a4e03-4895-4be1-e07e-2398cb0d0e9b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 224
        }
      },
      "source": [
        "#Code retreives VADER\n",
        "!wget https://raw.githubusercontent.com/cjhutto/vaderSentiment/master/vaderSentiment/vader_lexicon.txt"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-02-04 05:54:44--  https://raw.githubusercontent.com/cjhutto/vaderSentiment/master/vaderSentiment/vader_lexicon.txt\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.0.133, 151.101.64.133, 151.101.128.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.0.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 426687 (417K) [text/plain]\n",
            "Saving to: ‘vader_lexicon.txt.2’\n",
            "\n",
            "vader_lexicon.txt.2 100%[===================>] 416.69K  --.-KB/s    in 0.07s   \n",
            "\n",
            "2020-02-04 05:54:46 (5.74 MB/s) - ‘vader_lexicon.txt.2’ saved [426687/426687]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pT7xD4k2QiCH",
        "colab_type": "code",
        "outputId": "23099667-e103-457b-9716-6734b02b7299",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 632
        }
      },
      "source": [
        "#This code gets the train/dev/test files from github and imports them into Colab\n",
        "!wget https://raw.githubusercontent.com/dbamman/nlp20/master/HW_1/train.txt\n",
        "!wget https://raw.githubusercontent.com/dbamman/nlp20/master/HW_1/dev.txt\n",
        "!wget https://raw.githubusercontent.com/dbamman/nlp20/master/HW_1/test.txt.zip\n",
        "!unzip test.txt.zip"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-02-04 05:54:47--  https://raw.githubusercontent.com/dbamman/nlp20/master/HW_1/train.txt\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.0.133, 151.101.64.133, 151.101.128.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.0.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 1427184 (1.4M) [text/plain]\n",
            "Saving to: ‘train.txt.1’\n",
            "\n",
            "\rtrain.txt.1           0%[                    ]       0  --.-KB/s               \rtrain.txt.1         100%[===================>]   1.36M  --.-KB/s    in 0.09s   \n",
            "\n",
            "2020-02-04 05:54:47 (15.7 MB/s) - ‘train.txt.1’ saved [1427184/1427184]\n",
            "\n",
            "--2020-02-04 05:54:48--  https://raw.githubusercontent.com/dbamman/nlp20/master/HW_1/dev.txt\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.0.133, 151.101.64.133, 151.101.128.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.0.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 1474560 (1.4M) [text/plain]\n",
            "Saving to: ‘dev.txt.1’\n",
            "\n",
            "dev.txt.1           100%[===================>]   1.41M  --.-KB/s    in 0.1s    \n",
            "\n",
            "2020-02-04 05:54:48 (14.5 MB/s) - ‘dev.txt.1’ saved [1474560/1474560]\n",
            "\n",
            "--2020-02-04 05:54:49--  https://raw.githubusercontent.com/dbamman/nlp20/master/HW_1/test.txt.zip\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.0.133, 151.101.64.133, 151.101.128.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.0.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 13100860 (12M) [application/zip]\n",
            "Saving to: ‘test.txt.zip.1’\n",
            "\n",
            "test.txt.zip.1      100%[===================>]  12.49M  67.3MB/s    in 0.2s    \n",
            "\n",
            "2020-02-04 05:54:50 (67.3 MB/s) - ‘test.txt.zip.1’ saved [13100860/13100860]\n",
            "\n",
            "Archive:  test.txt.zip\n",
            "replace test.txt? [y]es, [n]o, [A]ll, [N]one, [r]ename: N\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QgwgKmYWLlc8",
        "colab_type": "code",
        "outputId": "cab638ef-e64a-40f1-f6f0-a083ac2ea324",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 207
        }
      },
      "source": [
        "#This cell trains two models: one on the dumb features and one on your fancy\n",
        "#features.  It will store the test set predictions in a csv.\n",
        "#The weights will be stored in a text file. \n",
        "#To access the files, click on the folder icon in the left sidebar.\n",
        "#You can preview the files in Colab by double clicking or download the files by \n",
        "#right clicking and selecting Download.\n",
        "if __name__ == \"__main__\":\n",
        "  trainingFile = \"./train.txt\"\n",
        "  evaluationFile = \"./dev.txt\"\n",
        "  testFile = \"./test.txt\"\n",
        "\n",
        "  for feature_method in [dumb_featurize, fancy_featurize]:\n",
        "    sentiment_classifier = SentimentClassifier(feature_method)\n",
        "    trainX, trainY = sentiment_classifier.process(trainingFile, training=True)\n",
        "    devX, devY = sentiment_classifier.process(evaluationFile, training=False)\n",
        "    testX, idsX = sentiment_classifier.load_test(testFile)\n",
        "    sentiment_classifier.evaluate(trainX, trainY, devX, devY)\n",
        "    sentiment_classifier.printWeights()\n",
        "    sentiment_classifier.predict(testX, idsX)\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Method: dumb_featurize, Features: 2, Train accuracy: 0.604, Dev accuracy: 0.611\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
            "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
            "\n",
            "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
            "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
            "Please also refer to the documentation for alternative solver options:\n",
            "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
            "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Method: fancy_featurize, Features: 4597, Train accuracy: 0.963, Dev accuracy: 0.809\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jaxHuhr5JBSo",
        "colab_type": "code",
        "outputId": "6f97c845-27d8-4587-f824-2b0d05804d8d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mX5wnnynJB4e",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}