word2vec with gensim. bad accuracy. less data

Author: shubham1810

class_15/.ipynb_checkpoints/Untitled-checkpoint.ipynb

@@ -0,0 +1,156 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "from matplotlib import pyplot as plt\n",
+    "import pickle as pk\n",
+    "%matplotlib inline\n",
+    "import gensim"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "f = open('./data.txt')\n",
+    "d = f.read()\n",
+    "f.close()\n",
+    "\n",
+    "data = d[1260:]\n",
+    "data = data.lower().decode('utf-8')\n",
+    "import re\n",
+    "\n",
+    "p = re.sub('[^A-Za-z.]+', ' ', data)\n",
+    "ds = p.split('.')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "u'it is often enough and always with great surprise intimated to me that there is something both ordinary and unusual in all my writings from the birth of tragedy to the recently published prelude to a philosophy of the future they all contain i have been told snares and nets for short sighted birds and something that is almost a constant subtle incitement to an overturning of habitual opinions and of approved customs'"
+      ]
+     },
+     "execution_count": 24,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "ds[0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "metadata": {
+    "collapsed": false,
+    "scrolled": false
+   },
+   "outputs": [],
+   "source": [
+    "for ix in range(len(ds)):\n",
+    "    ds[ix] = ds[ix].split()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 29,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "1462\n"
+     ]
+    }
+   ],
+   "source": [
+    "print len(ds)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 30,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "model = gensim.models.Word2Vec(ds, min_count=2)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 42,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[(u'an', 0.999976396560669),\n",
+       " (u'a', 0.9999758005142212),\n",
+       " (u'from', 0.9999721050262451),\n",
+       " (u'that', 0.9999714493751526),\n",
+       " (u'and', 0.9999709725379944),\n",
+       " (u'of', 0.9999709725379944),\n",
+       " (u'this', 0.9999706745147705),\n",
+       " (u'not', 0.9999699592590332),\n",
+       " (u'are', 0.9999694228172302),\n",
+       " (u'which', 0.9999691247940063)]"
+      ]
+     },
+     "execution_count": 42,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "model.similar_by_word('to')"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 2",
+   "language": "python",
+   "name": "python2"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}