re-run old notebooks, add current requirements file

Author: machow

dgrtwo-translations/2019-02-19-us_phds/us_phds_pandas.ipynb

@@ -1,5 +1,19 @@
 {
  "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Tidy Tuesday in Python: 2019-02-19 - US Phds\n",
+    "\n",
+    "This analysis is a python translation of Dave Robinson's R [code](https://github.com/dgrtwo/data-screencasts/blob/master/us_phds.Rmd) and corresponding [screencast](https://youtu.be/KzRP40PzopY).\n",
+    "\n",
+    "It compares the code for these python approaches side-by-side:\n",
+    "\n",
+    "* [siuba](http://github.com/machow/siuba) \n",
+    "* [pandas](https://github.com/pandas-dev/pandas)"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 1,
@@ -187,21 +201,16 @@
     }
    ],
    "source": [
-    "\n",
-    "renamed = (\n",
-    "    major_fields_raw\n",
-    "    >> rename(field=\"Field of study\")\n",
-    "    # >> gather(\"key\", \"value\", -_.field)\n",
-    ")\n",
+    "renamed = rename(major_fields_raw, field = \"Field of study\")\n",
     "\n",
     "melted_majors = renamed.melt(id_vars = \"field\", var_name = \"key\")\n",
     "\n",
     "major_fields = (\n",
     "    melted_majors\n",
     "    >> mutate(\n",
-    "        year=if_else(_.key.str.contains(\"Unnamed: \").fillna(False), np.nan, _.key),\n",
+    "        year=_.key.replace(\"Unnamed\", np.nan, regex = True),\n",
     "        type=_.value.where(_.value.str.contains(\"Number|Percent\"), np.nan),\n",
-    "        value=_.value.apply(pd.to_numeric, args=(\"coerce\",)),\n",
+    "        value=lambda d: pd.to_numeric(d.value, \"coerce\")\n",
     "    )\n",
     "    >> mutate(year=_.year.ffill(), type=_.type.ffill())\n",
     "    >> select(-_.key)\n",
@@ -221,7 +230,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 19,
    "metadata": {
     "pytabs": {
      "class": "pytabs-4",
@@ -344,15 +353,15 @@
        "[3543 rows x 3 columns]"
       ]
      },
-     "execution_count": 4,
+     "execution_count": 19,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
     "fine_fields = (\n",
     "    pd.read_excel(\"./data/sed17-sr-tab013.xlsx\", skiprows=3)\n",
-    "    >> rename(field = _[\"Fine field of study\"])\n",
+    "    >> rename(field = \"Fine field of study\")\n",
     "    >> gather(\"year\", \"number\", -_.field)\n",
     "    >> filter(_.number != 'na')\n",
     "    >> mutate(\n",
@@ -478,13 +487,10 @@
     "    >> rename(field=\"Unnamed: 0\")\n",
     "    >> filter(~_.field.isin(sex))\n",
     "    >> mutate(\n",
-    "        field=case_when(\n",
-    "            {\n",
-    "                _.field == \"Life sciencesb\": \"Life sciences\",\n",
-    "                _.field == \"Otherc\": \"Other\",\n",
-    "                True: _.field,\n",
-    "            }\n",
-    "        )\n",
+    "        field=_.field.rename({\n",
+    "            \"Life sciencesb\": \"Life sciences\",\n",
+    "            \"Otherc\": \"Other\"\n",
+    "        })\n",
     "    )\n",
     ")\n",
     "\n",
@@ -670,27 +676,16 @@
     "    >> rename(field=\"Sex and major field of study\")\n",
     "    >> select(-_.contains(\"change\"))\n",
     "    >> mutate(\n",
-    "        field=case_when(\n",
-    "            {\n",
-    "                _.field == \"All doctorate recipientsa\": \"All\",\n",
-    "                _.field == \"Otherb\": \"Other\",\n",
-    "                True: _.field,\n",
-    "            }\n",
-    "        ),\n",
-    "        sex=_.field.map({\"Male\": \"Male\", \"Female\": \"Female\", \"All\": \"All\"}),\n",
-    "        broad_field=\n",
-    "            if_else(_.field.isin(broad_fields), _.field, np.nan)\n",
-    "    )\n",
-    "\n",
-    "    >> mutate(\n",
-    "        broad_field = _.broad_field.apply(lambda x: x if x != 'nan' else np.nan).ffill(),\n",
-    "        sex = _.sex.ffill()\n",
+    "        field=_.field.replace({\n",
+    "            \"All doctorate recipientsa\": \"All\",\n",
+    "            \"Otherb\": \"Other\"\n",
+    "            }),\n",
+    "        sex=_.field.where(_.field.isin([\"Male\", \"Female\", \"All\"])),\n",
+    "        broad_field= _.field.where(_.field.isin(broad_fields))\n",
     "    )\n",
+    "    >> mutate(sex = _.sex.ffill(), broad_field = _.broad_field.ffill())\n",
     "    >> gather(\"year\", \"number\", -_.sex, -_.broad_field, -_.field)\n",
-    "    >> filter(\n",
-    "        ~_.field.isin(sex),\n",
-    "        _.number.notna()\n",
-    "    )\n",
+    "    >> filter(~_.field.isin(sex), _.number.notna())\n",
     "    >> mutate(year = _.year.astype(int))\n",
     ")\n",
     "\n",

dgrtwo-translations/2019-02-19-us_phds/us_phds_siuba-tabbed.ipynb

@@ -34,7 +34,7 @@ templates:
 
 2019-03-12-board-games/tabbed-analysis.html: 2019-03-12-board-games
 	cd "$<" && nbtabs convert -e --out tabbed-analysis.ipynb board-games-siuba.Rmd board-games.Rmd
-	jupyter nbconvert --to html --template templates/nbtabs-full.tpl "$</tabbed-analysis.ipynb"
+	jupyter nbconvert --to html --template templates/nbtabs-full.tpl "$</tabbed.ipynb"
 
 
 

dgrtwo-translations/2019-02-19-us_phds/us_phds_siuba.ipynb

@@ -0,0 +1,161 @@
+aiohttp==3.6.2
+altair==4.0.1
+appdirs==1.4.3
+appnope==0.1.0
+argh==0.26.2
+astor==0.8.1
+async-generator==1.10
+async-timeout==3.0.1
+attrs==19.3.0
+backcall==0.1.0
+base58==2.0.0
+black==19.10b0
+bleach==3.1.0
+blinker==1.4
+bokeh==1.4.0
+boto3==1.12.6
+botocore==1.15.6
+certifi==2019.11.28
+cffi==1.13.2
+chardet==3.0.4
+Click==7.0
+colorama==0.4.3
+cycler==0.10.0
+dash==1.9.0
+dash-core-components==1.8.0
+dash-html-components==1.0.2
+dash-renderer==1.2.4
+dash-table==4.6.0
+decorator==4.4.1
+defusedxml==0.6.0
+descartes==1.1.0
+-e git+https://github.com/django-extensions/django-extensions.git@0ad61ae76b121344ccd2ad4bbf4af4e4b5b8f70e#egg=django_extensions
+docutils==0.15.2
+entrypoints==0.3
+enum-compat==0.0.3
+Flask==1.1.1
+Flask-Compress==1.4.0
+future==0.18.2
+gitdb2==2.0.6
+GitPython==3.0.5
+idna==2.8
+idna-ssl==1.1.0
+importlib-metadata==1.4.0
+ipykernel==5.1.3
+ipython==7.11.1
+ipython-genutils==0.2.0
+ipywidgets==7.5.1
+itsdangerous==1.1.0
+jedi==0.15.2
+Jinja2==2.10.3
+jmespath==0.9.5
+joblib==0.14.1
+json5==0.8.5
+jsonschema==3.2.0
+jupyter-client==5.3.4
+jupyter-contrib-core==0.3.3
+jupyter-contrib-nbextensions==0.5.1
+jupyter-core==4.6.1
+jupyter-highlight-selected-word==0.2.0
+jupyter-latex-envs==1.4.6
+jupyter-nbextensions-configurator==0.4.1
+jupyter-server==0.1.1
+jupyter-server-proxy==1.2.0
+jupyterlab==1.2.6
+jupyterlab-code-formatter==1.0.3
+jupyterlab-dash==0.1.0a3
+jupyterlab-git==0.9.0
+jupyterlab-pygments==0.1.0
+jupyterlab-quickopen==0.3.0
+jupyterlab-server==1.0.6
+jupytext==1.3.2
+keyring==21.1.0
+kiwisolver==1.1.0
+lxml==4.5.0
+Markdown==3.1.1
+MarkupSafe==1.1.1
+matplotlib==3.1.2
+mistletoe==0.7.2
+mistune==0.8.4
+mizani==0.6.0
+more-itertools==8.1.0
+multidict==4.7.5
+multipy==0.16
+nbconvert==5.6.1
+nbdime==1.1.0
+nbformat==5.0.4
+-e [email protected]:machow/nbtabs.git@61199b91809a8f502ada774be67fc51e337d7d98#egg=nbtabs
+notebook==6.0.3
+numpy==1.18.1
+packaging==19.2
+palettable==3.3.0
+pandas==1.0.1
+pandocfilters==1.4.2
+parso==0.5.2
+pathspec==0.7.0
+pathtools==0.1.2
+patsy==0.5.1
+pexpect==4.8.0
+pickleshare==0.7.5
+Pillow==7.0.0
+pkginfo==1.5.0.1
+plotly==4.5.1
+plotnine==0.6.0
+pluggy==0.13.1
+prometheus-client==0.7.1
+prompt-toolkit==3.0.2
+protobuf==3.11.3
+psycopg2==2.8.4
+ptyprocess==0.6.0
+py==1.8.1
+pycparser==2.19
+pydeck==0.2.1
+Pygments==2.5.2
+pyparsing==2.4.6
+pyrsistent==0.15.7
+pytest==5.3.4
+python-dateutil==2.8.0
+pytz==2019.3
+PyYAML==5.3
+pyzmq==18.1.1
+qgrid==1.2.0
+readme-renderer==24.0
+regex==2020.1.8
+requests==2.22.0
+requests-toolbelt==0.9.1
+retrying==1.3.3
+rpy2==3.2.4
+s3transfer==0.3.3
+scikit-learn==0.22.1
+scipy==1.4.1
+Send2Trash==1.5.0
+simpervisor==0.3
+simplegeneric==0.8.1
+-e [email protected]:machow/siuba.git@42872e5cb4068e6bd1be53d04619ea2ec5bab186#egg=siuba
+six==1.14.0
+smmap2==2.0.5
+SQLAlchemy==1.3.13
+statsmodels==0.11.0
+streamlit==0.56.0
+terminado==0.8.3
+testpath==0.4.4
+toml==0.10.0
+toolz==0.10.0
+tornado==5.1.1
+tqdm==4.42.1
+traitlets==4.3.3
+twine==3.1.1
+typed-ast==1.4.1
+typing-extensions==3.7.4.1
+tzlocal==2.0.0
+urllib3==1.25.8
+validators==0.14.2
+voila==0.1.20
+watchdog==0.10.2
+wcwidth==0.1.8
+webencodings==0.5.1
+Werkzeug==1.0.0
+widgetsnbextension==3.5.1
+xlrd==1.2.0
+yarl==1.4.2
+zipp==2.0.1