2-Parse-Quantlets-Folders/parse-folders-with-code.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parsing Quantlet Directories\n",
    "\n",
    "Objective: to parse Quantlet directories with code and metainfo files and create a dataframe for further analysis "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1. Preparations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "##### INSTALL DEPENDENCIES AND SAVE SESSION\n",
    "%pip install pandas==1.3.5\n",
    "%pip install levenshtein\n",
    "%pip freeze > session_info_sunflower.txt "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "##### IMPORT DEPENDENCIES\n",
    "\n",
    "import os \n",
    "import pickle \n",
    "import sys\n",
    "\n",
    "import importlib\n",
    "sys.path.append('../3-data-preprocessing')\n",
    "\n",
    "import preprocessing_utils # type: ignore\n",
    "importlib.reload(preprocessing_utils)\n",
    "\n",
    "import traverse_and_parse as tp  # noqa: E402\n",
    "importlib.reload(tp)\n",
    "\n",
    "##### CONSTANTS AND PATHS\n",
    "\n",
    "# VARIABLE PARAMETERS\n",
    "TODAY = '20231104'\n",
    "\n",
    "# PATHS \n",
    "PATH_TO_PARSE = '../../data/QuantLet'\n",
    "FILE_TYPES = ['m', 'py', 'r', 'R', 'M', 'ipynb']\n",
    "TEST_PATH = '../data/Q_test'\n",
    "OUTPUT_PATH = f\"../../data/preprocessed/Quantlet/{TODAY}\"\n",
    "\n",
    "# CONSTANTS\n",
    "IPY_CONVERT = False\n",
    "\n",
    "if not os.path.exists(OUTPUT_PATH):\n",
    "    os.mkdir(OUTPUT_PATH)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2. Traversing and Parsing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "##### COLLECT, PARSE, AND CONCAT\n",
    "\n",
    "output = tp.traverse_folder(PATH_TO_PARSE, FILE_TYPES)\n",
    "repos = tp.prepare_repos_df(output['repos'])\n",
    "\n",
    "repos.type_script = repos.type_script.str.lower()\n",
    "repos = repos[repos.type_script.isin(\n",
    "            ['py', 'r', 'm', 'ipynb']\n",
    "        )\n",
    "    ]\n",
    "repos_df = repos[~repos.folder_name.str.contains('.ipynb_checkpoints')]\n",
    "repos_df.loc[repos_df.type_script== 'ipynb', 'type_script']='py'\n",
    "repos_df = repos_df.reset_index(drop=True)\n",
    "print(repos_df.shape)\n",
    "\n",
    "NAME = f'{OUTPUT_PATH}/Quantlets_{TODAY}'\n",
    "\n",
    "##### SAVE THE DATAFRAME\n",
    "\n",
    "with open(f'{NAME}.pkl', 'wb') as file:\n",
    "    pickle.dump(repos_df, file)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "encode_code",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}