Tester

.gitignore

@@ -16,4 +16,7 @@ htmlcov/
 dist/
 build/
 *.egg-info/
-helpers/*
+helpers/*
+
+.env
+**/raw_data/

backend/Dockerfile

@@ -8,4 +8,6 @@ WORKDIR $CONTAINER_HOME
 ARG DB_NAME
 ENV DB_NAME $DB_NAME
 
-RUN pip install -r $CONTAINER_HOME/requirements.txt
+RUN pip install -r $CONTAINER_HOME/requirements.txt
+RUN pip install --upgrade certifi
+RUN python -m spacy download en_core_web_sm

backend/app.py

@@ -3,6 +3,21 @@
 from flask import Flask, render_template, request
 from flask_cors import CORS
 from helpers.MySQLDatabaseHandler import MySQLDatabaseHandler
+from collections import defaultdict
+from collections import Counter
+import json
+import math
+import numpy as np
+import pandas as pd
+from nltk.tokenize import TreebankWordTokenizer
+from nltk.tokenize import RegexpTokenizer
+from nltk.corpus import stopwords
+from IPython.core.display import HTML
+import sys
+from helpers.OnetCsvHandler import OnetCsvHandler
+from helpers import onet_recommender_tools as ort
+from helpers import onet_downloader
+from helpers.QueryHandler import QueryHandler
 
 # ROOT_PATH for linking with all your files. 
 # Feel free to use a config.py or settings.py with a global export variable
@@ -11,35 +26,62 @@
 # These are the DB credentials for your OWN MySQL
 # Don't worry about the deployment credentials, those are fixed
 # You can use a different DB name if you want to
-MYSQL_USER = "root"
-MYSQL_USER_PASSWORD = "MayankRao16Cornell.edu"
-MYSQL_PORT = 3306
-MYSQL_DATABASE = "kardashiandb"
+# MYSQL_USER = "root"
+# MYSQL_USER_PASSWORD = os.getenv("MYSQL_USER_PASSWORD")
+# MYSQL_PORT = 3306
+# MYSQL_DATABASE = "kardashiandb"
 
-mysql_engine = MySQLDatabaseHandler(MYSQL_USER,MYSQL_USER_PASSWORD,MYSQL_PORT,MYSQL_DATABASE)
+# mysql_engine = MySQLDatabaseHandler(MYSQL_USER,MYSQL_USER_PASSWORD,MYSQL_PORT,MYSQL_DATABASE)
 
-# Path to init.sql file. This file can be replaced with your own file for testing on localhost, but do NOT move the init.sql file
-mysql_engine.load_file_into_db()
+# # Path to init.sql file. This file can be replaced with your own file for testing on localhost, but do NOT move the init.sql file
+# mysql_engine.load_file_into_db('/Users/jensen615/cs4300/careerfinder/init.sql')
 
 app = Flask(__name__)
 CORS(app)
 
-# Sample search, the LIKE operator in this case is hard-coded, 
-# but if you decide to use SQLAlchemy ORM framework, 
-# there's a much better and cleaner way to do this
-def sql_search(episode):
-    query_sql = f"""SELECT * FROM episodes WHERE LOWER( title ) LIKE '%%{episode.lower()}%%' limit 10"""
-    keys = ["id","title","descr"]
-    data = mysql_engine.query_selector(query_sql)
-    return json.dumps([dict(zip(keys,i)) for i in data])
+TEST = False
+
+def get_data(download=True):
+    raw_data_dir = "/helpers/raw_data/"
+
+    if download:
+        onet_downloader.mkdir(raw_data_dir)
+
+        onet_downloader.download_interests(raw_data_dir)
+        onet_downloader.download_knowledge(raw_data_dir)
+        onet_downloader.download_values(raw_data_dir)
+        onet_downloader.download_cross_skills(raw_data_dir)
+
+    csv_handler = OnetCsvHandler()
+    csv_handler.generate_onet_dictionary(raw_data_dir)
+    return csv_handler.data()
+
+jobs = get_data(False if TEST else True)
+inv_idx = ort.inverted_index(jobs)
+job_idx_map = ort.job_to_idx(jobs)
+skills_idx_map = ort.skill_to_idx(inv_idx)
+n_docs = len(jobs.items())
+idf = ort.compute_idf(inv_idx, n_docs)
+doc_norms = ort.compute_doc_norms(inv_idx, idf, n_docs, job_idx_map)
+
+skillsTrainingCsvPath = "/helpers/querySkillsTraining.csv"
+skillsTrainingCsvPath2 = "/helpers/querySkillsTraining2.csv"
+queryHandler = QueryHandler()
+queryHandler.load_csv_training_data(skillsTrainingCsvPath)
+queryHandler.trainCNN()
 
 @app.route("/")
 def home():
-    return render_template('base.html',title="sample html")
+    return render_template('careerfinder.html', title="sample html")
+
+@app.route("/search")
+def career_search():
+    text = request.args.get("interest")
+    newText = queryHandler.query(text)
+    result = ort.top10_results(newText, jobs, inv_idx, idf, doc_norms, job_idx_map)
+    print(result)
+    return result
 
-@app.route("/episodes")
-def episodes_search():
-    text = request.args.get("title")
-    return sql_search(text)
+if TEST:
+    app.run(debug=True)
 
-# app.run(debug=True)

backend/app_legacy.py

@@ -0,0 +1,49 @@
+import json
+import os
+from flask import Flask, render_template, request
+from flask_cors import CORS
+from helpers.MySQLDatabaseHandler import MySQLDatabaseHandler
+from dotenv import load_dotenv
+
+load_dotenv()
+
+# ROOT_PATH for linking with all your files. 
+# Feel free to use a config.py or settings.py with a global export variable
+os.environ['ROOT_PATH'] = os.path.abspath(os.path.join("..",os.curdir))
+
+# These are the DB credentials for your OWN MySQL
+# Don't worry about the deployment credentials, those are fixed
+# You can use a different DB name if you want to
+# MYSQL_USER = "root"
+# MYSQL_USER_PASSWORD = os.getenv("MYSQL_USER_PASSWORD")
+# MYSQL_PORT = 3306
+# MYSQL_DATABASE = "kardashiandb"
+
+# mysql_engine = MySQLDatabaseHandler(MYSQL_USER,MYSQL_USER_PASSWORD,MYSQL_PORT,MYSQL_DATABASE)
+
+# Path to init.sql file. This file can be replaced with your own file for testing on localhost, but do NOT move the init.sql file
+# mysql_engine.load_file_into_db()
+
+app = Flask(__name__)
+CORS(app)
+
+# Sample search, the LIKE operator in this case is hard-coded, 
+# but if you decide to use SQLAlchemy ORM framework, 
+# there's a much better and cleaner way to do this
+def sql_search(episode):
+    query_sql = f"""SELECT * FROM episodes WHERE LOWER( title ) LIKE '%%{episode.lower()}%%' limit 10"""
+    keys = ["id","title","descr"]
+    # data = mysql_engine.query_selector(query_sql)
+    # return json.dumps([dict(zip(keys,i)) for i in data])
+    return None
+
+@app.route("/")
+def home():
+    return render_template('base.html',title="sample html")
+
+@app.route("/episodes")
+def episodes_search():
+    text = request.args.get("title")
+    return sql_search(text)
+
+# app.run(debug=True)