diff --git a/.gitignore b/.gitignore index 68c24744..ef68ca82 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,8 @@ __pycache__ # Exclude pycharm metadata - .idea/ \ No newline at end of file + .idea/ + +model/ +jarvis_virtualenv/ +*.log diff --git a/data/.gitignore b/data/.gitignore new file mode 100644 index 00000000..98e6ef67 --- /dev/null +++ b/data/.gitignore @@ -0,0 +1 @@ +*.db diff --git a/requirements.txt b/requirements.txt index a9327dd4..39c4adb8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,13 +21,13 @@ lxml==4.6.3 mock==3.0.5 more-itertools==7.0.0 nltk==3.4.5 -numpy==1.16.4 patch==1.16 platformdirs==2.4.0 playsound==1.2.2 psutil==5.6.6 PTable==0.9.2 PyAudio==0.2.11 +pybind11==2.9.1 pycairo==1.20.1 PyGObject==3.42.0 pymongo==3.10.1 @@ -49,3 +49,4 @@ wikipedia==1.4.0 wolframalpha==3.0.1 word2number==1.1 xmltodict==0.12.0 +sounddevice==0.4.4 diff --git a/run_jarvis.sh b/run_jarvis.sh old mode 100644 new mode 100755 index 67b46182..fb44aad7 --- a/run_jarvis.sh +++ b/run_jarvis.sh @@ -1,15 +1,6 @@ #!/usr/bin/env bash -# -------------------------------- -# Start MongoDB service -# -------------------------------- -sudo systemctl start mongodb # -------------------------------- # Start Jarvis service with virtualenv # -------------------------------- ./jarvis_virtualenv/bin/python ./src/jarvis/start.py - -# -------------------------------- -# Stop MongoDB service -# -------------------------------- -sudo systemctl stop mongodb \ No newline at end of file diff --git a/run_tests.sh b/run_tests.sh index 5e3a91f4..e131d45f 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -5,20 +5,10 @@ export PYTHONPATH="${PYTHONPATH}:./src/jarvis" source jarvis_virtualenv/bin/activate -# -------------------------------- -# Start MongoDB service -# -------------------------------- -sudo systemctl start mongodb - # -------------------------------- # Run unittests # -------------------------------- python -m unittest discover -s ./src -p "*tests.py" exit_code=($?) -# -------------------------------- -# Stop MongoDB service -# -------------------------------- -sudo systemctl stop mongodb - exit $exit_code \ No newline at end of file diff --git a/setup.sh b/setup.sh index 4cf114e8..4cd0611c 100755 --- a/setup.sh +++ b/setup.sh @@ -25,18 +25,16 @@ fi #----------------------------------- # System dependencies installation #----------------------------------- -sudo apt-get update && / -sudo apt-get install build-essential && / -sudo apt-get install python3-dev && / -sudo apt-get install python3-setuptools && / -sudo apt-get install python3-pip && / -sudo apt-get install python3-venv && / -sudo apt-get install portaudio19-dev python3-pyaudio python3-pyaudio && / -sudo apt-get install libasound2-plugins libsox-fmt-all libsox-dev libxml2-dev libxslt-dev sox ffmpeg && / -sudo apt-get install espeak && / -sudo apt-get install libcairo2-dev libgirepository1.0-dev gir1.2-gtk-3.0 && / -sudo apt install mongodb && / -sudo apt-get install gnupg +sudo apt-get update && \ +sudo apt-get install build-essential \ + python3-dev python3-setuptools python3-pip python3-venv \ + portaudio19-dev python3-pyaudio python3-pyaudio \ + python3-numpy \ + libasound2-plugins libsox-fmt-all libsox-dev libxml2-dev libxslt-dev sox ffmpeg \ + libgstreamer1.0-dev libgstreamer1.0-0 \ + espeak \ + libcairo2-dev libgirepository1.0-dev gir1.2-gtk-3.0 \ + gnupg # Reload local package database sudo apt-get update @@ -73,7 +71,7 @@ echo "${green} ${activated_python_version} activated!${reset}" # Install python requirements pip3 install --upgrade cython pip3 install wheel -python setup.py bdist_wheel +python3 setup.py bdist_wheel pip3 install -r $JARVIS_DIR/requirements.txt pip3 install -U scikit-learn diff --git a/src/jarvis/jarvis/__init__.py b/src/jarvis/jarvis/__init__.py index b77fb60c..01270a9b 100644 --- a/src/jarvis/jarvis/__init__.py +++ b/src/jarvis/jarvis/__init__.py @@ -24,10 +24,12 @@ from jarvis import settings from jarvis.settings import ROOT_LOG_CONF -from jarvis.utils.mongoDB import db -from jarvis.utils.startup import configure_MongoDB +from jarvis.utils.startup import configure_defaults from jarvis.enumerations import InputMode import jarvis.engines as engines +from jarvis.utils.settings_database import settingsDB +from jarvis.utils.skills_registry import skills_registry +from jarvis.skills.registry import skill_objects, CONTROL_SKILLS, ENABLED_BASIC_SKILLS # ---------------------------------------------------------------------------------------------------------------------- # Create a Console & Rotating file logger @@ -41,16 +43,20 @@ f.close() # ---------------------------------------------------------------------------------------------------------------------- -# Configuare MongoDB, load skills and settings +# Configuare defaults, load skills and settings # ---------------------------------------------------------------------------------------------------------------------- -configure_MongoDB(db, settings) +configure_defaults(settings) +skills_registry.skill_objects = skill_objects +skills_registry.basic_skills = ENABLED_BASIC_SKILLS +skills_registry.control_skills = CONTROL_SKILLS # ---------------------------------------------------------------------------------------------------------------------- # Get assistant settings # ---------------------------------------------------------------------------------------------------------------------- -input_mode = db.get_documents(collection='general_settings')[0]['input_mode'] -response_in_speech = db.get_documents(collection='general_settings')[0]['response_in_speech'] -assistant_name = db.get_documents(collection='general_settings')[0]['assistant_name'] +generalSettings = settingsDB.getGeneralSettings() +input_mode = generalSettings.input_mode +response_in_speech = generalSettings.response_in_speech +assistant_name = generalSettings.assistant_name # ---------------------------------------------------------------------------------------------------------------------- # Create assistant input and output engine instances diff --git a/src/jarvis/jarvis/core/console.py b/src/jarvis/jarvis/core/console.py index d138af86..56b39ab5 100644 --- a/src/jarvis/jarvis/core/console.py +++ b/src/jarvis/jarvis/core/console.py @@ -26,9 +26,9 @@ import logging from jarvis import settings -from jarvis.utils.mongoDB import db +from jarvis.utils.settings_database import settingsDB from jarvis.utils.console import jarvis_logo, start_text, OutputStyler, headerize -from jarvis.enumerations import MongoCollections, InputMode +from jarvis.enumerations import InputMode class ConsoleManager: @@ -90,15 +90,13 @@ def console_output(self, text='', debug_log=None, info_log=None, warn_log=None, # ---------------------------------------------------------------------------------------------------------- # General info sector # ---------------------------------------------------------------------------------------------------------- - settings_documents = db.get_documents(collection=MongoCollections.GENERAL_SETTINGS.value) - if settings_documents: - settings_ = settings_documents[0] - print(OutputStyler.HEADER + headerize('GENERAL INFO') + OutputStyler.ENDC) - enabled = OutputStyler.GREEN + 'ENABLED' + OutputStyler.ENDC if settings_['response_in_speech'] else OutputStyler.WARNING + 'NOT ENABLED' + OutputStyler.ENDC - print(OutputStyler.BOLD + 'RESPONSE IN SPEECH: ' + enabled) - print(OutputStyler.BOLD + 'INPUT MODE: ' + OutputStyler.GREEN + '{0}'.format(settings_['input_mode'].upper() + OutputStyler.ENDC) + OutputStyler.ENDC) - if settings_['input_mode'] == InputMode.VOICE.value: - print(OutputStyler.BOLD + 'NOTE: ' + OutputStyler.GREEN + "Include " + "'{0}'".format(settings_['assistant_name'].upper()) + " in you command to enable assistant" + OutputStyler.ENDC + OutputStyler.ENDC) + settings_ = settingsDB.getGeneralSettings() + print(OutputStyler.HEADER + headerize('GENERAL INFO') + OutputStyler.ENDC) + enabled = OutputStyler.GREEN + 'ENABLED' + OutputStyler.ENDC if settings_.response_in_speech else OutputStyler.WARNING + 'NOT ENABLED' + OutputStyler.ENDC + print(OutputStyler.BOLD + 'RESPONSE IN SPEECH: ' + enabled) + print(OutputStyler.BOLD + 'INPUT MODE: ' + OutputStyler.GREEN + '{0}'.format(settings_.input_mode.upper() + OutputStyler.ENDC) + OutputStyler.ENDC) + if settings_.input_mode == InputMode.VOICE.value: + print(OutputStyler.BOLD + 'NOTE: ' + OutputStyler.GREEN + "Include " + "'{0}'".format(settings_.assistant_name.upper()) + " in you command to enable assistant" + OutputStyler.ENDC + OutputStyler.ENDC) # ---------------------------------------------------------------------------------------------------------- # System info sector diff --git a/src/jarvis/jarvis/core/processor.py b/src/jarvis/jarvis/core/processor.py index 6d54f13f..aba24568 100644 --- a/src/jarvis/jarvis/core/processor.py +++ b/src/jarvis/jarvis/core/processor.py @@ -26,10 +26,10 @@ from sklearn.metrics.pairwise import cosine_similarity from jarvis.skills.analyzer import SkillAnalyzer -from jarvis.skills.registry import skill_objects +from jarvis.utils.skills_registry import skills_registry from jarvis.core.nlp import ResponseCreator from jarvis.skills.collection.activation import ActivationSkills -from jarvis.utils.mongoDB import db +from jarvis.utils.history_database import historyDB, History from jarvis.skills.collection.wolframalpha import WolframSkills @@ -53,7 +53,7 @@ def run(self): - STEP 2: Matches the input with a skill - STEP 3: Create a response - STEP 4: Execute matched skill - - STEP 5: Insert user transcript and response in history collection (in MongoDB) + - STEP 5: Insert user transcript and response in history collection """ @@ -100,13 +100,10 @@ def run(self): # -------------------------------------------------------------------------------------------------------------- # Add new record to history # -------------------------------------------------------------------------------------------------------------- - - record = {'user_transcript': transcript, - 'response': response if response else '--', - 'executed_skill': skill_to_execute if skill_to_execute else '--' - } - - db.insert_many_documents('history', [record]) + historyDB.addHistory(History( + user_transcript=transcript, + response=response, + executed_skill='--' if not skill_to_execute else skill_to_execute['skill']['name'])) def _execute_skill(self, skill): if skill: @@ -115,7 +112,7 @@ def _execute_skill(self, skill): try: ActivationSkills.enable_assistant() skill_func_name = skill.get('skill').get('func') - skill_func = skill_objects[skill_func_name] + skill_func = skills_registry.skill_objects[skill_func_name] skill_func(**skill) except Exception as e: self.console_manager.console_output(error_log="Failed to execute skill {0} with message: {1}" diff --git a/src/jarvis/jarvis/enumerations.py b/src/jarvis/jarvis/enumerations.py index cca25e8b..f05c09a9 100644 --- a/src/jarvis/jarvis/enumerations.py +++ b/src/jarvis/jarvis/enumerations.py @@ -26,9 +26,3 @@ class InputMode(Enum): VOICE = 'voice' TEXT = 'text' - - -class MongoCollections(Enum): - GENERAL_SETTINGS = 'general_settings' - CONTROL_SKILLS = 'control_skills' - ENABLED_BASIC_SKILLS = 'enabled_basic_skills' diff --git a/src/jarvis/jarvis/settings.py b/src/jarvis/jarvis/settings.py index 98f5a0c6..0b50326d 100644 --- a/src/jarvis/jarvis/settings.py +++ b/src/jarvis/jarvis/settings.py @@ -65,6 +65,12 @@ """ +DATABASE_SETTINGS = { + 'base_dir': './data/', + 'history_filename': 'history.db', + 'settings_filename': 'settings.db', + 'skills_filename': 'skills.db' +} DEFAULT_GENERAL_SETTINGS = { 'assistant_name': 'Jarvis', diff --git a/src/jarvis/jarvis/skills/analyzer.py b/src/jarvis/jarvis/skills/analyzer.py index 943330b3..8b24c21e 100644 --- a/src/jarvis/jarvis/skills/analyzer.py +++ b/src/jarvis/jarvis/skills/analyzer.py @@ -21,8 +21,7 @@ # SOFTWARE. import logging from jarvis.utils.mapping import math_symbols_mapping -from jarvis.utils.mongoDB import db - +from jarvis.utils.skills_registry import skills_registry class SkillAnalyzer: def __init__(self, weight_measure, similarity_measure, args, sensitivity): @@ -35,9 +34,7 @@ def __init__(self, weight_measure, similarity_measure, args, sensitivity): @property def skills(self): - return db.get_documents(collection='control_skills')\ - + db.get_documents(collection='enabled_basic_skills')\ - + db.get_documents(collection='learned_skills') + return skills_registry.basic_skills + skills_registry.control_skills + skills_registry.learned_skills @property def tags(self): diff --git a/src/jarvis/jarvis/skills/collection/activation.py b/src/jarvis/jarvis/skills/collection/activation.py index 27f413dc..c5d7cb53 100644 --- a/src/jarvis/jarvis/skills/collection/activation.py +++ b/src/jarvis/jarvis/skills/collection/activation.py @@ -26,8 +26,8 @@ from jarvis.skills.skill import AssistantSkill from jarvis.utils.startup import play_activation_sound -from jarvis.utils.mongoDB import db -from jarvis.enumerations import InputMode, MongoCollections +from jarvis.utils.settings_database import settingsDB +from jarvis.enumerations import InputMode class ActivationSkills(AssistantSkill): @@ -38,8 +38,8 @@ def enable_assistant(cls, **kwargs): Plays activation sound and creates the assistant response according to the day hour. """ - input_mode = db.get_documents(collection=MongoCollections.GENERAL_SETTINGS.value)[0]['input_mode'] - if input_mode == InputMode.VOICE.value: + generalSettings = settingsDB.getGeneralSettings() + if generalSettings.input_mode == InputMode.VOICE.value: play_activation_sound() @classmethod diff --git a/src/jarvis/jarvis/skills/collection/configuration.py b/src/jarvis/jarvis/skills/collection/configuration.py index 211aa0e7..e9662156 100644 --- a/src/jarvis/jarvis/skills/collection/configuration.py +++ b/src/jarvis/jarvis/skills/collection/configuration.py @@ -24,14 +24,15 @@ from jarvis.skills.skill import AssistantSkill from jarvis import settings -from jarvis.utils.mongoDB import db -from jarvis.enumerations import InputMode, MongoCollections +from jarvis.utils.settings_database import settingsDB +from jarvis.enumerations import InputMode from jarvis.utils import console from jarvis.utils import input -input_mode = db.get_documents(collection='general_settings')[0]['input_mode'] -response_in_speech = db.get_documents(collection='general_settings')[0]['response_in_speech'] -assistant_name = db.get_documents(collection='general_settings')[0]['assistant_name'] +generalSettings = settingsDB.getGeneralSettings() +input_mode = generalSettings.input_mode +response_in_speech = generalSettings.response_in_speech +assistant_name = generalSettings.assistant_name class ConfigurationSkills(AssistantSkill): @@ -70,7 +71,10 @@ def configure_assistant(cls, **kwargs): cls.response('Do you want to save new settings? ', refresh_console=False) save = input.check_input_to_continue() if save: - db.update_collection(collection=MongoCollections.GENERAL_SETTINGS.value, documents=[new_settings]) + generalSettings.input_mode = new_input_mode + generalSettings.assistant_name = new_assistant_name + generalSettings.response_in_speech = new_response_in_speech + settingsDB.updateGeneralSettings(generalSettings) import jarvis importlib.reload(jarvis) diff --git a/src/jarvis/jarvis/skills/collection/history.py b/src/jarvis/jarvis/skills/collection/history.py index ef4d1a43..7a96d7d8 100644 --- a/src/jarvis/jarvis/skills/collection/history.py +++ b/src/jarvis/jarvis/skills/collection/history.py @@ -23,7 +23,7 @@ import re from jarvis.skills.skill import AssistantSkill -from jarvis.utils.mongoDB import db +from jarvis.utils.history_database import historyDB header = """ ----------------------------------------------------------------------------------------------- @@ -52,19 +52,18 @@ def show_history_log(cls, voice_transcript, skill): limit = cls._extract_history_limit(voice_transcript, skill) limit = limit if limit else cls.default_limit - documents = db.get_documents(collection='history', limit=limit) - response = cls._create_response(documents) + records = historyDB.getHistory(limit=limit) + response = cls._create_response(records) cls.console(response) @classmethod - def _create_response(cls, documents): + def _create_response(cls, records): response = '' try: - for document in documents: - response += response_base.format(document.get('user_transcript', '--'), - document.get('response', '--'), - document.get('executed_skill').get('skill').get('name') if - document.get('executed_skill') else '--' + for record in records: + response += response_base.format(record.user_transcript, + record.response, + record.executed_skill ) except Exception as e: cls.console(error_log=e) diff --git a/src/jarvis/jarvis/skills/collection/info.py b/src/jarvis/jarvis/skills/collection/info.py index 8a208ffd..97aa85c3 100644 --- a/src/jarvis/jarvis/skills/collection/info.py +++ b/src/jarvis/jarvis/skills/collection/info.py @@ -21,7 +21,7 @@ # SOFTWARE. from jarvis.skills.skill import AssistantSkill -from jarvis.utils.mongoDB import db +from jarvis.utils.skills_registry import skills_registry from jarvis.utils.console import headerize basic_skills_format = """ @@ -94,7 +94,8 @@ def _create_skill_response(cls, response): # -------------------------------------------------------------------------------------------------------------- # For existing skills (basic skills) # -------------------------------------------------------------------------------------------------------------- - basic_skills = db.get_documents(collection='enabled_basic_skills') + # basic_skills = db.get_documents(collection='enabled_basic_skills') + basic_skills = skills_registry.basic_skills response = response + basic_skills_format for skill_id, skill in enumerate(basic_skills, start=1): response = response + basic_skills_body_format.format(skill_id, @@ -106,7 +107,7 @@ def _create_skill_response(cls, response): # -------------------------------------------------------------------------------------------------------------- # For learned skills (created from 'learn' skill) # -------------------------------------------------------------------------------------------------------------- - skills = db.get_documents(collection='learned_skills') + skills = skills_registry.learned_skills response = response + learned_skills_format for skill_id, skill in enumerate(skills, start=1): response = response + learned_skills_body_format.format(skill_id, diff --git a/src/jarvis/jarvis/skills/collection/remember.py b/src/jarvis/jarvis/skills/collection/remember.py index a8e9cca9..04a72f7a 100644 --- a/src/jarvis/jarvis/skills/collection/remember.py +++ b/src/jarvis/jarvis/skills/collection/remember.py @@ -22,7 +22,7 @@ from jarvis.skills.skill import AssistantSkill -from jarvis.utils.mongoDB import db +from jarvis.utils.skills_database import skillsDB from jarvis.utils import input header = """ @@ -50,11 +50,11 @@ def remember(cls, **kwargs): 'func': cls.tell_response.__name__, 'response': response, 'tags': tags, - }, + } cls.response('Add more? ', refresh_console=False) continue_add = input.check_input_to_continue() - db.insert_many_documents(collection='learned_skills', documents=new_skill) + skillsDB.addLearnedSkill(new_skill) @classmethod def tell_response(cls, **kwargs): @@ -62,12 +62,12 @@ def tell_response(cls, **kwargs): @classmethod def clear_learned_skills(cls, **kwargs): - if db.is_collection_empty(collection='learned_skills'): + if skillsDB.hasLearnedSkills(): cls.response("I can't find learned skills in my database") else: cls.response('I found learned skills..') cls.response('Are you sure to remove learned skills? ', refresh_console=False) user_answer = input.check_input_to_continue() if user_answer: - db.drop_collection(collection='learned_skills') + skillsDB.clearLearnedSkills() cls.response("Perfect I have deleted them all") diff --git a/src/jarvis/jarvis/skills/registry.py b/src/jarvis/jarvis/skills/registry.py index a96a476d..c8cff7cd 100644 --- a/src/jarvis/jarvis/skills/registry.py +++ b/src/jarvis/jarvis/skills/registry.py @@ -38,6 +38,7 @@ from jarvis.skills.collection.math import MathSkills from jarvis.utils.mapping import math_tags from jarvis.skills.collection.configuration import ConfigurationSkills +from jarvis.utils.skills_registry import skills_registry # All available assistant skills # Keys description: diff --git a/src/jarvis/jarvis/utils/history_database.py b/src/jarvis/jarvis/utils/history_database.py new file mode 100644 index 00000000..4253e03a --- /dev/null +++ b/src/jarvis/jarvis/utils/history_database.py @@ -0,0 +1,91 @@ +# MIT License + +# Copyright (c) 2019 Georgios Papachristou + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import sqlite3 +from time import time +from os.path import isdir, dirname + +from jarvis.settings import DATABASE_SETTINGS + +class History: + def __init__(self, user_transcript = '--', response = '--', executed_skill = '--', timestamp=0): + self.user_transcript = user_transcript + self.response = response + self.executed_skill = executed_skill + self.timestamp = timestamp if timestamp != 0 else int(time() * 1000) + + def __str__(self): + return "user_transcript=%s, response=%s, executed_skill=%s, timestamp=%s" % ( + self.user_transcript, self.response, self.executed_skill, self.timestamp) + + def __repr__(self): + return self.__str__() + +class HistoryDatabase: + def __init__(self, database='history.db'): + self.database = database + self._ensure_database_exists() + + def _ensure_database_exists(self): + if not isdir(dirname(self.database)): + raise IOError("Database base dir %s does not exist" % dirname(self.database)) + + conn = sqlite3.connect(self.database) + conn.execute( + """create table if not exists history ( + timestamp INTEGER PRIMARY KEY, + user_transcript TEXT, + response TEXT, + executed_skill TEXT + ); """) + conn.commit() + conn.close() + + def getHistory(self, limit = 3): + conn = sqlite3.connect(self.database) + cursor = conn.cursor() + rows = cursor.execute("select * from history order by timestamp DESC limit ?", (limit,)).fetchall() + conn.close() + + response = [] + for row in rows: + response.append(History( + timestamp=row[0], + user_transcript=row[1], + response = row[2], + executed_skill = row[3])) + + print("Returned %d history items" % len(response)) + return response + + def addHistory(self, history): + conn = sqlite3.connect(self.database) + conn.execute( + ''' + insert into history + (timestamp, user_transcript, response, executed_skill) values + (?, ?, ?, ?) + ''', (history.timestamp, history.user_transcript, history.response, history.executed_skill,)) + conn.commit() + conn.close() + +historyDB = HistoryDatabase(DATABASE_SETTINGS['base_dir'] + DATABASE_SETTINGS['history_filename']) diff --git a/src/jarvis/jarvis/utils/mongoDB.py b/src/jarvis/jarvis/utils/mongoDB.py deleted file mode 100644 index 0f78940a..00000000 --- a/src/jarvis/jarvis/utils/mongoDB.py +++ /dev/null @@ -1,80 +0,0 @@ -# MIT License - -# Copyright (c) 2019 Georgios Papachristou - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import logging -from pymongo import MongoClient, DESCENDING - - -class MongoDB: - """ - This class encapsulates methods related to MongoDB - """ - - def __init__(self, host='localhost', port=27017): - self.client = MongoClient(host, port) - self.database = self.client['jarvis'] - - def get_documents(self, collection, key=None, limit=None): - collection_obj = self.database[collection] - try: - result = collection_obj.find(key).sort('_id', DESCENDING) - return list(result.limit(limit) if limit else result) - except Exception as e: - logging.error(e) - - def insert_many_documents(self, collection, documents): - collection_obj = self.database[collection] - try: - collection_obj.insert_many(documents) - except Exception as e: - logging.error(e) - - def drop_collection(self, collection): - collection_obj = self.database[collection] - try: - collection_obj.drop() - except Exception as e: - logging.error(e) - - def update_collection(self, collection, documents): - self.drop_collection(collection) - self.insert_many_documents(collection, documents) - - def update_document(self, collection, query, new_value, upsert=True): - collection_obj = self.database[collection] - try: - collection_obj.update_one(query, {'$set': new_value}, upsert) - except Exception as e: - logging.error(e) - - def is_collection_empty(self, collection): - collection_obj = self.database[collection] - try: - return collection_obj.estimated_document_count() == 0 - except Exception as e: - logging.error(e) - - -# ---------------------------------------------------------------------------------------------------------------------- -# Create MongoDB connection instance -# ---------------------------------------------------------------------------------------------------------------------- -db = MongoDB() diff --git a/src/jarvis/jarvis/utils/settings_database.py b/src/jarvis/jarvis/utils/settings_database.py new file mode 100644 index 00000000..15379919 --- /dev/null +++ b/src/jarvis/jarvis/utils/settings_database.py @@ -0,0 +1,91 @@ +# MIT License + +# Copyright (c) 2019 Georgios Papachristou + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import sqlite3 +import json +from os.path import isdir, dirname + +from jarvis.settings import DATABASE_SETTINGS + +class GeneralSettings: + def __init__(self, input_mode='voice', assistant_name='jarvis', response_in_speech='true'): + self.input_mode = input_mode + self.assistant_name = assistant_name + self.response_in_speech = response_in_speech + def __str__(self): + return "input_mode=%s, assistant_name=%s, response_in_speech=%s" % ( + self.input_mode, self.assistant_name, self.response_in_speech) + +class SettingsDatabase: + def __init__(self, database='settings.db'): + self.database = database + self._ensure_database_exists() + + def _ensure_database_exists(self): + if not isdir(dirname(self.database)): + raise IOError("Database base dir %s does not exist" % dirname(self.database)) + + conn = sqlite3.connect(self.database) + conn.execute( + """create table if not exists collections ( + name VARCHAR(255) PRIMARY KEY, + data BLOB + ); """) + conn.commit() + conn.close() + + def hasGeneralSettings(self): + conn = sqlite3.connect(self.database) + cursor = conn.cursor() + rows = cursor.execute("select data from collections where name = 'general_settings'").fetchall() + conn.close() + + return len(rows) > 0 + + def getGeneralSettings(self): + conn = sqlite3.connect(self.database) + cursor = conn.cursor() + rows = cursor.execute("select data from collections where name = 'general_settings'").fetchall() + conn.close() + + if len(rows) < 1: + return GeneralSettings() + + settings = json.loads(rows[0][0].decode('utf-8')) + return GeneralSettings( + input_mode = settings['input_mode'], + response_in_speech = settings['response_in_speech'], + assistant_name = settings['assistant_name'] + ) + + def updateGeneralSettings(self, settings): + blob = bytes(json.dumps(settings.__dict__), 'utf-8') + conn = sqlite3.connect(self.database) + conn.execute( + ''' + insert into collections(name, data) values ('general_settings', ?) + on conflict(name) do update set data=excluded.data; + ''', (blob,)) + conn.commit() + conn.close() + +settingsDB = SettingsDatabase(DATABASE_SETTINGS['base_dir'] + DATABASE_SETTINGS['settings_filename']) diff --git a/src/jarvis/jarvis/utils/skills_database.py b/src/jarvis/jarvis/utils/skills_database.py new file mode 100644 index 00000000..6a9c496a --- /dev/null +++ b/src/jarvis/jarvis/utils/skills_database.py @@ -0,0 +1,86 @@ +# MIT License + +# Copyright (c) 2019 Georgios Papachristou + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import sqlite3 +import json +from os.path import isdir, dirname + +from jarvis.settings import DATABASE_SETTINGS + +class SkillsDatabase: + def __init__(self, database='skills.db'): + self.database = database + self._ensure_database_exists() + + def _ensure_database_exists(self): + if not isdir(dirname(self.database)): + raise IOError("Database base dir %s does not exist" % dirname(self.database)) + + conn = sqlite3.connect(self.database) + conn.execute( + """create table if not exists collections ( + data BLOB + ); """) + conn.commit() + conn.close() + + def hasLearnedSkills(self): + conn = sqlite3.connect(self.database) + cursor = conn.cursor() + rows = cursor.execute("select count(*) from collections").fetchall() + conn.close() + + return rows[0][0] > 0 + + def getLearnedSkills(self): + conn = sqlite3.connect(self.database) + cursor = conn.cursor() + rows = cursor.execute("select data from collections").fetchall() + conn.close() + + response = [] + for row in rows: + skill = json.loads(row[0].decode('utf-8')) + response.append(skill) + return response + + def addLearnedSkill(self, skill): + blob = bytes(json.dumps(skill), 'utf-8') + conn = sqlite3.connect(self.database) + conn.execute( + ''' + insert into collections(data) values (?) + ''', (blob,)) + conn.commit() + conn.close() + + def clearLearnedSkills(self): + conn = sqlite3.connect(self.database) + conn.execute( + ''' + drop from collections(data) + ''') + conn.commit() + conn.close() + + +skillsDB = SkillsDatabase(DATABASE_SETTINGS['base_dir'] + DATABASE_SETTINGS['skills_filename']) diff --git a/src/jarvis/jarvis/utils/skills_registry.py b/src/jarvis/jarvis/utils/skills_registry.py new file mode 100644 index 00000000..78d3662b --- /dev/null +++ b/src/jarvis/jarvis/utils/skills_registry.py @@ -0,0 +1,65 @@ +# MIT License + +# Copyright (c) 2019 Georgios Papachristou + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import logging +from jarvis.utils.skills_database import skillsDB + +class SkillsRegistry: + def __init__(self): + self.skill_objects = [] + self.basic_skills = [] + self.control_skills = [] + + @property + def skill_objects(self): + return self._skill_objects.copy() + + @skill_objects.setter + def skill_objects(self, skill_objects): + # logging.warn("setting skill_objects = %d" %len(skill_objects)) + self._skill_objects = skill_objects.copy() + + @property + def basic_skills(self): + # logging.warn("getting basic_skills %s" % type(self._basic_skills)) + return self._basic_skills.copy() + + @basic_skills.setter + def basic_skills(self, basic_skills): + # logging.warn("setting basic_skills = %d, %s" % (len(basic_skills), type(basic_skills))) + self._basic_skills = basic_skills.copy() + + @property + def control_skills(self): + # logging.warn("getting control_skills %s" % type(self._control_skills)) + return self._control_skills.copy() + + @control_skills.setter + def control_skills(self, control_skills): + # logging.warn("setting control_skills = %d" % len(control_skills)) + self._control_skills = control_skills.copy() + + @property + def learned_skills(self): + return skillsDB.getLearnedSkills() + +skills_registry = SkillsRegistry() diff --git a/src/jarvis/jarvis/utils/startup.py b/src/jarvis/jarvis/utils/startup.py index 61f9f834..99ad15b7 100644 --- a/src/jarvis/jarvis/utils/startup.py +++ b/src/jarvis/jarvis/utils/startup.py @@ -27,9 +27,8 @@ from playsound import playsound from jarvis.utils import console -from jarvis.enumerations import MongoCollections from jarvis.core.console import ConsoleManager - +from jarvis.utils.settings_database import settingsDB, GeneralSettings def play_activation_sound(): """ @@ -56,14 +55,14 @@ def internet_connectivity_check(url='http://www.google.com/', timeout=2): return False -def configure_MongoDB(db, settings): +def configure_defaults(settings): # ------------------------------------------------------------------------------------------------------------------ # Load settings # ------------------------------------------------------------------------------------------------------------------ # Only in first time or if 'general_settings' collection is deleted - if db.is_collection_empty(collection=MongoCollections.GENERAL_SETTINGS.value): + if not settingsDB.hasGeneralSettings(): console.print_console_header() print('First time configuration..') console.print_console_header() @@ -80,7 +79,11 @@ def configure_MongoDB(db, settings): } try: - db.update_collection(collection=MongoCollections.GENERAL_SETTINGS.value, documents=[new_settings]) + settingsDB.updateGeneralSettings(GeneralSettings( + input_mode=default_input_mode, + assistant_name=default_assistant_name, + response_in_speech=default_response_in_speech + )) console.print_console_header('Assistant Name') print('Assistant name- {0} configured successfully!'.format(default_assistant_name.lower())) print('Input mode - {0} configured successfully!'.format(default_input_mode)) @@ -89,16 +92,3 @@ def configure_MongoDB(db, settings): except Exception as e: logging.error('Failed to configure assistant settings with error message {0}'.format(e)) - - # ------------------------------------------------------------------------------------------------------------------ - # Load skills - # ------------------------------------------------------------------------------------------------------------------ - - from jarvis.skills.registry import CONTROL_SKILLS, ENABLED_BASIC_SKILLS - - all_skills = { - MongoCollections.CONTROL_SKILLS.value: CONTROL_SKILLS, - MongoCollections.ENABLED_BASIC_SKILLS.value: ENABLED_BASIC_SKILLS, - } - for collection, documents in all_skills.items(): - db.update_collection(collection, documents) diff --git a/src/tests/skill_analyzer_tests.py b/src/tests/skill_analyzer_tests.py index f252590e..c2510dd1 100644 --- a/src/tests/skill_analyzer_tests.py +++ b/src/tests/skill_analyzer_tests.py @@ -25,35 +25,14 @@ from sklearn.metrics.pairwise import cosine_similarity from jarvis import settings -from jarvis.skills.registry import CONTROL_SKILLS, BASIC_SKILLS, ENABLED_BASIC_SKILLS -from jarvis.enumerations import MongoCollections +from jarvis.skills.registry import BASIC_SKILLS from jarvis.skills.analyzer import SkillAnalyzer -from jarvis.utils.mongoDB import db class TestSkillMatching(unittest.TestCase): def setUp(self): - all_skills = { - MongoCollections.CONTROL_SKILLS.value: CONTROL_SKILLS, - MongoCollections.ENABLED_BASIC_SKILLS.value: ENABLED_BASIC_SKILLS, - } - for collection, documents in all_skills.items(): - db.update_collection(collection, documents) - - default_assistant_name = settings.DEFAULT_GENERAL_SETTINGS['assistant_name'] - default_input_mode = settings.DEFAULT_GENERAL_SETTINGS['input_mode'] - default_response_in_speech = settings.DEFAULT_GENERAL_SETTINGS['response_in_speech'] - - default_settings = { - 'assistant_name': default_assistant_name, - 'input_mode': default_input_mode, - 'response_in_speech': default_response_in_speech, - } - - db.update_collection(collection=MongoCollections.GENERAL_SETTINGS.value, documents=[default_settings]) - self.skill_analyzer = SkillAnalyzer( weight_measure=TfidfVectorizer, similarity_measure=cosine_similarity,