diff --git a/.gitignore b/.gitignore index 3299b04..9d38353 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,117 @@ -*~ -*.pyc -/.coverage -/AnkiServer.egg-info -/development.ini -/server.log -/collections -/session.db -/auth.db -/dist -/build +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +test/ +downloads/ +eggs/ +.eggs/ +bin/ +lib/ +lib64/ +include/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +*.txt +*.doc +101.py +Pipfile +Pipfile.lock +*.jpg +*.gif +img +auth.db +files.txt diff --git a/AnkiServer/apps/rest_app.py b/AnkiServer/apps/rest_app.py index 5aea698..27eac9f 100644 --- a/AnkiServer/apps/rest_app.py +++ b/AnkiServer/apps/rest_app.py @@ -116,8 +116,9 @@ def add_handler(self, type, name, handler): - 'handler' is a callable that takes (collection, data, ids). """ - if self.handlers[type].has_key(name): - raise "Handler already for %(type)s/%(name)s exists!" + #if self.handlers[type].has_key(name): + if name in self.handlers[type]: + raise #"Handler already for %(type)s/%(name)s exists!" self.handlers[type][name] = handler def add_handler_group(self, type, group): @@ -248,7 +249,7 @@ def _parseRequestBody(self, req): try: data = json.loads(req.body) - except JSONDecodeError, e: + except JSONDecodeError as e: logging.error(req.path+': Unable to parse JSON: '+str(e), exc_info=True) raise HTTPBadRequest() @@ -285,7 +286,7 @@ def __call__(self, req): # get the collection path collection_path = self._getCollectionPath(ids[0]) - print collection_path + print(collection_path) # get the handler function handler, hasReturnValue = self._getHandler(type, name) @@ -308,10 +309,10 @@ def __call__(self, req): col = self.collection_manager.get_collection(collection_path, self.setup_new_collection) handler_request = RestHandlerRequest(self, data, ids, session) output = col.execute(self._execute_handler, [handler_request, handler], {}, hasReturnValue) - except HTTPError, e: + except HTTPError as e: # we pass these on through! raise - except Exception, e: + except Exception as e: logging.error(e) return HTTPInternalServerError() @@ -355,7 +356,8 @@ def latest_notes(self, col, req): # TODO: use SQLAlchemy objects to do this sql = "SELECT n.id FROM notes AS n"; args = [] - if req.data.has_key('updated_since'): + if 'updated_since' in req.data: + #if req.data.has_key('updated_since'): sql += ' WHERE n.mod > ?' args.append(req.data['updated_since']) sql += ' ORDER BY n.mod DESC' diff --git a/AnkiServer/apps/sync_app.py b/AnkiServer/apps/sync_app.py index 658ecf3..c9b4247 100644 --- a/AnkiServer/apps/sync_app.py +++ b/AnkiServer/apps/sync_app.py @@ -20,6 +20,7 @@ from webob import Response import os +import io import hashlib import logging import random @@ -35,20 +36,14 @@ from anki.utils import intTime, checksum, isMac from anki.consts import SYNC_ZIP_SIZE, SYNC_ZIP_COUNT +logging.basicConfig(filename='server.log', filemode='w', format='%(name)s - %(levelname)s - %(message)s',level=logging.DEBUG) + try: import simplejson as json except ImportError: import json -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO - -try: - from pysqlite2 import dbapi2 as sqlite -except ImportError: - from sqlite3 import dbapi2 as sqlite +from sqlite3 import dbapi2 as sqlite class SyncCollectionHandler(Syncer): operations = ['meta', 'applyChanges', 'start', 'chunk', 'applyChunk', 'sanityCheck2', 'finish'] @@ -130,7 +125,7 @@ def _check_zip_data(zip_data): max_zip_size = 100*1024*1024 max_meta_file_size = 100000 - file_buffer = StringIO(zip_data) + file_buffer = io.BytesIO(zip_data) zip_file = zipfile.ZipFile(file_buffer, 'r') meta_file_size = zip_file.getinfo("_meta").file_size @@ -152,7 +147,7 @@ def _adopt_media_changes_from_zip(self, zip_data): according to the data in zip file zipData. """ - file_buffer = StringIO(zip_data) + file_buffer = io.BytesIO(zip_data) zip_file = zipfile.ZipFile(file_buffer, 'r') # Get meta info first. @@ -241,7 +236,7 @@ def downloadFiles(self, files): flist = {} cnt = 0 sz = 0 - f = StringIO() + f = io.BytesIO() z = zipfile.ZipFile(f, "w", compression=zipfile.ZIP_DEFLATED) for fname in files: @@ -436,7 +431,7 @@ def _decode_data(self, data, compression=0): import gzip if compression: - buf = gzip.GzipFile(mode="rb", fileobj=StringIO(data)) + buf = gzip.GzipFile(mode="rb", fileobj=io.BytesIO(data)) data = buf.read() buf.close() @@ -562,12 +557,12 @@ def __call__(self, req): if url in SyncCollectionHandler.operations + SyncMediaHandler.operations: # 'meta' passes the SYNC_VER but it isn't used in the handler if url == 'meta': - if session.skey == None and req.POST.has_key('s'): + if session.skey == None and 's' in req.POST: session.skey = req.POST['s'] - if data.has_key('v'): + if 'v' in data: session.version = data['v'] del data['v'] - if data.has_key('cv'): + if 'cv' in data: session.client_version = data['cv'] self.session_manager.save(hkey, session) session = self.session_manager.load(hkey, self.create_session) @@ -752,17 +747,24 @@ def authenticate(self, username, password): salt = db_hash[-16:] hashobj = hashlib.sha256() - hashobj.update(username+password+salt) + hashobj.update((username+password+salt).encode()) conn.close() + loggin.info(db_ret,db_hash) + + passhash = hashobj.hexdigest() + salt + + if passhash == db_hash: + return True + else: + return False - return (db_ret != None and hashobj.hexdigest()+salt == db_hash) # Our entry point def make_app(global_conf, **local_conf): - if local_conf.has_key('session_db_path'): + if 'session_db_path' in local_conf: local_conf['session_manager'] = SqliteSessionManager(local_conf['session_db_path']) - if local_conf.has_key('auth_db_path'): + if 'auth_db_path' in local_conf: local_conf['user_manager'] = SqliteUserManager(local_conf['auth_db_path']) return SyncApp(**local_conf) @@ -773,10 +775,10 @@ def main(): ankiserver = SyncApp() httpd = make_server('', 8001, ankiserver) try: - print "Starting..." + print( "Starting...") httpd.serve_forever() except KeyboardInterrupt: - print "Exiting ..." + print( "Exiting ...") finally: shutdown() diff --git a/AnkiServer/collection.py b/AnkiServer/collection.py index 4324dac..95094f8 100644 --- a/AnkiServer/collection.py +++ b/AnkiServer/collection.py @@ -69,7 +69,7 @@ def __create_collection(self): dirname = os.path.dirname(self.path) try: os.makedirs(dirname) - except OSError, exc: + except OSError as exc: if exc.errno == errno.EEXIST: pass else: diff --git a/AnkiServer/threading.py b/AnkiServer/threading.py index 5632f22..88c6b3b 100644 --- a/AnkiServer/threading.py +++ b/AnkiServer/threading.py @@ -23,7 +23,7 @@ from AnkiServer.collection import CollectionWrapper, CollectionManager from threading import Thread -from Queue import Queue +from queue import Queue import time, logging @@ -93,7 +93,7 @@ def _run(self): try: ret = self.wrapper.execute(func, args, kw, return_queue) - except Exception, e: + except Exception as e: logging.error('CollectionThread[%s]: Unable to %s(*%s, **%s): %s', self.path, func_name, repr(args), repr(kw), e, exc_info=True) # we return the Exception which will be raise'd on the other end @@ -101,7 +101,7 @@ def _run(self): if return_queue is not None: return_queue.put(ret) - except Exception, e: + except Exception as e: logging.error('CollectionThread[%s]: Thread crashed! Exception: %s', self.path, e, exc_info=True) finally: self.wrapper.close() diff --git a/AnkiServer/users.py b/AnkiServer/users.py new file mode 100644 index 0000000..1a2c807 --- /dev/null +++ b/AnkiServer/users.py @@ -0,0 +1,164 @@ +# -*- coding: utf-8 -*- +import binascii +import hashlib +import logging +import os +import sqlite3 as sqlite + + +class SimpleUserManager: + """A simple user manager that always allows any user.""" + + def __init__(self, collection_path=''): + self.collection_path = collection_path + + def authenticate(self, username, password): + """ + Returns True if this username is allowed to connect with this password. + False otherwise. Override this to change how users are authenticated. + """ + + return True + + def userdir(self, username): + """ + Returns the directory name for the given user. By default, this is just + the username. Override this to adjust the mapping between users and + their directory. + """ + + return username + + def _create_user_dir(self, username): + user_dir_path = os.path.join(self.collection_path, username) + if not os.path.isdir(user_dir_path): + logging.info("Creating collection directory for user '{}' at {}" + .format(username, user_dir_path)) + os.makedirs(user_dir_path) + + +class SqliteUserManager(SimpleUserManager): + """Authenticates users against a SQLite database.""" + + def __init__(self, auth_db_path, collection_path=None): + SimpleUserManager.__init__(self, collection_path) + self.auth_db_path = os.path.realpath(auth_db_path) + + def auth_db_exists(self): + return os.path.isfile(self.auth_db_path) + + def user_list(self): + if not self.auth_db_exists(): + raise ValueError("Auth DB {} doesn't exist".format(self.auth_db_path)) + + conn = sqlite.connect(self.auth_db_path) + cursor = conn.cursor() + cursor.execute("SELECT user FROM auth") + rows = cursor.fetchall() + conn.commit() + conn.close() + + return [row[0] for row in rows] + + def user_exists(self, username): + users = self.user_list() + return username in users + + def del_user(self, username): + if not self.auth_db_exists(): + raise ValueError("Auth DB {} doesn't exist".format(self.auth_db_path)) + + conn = sqlite.connect(self.auth_db_path) + cursor = conn.cursor() + logging.info("Removing user '{}' from auth db".format(username)) + cursor.execute("DELETE FROM auth WHERE user=?", (username,)) + conn.commit() + conn.close() + + def add_user(self, username, password): + self._add_user_to_auth_db(username, password) + self._create_user_dir(username) + + def add_users(self, users_data): + for username, password in users_data: + self.add_user(username, password) + + def _add_user_to_auth_db(self, username, password): + if not self.auth_db_exists(): + self.create_auth_db() + + pass_hash = self._create_pass_hash(username, password) + + conn = sqlite.connect(self.auth_db_path) + cursor = conn.cursor() + logging.info("Adding user '{}' to auth db.".format(username)) + cursor.execute("INSERT INTO auth VALUES (?, ?)", + (username, pass_hash)) + conn.commit() + conn.close() + + def set_password_for_user(self, username, new_password): + if not self.auth_db_exists(): + raise ValueError("Auth DB {} doesn't exist".format(self.auth_db_path)) + elif not self.user_exists(username): + raise ValueError("User {} doesn't exist".format(username)) + + hash = self._create_pass_hash(username, new_password) + + conn = sqlite.connect(self.auth_db_path) + cursor = conn.cursor() + cursor.execute("UPDATE auth SET hash=? WHERE user=?", (hash, username)) + conn.commit() + conn.close() + + logging.info("Changed password for user {}".format(username)) + + def authenticate(self, username, password): + """Returns True if this username is allowed to connect with this password. False otherwise.""" + + conn = sqlite.connect(self.auth_db_path) + cursor = conn.cursor() + param = (username,) + cursor.execute("SELECT hash FROM auth WHERE user=?", param) + db_hash = cursor.fetchone() + conn.close() + + if db_hash is None: + logging.info("Authentication failed for nonexistent user {}." + .format(username)) + return False + + expected_value = str(db_hash[0]) + salt = self._extract_salt(expected_value) + + hashobj = hashlib.sha256() + hashobj.update((username + password + salt).encode()) + actual_value = hashobj.hexdigest() + salt + + if actual_value == expected_value: + logging.info("Authentication succeeded for user {}".format(username)) + return True + else: + logging.info("Authentication failed for user {}".format(username)) + return False + + @staticmethod + def _extract_salt(hash): + return hash[-16:] + + @staticmethod + def _create_pass_hash(username, password): + salt = binascii.b2a_hex(os.urandom(8)) + pass_hash = (hashlib.sha256((username + password).encode() + salt).hexdigest() + + salt.decode()) + return pass_hash + + def create_auth_db(self): + conn = sqlite.connect(self.auth_db_path) + cursor = conn.cursor() + logging.info("Creating auth db at {}." + .format(self.auth_db_path)) + cursor.execute("""CREATE TABLE IF NOT EXISTS auth + (user VARCHAR PRIMARY KEY, hash VARCHAR)""") + conn.commit() + conn.close() diff --git a/ankiserverctl.py b/ankiserverctl.py index ac8d47c..1ad8ecc 100755 --- a/ankiserverctl.py +++ b/ankiserverctl.py @@ -1,169 +1,110 @@ -#!/usr/bin/env python - -import os -import sys -import signal -import subprocess -import binascii -import getpass -import hashlib -import sqlite3 - -SERVERCONFIG = "production.ini" -AUTHDBPATH = "auth.db" -PIDPATH = "/tmp/ankiserver.pid" -COLLECTIONPATH = "collections/" - -def usage(): - print "usage: "+sys.argv[0]+" []" - print - print "Commands:" - print " start [configfile] - start the server" - print " debug [configfile] - start the server in debug mode" - print " stop - stop the server" - print " adduser - add a new user" - print " deluser - delete a user" - print " lsuser - list users" - print " passwd - change password of a user" - -def startsrv(configpath, debug): - if not configpath: - configpath = SERVERCONFIG - - # We change to the directory containing the config file - # so that all the paths will be relative to it. - configdir = os.path.dirname(configpath) - if configdir != '': - os.chdir(configdir) - configpath = os.path.basename(configpath) - - if debug: - # Start it in the foreground and wait for it to complete. - subprocess.call( ["paster", "serve", configpath], shell=False) - return - - devnull = open(os.devnull, "w") - pid = subprocess.Popen( ["paster", "serve", configpath], - stdout=devnull, - stderr=devnull).pid - - with open(PIDPATH, "w") as pidfile: - pidfile.write(str(pid)) - -def stopsrv(): - if os.path.isfile(PIDPATH): - try: - with open(PIDPATH) as pidfile: - pid = int(pidfile.read()) - - os.kill(pid, signal.SIGKILL) - os.remove(PIDPATH) - except Exception, error: - print >>sys.stderr, sys.argv[0]+": Failed to stop server: "+error.message - else: - print >>sys.stderr, sys.argv[0]+": The server is not running" - -def adduser(username): - if username: - print "Enter password for "+username+": " - - password = getpass.getpass() - salt = binascii.b2a_hex(os.urandom(8)) - hash = hashlib.sha256(username+password+salt).hexdigest()+salt - - conn = sqlite3.connect(AUTHDBPATH) - cursor = conn.cursor() - - cursor.execute( "CREATE TABLE IF NOT EXISTS auth " - "(user VARCHAR PRIMARY KEY, hash VARCHAR)") - - cursor.execute("INSERT INTO auth VALUES (?, ?)", (username, hash)) - - if not os.path.isdir(COLLECTIONPATH+username): - os.makedirs(COLLECTIONPATH+username) - - conn.commit() - conn.close() - else: - usage() - -def deluser(username): - if username and os.path.isfile(AUTHDBPATH): - conn = sqlite3.connect(AUTHDBPATH) - cursor = conn.cursor() - - cursor.execute("DELETE FROM auth WHERE user=?", (username,)) - - conn.commit() - conn.close() - elif not username: - usage() - else: - print >>sys.stderr, sys.argv[0]+": Database file does not exist" - -def lsuser(): - conn = sqlite3.connect(AUTHDBPATH) - cursor = conn.cursor() - - cursor.execute("SELECT user FROM auth") - - row = cursor.fetchone() - - while row is not None: - print row[0] - - row = cursor.fetchone() - - conn.close() - -def passwd(username): - if os.path.isfile(AUTHDBPATH): - print "Enter password for "+username+": " - - password = getpass.getpass() - salt = binascii.b2a_hex(os.urandom(8)) - hash = hashlib.sha256(username+password+salt).hexdigest()+salt - - conn = sqlite3.connect(AUTHDBPATH) - cursor = conn.cursor() - - cursor.execute("UPDATE auth SET hash=? WHERE user=?", (hash, username)) - - conn.commit() - conn.close() - else: - print >>sys.stderr, sys.argv[0]+": Database file does not exist" - -def main(): - argc = len(sys.argv) - exitcode = 0 - - if argc < 2: - usage() - exitcode = 1 - else: - if argc < 3: - sys.argv.append(None) - - if sys.argv[1] == "start": - startsrv(sys.argv[2], False) - elif sys.argv[1] == "debug": - startsrv(sys.argv[2], True) - elif sys.argv[1] == "stop": - stopsrv() - elif sys.argv[1] == "adduser": - adduser(sys.argv[2]) - elif sys.argv[1] == "deluser": - deluser(sys.argv[2]) - elif sys.argv[1] == "lsuser": - lsuser() - elif sys.argv[1] == "passwd": - passwd(sys.argv[2]) - else: - usage() - exitcode = 1 - - sys.exit(exitcode) - -if __name__ == "__main__": - main() +#!/usr/bin/env python3 +import os +import sys +import click +import getpass +import subprocess + +from AnkiServer.users import SqliteUserManager + +SERVERCONFIG = "production.ini" +AUTHDBPATH = "auth.db" +PIDPATH = "/tmp/ankiserver.pid" +COLLECTIONPATH = "collections/" + + +@click.command('start',short_help="start the server") +@click.argument('configpath',type=click.Path(),required=False) +@click.option('--debug',default=True) +def startsrv(configpath, debug): + if not configpath: + configpath = SERVERCONFIG + + # We change to the directory containing the config file + # so that all the paths will be relative to it. + configdir = os.path.dirname(configpath) + if configdir != '': + os.chdir(configdir) + configpath = os.path.basename(configpath) + + if debug: + # Start it in the foreground and wait for it to complete. + subprocess.call( ["paster", "serve", configpath], shell=False) + return + + devnull = open(os.devnull, "w") + pid = subprocess.Popen( ["paster", "serve", configpath], + stdout=devnull, + stderr=devnull).pid + + with open(PIDPATH, "w") as pidfile: + pidfile.write(str(pid)) + +@click.command('stop',short_help="stop the server") +def stopsrv(): + if os.path.isfile(PIDPATH): + try: + with open(PIDPATH) as pidfile: + pid = int(pidfile.read()) + + os.kill(pid, signal.SIGKILL) + os.remove(PIDPATH) + except Exception as error: + print(": Failed to stop server: "+error.message) + else: + print(": The server is not running") + +@click.command('adduser',short_help="add a user") +@click.argument('username') +def adduser(username): + password = getpass.getpass("Enter password for {}: ".format(username)) + + user_manager = SqliteUserManager(AUTHDBPATH, COLLECTIONPATH) + user_manager.add_user(username, password) + +@click.command('deluser',short_help="delete a user") +@click.argument('username') +def deluser(username): + user_manager = SqliteUserManager(AUTHDBPATH, COLLECTIONPATH) + try: + user_manager.del_user(username) + except ValueError as error: + print("Could not delete user {}: {}".format(username, error), file=sys.stderr) + +@click.command('lsuser',short_help="list all user") +def lsuser(): + user_manager = SqliteUserManager(AUTHDBPATH, COLLECTIONPATH) + try: + users = user_manager.user_list() + for username in users: + print(username) + except ValueError as error: + print("Could not list users: {}".format(error), file=sys.stderr) + +@click.command('passwd',short_help="change password for a user") +@click.argument('username') +def passwd(username): + user_manager = SqliteUserManager(AUTHDBPATH, COLLECTIONPATH) + + if username not in user_manager.user_list(): + print("User {} doesn't exist".format(username)) + return + + password = getpass.getpass("Enter password for {}: ".format(username)) + try: + user_manager.set_password_for_user(username, password) + except ValueError as error: + print("Could not set password for user {}: {}".format(username, error), file=sys.stderr) + +@click.group() +def main(): + pass + +main.add_command(startsrv) +main.add_command(stopsrv) +main.add_command(adduser) +main.add_command(deluser) +main.add_command(lsuser) +main.add_command(passwd) +if __name__ == "__main__": + main() diff --git a/logging.conf b/logging.conf index fb94bf0..a41d05c 100644 --- a/logging.conf +++ b/logging.conf @@ -10,8 +10,8 @@ keys=normal,email [logger_root] level=INFO -handlers=screen -#handlers=file +#handlers=screen +handlers=file #handlers=file,email [handler_file] diff --git a/pip-selfcheck.json b/pip-selfcheck.json new file mode 100644 index 0000000..ae7e839 --- /dev/null +++ b/pip-selfcheck.json @@ -0,0 +1 @@ +{"last_check":"2018-11-18T06:31:37Z","pypi_version":"18.1"} \ No newline at end of file diff --git a/production.ini b/production.ini new file mode 100644 index 0000000..9e915bd --- /dev/null +++ b/production.ini @@ -0,0 +1,31 @@ + +[server:main] +use = egg:AnkiServer#server +#host = 0.0.0.0 +host = 192.168.56.101 +port = 27701 + +[filter-app:main] +use = egg:Paste#translogger +next = real + +[app:real] +use = egg:Paste#urlmap +/ = rest_app +/msync = sync_app +/sync = sync_app + +[app:rest_app] +use = egg:AnkiServer#rest_app +data_root = ./collections +allowed_hosts =0.0.0.0 +logging.config_file = logging.conf + +[app:sync_app] +use = egg:AnkiServer#sync_app +data_root = ./collections +base_url = /sync/ +base_media_url = /msync/ +session_db_path = ./session.db +auth_db_path = ./auth.db + diff --git a/session.db b/session.db new file mode 100644 index 0000000..75fec83 Binary files /dev/null and b/session.db differ