my.core: experimental support for apsw, use it for my.bluemaestro

apsw seems to be about 30% faster than python's sqlite adapter

Author: karlicoss

src/my/bluemaestro.py

@@ -9,6 +9,10 @@
 
 from __future__ import annotations
 
+REQUIRES = [
+    'apsw',  # faster sqlite adapter
+]
+
 import calendar
 
 # todo most of it belongs to DAL... but considering so few people use it I didn't bother for now
@@ -31,7 +35,7 @@
 )
 from my.core.cachew import mcachew
 from my.core.pandas import DataFrameT, as_dataframe
-from my.core.sqlite import SqliteTool, sqlite_connect_immutable
+from my.core.sqlite import SqliteTool, sqlite_connection
 
 
 class config(Protocol):
@@ -99,7 +103,7 @@ def measurements() -> Iterable[Res[Measurement]]:
         tot = 0
         new = 0
         # todo assert increasing timestamp?
-        with sqlite_connect_immutable(path) as db:
+        with sqlite_connection(path, immutable=True, _via_apsw=True) as db:
             tool = SqliteTool(db)
             old_format = 'data' in tool.get_table_names()
 
@@ -136,7 +140,7 @@ def measurements() -> Iterable[Res[Measurement]]:
                 # UPD: fucking hell, so you can set the reference date in the settings (calcReferenceUnix field in meta db)
                 # but it's not set by default.
 
-                log_tables = [c[0] for c in db.execute('SELECT name FROM sqlite_sequence WHERE name LIKE "%_log"')]
+                log_tables = [row[0] for row in db.execute('SELECT name FROM sqlite_sequence WHERE name LIKE "%_log"')]
                 # 'omnibus' appears in bmLogger app, but seems that it contains the same data that last export contains?
                 # not sure what's the point :shrug:
                 log_tables = [t for t in log_tables if 'omnibus' not in t]

src/my/core/sqlite.py

@@ -8,6 +8,7 @@
 from tempfile import TemporaryDirectory
 from typing import Any, Callable, Literal, Union, overload
 
+from . import warnings
 from .common import PathIsh
 from .compat import assert_never
 
@@ -44,13 +45,19 @@ def dict_factory(cursor, row):
 
 
 @contextmanager
-def sqlite_connection(db: PathIsh, *, immutable: bool = False, row_factory: Factory | None = None) -> Iterator[sqlite3.Connection]:
-    dbp = f'file:{db}'
+def sqlite_connection(
+    db: PathIsh,
+    *,
+    immutable: bool = False,
+    row_factory: Factory | None = None,
+    _via_apsw: bool = False,  # NOTE: experimental for now
+) -> Iterator[sqlite3.Connection]:
+    uri = f'file:{db}'
     # https://www.sqlite.org/draft/uri.html#uriimmutable
     if immutable:
         # assert results in nicer error than sqlite3.OperationalError
         assert Path(db).exists(), db
-        dbp = f'{dbp}?immutable=1'
+        uri = f'{uri}?immutable=1'
     row_factory_: Any = None
     if row_factory is not None:
         if callable(row_factory):
@@ -62,7 +69,25 @@ def sqlite_connection(db: PathIsh, *, immutable: bool = False, row_factory: Fact
         else:
             assert_never(row_factory)  # ty: ignore[type-assertion-failure]
 
-    conn = sqlite3.connect(dbp, uri=True)
+
+    if _via_apsw:
+        try:
+            # for now, defensive, will see later how to do it properly
+            import apsw  # type: ignore[import-not-found,unused-ignore]  # ty: ignore[unresolved-import]
+        except ImportError:
+            warnings.high('apsw is not installed, falling back to sqlite')
+        else:
+            assert row_factory is None, "row_factory is not supported with apsw (yet?)"
+
+            apsw_conn = apsw.Connection(uri, flags=apsw.SQLITE_OPEN_READONLY | apsw.SQLITE_OPEN_URI)
+            try:
+                with apsw_conn:
+                    yield apsw_conn  # type: ignore[misc,unused-ignore]
+            finally:
+                apsw_conn.close()
+            return
+
+    conn = sqlite3.connect(uri, uri=True)
     try:
         conn.row_factory = row_factory_
         with conn:

tox.ini

@@ -63,6 +63,7 @@ deps =
     ijson  # optional dependency for various modules
 commands =
     {envpython} -m my.core module install \
+        my.bluemaestro \
         ## tz/location
         my.location.google \
         my.time.tz.via_location \
@@ -125,6 +126,8 @@ commands =
         {posargs}
 
 
+# TODO ugh.  my.bluemaestro uses apsw, which only supports >=3.11 python syntax?
+# so mypy chokes over it... think how to handle it later
 [testenv:typecheck-all_base]
 hpi_modules =
     my.arbtt                  \