Add an initial set of performance benchmarks (#199)

* Add an initial set of performance benchmarks

* make it work with pre-existing dbs too

* Add whether cython is enabled

---------

Co-authored-by: Martin Gallwey <mgy5@3ds.com>

Author: TheDistributor

.circleci/config.yml

@@ -73,10 +73,88 @@ jobs:
       - after_failure:
           when : "on_fail"
 
+  perf_bench:
+    description: "Run insert/select performance benchmarks on master and on this branch, print the diff"
+    docker:
+      - image: nuodb/nuodb:latest
+        user: root
+    resource_class: xlarge
+    environment:
+      TZ : America/New_York
+      NUO_SET_TLS : disable
+      NUOCMD_CLIENT_KEY : ""
+      NUOCMD_VERIFY_SERVER : ""
+      NUOCMD_PLUGINS : ""
+    steps:
+      - checkout
+      - run:
+          name: Install build tools
+          command: |
+            PYVER=$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')
+            dnf install -y git make gcc "python${PYVER}-devel"
+      - run:
+          name: Install pip
+          command: |
+            curl https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py
+            python3 /tmp/get-pip.py --user
+      - run:
+          name: Make artifact directories
+          command: mkdir -p artifacts results
+      - run:
+          name: Start NuoDB Admin
+          command: |
+            sudo -u nuodb /opt/nuodb/etc/nuoadmin tls $NUO_SET_TLS
+            sudo -u nuodb /opt/nuodb/etc/nuoadmin tls status
+            sudo -u nuodb /opt/nuodb/etc/nuoadmin start
+            sudo -u nuodb /opt/nuodb/bin/nuocmd --show-json get effective-license
+      # Run master then this branch on the same runner and let compare.py
+      # print a delta table.  Hardware noise mostly cancels because both
+      # runs share the container; xlarge gives us dedicated cores.
+      - run:
+          name: Baseline benchmarks on master
+          command: |
+            GIT_SSH_COMMAND="ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+                git fetch --no-tags --depth=1 origin master
+            git worktree add /tmp/base FETCH_HEAD
+            # Copy the perf suite + conftest hooks from this branch so the
+            # master worktree's driver is exercised by the same benchmarks.
+            cp -a tests/perf /tmp/base/tests/
+            cp tests/conftest.py /tmp/base/tests/conftest.py
+            cd /tmp/base
+            make PYTHON=python3 install
+            $HOME/.local/bin/pip install pytest-benchmark
+            python3 -m pytest tests/perf --run-perf --benchmark-only \
+                --benchmark-json=/tmp/master.json \
+                --benchmark-columns=min,mean,median,stddev,rounds
+      - run:
+          name: Reset DB
+          command: |
+            sudo -u nuodb /opt/nuodb/bin/nuocmd shutdown database \
+                --db-name pynuodb_test 2>/dev/null || true
+      - run:
+          name: Branch benchmarks + diff vs master
+          command: |
+            make PYTHON=python3 install
+            $HOME/.local/bin/pip install pytest-benchmark
+            python3 -m pytest tests/perf --run-perf --benchmark-only \
+                --benchmark-json=artifacts/branch.json \
+                --benchmark-columns=min,mean,median,stddev,rounds
+            python3 tests/perf/compare.py \
+                /tmp/master.json artifacts/branch.json \
+                | tee artifacts/perf_diff.txt
+      - store_artifacts:
+          path: artifacts
+      - after_failure:
+          when : "on_fail"
+
 workflows:
   build-project:
     jobs:
       - build_n_run:
           name: "Build and run regression tests"
           context:
             - common-config
+      - perf_bench:
+          name: "Run performance benchmarks, comparing results to master"
+          context:
+            - common-config

test-performance/timesInsert.py

@@ -1,6 +1,7 @@
 mock>=1.0
 nose>=1.3
 pytest>=2.7
+pytest-benchmark>=4.0
 coverage>=3.7
 pytest-cov>=1.8.1
 coveralls>=0.5

test_requirements.txt

@@ -30,6 +30,52 @@
 
 from . import nuocmd, cvtjson
 
+
+def pytest_addoption(parser):
+    parser.addoption("--run-perf", action="store_true", default=False,
+                     help="run performance benchmarks under tests/perf")
+    # Bypass the nuocmd-driven discovery/lifecycle fixtures and connect
+    # straight to an already-running database.  Handy for local perf runs
+    # where you don't want (or can't) shell out to nuocmd.
+    parser.addoption("--use-existing-db", action="store_true", default=False,
+                     help="skip nuocmd discovery; use --db-* options to connect")
+    parser.addoption("--db-host", default="localhost:48004",
+                     help="SQL host:port when --use-existing-db is set")
+    parser.addoption("--db-name", default=DATABASE_NAME,
+                     help="database name when --use-existing-db is set")
+    parser.addoption("--db-user", default=DBA_USER,
+                     help="user when --use-existing-db is set")
+    parser.addoption("--db-password", default=DBA_PASSWORD,
+                     help="password when --use-existing-db is set")
+    parser.addoption("--db-schema", default="test",
+                     help="schema when --use-existing-db is set")
+
+
+def pytest_configure(config):
+    config.addinivalue_line(
+        "markers",
+        "perf: performance benchmark; skipped unless --run-perf is passed")
+
+
+def pytest_report_header(config):
+    import pynuodb
+    try:
+        from pynuodb import _fetch
+        ext = "cython: %s" % _fetch.__file__
+    except ImportError:
+        ext = "cython: NOT loaded (pure-Python fallback)"
+    return "pynuodb: %s (%s)\n%s" % (
+        pynuodb.__version__, pynuodb.__file__, ext)
+
+
+def pytest_collection_modifyitems(config, items):
+    if config.getoption("--run-perf"):
+        return
+    skip = pytest.mark.skip(reason="need --run-perf to run performance tests")
+    for item in items:
+        if "perf" in item.keywords:
+            item.add_marker(skip)
+
 _log = logging.getLogger("pynuodbtest")
 
 DB_OPTIONS = []  # type: List[str]
@@ -286,19 +332,33 @@ def te(ap, db):
 
 
 @pytest.fixture(scope='session')
-def database(ap, db, te):
-    # type: (AP_FIXTURE, DB_FIXTURE, TE_FIXTURE) -> DATABASE_FIXTURE
+def database(request):
+    # type: (pytest.FixtureRequest) -> DATABASE_FIXTURE
     import pynuodb
+
+    if request.config.getoption("--use-existing-db"):
+        connect_args = {
+            'database': request.config.getoption("--db-name"),
+            'host':     request.config.getoption("--db-host"),
+            'user':     request.config.getoption("--db-user"),
+            'password': request.config.getoption("--db-password"),
+            'options':  {'schema': request.config.getoption("--db-schema")},
+        }  # type: DATABASE_FIXTURE
+    else:
+        ap = request.getfixturevalue('ap')
+        db = request.getfixturevalue('db')
+        request.getfixturevalue('te')
+        connect_args = {'database': db[0],
+                        'host': ap[1],
+                        'user': db[1],
+                        'password': db[2],
+                        'options': {'schema': 'test'}}
+
     end = time.time() + 30
     conn = None
-    _log.info("Creating a SQL connection to %s as user %s with schema 'test'",
-              db[0], db[1])
-
-    connect_args = {'database': db[0],
-                    'host': ap[1],
-                    'user': db[1],
-                    'password': db[2],
-                    'options': {'schema': 'test'}}  # type: DATABASE_FIXTURE
+    _log.info("Creating a SQL connection to %s as user %s",
+              connect_args['database'], connect_args['user'])
+
     system_information = {'effective_version': 0}
 
     try:
@@ -323,6 +383,6 @@ def database(ap, db, te):
         if conn:
             conn.close()
 
-    _log.info("Database %s is available", db[0])
+    _log.info("Database %s is available", connect_args['database'])
 
     return {'connect_args': connect_args, 'system_information': system_information}

tests/conftest.py

@@ -31,11 +31,17 @@ class NuoBase(object):
     lower_func = 'lower'  # For stored procedure test
 
     @pytest.fixture(autouse=True)
-    def _setup(self, database):
+    def _setup(self, database, request):
         # Preserve the options we'll need to create a connection to the DB
         self.connect_args = database['connect_args']
         self.system_information = database['system_information']
 
+        # In --use-existing-db mode we trust the caller and skip the
+        # nuocmd process check: the successful connection made in the
+        # `database` fixture is already proof the DB and a TE are up.
+        if request.config.getoption("--use-existing-db"):
+            return
+
         # Verify the database is up and has a running TE
         dbname = self.connect_args['database']
         (ret, out) = nuocmd(['--show-json', 'get', 'processes',

tests/nuodb_base.py

@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*-
+"""Compare two pytest-benchmark JSON files (master vs branch).
+
+Prints a table of master min, branch min, and the absolute + percentage
+delta per test.  Exits non-zero if any test regressed by more than
+--fail-threshold (default 10%), so CI turns a real regression into a
+failed build.  Improvements never fail the build.
+"""
+from __future__ import print_function
+
+import argparse
+import json
+import sys
+
+
+def _load(path):
+    with open(path) as f:
+        data = json.load(f)
+    return {b['name']: b['stats']['min'] for b in data['benchmarks']}
+
+
+def main(args):
+    master = _load(args.master)
+    branch = _load(args.branch)
+
+    print("%-40s %16s %16s %14s %10s" % ( "Test", "master min (ms)", "branch min (ms)", "delta (ms)", "delta %"))
+    print("-" * 100)
+
+    regressed = []
+    for name in sorted(set(master) & set(branch)):
+        m = master[name] * 1000.0
+        b = branch[name] * 1000.0
+        d = b - m
+        p = (d / m) * 100.0 if m else float('nan')
+        print("%-40s %16.3f %16.3f %+14.3f %+9.2f%%" % (name, m, b, d, p))
+        if p > args.fail_threshold:
+            regressed.append((name, p))
+
+    only_master = sorted(set(master) - set(branch))
+    only_branch = sorted(set(branch) - set(master))
+    if only_master:
+        print("\nOnly in master: %s" % ", ".join(only_master))
+    if only_branch:
+        print("Only in branch: %s" % ", ".join(only_branch))
+
+    print()
+    if regressed:
+        print("FAIL: %d test(s) regressed by more than %.2f%%:" % (len(regressed), args.fail_threshold))
+        for name, p in regressed:
+            print("  %s: %+.2f%%" % (name, p))
+        sys.exit(1)
+    print("OK: no test regressed by more than %.2f%%" % args.fail_threshold)
+
+
+def _parse_args():
+    p = argparse.ArgumentParser()
+    p.add_argument('master', help='pytest-benchmark JSON for master')
+    p.add_argument('branch', help='pytest-benchmark JSON for this branch')
+    p.add_argument('--fail-threshold', type=float, default=15.0,
+                   help='percent slowdown that fails the build (default 15)')
+    return p.parse_args()
+
+
+if __name__ == '__main__':
+    main(_parse_args())