[DO NOT MERGE] Blagoev/perf 6348 port tpcc postgress jsonb

.vscode/launch.json

@@ -0,0 +1,73 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Debug MongoDB driver",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "tpcc.py",
+            "console": "integratedTerminal",
+            //"args": "${command:pickArgs}"
+            "args": "--reset --clients=1 --duration=1 --scalefactor=2000 --warehouses=1 --config=mongodb.config mongodb --stop-on-error",
+            "cwd": "${workspaceFolder}/pytpcc/",
+            "python": "/home/linuka/python_envs/py-tpcc-env/bin/python"
+        },
+        {
+            "name": "Debug Postgresql JSONB driver",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "tpcc.py",
+            "console": "integratedTerminal",
+            "args": "--reset --clients=1 --duration=1 --scalefactor=2000 --warehouses=1 --ddl tpcc_jsonb.sql --config=postgresqljsonb.config postgresqljsonb",
+            "cwd": "${workspaceFolder}/pytpcc/",
+            "python": "/home/linuka/python_envs/py-tpcc-env/bin/python"
+        },
+        {
+            // To debug doDelivery, doOrderStatus, doPayment, doStockLevel, getNumberWH
+            "name": "Debug Postgresql JSONB driver No RESET No LOAD",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "tpcc.py",
+            "console": "integratedTerminal",
+            "args": "--no-load --clients=4 --duration=10 --warehouses=1 --ddl tpcc_jsonb.sql --config=postgresqljsonb.config postgresqljsonb",
+            "cwd": "${workspaceFolder}/pytpcc/",
+            "python": "/home/linuka/python_envs/py-tpcc-env/bin/python"
+        },
+        {
+            "name": "Debug Postgresql JSONB driver Perf",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "tpcc.py",
+            "console": "integratedTerminal",
+            "args": "--reset --clients=1 --duration=1 --scalefactor=20 --warehouses=1 --ddl tpcc_jsonb.sql --config=postgresqljsonb.config postgresqljsonb",
+            "cwd": "${workspaceFolder}/pytpcc/",
+            "python": "/home/linuka/python_envs/py-tpcc-env/bin/python"
+        },
+        {
+            "name": "Debug Postgresql driver",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "tpcc.py",
+            "console": "integratedTerminal",
+            //"args": "${command:pickArgs}"
+            "args": "--reset --clients=1 --duration=1 --scalefactor=2000 --warehouses=1 --config=postgresql.config postgresql",
+            "cwd": "${workspaceFolder}/pytpcc/",
+            "python": "/home/linuka/python_envs/py-tpcc-env/bin/python"
+        },
+        {
+            "name": "Debug Postgresql driver No RESET No LOAD",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "tpcc.py",
+            "console": "integratedTerminal",
+            //"args": "${command:pickArgs}"
+            "args": "--no-load --clients=1 --duration=1 --scalefactor=2000 --warehouses=1 --config=postgresql.config postgresql",
+            "cwd": "${workspaceFolder}/pytpcc/",
+            "python": "/home/linuka/python_envs/py-tpcc-env/bin/python"
+        }
+
+    ]
+}

README.md

@@ -16,3 +16,76 @@ The structure of the repo is:
 All the tests were run using [MongoDB Atlas](https://www.mongodb.com/cloud/atlas?jmp=VLDB2019).
 Use code `VLDB2019` to get $150 credit to get started with MongoDB Atlas.
 
+
+## Postgres JSONB Driver
+
+This branch contains a Postgres JSONB Driver.
+
+Steps to run the PostgreSQL JSONB Driver
+
+1. Start Postgres.
+
+```bash
+sudo systemctl start postgresql
+```
+
+2. Create ana activate a python env.
+
+```bash
+mkdir ~/python_envs
+cd ~/python_envs
+~/python_envs$ python -m venv py-tpcc-env
+source ~/python_envs/py-tpcc-env/bin/activate
+```
+
+3. Print your config.
+
+```bash
+cd ~/py-tpcc/pytpcc
+~/py-tpcc/pytpcc$ python ./tpcc.py --print-config postgresqljsonb > postgresqljsonb.config
+```
+
+3. Edit the configuraiton for Postgres in the postgresqljsonb.config. Add a password.
+
+```bash
+# PostgresqljsonbDriver Configuration File
+# Created 2025-03-18 23:00:45.340852
+[postgresqljsonb]
+
+# The name of the PostgreSQL database
+database             = tpcc
+
+# The host address of the PostgreSQL server
+host                 = localhost
+
+# The port number of the PostgreSQL server
+port                 = 5432
+
+# The username to connect to the PostgreSQL database
+user                 = postgres
+
+# The password to connect to the PostgreSQL database
+password             = <ADD_PASSWORD_HERE>
+```
+
+4. Run the PostgreSQL JSONB driver tests with resetting the database.
+
+```bash
+~/py-tpcc/pytpcc$ python ./tpcc.py --reset --clients=1 --duration=1 --warehouses=1 --ddl tpcc_jsonb.sql --config=postgresqljsonb.config postgresqljsonb --stop-on-error
+```
+
+5. Run the PostgreSQL JSONB driver tests with no load phase to use the data that is already loaded in the Postgres database.
+
+```bash
+~/py-tpcc/pytpcc$ python ./tpcc.py --no-load --clients=1 --duration=1 --warehouses=1 --ddl tpcc_jsonb.sql --config=postgresqljsonb.config postgresqljsonb --stop-on-error
+```
+
+6. If you need to connect to Postgres and check the database size
+
+```bash
+psql -U postgres # and type the password
+postgres=# \l+
+
+# For any SQL command first use the database
+\c tpcc;
+```

pytpcc/drivers/mongodbdriver.py

@@ -38,6 +38,7 @@
 from pprint import pformat
 from time import sleep
 import pymongo
+from pymongo.client_session import TransactionOptions
 
 import constants
 from .abstractdriver import AbstractDriver
@@ -233,6 +234,13 @@ def __init__(self, ddl):
         self.warehouses = 0
         self.shards = 1
 
+        # Get a list of all attributes and methods of pymongo
+        # attributes = dir(pymongo)
+        # # Print the attributes
+        # for attr in attributes:
+        #     print(attr)
+        # sys.exit()
+
         ## Create member mapping to collections
         for name in constants.ALL_TABLES:
             self.__dict__[name.lower()] = None
@@ -357,6 +365,8 @@ def loadTuples(self, tableName, tuples):
         if not tuples:
             return
         logging.debug("Loading %d tuples for tableName %s", len(tuples), tableName)
+        # print (tuples)
+        # sys.exit()
 
         assert tableName in TABLE_COLUMNS, "Table %s not found in TABLE_COLUMNS" % tableName
         columns = TABLE_COLUMNS[tableName]
@@ -407,7 +417,8 @@ def loadTuples(self, tableName, tuples):
             for t in tuples:
                 tuple_dicts.append(dict([(columns[i], t[i]) for i in num_columns]))
             ## FOR
-
+            # print (tuple_dicts[:5])
+            # sys.exit(-1)
             self.database[tableName].insert_many(tuple_dicts)
         ## IF
 

pytpcc/drivers/postgresqldriver.py

@@ -4,6 +4,7 @@
 
 import psycopg2
 import logging
+import traceback
 from pprint import pformat
 from time import sleep
 
@@ -187,7 +188,9 @@ def doDelivery(self, params):
                     ## FOR
                 self.conn.commit()
                 return (result,retries)
-            except:
+            except Exception as e:
+                #print("An error occurred:")
+                #traceback.print_exc()
                 self.conn.rollback()  # Rollback the transaction on error
                 retries += 1
                 sleep(retries * .1)
@@ -317,7 +320,9 @@ def doNewOrder(self, params):
                 ## Pack up values the client is missing (see TPC-C 2.4.3.5)
                 misc = [ (w_tax, d_tax, d_next_o_id, total) ]
                 return ([ customer_info, misc, item_data ], retries)
-            except:
+            except Exception as e:
+                #print("An error occurred:")
+                #traceback.print_exc()
                 self.conn.rollback()  # Rollback the transaction on error
                 retries += 1
                 sleep(retries * .1)
@@ -364,7 +369,9 @@ def doOrderStatus(self, params):
 
                 self.conn.commit()
                 return ([ customer, order, orderLines ],retries)
-            except:
+            except Exception as e:
+                #print("An error occurred:")
+                #traceback.print_exc()
                 self.conn.rollback()  # Rollback the transaction on error
                 retries += 1
                 sleep(retries * .1)
@@ -439,7 +446,9 @@ def doPayment(self, params):
 
                 # Hand back all the warehouse, district, and customer data
                 return ([ warehouse, district, customer ],retries)
-            except:
+            except Exception as e:
+                #print("An error occurred:")
+                #traceback.print_exc()
                 self.conn.rollback()  # Rollback the transaction on error
                 retries += 1
                 sleep(retries * .1)
@@ -466,7 +475,9 @@ def doStockLevel(self, params):
                 self.conn.commit()
 
                 return (int(result[0]),retries)
-            except:
+            except Exception as e:
+                #print("An error occurred:")
+                #traceback.print_exc()
                 self.conn.rollback()  # Rollback the transaction on error
                 retries += 1
                 sleep(retries * .1)