Changes and corrections to setup and run the baseline TOST Project

README.md

@@ -30,9 +30,9 @@ A PostgreSQL (aka Postgres) database is used to store the raw trip and vessel da
   1. Create a database in Postgre named `tost-db`
   2. Run the sql create statements in the `/database-scripts` directory in the given sequence in the filename prefix (v01-v06). Running all these create statements will create all the required table in the `tost-db` database
   3. When the tables are there, to fill in the tables from the raw AIS data in the available csv files, first download the csv files from the shared Google drive folder. Then, in the `/scripts` directory there are python scripts to read in the csv files and write in the database table. <br>
-  Change the username and password to your Postgres username and password in the _db_connection.py_ file
-
-  1. Running all the python scripts in the sequence _script1_ to _script9_, will 
+  Change the username and password to your Postgres username and password in the _db_connection.py_ file. 
+  4. In _script1.py_ and _script3.py_ set the `csv_file_path` variable to the directory where you stored the raw trip and vessel data csv files.
+  5. Running all the python scripts in the sequence _script1_ to _script9_, will 
 
 
 

backend/db/queries/segments.py

@@ -46,17 +46,25 @@ def save_segment_values_for_trip(segmentation_id, segment_number, trip_id, value
                                               distance_in_nm,
                                               duration_in_sec,
                                               interpolation_percentage)
-              VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s) ON CONFLICT DO NOTHING;"""
-  cursor.execute(query, (segmentation_id,
-                        segment_number,
-                        trip_id,
-                        values['min_speed_in_knots'],
-                        values['average_speed_in_knots'],
+              VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s,%s) ON CONFLICT DO NOTHING;"""
+
+  to_insert = (segmentation_id, segment_number, trip_id, values['min_speed_in_knots'],values['average_speed_in_knots'],
                         values['max_speed_in_knots'],
                         values['average_bearing_in_deg'],
                         values['travel_distance_in_nm'],
                         values['duration_in_seconds'],
-                        values['interpolation_percentage']))
+                        values['interpolation_percentage'])
+  cursor.execute(query,to_insert)
+  # cursor.execute(query, (segmentation_id,
+  #                       segment_number,
+  #                       trip_id,
+  #                       values['min_speed_in_knots'],
+  #                       values['average_speed_in_knots'],
+  #                       values['max_speed_in_knots'],
+  #                       values['average_bearing_in_deg'],
+  #                       values['travel_distance_in_nm'],
+  #                       values['duration_in_seconds'],
+  #                       values['interpolation_percentage']))
 
 # def update_segment_values_for_trip(segmentation_id, segment_number, trip_id, interpolation_percentage):
 #   cursor = create_cursor()

backend/db/queries/trips.py

@@ -25,7 +25,7 @@ def save_trip(trip_id, trip_df):
   Save trip dataframe to trip_v2 table
   '''
   cursor = create_cursor()
-  query = 'INSERT INTO trip_v2 (trip_id, timestamp, lng_lat, interpolated, speed_in_knots, bearing_in_deg, travel_distance_in_nm) VALUES (%s, to_timestamp(%s),%s, %s, %s, %s, %s) ON CONFLICT DO NOTHIN;'
+  query = 'INSERT INTO trip_v2 (trip_id, timestamp, lng_lat, interpolated, speed_in_knots, bearing_in_deg, travel_distance_in_nm) VALUES (%s, to_timestamp(%s),%s, %s, %s, %s, %s) ON CONFLICT DO NOTHING;'
   for index, row in trip_df.iterrows():
     lng_lat = "SRID=4326;POINT({} {})".format(row['lng'], row['lat'])
     timestamp = row['timestamp_in_seconds']

backend/db/scripts/script1_populate_trip_v2_table.py

@@ -1,5 +1,5 @@
 import sys
-sys.path.append('../../') # hack so we can access sibling folder https://stackoverflow.com/questions/4383571/importing-files-from-different-folder
+sys.path.append(os.path.dirname(os.path.abspath(__file__)) + "/../..") # hack so we can access sibling folder https://stackoverflow.com/questions/6323860/sibling-package-imports
 from db.db_connection import create_connection
 from db.queries.trips import load_all_trip_ids, load_raw_trip, save_trip
 from hampel_filter import hampel_filter_pandas
@@ -25,7 +25,8 @@ def populate_trip_table():
       trip_df = interpolate(trip_df)
       trip_df = calculate_additional_attributes(trip_df)
       save_trip(trip_id, trip_df)
-    except:
+    except Exception as e: 
+      print(e)
       print(trip_id)
 
   print('trip table have been correctly populated')

backend/db/scripts/script3_create_segment_values.py

@@ -20,7 +20,9 @@ def calculate_segment_values(segmentation_id):
         trip_on_segment_df = get_trip_df(trips_data_on_segment_df, trip_id)
         trip_segment_values = calculate_trip_values_on_segment(trip_on_segment_df)
         save_segment_values_for_trip(segmentation_id, segment['segment_number'], trip_id, trip_segment_values)
-      except:
+        print(f'Successfully calculated segment values for trip_id<{trip_id}> on segment<{segment["segment_number"]}>')
+      except Exception as e:
+        print(e)
         print(f'failed to calculate segment values for trip_id<{trip_id}> on segment<{segment["segment_number"]}>')
 
 def get_trip_df(trips_df, trip_id):