chore: add data_access support for encrypted endpoints

apps/astarte_appengine_api/lib/astarte_appengine_api/device/queries.ex

@@ -112,7 +112,8 @@ defmodule Astarte.AppEngine.API.Device.Queries do
           :allow_unset,
           :endpoint_id,
           :interface_id,
-          :explicit_timestamp
+          :explicit_timestamp,
+          :encrypted
         ]
 
     opts = [

apps/astarte_appengine_api/test/support/helpers/database.ex

@@ -193,6 +193,7 @@ defmodule Astarte.Helpers.Database do
         description varchar,
         doc varchar,
         required boolean,
+        encrypted boolean,
 
         PRIMARY KEY ((interface_id), endpoint_id)
     );
@@ -283,6 +284,8 @@ defmodule Astarte.Helpers.Database do
       stringarray_value list<varchar>,
       binaryblobarray_value list<blob>,
       datetimearray_value list<timestamp>,
+      encryptedblob_value blob,
+      encrypted_dek blob,
 
       PRIMARY KEY((device_id, interface_id), endpoint_id, path)
     );
@@ -311,6 +314,8 @@ defmodule Astarte.Helpers.Database do
       longintegerarray_value list<bigint>,
       string_value text,
       stringarray_value list<text>,
+      encryptedblob_value blob,
+      encrypted_dek blob,
 
       PRIMARY KEY((device_id, interface_id, endpoint_id, path), value_timestamp, reception_timestamp, reception_timestamp_submillis)
     );

apps/astarte_appengine_api/test/support/helpers/database_v2.ex

@@ -142,6 +142,7 @@ defmodule Astarte.Helpers.DatabaseV2 do
     description text,
     doc text,
     required boolean,
+    encrypted boolean,
 
     PRIMARY KEY ((interface_id), endpoint_id)
   )
@@ -182,6 +183,8 @@ defmodule Astarte.Helpers.DatabaseV2 do
     stringarray_value list<text>,
     binaryblobarray_value list<blob>,
     datetimearray_value list<timestamp>,
+    encryptedblob_value blob,
+    encrypted_dek blob,
 
     PRIMARY KEY((device_id, interface_id), endpoint_id, path)
   );
@@ -210,6 +213,9 @@ defmodule Astarte.Helpers.DatabaseV2 do
       longintegerarray_value list<bigint>,
       string_value text,
       stringarray_value list<text>,
+      encryptedblob_value blob,
+      encrypted_dek blob,
+
       PRIMARY KEY ((device_id, interface_id, endpoint_id, path), value_timestamp, reception_timestamp, reception_timestamp_submillis)
   )
   """

apps/astarte_data_updater_plant/test/support/database_test_helper.ex

@@ -125,6 +125,7 @@ defmodule Astarte.DataUpdaterPlant.DatabaseTestHelper do
         description varchar,
         doc varchar,
         required boolean,
+        encrypted boolean,
 
         PRIMARY KEY ((interface_id), endpoint_id)
     );

apps/astarte_data_updater_plant/test/support/helpers/database.ex

@@ -147,6 +147,7 @@ defmodule Astarte.Helpers.Database do
     description text,
     doc text,
     required boolean,
+    encrypted boolean,
 
     PRIMARY KEY ((interface_id), endpoint_id)
   )
@@ -187,6 +188,8 @@ defmodule Astarte.Helpers.Database do
     stringarray_value list<text>,
     binaryblobarray_value list<blob>,
     datetimearray_value list<timestamp>,
+    encryptedblob_value blob,
+    encrypted_dek blob,
 
     PRIMARY KEY((device_id, interface_id), endpoint_id, path)
   );
@@ -215,6 +218,9 @@ defmodule Astarte.Helpers.Database do
       longintegerarray_value list<bigint>,
       string_value text,
       stringarray_value list<text>,
+      encryptedblob_value blob,
+      encrypted_dek blob,
+
       PRIMARY KEY ((device_id, interface_id, endpoint_id, path), value_timestamp, reception_timestamp, reception_timestamp_submillis)
   )
   """

apps/astarte_housekeeping/test/support/helpers/database.ex

@@ -168,6 +168,7 @@ defmodule Astarte.Housekeeping.Helpers.Database do
     description text,
     doc text,
     required boolean,
+    encrypted boolean,
 
     PRIMARY KEY ((interface_id), endpoint_id)
   )

apps/astarte_pairing/test/support/helpers/database.ex

@@ -198,6 +198,7 @@ defmodule Astarte.Helpers.Database do
     description text,
     doc text,
     required boolean,
+    encrypted boolean,
 
     PRIMARY KEY ((interface_id), endpoint_id)
   )
@@ -238,6 +239,8 @@ defmodule Astarte.Helpers.Database do
     stringarray_value list<text>,
     binaryblobarray_value list<blob>,
     datetimearray_value list<timestamp>,
+    encryptedblob_value blob,
+    encrypted_dek blob,
 
     PRIMARY KEY((device_id, interface_id), endpoint_id, path)
   );
@@ -266,6 +269,9 @@ defmodule Astarte.Helpers.Database do
       longintegerarray_value list<bigint>,
       string_value text,
       stringarray_value list<text>,
+      encryptedblob_value blob,
+      encrypted_dek blob,
+
       PRIMARY KEY ((device_id, interface_id, endpoint_id, path), value_timestamp, reception_timestamp, reception_timestamp_submillis)
   )
   """

apps/astarte_realm_management/test/support/helpers/database.ex

@@ -140,6 +140,7 @@ defmodule Astarte.Helpers.Database do
     description text,
     doc text,
     required boolean,
+    encrypted boolean,
 
     PRIMARY KEY ((interface_id), endpoint_id)
   )
@@ -180,6 +181,8 @@ defmodule Astarte.Helpers.Database do
     stringarray_value list<text>,
     binaryblobarray_value list<blob>,
     datetimearray_value list<timestamp>,
+    encryptedblob_value blob,
+    encrypted_dek blob,
 
     PRIMARY KEY((device_id, interface_id), endpoint_id, path)
   );
@@ -208,6 +211,9 @@ defmodule Astarte.Helpers.Database do
       longintegerarray_value list<bigint>,
       string_value text,
       stringarray_value list<text>,
+      encryptedblob_value blob,
+      encrypted_dek blob,
+
       PRIMARY KEY ((device_id, interface_id, endpoint_id, path), value_timestamp, reception_timestamp, reception_timestamp_submillis)
   )
   """

apps/astarte_trigger_engine/test/support/helpers/database.ex

@@ -142,6 +142,7 @@ defmodule Astarte.Helpers.Database do
     description text,
     doc text,
     required boolean,
+    encrypted boolean,
 
     PRIMARY KEY ((interface_id), endpoint_id)
   )
@@ -182,6 +183,8 @@ defmodule Astarte.Helpers.Database do
     stringarray_value list<text>,
     binaryblobarray_value list<blob>,
     datetimearray_value list<timestamp>,
+    encryptedblob_value blob,
+    encrypted_dek blob,
 
     PRIMARY KEY((device_id, interface_id), endpoint_id, path)
   );
@@ -210,6 +213,9 @@ defmodule Astarte.Helpers.Database do
       longintegerarray_value list<bigint>,
       string_value text,
       stringarray_value list<text>,
+      encryptedblob_value blob,
+      encrypted_dek blob,
+
       PRIMARY KEY ((device_id, interface_id, endpoint_id, path), value_timestamp, reception_timestamp, reception_timestamp_submillis)
   )
   """

libs/astarte_data_access/lib/astarte_data_access/mappings.ex

@@ -43,7 +43,8 @@ defmodule Astarte.DataAccess.Mappings do
     :explicit_timestamp,
     :endpoint_id,
     :interface_id,
-    :required
+    :required,
+    :encrypted
   ]
 
   @spec fetch_interface_mappings(String.t(), binary, keyword) ::

libs/astarte_data_access/lib/astarte_data_access/realms/endpoint.ex

@@ -45,7 +45,8 @@ defmodule Astarte.DataAccess.Realms.Endpoint do
                       :reliability,
                       :retention,
                       :value_type,
-                      :required
+                      :required,
+                      :encrypted
                     ] ++ @required_fields
 
   @primary_key false
@@ -68,6 +69,7 @@ defmodule Astarte.DataAccess.Realms.Endpoint do
     field :retention, Retention
     field :value_type, ValueType
     field :required, :boolean
+    field :encrypted, :boolean
   end
 
   def changeset(endpoint, params \\ %{}) do

libs/astarte_data_access/lib/astarte_data_access/realms/individual_datastream.ex

@@ -53,5 +53,7 @@ defmodule Astarte.DataAccess.Realms.IndividualDatastream do
     field :longintegerarray_value, {:array, BigInt}
     field :string_value, :string
     field :stringarray_value, {:array, :string}
+    field :encryptedblob_value, :binary
+    field :encrypted_dek, :binary
   end
 end

libs/astarte_data_access/lib/astarte_data_access/realms/individual_property.ex

@@ -50,6 +50,8 @@ defmodule Astarte.DataAccess.Realms.IndividualProperty do
     field :stringarray_value, {:array, :string}
     field :binaryblobarray_value, {:array, :binary}
     field :datetimearray_value, {:array, DateTimeMs}
+    field :encryptedblob_value, :binary
+    field :encrypted_dek, :binary
   end
 
   def reception(individual_property) do

libs/astarte_data_access/mix.lock

@@ -1,5 +1,5 @@
 %{
-  "astarte_core": {:git, "https://github.com/astarte-platform/astarte_core.git", "a6efa2e87b7f121cc7656640cdff88c408e180f5", []},
+  "astarte_core": {:git, "https://github.com/astarte-platform/astarte_core.git", "af1174a792f43637972e76b4a2c3e290a11a7c2d", []},
   "b58": {:hex, :b58, "1.0.3", "d300d6ae5a3de956a54b9e8220e924e4fee1a349de983df2340fe61e0e464202", [:mix], [], "hexpm", "af62a98a8661fd89978cf3a3a4b5b2ebe82209de6ac6164f0b112e36af72fc59"},
   "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"},
   "castore": {:hex, :castore, "1.0.17", "4f9770d2d45fbd91dcf6bd404cf64e7e58fed04fadda0923dc32acca0badffa2", [:mix], [], "hexpm", "12d24b9d80b910dd3953e165636d68f147a31db945d2dcb9365e441f8b5351e5"},

libs/astarte_data_access/test/mappings_test.exs

@@ -56,6 +56,7 @@ defmodule Astarte.DataAccess.Mappings.XandraTest do
       database_retention_ttl: nil,
       description: nil,
       doc: nil,
+      encrypted: false,
       endpoint: "/foo/%{param}/timestampValue",
       endpoint_id: <<52, 108, 128, 228, 202, 153, 98, 116, 129, 246, 123, 28, 27, 229, 149, 33>>,
       expiry: 0,
@@ -74,6 +75,7 @@ defmodule Astarte.DataAccess.Mappings.XandraTest do
       database_retention_ttl: nil,
       description: nil,
       doc: nil,
+      encrypted: false,
       endpoint: "/foo/%{param}/stringValue",
       endpoint_id: <<57, 7, 212, 29, 91, 202, 50, 157, 158, 81, 76, 234, 42, 84, 169, 154>>,
       expiry: 0,
@@ -92,6 +94,7 @@ defmodule Astarte.DataAccess.Mappings.XandraTest do
       database_retention_ttl: 120,
       description: nil,
       doc: nil,
+      encrypted: false,
       endpoint: "/%{itemIndex}/value",
       endpoint_id: <<117, 1, 14, 27, 25, 158, 238, 252, 221, 53, 210, 84, 176, 226, 9, 36>>,
       expiry: 0,
@@ -110,6 +113,7 @@ defmodule Astarte.DataAccess.Mappings.XandraTest do
       database_retention_ttl: nil,
       description: nil,
       doc: nil,
+      encrypted: false,
       endpoint: "/foo/%{param}/blobValue",
       endpoint_id: <<122, 164, 76, 17, 34, 115, 71, 217, 230, 36, 74, 224, 41, 222, 222, 170>>,
       expiry: 0,
@@ -128,6 +132,7 @@ defmodule Astarte.DataAccess.Mappings.XandraTest do
       database_retention_ttl: nil,
       description: nil,
       doc: nil,
+      encrypted: false,
       endpoint: "/foo/%{param}/longValue",
       endpoint_id: <<239, 249, 87, 207, 3, 223, 222, 237, 151, 132, 168, 112, 142, 61, 140, 185>>,
       expiry: 0,
@@ -139,6 +144,25 @@ defmodule Astarte.DataAccess.Mappings.XandraTest do
       retention: :discard,
       type: nil,
       value_type: :longinteger
+    },
+    <<59, 57, 253, 58, 244, 150, 38, 255, 129, 246, 76, 45, 209, 80, 184, 100>> => %Mapping{
+      allow_unset: false,
+      database_retention_policy: :no_ttl,
+      database_retention_ttl: nil,
+      description: nil,
+      doc: nil,
+      encrypted: true,
+      endpoint: "/encrypted/value",
+      endpoint_id: <<59, 57, 253, 58, 244, 150, 38, 255, 129, 246, 76, 45, 209, 80, 184, 100>>,
+      expiry: 0,
+      explicit_timestamp: nil,
+      interface_id:
+        <<10, 13, 167, 125, 133, 181, 147, 217, 212, 210, 189, 38, 221, 24, 201, 175>>,
+      path: nil,
+      reliability: :unique,
+      retention: :discard,
+      type: nil,
+      value_type: :datetime
     }
   }
 

libs/astarte_data_access/test/support/database_test_helper.exs

@@ -213,6 +213,7 @@ defmodule Astarte.DataAccess.DatabaseTestHelper do
         description varchar,
         doc varchar,
         required boolean,
+        encrypted boolean,
 
         PRIMARY KEY ((interface_id), endpoint_id)
       );
@@ -267,6 +268,11 @@ defmodule Astarte.DataAccess.DatabaseTestHelper do
     INSERT INTO autotestrealm.endpoints (interface_id, endpoint_id, allow_unset, endpoint, expiry, interface_major_version, interface_minor_version, interface_name, interface_type, reliability, retention, value_type) VALUES
         (0a0da77d-85b5-93d9-d4d2-bd26dd18c9af, 346c80e4-ca99-6274-81f6-7b1c1be59521, False, '/foo/%{param}/timestampValue', 0, 1, 0, 'com.test.SimpleStreamTest', 2, 3, 1, 13);
     """,
+    # explicitly set endpoint encryption to True
+    """
+    INSERT INTO autotestrealm.endpoints (interface_id, endpoint_id, allow_unset, endpoint, expiry, interface_major_version, interface_minor_version, interface_name, interface_type, reliability, retention, value_type, encrypted) VALUES
+        (0a0da77d-85b5-93d9-d4d2-bd26dd18c9af, 3b39fd3a-f496-26ff-81f6-4c2dd150b864, False, '/encrypted/value', 0, 1, 0, 'com.test.SimpleStreamTest', 2, 3, 1, 13, True);
+    """,
     """
     INSERT INTO autotestrealm.endpoints (interface_id, endpoint_id, allow_unset, endpoint, expiry, interface_major_version, interface_minor_version, interface_name, interface_type, reliability, retention, value_type) VALUES
         (db576345-80b1-5358-f305-d77ec39b3d84, 7c9f14e8-4f2f-977f-c126-d5e1bb9876e7, False, '/string', 0, 1, 5, 'com.example.TestObject', 2, 2, 3, 7);
@@ -311,6 +317,8 @@ defmodule Astarte.DataAccess.DatabaseTestHelper do
       stringarray_value list<varchar>,
       binaryblobarray_value list<blob>,
       datetimearray_value list<timestamp>,
+      encryptedblob_value blob,
+      encrypted_dek blob,
 
       PRIMARY KEY((device_id, interface_id), endpoint_id, path)
     );
@@ -340,6 +348,8 @@ defmodule Astarte.DataAccess.DatabaseTestHelper do
       stringarray_value list<varchar>,
       binaryblobarray_value list<blob>,
       datetimearray_value list<timestamp>,
+      encryptedblob_value blob,
+      encrypted_dek blob,
 
       PRIMARY KEY((device_id, interface_id, endpoint_id, path), value_timestamp, reception_timestamp, reception_timestamp_submillis)
     );