improvement: add storage type option (#342)

---------

Co-authored-by: Zach Daniel <[email protected]>

Author: barnabasJ

.formatter.exs

@@ -43,6 +43,7 @@ spark_locals_without_parens = [
   skip_unique_indexes: 1,
   statement: 1,
   statement: 2,
+  storage_types: 1,
   table: 1,
   template: 1,
   unique: 1,

documentation/dsls/DSL:-AshPostgres.DataLayer.md

@@ -40,6 +40,7 @@ end
 |------|------|---------|------|
 | [`repo`](#postgres-repo){: #postgres-repo .spark-required} | `module \| (any, any -> any)` |  | The repo that will be used to fetch your data. See the `AshPostgres.Repo` documentation for more. Can also be a function that takes a resource and a type `:read \| :mutate` and returns the repo |
 | [`migrate?`](#postgres-migrate?){: #postgres-migrate? } | `boolean` | `true` | Whether or not to include this resource in the generated migrations with `mix ash.generate_migrations` |
+| [`storage_types`](#postgres-storage_types){: #postgres-storage_types } | `keyword` | `[]` | A keyword list of attribute names to the ecto type that should be used for that attribute. Only necessary if you need to override the defaults. |
 | [`migration_types`](#postgres-migration_types){: #postgres-migration_types } | `keyword` | `[]` | A keyword list of attribute names to the ecto migration type that should be used for that attribute. Only necessary if you need to override the defaults. |
 | [`migration_defaults`](#postgres-migration_defaults){: #postgres-migration_defaults } | `keyword` | `[]` | A keyword list of attribute names to the ecto migration default that should be used for that attribute. The string you use will be placed verbatim in the migration. Use fragments like `fragment(\\"now()\\")`, or for `nil`, use `\\"nil\\"`. |
 | [`calculations_to_sql`](#postgres-calculations_to_sql){: #postgres-calculations_to_sql } | `keyword` |  | A keyword list of calculations and their SQL representation. Used when creating unique indexes for identities over calculations |
@@ -114,7 +115,7 @@ index ["column", "column2"], unique: true, where: "thing = TRUE"
 | [`prefix`](#postgres-custom_indexes-index-prefix){: #postgres-custom_indexes-index-prefix } | `String.t` |  | specify an optional prefix for the index. |
 | [`where`](#postgres-custom_indexes-index-where){: #postgres-custom_indexes-index-where } | `String.t` |  | specify conditions for a partial index. |
 | [`include`](#postgres-custom_indexes-index-include){: #postgres-custom_indexes-index-include } | `list(String.t)` |  | specify fields for a covering index. This is not supported by all databases. For more information on PostgreSQL support, please read the official docs. |
-| [`nulls_distinct`](#postgres-custom_indexes-index-nulls_distinct){: #postgres-custom_indexes-index-nulls_distinct } | `boolean` | `true` | specify whether null values should be considered distinct for a unique index. |
+| [`nulls_distinct`](#postgres-custom_indexes-index-nulls_distinct){: #postgres-custom_indexes-index-nulls_distinct } | `boolean` | `true` | specify whether null values should be considered distinct for a unique index. Requires PostgreSQL 15 or later |
 | [`message`](#postgres-custom_indexes-index-message){: #postgres-custom_indexes-index-message } | `String.t` |  | A custom message to use for unique indexes that have been violated |
 | [`all_tenants?`](#postgres-custom_indexes-index-all_tenants?){: #postgres-custom_indexes-index-all_tenants? } | `boolean` | `false` | Whether or not the index should factor in the multitenancy attribute or not. |
 

lib/custom_index.ex

@@ -62,7 +62,8 @@ defmodule AshPostgres.CustomIndex do
     ],
     nulls_distinct: [
       type: :boolean,
-      doc: "specify whether null values should be considered distinct for a unique index. Requires PostgreSQL 15 or later",
+      doc:
+        "specify whether null values should be considered distinct for a unique index. Requires PostgreSQL 15 or later",
       default: true
     ],
     message: [

lib/data_layer.ex

@@ -287,6 +287,12 @@ defmodule AshPostgres.DataLayer do
         doc:
           "Whether or not to include this resource in the generated migrations with `mix ash.generate_migrations`"
       ],
+      storage_types: [
+        type: :keyword_list,
+        default: [],
+        doc:
+          "A keyword list of attribute names to the ecto type that should be used for that attribute. Only necessary if you need to override the defaults."
+      ],
       migration_types: [
         type: :keyword_list,
         default: [],

lib/data_layer/info.ex

@@ -78,6 +78,11 @@ defmodule AshPostgres.DataLayer.Info do
     Extension.get_opt(resource, [:postgres], :migration_types, [])
   end
 
+  @doc "A keyword list of customized storage types"
+  def storage_types(resource) do
+    Extension.get_opt(resource, [:postgres], :storage_types, [])
+  end
+
   @doc "A keyword list of customized migration defaults"
   def migration_defaults(resource) do
     Extension.get_opt(resource, [:postgres], :migration_defaults, [])

lib/sql_implementation.ex

@@ -17,6 +17,30 @@ defmodule AshPostgres.SqlImplementation do
   def require_extension_for_citext, do: {true, "citext"}
 
   @impl true
+  def storage_type(resource, field) do
+    case AshPostgres.DataLayer.Info.storage_types(resource)[field] do
+      nil ->
+        nil
+
+      {:array, type} ->
+        parameterized_type({:array, Ash.Type.get_type(type)}, [], false)
+
+      {:array, type, constraints} ->
+        parameterized_type({:array, Ash.Type.get_type(type)}, constraints, false)
+
+      {type, constraints} ->
+        parameterized_type(type, constraints, false)
+
+      type ->
+        parameterized_type(type, [], false)
+    end
+  end
+
+  @impl true
+  def expr(_query, [], _bindings, _embedded?, acc, type) when type in [:map, :jsonb] do
+    {:ok, Ecto.Query.dynamic(fragment("'[]'::jsonb")), acc}
+  end
+
   def expr(
         query,
         %like{arguments: [arg1, arg2], embedded?: pred_embedded?},

mix.exs

@@ -163,7 +163,7 @@ defmodule AshPostgres.MixProject do
   defp deps do
     [
       {:ash, ash_version("~> 3.0 and >= 3.0.15")},
-      {:ash_sql, ash_sql_version("~> 0.2 and >= 0.2.6")},
+      {:ash_sql, ash_sql_version("~> 0.2 and >= 0.2.8")},
       {:igniter, "~> 0.2.6"},
       {:ecto_sql, "~> 3.9"},
       {:ecto, "~> 3.9"},

mix.lock

@@ -1,12 +1,12 @@
 %{
   "ash": {:hex, :ash, "3.0.16", "8eaebd5a9f3ee404937ac811a240799613b0619026e097436132d60eaf18ed16", [:mix], [{:comparable, "~> 1.0", [hex: :comparable, repo: "hexpm", optional: false]}, {:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:ecto, "~> 3.7", [hex: :ecto, repo: "hexpm", optional: false]}, {:ets, "~> 0.8", [hex: :ets, repo: "hexpm", optional: false]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: false]}, {:picosat_elixir, "~> 0.2", [hex: :picosat_elixir, repo: "hexpm", optional: true]}, {:plug, ">= 0.0.0", [hex: :plug, repo: "hexpm", optional: true]}, {:reactor, ">= 0.8.1 and < 1.0.0-0", [hex: :reactor, repo: "hexpm", optional: false]}, {:simple_sat, ">= 0.1.1 and < 1.0.0-0", [hex: :simple_sat, repo: "hexpm", optional: true]}, {:spark, ">= 2.1.18 and < 3.0.0-0", [hex: :spark, repo: "hexpm", optional: false]}, {:splode, "~> 0.2", [hex: :splode, repo: "hexpm", optional: false]}, {:stream_data, "~> 1.0", [hex: :stream_data, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.1", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "36c0d7653f7fb1d13cc03e1cc7ea7f6b9aadd278b9c9375ff5f0636ed0d7a785"},
-  "ash_sql": {:hex, :ash_sql, "0.2.7", "56bfddcb4cf3edbbf702e2b665497309e43672fbf449ef049f4805211b9cd1b7", [:mix], [{:ash, "~> 3.0", [hex: :ash, repo: "hexpm", optional: false]}, {:ecto, "~> 3.9", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "~> 3.9", [hex: :ecto_sql, repo: "hexpm", optional: false]}], "hexpm", "14622713cc08ede8fd0d2618b1718d759a6ee28839b8f738e6ee084703bd9437"},
+  "ash_sql": {:hex, :ash_sql, "0.2.8", "ad20dc5487b68a095a12f84918d8ffab4bef12de59b7c5cf962a7c1dc03f5d81", [:mix], [{:ash, "~> 3.0", [hex: :ash, repo: "hexpm", optional: false]}, {:ecto, "~> 3.9", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "~> 3.9", [hex: :ecto_sql, repo: "hexpm", optional: false]}], "hexpm", "d120568a19f3d72a634c3bfa9ce5fa0fae1d3c44ee4cdb8c78f151e490bf849b"},
   "benchee": {:hex, :benchee, "1.3.1", "c786e6a76321121a44229dde3988fc772bca73ea75170a73fd5f4ddf1af95ccf", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}, {:statistex, "~> 1.0", [hex: :statistex, repo: "hexpm", optional: false]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "76224c58ea1d0391c8309a8ecbfe27d71062878f59bd41a390266bf4ac1cc56d"},
   "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"},
   "castore": {:hex, :castore, "1.0.8", "dedcf20ea746694647f883590b82d9e96014057aff1d44d03ec90f36a5c0dc6e", [:mix], [], "hexpm", "0b2b66d2ee742cb1d9cb8c8be3b43c3a70ee8651f37b75a8b982e036752983f1"},
   "comparable": {:hex, :comparable, "1.0.0", "bb669e91cedd14ae9937053e5bcbc3c52bb2f22422611f43b6e38367d94a495f", [:mix], [{:typable, "~> 0.1", [hex: :typable, repo: "hexpm", optional: false]}], "hexpm", "277c11eeb1cd726e7cd41c6c199e7e52fa16ee6830b45ad4cdc62e51f62eb60c"},
   "credo": {:hex, :credo, "1.7.7", "771445037228f763f9b2afd612b6aa2fd8e28432a95dbbc60d8e03ce71ba4446", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "8bc87496c9aaacdc3f90f01b7b0582467b69b4bd2441fe8aae3109d843cc2f2e"},
-  "db_connection": {:hex, :db_connection, "2.6.0", "77d835c472b5b67fc4f29556dee74bf511bbafecdcaf98c27d27fa5918152086", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c2f992d15725e721ec7fbc1189d4ecdb8afef76648c746a8e1cad35e3b8a35f3"},
+  "db_connection": {:hex, :db_connection, "2.7.0", "b99faa9291bb09892c7da373bb82cba59aefa9b36300f6145c5f201c7adf48ec", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "dcf08f31b2701f857dfc787fbad78223d61a32204f217f15e881dd93e4bdd3ff"},
   "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"},
   "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"},
   "dialyxir": {:hex, :dialyxir, "1.4.3", "edd0124f358f0b9e95bfe53a9fcf806d615d8f838e2202a9f430d59566b6b53b", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "bf2cfb75cd5c5006bec30141b131663299c661a864ec7fbbc72dfa557487a986"},

priv/resource_snapshots/test_repo/authors/20240705113722.json

@@ -0,0 +1,82 @@
+{
+  "attributes": [
+    {
+      "allow_nil?": false,
+      "default": "fragment(\"gen_random_uuid()\")",
+      "generated?": false,
+      "primary_key?": true,
+      "references": null,
+      "size": null,
+      "source": "id",
+      "type": "uuid"
+    },
+    {
+      "allow_nil?": true,
+      "default": "nil",
+      "generated?": false,
+      "primary_key?": false,
+      "references": null,
+      "size": null,
+      "source": "first_name",
+      "type": "text"
+    },
+    {
+      "allow_nil?": true,
+      "default": "nil",
+      "generated?": false,
+      "primary_key?": false,
+      "references": null,
+      "size": null,
+      "source": "last_name",
+      "type": "text"
+    },
+    {
+      "allow_nil?": true,
+      "default": "nil",
+      "generated?": false,
+      "primary_key?": false,
+      "references": null,
+      "size": null,
+      "source": "bio",
+      "type": "map"
+    },
+    {
+      "allow_nil?": true,
+      "default": "nil",
+      "generated?": false,
+      "primary_key?": false,
+      "references": null,
+      "size": null,
+      "source": "bios",
+      "type": "jsonb"
+    },
+    {
+      "allow_nil?": true,
+      "default": "nil",
+      "generated?": false,
+      "primary_key?": false,
+      "references": null,
+      "size": null,
+      "source": "badges",
+      "type": [
+        "array",
+        "text"
+      ]
+    }
+  ],
+  "base_filter": null,
+  "check_constraints": [],
+  "custom_indexes": [],
+  "custom_statements": [],
+  "has_create_action": true,
+  "hash": "D0080403BD79419DCA499838D2BE1F8AE2744F28A2ADC775B74D8EDA0EC11DB1",
+  "identities": [],
+  "multitenancy": {
+    "attribute": null,
+    "global": null,
+    "strategy": null
+  },
+  "repo": "Elixir.AshPostgres.TestRepo",
+  "schema": null,
+  "table": "authors"
+}

priv/test_repo/migrations/20240705113722_migrate_resources33.exs

@@ -0,0 +1,21 @@
+defmodule AshPostgres.TestRepo.Migrations.MigrateResources33 do
+  @moduledoc """
+  Updates resources based on their most recent snapshots.
+
+  This file was autogenerated with `mix ash_postgres.generate_migrations`
+  """
+
+  use Ecto.Migration
+
+  def up do
+    alter table(:authors) do
+      add(:bios, :jsonb)
+    end
+  end
+
+  def down do
+    alter table(:authors) do
+      remove(:bios)
+    end
+  end
+end

test/storage_types_test.exs

@@ -0,0 +1,45 @@
+defmodule AshPostgres.StorageTypesTest do
+  use AshPostgres.RepoCase, async: false
+
+  alias Ash.BulkResult
+  alias AshPostgres.Test.Author
+
+  require Ash.Query
+
+  test "can save {:array, :map} as jsonb" do
+    %{id: id} =
+      Author
+      |> Ash.Changeset.for_create(
+        :create,
+        %{bios: [%{title: "bio1"}, %{title: "bio2"}]}
+      )
+      |> Ash.create!()
+
+    # testing empty list edge case
+    %BulkResult{records: [author]} =
+      Author
+      |> Ash.Query.filter(id == ^id)
+      |> Ash.bulk_update(:update, %{bios: []},
+        return_errors?: true,
+        notify?: true,
+        strategy: [:atomic, :stream, :atomic_batches],
+        allow_stream_with: :full_read,
+        return_records?: true
+      )
+
+    assert author.bios == []
+
+    %BulkResult{records: [author]} =
+      Author
+      |> Ash.Query.filter(id == ^id)
+      |> Ash.bulk_update(:update, %{bios: [%{a: 1}]},
+        return_errors?: true,
+        notify?: true,
+        strategy: [:atomic, :stream, :atomic_batches],
+        allow_stream_with: :full_read,
+        return_records?: true
+      )
+
+    assert author.bios == [%{"a" => 1}]
+  end
+end

test/support/resources/author.ex

@@ -18,13 +18,17 @@ defmodule AshPostgres.Test.Author do
   postgres do
     table("authors")
     repo(AshPostgres.TestRepo)
+
+    migration_types bios: :jsonb
+    storage_types(bios: :jsonb)
   end
 
   attributes do
     uuid_primary_key(:id, writable?: true)
     attribute(:first_name, :string, public?: true)
     attribute(:last_name, :string, public?: true)
     attribute(:bio, AshPostgres.Test.Bio, public?: true)
+    attribute(:bios, {:array, :map}, public?: true)
     attribute(:badges, {:array, :atom}, public?: true)
   end