Implement support for automatically mapping structs

README.md

@@ -54,6 +54,17 @@ These are the currently available features:
 * [Line](https://github.com/crashtech/torque-postgresql/wiki/Line)
 * [Segment](https://github.com/crashtech/torque-postgresql/wiki/Segment)
 
+## Structs
+
+If you are using `create type X as (field_name Y, other_field_name Z)`, torque-postgresql will
+automatically map a subclass of ::Torque::Struct to that type using the singular-form ActiveRecord
+table naming rules.
+
+EG if you have a type named my_struct, columns of that struct in your app
+will be automatically mapped to instances of `class MyStruct < Torque::Struct`, if it is defined.
+
+Nesting is supported; (eg structs can have fields that are themselves structs/arrays of structs).
+
 ## Querying
 
 * [Arel](https://github.com/crashtech/torque-postgresql/wiki/Arel)
@@ -83,3 +94,4 @@ Finally, fix and send a pull request.
 ## License
 
 Copyright © 2017- Carlos Silva. See [The MIT License](MIT-LICENSE) for further details.
+

lib/torque/postgresql.rb

@@ -28,4 +28,6 @@
 require 'torque/postgresql/reflection'
 require 'torque/postgresql/schema_cache'
 
+require 'torque/struct'
+
 require 'torque/postgresql/railtie' if defined?(Rails)

lib/torque/postgresql/adapter/database_statements.rb

@@ -43,6 +43,32 @@ def create_enum(name, *)
           load_additional_types([oid])
         end
 
+        # Given a name and a hash of fieldname->type, creates an enum type.
+        def create_struct(name, fields)
+          # TODO: Support macro types like e.g. :timestamp
+          sql_values = fields.map do |k,v|
+            "#{k} #{v}"
+          end.join(", ")
+          query = <<~SQL
+            DO $$
+            BEGIN
+                IF NOT EXISTS (
+                  SELECT 1 FROM pg_type t
+                  WHERE t.typname = '#{name}'
+                ) THEN
+                    CREATE TYPE \"#{name}\" AS (#{sql_values});
+                END IF;
+            END
+            $$;
+          SQL
+          exec_query(query)
+
+          # Since we've created a new type, type map needs to be rebooted to include
+          # the new ones, both normal and array one
+          oid = query_value("SELECT #{quote(name)}::regtype::oid", "SCHEMA").to_i
+          load_additional_types([oid])
+        end
+
         # Change some of the types being mapped
         def initialize_type_map(m = type_map)
           super
@@ -73,20 +99,28 @@ def torque_load_additional_types(oids = nil)
             INNER JOIN  pg_type a ON (a.oid = t.typarray)
             LEFT JOIN   pg_catalog.pg_namespace n ON n.oid = t.typnamespace
             WHERE       n.nspname NOT IN ('pg_catalog', 'information_schema')
-            AND     t.typtype IN ( 'e' )
+            AND     t.typtype IN ( 'e', 'c' )
             #{filter}
             AND     NOT EXISTS(
                       SELECT 1 FROM pg_catalog.pg_type el
                         WHERE el.oid = t.typelem AND el.typarray = t.oid
                       )
             AND     (t.typrelid = 0 OR (
-                      SELECT c.relkind = 'c' FROM pg_catalog.pg_class c
+                      SELECT c.relkind IN ('c', 'r') FROM pg_catalog.pg_class c
                         WHERE c.oid = t.typrelid
                       ))
           SQL
 
           execute_and_clear(query, 'SCHEMA', []) do |records|
-            records.each { |row| OID::Enum.create(row, type_map) }
+            records.each do |row|
+              if row['typtype'] == 'e'
+                OID::Enum.create(row, type_map)
+              elsif row['typtype'] == 'c'
+                OID::Struct.create(self, row, type_map)
+              else
+                raise "Invalid typetyp #{row['typtype'].inspect}, expected e (enum) or c (struct); #{row.inspect}"
+              end
+            end
           end
         end
 
@@ -101,6 +135,7 @@ def user_defined_types(*categories)
             SELECT t.typname AS name,
                    CASE t.typtype
                      WHEN 'e' THEN 'enum'
+                     WHEN 'c' THEN 'struct'
                      END     AS type
             FROM pg_type t
                    LEFT JOIN pg_catalog.pg_namespace n ON n.oid = t.typnamespace

lib/torque/postgresql/adapter/oid.rb

@@ -6,6 +6,7 @@
 require_relative 'oid/line'
 require_relative 'oid/range'
 require_relative 'oid/segment'
+require_relative 'oid/struct'
 
 module Torque
   module PostgreSQL

lib/torque/postgresql/adapter/oid/struct.rb

@@ -0,0 +1,195 @@
+
+# frozen_string_literal: true
+
+module Torque
+  module PostgreSQL
+    module Adapter
+      module OID
+        class Struct < ActiveModel::Type::Value
+          attr_reader :name
+          include ActiveRecord::ConnectionAdapters::Quoting
+          include ActiveRecord::ConnectionAdapters::PostgreSQL::Quoting
+
+          AvailableType = ::Struct.new(:type_map, :name, :oid, :arr_oid, :klass, :array_klass, :registered, keyword_init: true)
+
+          def self.for_type(name, klass: nil)
+            typ = _type_by_name(name)
+            return typ if !klass
+
+            raise "No type registered to #{name}" unless typ
+            return nil unless typ
+
+            if typ.registered
+              if typ.klass.klass != klass
+                if defined?(Rails) && !Rails.application.config.cache_classes && typ.klass.klass.name == klass.name
+                  typ.klass.klass = klass # Rails constant reloading
+                else
+                  raise "Class mismatch; #{name} already registered for #{typ.klass.klass.name}"
+                end
+              end
+            else
+              typ.klass.klass = klass
+              typ.type_map.register_type(typ.oid,     typ.klass)
+              typ.type_map.register_type(typ.arr_oid, typ.array_klass)
+              typ.registered = true
+            end
+
+            typ.name == name ? typ.klass : typ.array_klass
+          end
+
+          def self.register!(type_map, name, oid, arr_oid, klass, array_klass)
+            raise ArgumentError, "Already Registered" if _type_by_name(name)
+            available_types << AvailableType.new(
+              type_map: type_map,
+              name: name,
+              oid: oid,
+              arr_oid: arr_oid,
+              klass: klass,
+              array_klass: array_klass,
+            )
+          end
+
+          def self.available_types
+            @registry ||= []
+          end
+
+          def self._type_by_name(name)
+            available_types.find {|a| a.name == name || a.name + '[]' == name}
+          end
+
+          def self.create(connection, row, type_map)
+            name    = row['typname']
+            return if _type_by_name(name)
+
+            oid     = row['oid'].to_i
+            arr_oid = row['typarray'].to_i
+            type = Struct.new(connection, name)
+            arr_type = ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Array.new(type)
+
+            register!(type_map, name, oid, arr_oid, type, arr_type)
+          end
+
+          def initialize(connection, name)
+            @connection = connection # The connection we're attached to
+            @name = name
+
+            @pg_encoder = PG::TextEncoder::Record.new name: name
+            @pg_decoder = PG::TextDecoder::Record.new name: name
+            super()
+          end
+
+          def deserialize(value)
+            return unless value.present?
+            return super(value) unless klass
+            return value if value.is_a? klass
+            fields = PG::TextDecoder::Record.new.decode(value)
+            field_names = klass.columns.map(&:name)
+            attributes = Hash[field_names.zip(fields)]
+            field_names.each { |field| attributes[field] = klass.type_for_attribute(field).deserialize(attributes[field]) }
+            build_from_attrs(attributes, from_database: true)
+          end
+
+          def serialize(value)
+            return if value.blank?
+            return super(value) unless klass
+            value = cast_value(value)
+            if value.nil?
+              "NULL"
+            else
+              casted_values = klass.columns.map do |col|
+                col_value = value[col.name]
+                serialized = klass.type_for_attribute(col.name).serialize(col_value)
+                begin
+                  @connection.type_cast(serialized)
+                rescue TypeError => e
+                  if klass.type_for_attribute(col.name).class == ActiveModel::Type::Value
+                    # attribute :nested, NestedStruct.database_type
+                    col = klass.columns.find {|c| c.name == col.name }
+
+                    available_custom_type = self.class._type_by_name(col.sql_type)
+                    if available_custom_type && !available_custom_type.registered
+                      hint = "add `attribute :#{col.name}, #{col.sql_type.classify}.database_#{col.array ? 'array_' : ''}type`"
+                      raise e, "#{e} (in #{klass.name}, #{hint}`", $!.backtrace
+                    end
+                    raise
+                  else
+                    raise
+                  end
+                end
+              end
+              PG::TextEncoder::Record.new.encode(casted_values)
+            end
+          end
+
+          def assert_valid_value(value)
+            cast_value(value)
+          end
+
+          def type_cast_for_schema(value)
+            # TODO: Check default values for struct types work
+            serialize(value)
+          end
+
+          def ==(other)
+            self.class == other.class &&
+              other.klass == klass &&
+              other.type == type
+          end
+
+          def klass=(value)
+            raise ArgumentError, "Not a valid struct class" unless validate_klass(value)
+            @klass = value
+          end
+
+          def klass
+            @klass ||= validate_klass(name.to_s.camelize.singularize) || validate_klass(name.to_s.camelize.pluralize)
+            return nil unless @klass
+            if @klass.ancestors.include?(::ActiveRecord::Base)
+              return @klass if @klass.table_name == name
+            end
+            return nil unless @klass.ancestors.include?(::Torque::Struct)
+            @klass
+          end
+
+          def type_cast(value)
+            value
+          end
+
+          private
+
+            def validate_klass_name(class_name)
+              validate_klass class_name.safe_constantize
+            end
+
+            def validate_klass(klass)
+              if klass && klass.ancestors.include?(::Torque::Struct)
+                klass
+              elsif klass && klass.ancestors.include?(::ActiveRecord::Base)
+                klass.table_name == name ? klass : nil
+              else
+                false
+              end
+            end
+
+            def cast_value(value)
+              return if value.blank?
+              return if klass.blank?
+              return value if value.is_a?(klass)
+              build_from_attrs(value, from_database: false)
+            end
+
+            def build_from_attrs(attributes, from_database:)
+              klass.define_attribute_methods
+              if from_database
+                attributes = klass.attributes_builder.build_from_database(attributes, {})
+                klass.allocate.init_with_attributes(attributes)
+              else
+                klass.new(attributes)
+              end
+            end
+
+        end
+      end
+    end
+  end
+end

lib/torque/postgresql/inheritance.rb

@@ -60,6 +60,8 @@ def physically_inherited?
           ).present?
         rescue ActiveRecord::ConnectionNotEstablished
           false
+        rescue ActiveRecord::NoDatabaseError
+          false
         end
 
         # Get the list of all tables directly or indirectly dependent of the

lib/torque/postgresql/migration/command_recorder.rb

@@ -25,6 +25,14 @@ def invert_create_enum(args)
           [:drop_type, [args.first]]
         end
 
+        # Records the creation of the struct to be reverted.
+        def create_struct(*args, &block)
+          record(:create_struct, args, &block)
+        end
+        def invert_create_struct(*args)
+          [:drop_type, [args.first]]
+        end
+
       end
 
       ActiveRecord::Migration::CommandRecorder.include CommandRecorder

lib/torque/struct.rb

@@ -0,0 +1,101 @@
+
+require "active_support/concern"
+require "torque/postgresql/adapter"
+
+module Torque
+  class BaseStruct
+    def to_s
+      # Avoid printing excessive volumes
+      "#<#{self.class.name}>"
+    end
+
+    def _run_find_callbacks
+    end
+    def _run_initialize_callbacks
+    end
+
+    class << self
+      def connection
+        # Lets you overwrite `connection` per-class
+        ActiveRecord::Base.connection
+      end
+      class_attribute :primary_key
+      self.primary_key = "id"
+
+      def base_class?
+        self == BaseStruct || self == Struct
+      end
+
+      def base_class
+        BaseStruct
+      end
+
+      def table_name
+        nil
+      end
+
+      def abstract_class?
+        base_class?
+      end
+    end
+  end
+  class Struct < BaseStruct
+    include ActiveRecord::Core
+    include ActiveRecord::Persistence
+    include ActiveRecord::ModelSchema
+    include ActiveRecord::Attributes
+    include ActiveRecord::AttributeMethods
+    include ActiveRecord::Serialization
+    include ActiveRecord::AttributeAssignment
+    self.pluralize_table_names = false
+    def initialize(attributes = nil)
+      @attributes = self.class.attributes_builder.build_from_database
+      assign_attributes(attributes) if attributes
+      self.class.define_attribute_methods
+      yield self if block_given?
+    end
+
+    class << self
+
+      # ActiveRecord modules call `superclass.foo`, so we need an extra layer of inheritance
+      def database_type
+        ::Torque::PostgreSQL::Adapter::OID::Struct.for_type(table_name, klass: self)
+      end
+
+      def database_array_type
+        ::Torque::PostgreSQL::Adapter::OID::Struct.for_type(table_name + "[]", klass: self)
+      end
+
+      def table_exists?
+        ::Torque::PostgreSQL::Adapter::OID::Struct.for_type(table_name).present?
+      end
+
+      def type_name
+        table_name
+      end
+      def type_name=(value)
+        @type_name = value
+      end
+      def table_name
+        return @type_name if @type_name
+        if self === Struct
+          nil
+        else
+          self.name.underscore
+        end
+      end
+    end
+  end
+
+  class ActiveRecord::Base
+    class << self
+      def database_type
+        ::Torque::PostgreSQL::Adapter::OID::Struct.for_type(table_name, klass: self)
+      end
+
+      def database_array_type
+        ::Torque::PostgreSQL::Adapter::OID::Struct.for_type(table_name + "[]", klass: self)
+      end
+    end
+  end
+end

spec/models/inner_struct.rb

@@ -0,0 +1,5 @@
+require 'torque/postgresql'
+require_relative './question'
+class InnerStruct < Torque::Struct
+  attribute :question, Question.database_type
+end

spec/models/nested.rb

@@ -0,0 +1,5 @@
+require_relative './nested_struct'
+require 'torque/postgresql'
+class Nested < ActiveRecord::Base
+  attribute :nested, NestedStruct.database_type
+end

spec/models/nested_struct.rb

@@ -0,0 +1,4 @@
+require 'torque/postgresql'
+class NestedStruct < Torque::Struct
+  attribute :ary, InnerStruct.database_array_type
+end

spec/schema.rb

@@ -19,6 +19,7 @@
   # These are extensions that must be enabled in order to support this database
   enable_extension "pgcrypto"
   enable_extension "plpgsql"
+  enable_extension "hstore"
 
   # Custom types defined in this database.
   # Note that some types may not work with other database engines. Be careful if changing database.
@@ -144,6 +145,32 @@
     t.integer  "activated"
   end
 
+  create_struct "inner_struct", {
+    num: "smallint",
+    num_ary: "smallint[]",
+    str: "character varying(255)",
+    str_ary: "character varying(255)[]",
+    timestamp: "timestamp with time zone",
+    timestamp_ary: "timestamp with time zone[]",
+    hsh: "hstore",
+    json: "jsonb",
+    question: "questions",
+    question_ary: "questions[]"
+  }
+
+  create_struct "unregistered_struct", {
+    num: "smallint",
+  }
+
+  create_struct "nested_struct", {
+    ary: "inner_struct[]",
+    unregistered: "unregistered_struct[]" # for testing that unhandled UDT's do not break anything
+  }
+
+  create_table "nesteds", force: :cascade do |t|
+    t.column "nested", "nested_struct"
+  end
+
   create_table "activity_post_samples", force: :cascade, inherits: :activity_posts
 
   create_table "question_selects", force: :cascade, inherits: :questions do |t|

spec/tests/nested_spec.rb

@@ -0,0 +1,40 @@
+require 'spec_helper'
+
+RSpec.describe 'Composite Types' do
+
+  it "can save and load composite types" do
+    start = DateTime.now.utc
+    question = Question.create!(title: "single question")
+    questions = [Question.create!(title: "some question"), Question.create!(title: "some other question")]
+
+    instance = Nested.new
+
+    instance.nested = NestedStruct.new
+    instance.nested.ary = [InnerStruct.new, InnerStruct.new]
+
+    instance.nested.ary[0].num = 2
+    instance.nested.ary[0].num_ary = [3]
+    instance.nested.ary[0].str = "string contents"
+    instance.nested.ary[0].str_ary = ["string array contents", '", with quotes and commas,"']
+    instance.nested.ary[0].timestamp = start
+    instance.nested.ary[0].timestamp_ary = [start + 1.minute, start + 2.minutes]
+    instance.nested.ary[0].hsh = {"foo" => "bar"}
+    instance.nested.ary[0].json = [nil, {sym: 4}]
+    instance.nested.ary[0].question = question
+    instance.nested.ary[0].question_ary = questions
+    instance.save!
+    instance = Nested.find(instance.id)
+
+    expect(instance.nested.ary.length).to eq(2)
+    expect(instance.nested.ary[0].num).to eq(2)
+    expect(instance.nested.ary[0].num_ary).to eq([3])
+    expect(instance.nested.ary[0].str).to eq("string contents")
+    expect(instance.nested.ary[0].str_ary).to eq(["string array contents", '", with quotes and commas,"'])
+    expect(instance.nested.ary[0].timestamp.to_i).to eq(start.to_i)
+    expect(instance.nested.ary[0].timestamp_ary.map(&:to_i)).to eq([(start + 1.minute).to_i, (start + 2.minutes).to_i])
+    expect(instance.nested.ary[0].hsh).to eq({"foo" => "bar"})
+    expect(instance.nested.ary[0].json).to eq([nil, {"sym" => 4}])
+    expect(instance.nested.ary[0].question).to eq(question)
+    expect(instance.nested.ary[0].question_ary).to eq(questions)
+  end
+end