diff --git a/README.md b/README.md index 4939376..c8bc808 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,17 @@ These are the currently available features: * [Line](https://github.com/crashtech/torque-postgresql/wiki/Line) * [Segment](https://github.com/crashtech/torque-postgresql/wiki/Segment) +## Structs + +If you are using `create type X as (field_name Y, other_field_name Z)`, torque-postgresql will +automatically map a subclass of ::Torque::Struct to that type using the singular-form ActiveRecord +table naming rules. + +EG if you have a type named my_struct, columns of that struct in your app +will be automatically mapped to instances of `class MyStruct < Torque::Struct`, if it is defined. + +Nesting is supported; (eg structs can have fields that are themselves structs/arrays of structs). + ## Querying * [Arel](https://github.com/crashtech/torque-postgresql/wiki/Arel) @@ -83,3 +94,4 @@ Finally, fix and send a pull request. ## License Copyright © 2017- Carlos Silva. See [The MIT License](MIT-LICENSE) for further details. + diff --git a/lib/torque/postgresql.rb b/lib/torque/postgresql.rb index 2ee6f53..22c7695 100644 --- a/lib/torque/postgresql.rb +++ b/lib/torque/postgresql.rb @@ -28,4 +28,6 @@ require 'torque/postgresql/reflection' require 'torque/postgresql/schema_cache' +require 'torque/struct' + require 'torque/postgresql/railtie' if defined?(Rails) diff --git a/lib/torque/postgresql/adapter/database_statements.rb b/lib/torque/postgresql/adapter/database_statements.rb index 536df32..ef083c5 100644 --- a/lib/torque/postgresql/adapter/database_statements.rb +++ b/lib/torque/postgresql/adapter/database_statements.rb @@ -43,6 +43,32 @@ def create_enum(name, *) load_additional_types([oid]) end + # Given a name and a hash of fieldname->type, creates an enum type. + def create_struct(name, fields) + # TODO: Support macro types like e.g. :timestamp + sql_values = fields.map do |k,v| + "#{k} #{v}" + end.join(", ") + query = <<~SQL + DO $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_type t + WHERE t.typname = '#{name}' + ) THEN + CREATE TYPE \"#{name}\" AS (#{sql_values}); + END IF; + END + $$; + SQL + exec_query(query) + + # Since we've created a new type, type map needs to be rebooted to include + # the new ones, both normal and array one + oid = query_value("SELECT #{quote(name)}::regtype::oid", "SCHEMA").to_i + load_additional_types([oid]) + end + # Change some of the types being mapped def initialize_type_map(m = type_map) super @@ -73,20 +99,28 @@ def torque_load_additional_types(oids = nil) INNER JOIN pg_type a ON (a.oid = t.typarray) LEFT JOIN pg_catalog.pg_namespace n ON n.oid = t.typnamespace WHERE n.nspname NOT IN ('pg_catalog', 'information_schema') - AND t.typtype IN ( 'e' ) + AND t.typtype IN ( 'e', 'c' ) #{filter} AND NOT EXISTS( SELECT 1 FROM pg_catalog.pg_type el WHERE el.oid = t.typelem AND el.typarray = t.oid ) AND (t.typrelid = 0 OR ( - SELECT c.relkind = 'c' FROM pg_catalog.pg_class c + SELECT c.relkind IN ('c', 'r') FROM pg_catalog.pg_class c WHERE c.oid = t.typrelid )) SQL execute_and_clear(query, 'SCHEMA', []) do |records| - records.each { |row| OID::Enum.create(row, type_map) } + records.each do |row| + if row['typtype'] == 'e' + OID::Enum.create(row, type_map) + elsif row['typtype'] == 'c' + OID::Struct.create(self, row, type_map) + else + raise "Invalid typetyp #{row['typtype'].inspect}, expected e (enum) or c (struct); #{row.inspect}" + end + end end end @@ -101,6 +135,7 @@ def user_defined_types(*categories) SELECT t.typname AS name, CASE t.typtype WHEN 'e' THEN 'enum' + WHEN 'c' THEN 'struct' END AS type FROM pg_type t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = t.typnamespace diff --git a/lib/torque/postgresql/adapter/oid.rb b/lib/torque/postgresql/adapter/oid.rb index e6cb3cf..b329ed6 100644 --- a/lib/torque/postgresql/adapter/oid.rb +++ b/lib/torque/postgresql/adapter/oid.rb @@ -6,6 +6,7 @@ require_relative 'oid/line' require_relative 'oid/range' require_relative 'oid/segment' +require_relative 'oid/struct' module Torque module PostgreSQL diff --git a/lib/torque/postgresql/adapter/oid/struct.rb b/lib/torque/postgresql/adapter/oid/struct.rb new file mode 100644 index 0000000..1224bbb --- /dev/null +++ b/lib/torque/postgresql/adapter/oid/struct.rb @@ -0,0 +1,195 @@ + +# frozen_string_literal: true + +module Torque + module PostgreSQL + module Adapter + module OID + class Struct < ActiveModel::Type::Value + attr_reader :name + include ActiveRecord::ConnectionAdapters::Quoting + include ActiveRecord::ConnectionAdapters::PostgreSQL::Quoting + + AvailableType = ::Struct.new(:type_map, :name, :oid, :arr_oid, :klass, :array_klass, :registered, keyword_init: true) + + def self.for_type(name, klass: nil) + typ = _type_by_name(name) + return typ if !klass + + raise "No type registered to #{name}" unless typ + return nil unless typ + + if typ.registered + if typ.klass.klass != klass + if defined?(Rails) && !Rails.application.config.cache_classes && typ.klass.klass.name == klass.name + typ.klass.klass = klass # Rails constant reloading + else + raise "Class mismatch; #{name} already registered for #{typ.klass.klass.name}" + end + end + else + typ.klass.klass = klass + typ.type_map.register_type(typ.oid, typ.klass) + typ.type_map.register_type(typ.arr_oid, typ.array_klass) + typ.registered = true + end + + typ.name == name ? typ.klass : typ.array_klass + end + + def self.register!(type_map, name, oid, arr_oid, klass, array_klass) + raise ArgumentError, "Already Registered" if _type_by_name(name) + available_types << AvailableType.new( + type_map: type_map, + name: name, + oid: oid, + arr_oid: arr_oid, + klass: klass, + array_klass: array_klass, + ) + end + + def self.available_types + @registry ||= [] + end + + def self._type_by_name(name) + available_types.find {|a| a.name == name || a.name + '[]' == name} + end + + def self.create(connection, row, type_map) + name = row['typname'] + return if _type_by_name(name) + + oid = row['oid'].to_i + arr_oid = row['typarray'].to_i + type = Struct.new(connection, name) + arr_type = ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Array.new(type) + + register!(type_map, name, oid, arr_oid, type, arr_type) + end + + def initialize(connection, name) + @connection = connection # The connection we're attached to + @name = name + + @pg_encoder = PG::TextEncoder::Record.new name: name + @pg_decoder = PG::TextDecoder::Record.new name: name + super() + end + + def deserialize(value) + return unless value.present? + return super(value) unless klass + return value if value.is_a? klass + fields = PG::TextDecoder::Record.new.decode(value) + field_names = klass.columns.map(&:name) + attributes = Hash[field_names.zip(fields)] + field_names.each { |field| attributes[field] = klass.type_for_attribute(field).deserialize(attributes[field]) } + build_from_attrs(attributes, from_database: true) + end + + def serialize(value) + return if value.blank? + return super(value) unless klass + value = cast_value(value) + if value.nil? + "NULL" + else + casted_values = klass.columns.map do |col| + col_value = value[col.name] + serialized = klass.type_for_attribute(col.name).serialize(col_value) + begin + @connection.type_cast(serialized) + rescue TypeError => e + if klass.type_for_attribute(col.name).class == ActiveModel::Type::Value + # attribute :nested, NestedStruct.database_type + col = klass.columns.find {|c| c.name == col.name } + + available_custom_type = self.class._type_by_name(col.sql_type) + if available_custom_type && !available_custom_type.registered + hint = "add `attribute :#{col.name}, #{col.sql_type.classify}.database_#{col.array ? 'array_' : ''}type`" + raise e, "#{e} (in #{klass.name}, #{hint}`", $!.backtrace + end + raise + else + raise + end + end + end + PG::TextEncoder::Record.new.encode(casted_values) + end + end + + def assert_valid_value(value) + cast_value(value) + end + + def type_cast_for_schema(value) + # TODO: Check default values for struct types work + serialize(value) + end + + def ==(other) + self.class == other.class && + other.klass == klass && + other.type == type + end + + def klass=(value) + raise ArgumentError, "Not a valid struct class" unless validate_klass(value) + @klass = value + end + + def klass + @klass ||= validate_klass(name.to_s.camelize.singularize) || validate_klass(name.to_s.camelize.pluralize) + return nil unless @klass + if @klass.ancestors.include?(::ActiveRecord::Base) + return @klass if @klass.table_name == name + end + return nil unless @klass.ancestors.include?(::Torque::Struct) + @klass + end + + def type_cast(value) + value + end + + private + + def validate_klass_name(class_name) + validate_klass class_name.safe_constantize + end + + def validate_klass(klass) + if klass && klass.ancestors.include?(::Torque::Struct) + klass + elsif klass && klass.ancestors.include?(::ActiveRecord::Base) + klass.table_name == name ? klass : nil + else + false + end + end + + def cast_value(value) + return if value.blank? + return if klass.blank? + return value if value.is_a?(klass) + build_from_attrs(value, from_database: false) + end + + def build_from_attrs(attributes, from_database:) + klass.define_attribute_methods + if from_database + attributes = klass.attributes_builder.build_from_database(attributes, {}) + klass.allocate.init_with_attributes(attributes) + else + klass.new(attributes) + end + end + + end + end + end + end +end diff --git a/lib/torque/postgresql/inheritance.rb b/lib/torque/postgresql/inheritance.rb index 8c49296..9618894 100644 --- a/lib/torque/postgresql/inheritance.rb +++ b/lib/torque/postgresql/inheritance.rb @@ -60,6 +60,8 @@ def physically_inherited? ).present? rescue ActiveRecord::ConnectionNotEstablished false + rescue ActiveRecord::NoDatabaseError + false end # Get the list of all tables directly or indirectly dependent of the diff --git a/lib/torque/postgresql/migration/command_recorder.rb b/lib/torque/postgresql/migration/command_recorder.rb index e6bf830..f54c0d5 100644 --- a/lib/torque/postgresql/migration/command_recorder.rb +++ b/lib/torque/postgresql/migration/command_recorder.rb @@ -25,6 +25,14 @@ def invert_create_enum(args) [:drop_type, [args.first]] end + # Records the creation of the struct to be reverted. + def create_struct(*args, &block) + record(:create_struct, args, &block) + end + def invert_create_struct(*args) + [:drop_type, [args.first]] + end + end ActiveRecord::Migration::CommandRecorder.include CommandRecorder diff --git a/lib/torque/struct.rb b/lib/torque/struct.rb new file mode 100644 index 0000000..50de948 --- /dev/null +++ b/lib/torque/struct.rb @@ -0,0 +1,101 @@ + +require "active_support/concern" +require "torque/postgresql/adapter" + +module Torque + class BaseStruct + def to_s + # Avoid printing excessive volumes + "#<#{self.class.name}>" + end + + def _run_find_callbacks + end + def _run_initialize_callbacks + end + + class << self + def connection + # Lets you overwrite `connection` per-class + ActiveRecord::Base.connection + end + class_attribute :primary_key + self.primary_key = "id" + + def base_class? + self == BaseStruct || self == Struct + end + + def base_class + BaseStruct + end + + def table_name + nil + end + + def abstract_class? + base_class? + end + end + end + class Struct < BaseStruct + include ActiveRecord::Core + include ActiveRecord::Persistence + include ActiveRecord::ModelSchema + include ActiveRecord::Attributes + include ActiveRecord::AttributeMethods + include ActiveRecord::Serialization + include ActiveRecord::AttributeAssignment + self.pluralize_table_names = false + def initialize(attributes = nil) + @attributes = self.class.attributes_builder.build_from_database + assign_attributes(attributes) if attributes + self.class.define_attribute_methods + yield self if block_given? + end + + class << self + + # ActiveRecord modules call `superclass.foo`, so we need an extra layer of inheritance + def database_type + ::Torque::PostgreSQL::Adapter::OID::Struct.for_type(table_name, klass: self) + end + + def database_array_type + ::Torque::PostgreSQL::Adapter::OID::Struct.for_type(table_name + "[]", klass: self) + end + + def table_exists? + ::Torque::PostgreSQL::Adapter::OID::Struct.for_type(table_name).present? + end + + def type_name + table_name + end + def type_name=(value) + @type_name = value + end + def table_name + return @type_name if @type_name + if self === Struct + nil + else + self.name.underscore + end + end + end + end + + class ActiveRecord::Base + class << self + def database_type + ::Torque::PostgreSQL::Adapter::OID::Struct.for_type(table_name, klass: self) + end + + def database_array_type + ::Torque::PostgreSQL::Adapter::OID::Struct.for_type(table_name + "[]", klass: self) + end + end + end +end diff --git a/spec/models/inner_struct.rb b/spec/models/inner_struct.rb new file mode 100644 index 0000000..ef76a4f --- /dev/null +++ b/spec/models/inner_struct.rb @@ -0,0 +1,5 @@ +require 'torque/postgresql' +require_relative './question' +class InnerStruct < Torque::Struct + attribute :question, Question.database_type +end diff --git a/spec/models/nested.rb b/spec/models/nested.rb new file mode 100644 index 0000000..308773b --- /dev/null +++ b/spec/models/nested.rb @@ -0,0 +1,5 @@ +require_relative './nested_struct' +require 'torque/postgresql' +class Nested < ActiveRecord::Base + attribute :nested, NestedStruct.database_type +end diff --git a/spec/models/nested_struct.rb b/spec/models/nested_struct.rb new file mode 100644 index 0000000..853b2af --- /dev/null +++ b/spec/models/nested_struct.rb @@ -0,0 +1,4 @@ +require 'torque/postgresql' +class NestedStruct < Torque::Struct + attribute :ary, InnerStruct.database_array_type +end diff --git a/spec/schema.rb b/spec/schema.rb index d8f06ca..0abd6d9 100644 --- a/spec/schema.rb +++ b/spec/schema.rb @@ -19,6 +19,7 @@ # These are extensions that must be enabled in order to support this database enable_extension "pgcrypto" enable_extension "plpgsql" + enable_extension "hstore" # Custom types defined in this database. # Note that some types may not work with other database engines. Be careful if changing database. @@ -144,6 +145,32 @@ t.integer "activated" end + create_struct "inner_struct", { + num: "smallint", + num_ary: "smallint[]", + str: "character varying(255)", + str_ary: "character varying(255)[]", + timestamp: "timestamp with time zone", + timestamp_ary: "timestamp with time zone[]", + hsh: "hstore", + json: "jsonb", + question: "questions", + question_ary: "questions[]" + } + + create_struct "unregistered_struct", { + num: "smallint", + } + + create_struct "nested_struct", { + ary: "inner_struct[]", + unregistered: "unregistered_struct[]" # for testing that unhandled UDT's do not break anything + } + + create_table "nesteds", force: :cascade do |t| + t.column "nested", "nested_struct" + end + create_table "activity_post_samples", force: :cascade, inherits: :activity_posts create_table "question_selects", force: :cascade, inherits: :questions do |t| diff --git a/spec/tests/nested_spec.rb b/spec/tests/nested_spec.rb new file mode 100644 index 0000000..d087771 --- /dev/null +++ b/spec/tests/nested_spec.rb @@ -0,0 +1,40 @@ +require 'spec_helper' + +RSpec.describe 'Composite Types' do + + it "can save and load composite types" do + start = DateTime.now.utc + question = Question.create!(title: "single question") + questions = [Question.create!(title: "some question"), Question.create!(title: "some other question")] + + instance = Nested.new + + instance.nested = NestedStruct.new + instance.nested.ary = [InnerStruct.new, InnerStruct.new] + + instance.nested.ary[0].num = 2 + instance.nested.ary[0].num_ary = [3] + instance.nested.ary[0].str = "string contents" + instance.nested.ary[0].str_ary = ["string array contents", '", with quotes and commas,"'] + instance.nested.ary[0].timestamp = start + instance.nested.ary[0].timestamp_ary = [start + 1.minute, start + 2.minutes] + instance.nested.ary[0].hsh = {"foo" => "bar"} + instance.nested.ary[0].json = [nil, {sym: 4}] + instance.nested.ary[0].question = question + instance.nested.ary[0].question_ary = questions + instance.save! + instance = Nested.find(instance.id) + + expect(instance.nested.ary.length).to eq(2) + expect(instance.nested.ary[0].num).to eq(2) + expect(instance.nested.ary[0].num_ary).to eq([3]) + expect(instance.nested.ary[0].str).to eq("string contents") + expect(instance.nested.ary[0].str_ary).to eq(["string array contents", '", with quotes and commas,"']) + expect(instance.nested.ary[0].timestamp.to_i).to eq(start.to_i) + expect(instance.nested.ary[0].timestamp_ary.map(&:to_i)).to eq([(start + 1.minute).to_i, (start + 2.minutes).to_i]) + expect(instance.nested.ary[0].hsh).to eq({"foo" => "bar"}) + expect(instance.nested.ary[0].json).to eq([nil, {"sym" => 4}]) + expect(instance.nested.ary[0].question).to eq(question) + expect(instance.nested.ary[0].question_ary).to eq(questions) + end +end