diff --git a/docker-compose.yaml b/docker-compose.yaml index 5e18cb6..17f0bd2 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,13 +1,17 @@ version: "2.2" services: postgres: - image: postgres:13 + # image: postgres:13 + build: + dockerfile: ./postgres.Dockerfile ports: - "65432:5432" environment: - POSTGRES_USER=postgres - POSTGRES_PASSWORD=memex - POSTGRES_DB=memex + networks: + - memex1 volumes: - /dev/urandom:/dev/random # Required to get non-blocking entropy source - postgres-db:/var/lib/postgresql/data @@ -23,7 +27,7 @@ services: "postgres", "-c", "select 1", - "memex", + "memex" ] interval: 10s timeout: 10s @@ -32,3 +36,7 @@ services: volumes: postgres-db: + + +networks: + memex1: diff --git a/memex/config/config.exs b/memex/config/config.exs index 5bc99a3..044834f 100644 --- a/memex/config/config.exs +++ b/memex/config/config.exs @@ -26,7 +26,8 @@ config :memex, ecto_repos: [Memex.Repo] config :memex, Memex.Repo, url: System.get_env("POSTGRES_DSN"), pool_size: 5, - timeout: 60_000 + timeout: 60_000, + types: Memex.PostgrexTypes config :elixir, :time_zone_database, Tzdata.TimeZoneDatabase diff --git a/memex/lib/memex/ai/sentence_transformers.ex b/memex/lib/memex/ai/sentence_transformers.ex new file mode 100644 index 0000000..cb3a2a8 --- /dev/null +++ b/memex/lib/memex/ai/sentence_transformers.ex @@ -0,0 +1,114 @@ +defmodule Memex.Ai.SentenceTransformers do + alias Bumblebee.Shared + + def embed(text) when is_binary(text) do + embedding = + Nx.Serving.batched_run(__MODULE__, [text]) + |> Nx.to_flat_list() + end + + """ + {:ok, _} = + Supervisor.start_link( + [ + PhoenixDemo.Endpoint, + {Nx.Serving, serving: Memex.Ai.SentenceTransformers.serving(), name: Memex.Ai.SentenceTransformers, batch_timeout: 100} + ], + strategy: :one_for_one + ) + """ + + def serving() do + model_name = "sentence-transformers/all-MiniLM-L6-v2" + {:ok, model_info} = Bumblebee.load_model({:hf, model_name}) + {:ok, tokenizer} = Bumblebee.load_tokenizer({:hf, model_name}) + + Memex.Ai.SentenceTransformers.sentence_embeddings(model_info, tokenizer, + compile: [batch_size: 10, sequence_length: 128], + defn_options: [compiler: EXLA] + ) + end + + def test(query \\ "TicketSwap") do + embedding = Nx.Serving.batched_run(__MODULE__, [query]) + + embeddings = + Nx.Serving.batched_run(__MODULE__, [ + "pgvector/pgvector: Open-source vector similarity search for Postgres MacBook Pro: https://github.com/pgvector/pgvector", + "Install Jupyter Notebook | Learn How to Install and Use Jupyter Notebook", + "A picture of London at night", + "Fastai on Apple M1 - Deep Learning - fast.ai Course Forums: https://forums.fast.ai/t/fastai-on-apple-m1/86059/50" + ]) + + Bumblebee.Utils.Nx.cosine_similarity(embedding, embeddings) + end + + def sentence_embeddings(model_info, tokenizer, opts \\ []) do + %{model: model, params: params, spec: spec} = model_info + Shared.validate_architecture!(spec, :base) + opts = Keyword.validate!(opts, [:compile, defn_options: []]) + + compile = opts[:compile] + defn_options = opts[:defn_options] + + batch_size = compile[:batch_size] + sequence_length = compile[:sequence_length] + + if compile != nil and (batch_size == nil or sequence_length == nil) do + raise ArgumentError, + "expected :compile to be a keyword list specifying :batch_size and :sequence_length, got: #{inspect(compile)}" + end + + {_init_fun, predict_fun} = Axon.build(model) + + scores_fun = fn params, input -> + outputs = predict_fun.(params, input) + outputs.pooled_state + end + + Nx.Serving.new( + fn -> + scores_fun = + Shared.compile_or_jit(scores_fun, defn_options, compile != nil, fn -> + inputs = %{ + "input_ids" => Nx.template({batch_size, sequence_length}, :s64), + "token_type_ids" => Nx.template({batch_size, sequence_length}, :s64), + "attention_mask" => Nx.template({batch_size, sequence_length}, :s64) + } + + [params, inputs] + end) + + fn inputs -> + inputs = Shared.maybe_pad(inputs, batch_size) + scores_fun.(params, inputs) + end + end, + batch_size: batch_size + ) + |> Nx.Serving.client_preprocessing(fn input -> + {texts, multi?} = Shared.validate_serving_input!(input, &is_binary/1, "a string") + + inputs = Bumblebee.apply_tokenizer(tokenizer, texts) + + {Nx.Batch.concatenate([inputs]), multi?} + end) + |> Nx.Serving.client_postprocessing(fn scores, metadata, multi? -> + scores |> IO.inspect(label: "103") + multi? |> IO.inspect(label: "98") + metadata |> IO.inspect(label: "99") + # Mean Pooling - Take attention mask into account for correct averaging + # def mean_pooling(model_output, attention_mask): + # token_embeddings = model_output[0] #First element of model_output contains all token embeddings + # input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float() + # return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9) + + token_embeddings = scores[0] + input_mask_expanded = Nx.unsqueece(token_embeddings, -1) + Nx.size(token_embeddings) + + scores + |> Shared.normalize_output(multi?) + end) + end +end diff --git a/memex/lib/memex/application.ex b/memex/lib/memex/application.ex index ee121ed..b0a4112 100644 --- a/memex/lib/memex/application.ex +++ b/memex/lib/memex/application.ex @@ -18,7 +18,11 @@ defmodule Memex.Application do {Memex.Repo, []}, # Start a worker by calling: Memex.Worker.start_link(arg) # {Memex.Worker, arg} - {Memex.Scheduler, []} + {Memex.Scheduler, []}, + {Nx.Serving, + serving: Memex.Ai.SentenceTransformers.serving(), + name: Memex.Ai.SentenceTransformers, + batch_timeout: 100} ] # See https://hexdocs.pm/elixir/Supervisor.html diff --git a/memex/lib/memex/search/postgres.ex b/memex/lib/memex/search/postgres.ex index 1fab088..8159a1f 100644 --- a/memex/lib/memex/search/postgres.ex +++ b/memex/lib/memex/search/postgres.ex @@ -8,6 +8,7 @@ defmodule Memex.Search.Postgres do from(d in Document) |> add_search(query) |> add_select(query) + # |> add_vector_search(query) |> add_filters(prepare_filters(query)) |> add_limit(query) |> add_order_by(query.order_by) @@ -29,6 +30,12 @@ defmodule Memex.Search.Postgres do from(q in q, where: fragment("? @@ to_tsquery('simple', ?)", q.search, ^to_tsquery(query))) end + defp add_vector_search(q, %Query{query: ""} = _query), do: q + + defp add_vector_search(q, %Query{} = query) do + from(q in q, where: fragment("? <=> ? > 0.6", q.search_embedding, ^to_vector(query))) + end + defp add_select(q, %Query{select: :hits_with_highlights} = _query) do from(q in q, select: q) end @@ -154,6 +161,10 @@ defmodule Memex.Search.Postgres do |> String.trim() end + defp to_vector(%Query{} = query) do + Memex.Ai.SentenceTransformers.embed(query.query) + end + defp format_results(results, %Query{select: :hits_with_highlights} = query) do results |> Enum.map(&put_in(&1, ["hit", "_formatted"], format_hit(&1["hit"], query))) diff --git a/memex/lib/postgrex_types.ex b/memex/lib/postgrex_types.ex new file mode 100644 index 0000000..bea5ed6 --- /dev/null +++ b/memex/lib/postgrex_types.ex @@ -0,0 +1,5 @@ +Postgrex.Types.define( + Memex.PostgrexTypes, + [Pgvector.Extensions.Vector] ++ Ecto.Adapters.Postgres.extensions(), + [] +) diff --git a/memex/mix.exs b/memex/mix.exs index 140f264..2311f92 100644 --- a/memex/mix.exs +++ b/memex/mix.exs @@ -47,6 +47,11 @@ defmodule Memex.MixProject do # Type `mix help deps` for examples and options. defp deps do [ + {:axon, "~> 0.3.0"}, + {:axon_onnx, path: "./axon_onnx"}, + {:bumblebee, "~> 0.1.0"}, + {:exla, "~> 0.4.0"}, + {:nx, "~> 0.4.0"}, {:con_cache, "~> 0.13"}, {:earmark, "~> 1.4.16"}, {:ecto_sql, "~> 3.4"}, @@ -58,6 +63,7 @@ defmodule Memex.MixProject do {:gettext, "~> 0.11"}, {:git_diff, "~> 0.6.2"}, {:hackney, "~> 1.16.0"}, + {:heroicons, "~> 0.2.2"}, {:jason, "~> 1.0"}, {:money, "~> 1.9"}, {:month, "~> 2.1"}, @@ -68,12 +74,15 @@ defmodule Memex.MixProject do {:phoenix, "~> 1.6.0"}, {:plug_cowboy, "~> 2.0"}, {:postgrex, "~> 0.15.9"}, + {:pgvector, "~> 0.1.0"}, + {:rustler, ">= 0.0.0", optional: true}, {:surface, "~> 0.9.0"}, {:surface_catalogue, "~> 0.5.0"}, {:surface_formatter, "~> 0.7.5", only: :dev}, {:telemetry_metrics, "~> 0.6"}, {:telemetry_poller, "~> 0.5"}, {:tesla, "~> 1.4.0"}, + {:tokenizers, "~> 0.2.0"}, {:tzdata, "~> 1.0"} ] end diff --git a/memex/mix.lock b/memex/mix.lock index e565928..60dccc0 100644 --- a/memex/mix.lock +++ b/memex/mix.lock @@ -1,7 +1,11 @@ %{ + "axon": {:hex, :axon, "0.3.1", "d2f678871d439ff623c50eecb255e2e89de9e2cd5d4bb7a40d9a80f3a46b86d1", [:mix], [{:kino, "~> 0.7", [hex: :kino, repo: "hexpm", optional: true]}, {:nx, "~> 0.4.0", [hex: :nx, repo: "hexpm", optional: false]}, {:table_rex, "~> 3.1.1", [hex: :table_rex, repo: "hexpm", optional: true]}], "hexpm", "01e0c085a3f26d0cccd44e29a2f308c537ad71d573bb853130f1fd484caccd55"}, + "axon_onnx": {:hex, :axon_onnx, "0.3.0", "2d4facd98742fcb2a08477b8d06a82c5d4563bc831ab5ba91ac7214426e9c817", [:mix], [{:axon, "~> 0.3.0", [hex: :axon, repo: "hexpm", optional: false]}, {:nx, "~> 0.4.0", [hex: :nx, repo: "hexpm", optional: false]}, {:protox, "~> 1.6.10", [hex: :protox, repo: "hexpm", optional: false]}], "hexpm", "87e08c82bb4c754092f2988d513806859f98e9b68b911273f5b39f6596cd11fc"}, + "bumblebee": {:hex, :bumblebee, "0.1.0", "285cfa58dd82d8361165fdf6217565bb79aa0218937094eb007195190d717c7b", [:mix], [{:axon, "~> 0.3.1", [hex: :axon, repo: "hexpm", optional: false]}, {:castore, "~> 0.1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.4.0", [hex: :jason, repo: "hexpm", optional: false]}, {:nx, "~> 0.4.1", [hex: :nx, repo: "hexpm", optional: false]}, {:nx_image, "~> 0.1.0", [hex: :nx_image, repo: "hexpm", optional: false]}, {:progress_bar, "~> 2.0", [hex: :progress_bar, repo: "hexpm", optional: false]}, {:tokenizers, "~> 0.2.0", [hex: :tokenizers, repo: "hexpm", optional: false]}, {:unpickler, "~> 0.1.0", [hex: :unpickler, repo: "hexpm", optional: false]}], "hexpm", "9e5de46e5dd53ac7b9c2e5a0bd305f314b2a31f4035935ef1071530e29044c7c"}, "calendar_interval": {:hex, :calendar_interval, "0.2.0", "2b253b1e37ee1d4344639a3cbfb12abd0e996e4a8181537eb33c3e93fdfaffd9", [:mix], [], "hexpm", "c13d5e0108e61808a38f622987e1c5e881d96d28945213d3efe6dd06c28ba7b0"}, - "castore": {:hex, :castore, "0.1.18", "deb5b9ab02400561b6f5708f3e7660fc35ca2d51bfc6a940d2f513f89c2975fc", [:mix], [], "hexpm", "61bbaf6452b782ef80b33cdb45701afbcf0a918a45ebe7e73f1130d661e66a06"}, + "castore": {:hex, :castore, "0.1.20", "62a0126cbb7cb3e259257827b9190f88316eb7aa3fdac01fd6f2dfd64e7f46e9", [:mix], [], "hexpm", "a020b7650529c986c454a4035b6b13a328e288466986307bea3aadb4c95ac98a"}, "certifi": {:hex, :certifi, "2.5.2", "b7cfeae9d2ed395695dd8201c57a2d019c0c43ecaf8b8bcb9320b40d6662f340", [:rebar3], [{:parse_trans, "~>3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm", "3b3b5f36493004ac3455966991eaf6e768ce9884693d9968055aeeeb1e575040"}, + "complex": {:hex, :complex, "0.4.2", "923e5db0be13dbb3ea00cf8459d9f75f3afdd9ff5a82742ded21064330d28273", [:mix], [], "hexpm", "069a085ef820ce675a2619fd125b963ff4514af2102c7f7d7965128e5ec0a429"}, "con_cache": {:hex, :con_cache, "0.14.0", "863acb90fa08017be3129074993af944cf7a4b6c3ee7c06c5cd0ed6b94fbc223", [:mix], [], "hexpm", "50887a8949377d0b707a3c6653b7610de06074751b52d0f267f52135f391aece"}, "connection": {:hex, :connection, "1.1.0", "ff2a49c4b75b6fb3e674bfc5536451607270aac754ffd1bdfe175abe4a6d7a68", [:mix], [], "hexpm", "722c1eb0a418fbe91ba7bd59a47e28008a189d47e37e0e7bb85585a016b2869c"}, "cowboy": {:hex, :cowboy, "2.9.0", "865dd8b6607e14cf03282e10e934023a1bd8be6f6bacf921a7e2a96d800cd452", [:make, :rebar3], [{:cowlib, "2.11.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "2c729f934b4e1aa149aff882f57c6372c15399a20d54f65c8d67bef583021bde"}, @@ -9,6 +13,7 @@ "cowlib": {:hex, :cowlib, "2.11.0", "0b9ff9c346629256c42ebe1eeb769a83c6cb771a6ee5960bd110ab0b9b872063", [:make, :rebar3], [], "hexpm", "2b3e9da0b21c4565751a6d4901c20d1b4cc25cbb7fd50d91d2ab6dd287bc86a9"}, "db_connection": {:hex, :db_connection, "2.4.2", "f92e79aff2375299a16bcb069a14ee8615c3414863a6fef93156aee8e86c2ff3", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "4fe53ca91b99f55ea249693a0229356a08f4d1a7931d8ffa79289b145fe83668"}, "decimal": {:hex, :decimal, "2.0.0", "a78296e617b0f5dd4c6caf57c714431347912ffb1d0842e998e9792b5642d697", [:mix], [], "hexpm", "34666e9c55dea81013e77d9d87370fe6cb6291d1ef32f46a1600230b1d44f577"}, + "dll_loader_helper": {:hex, :dll_loader_helper, "0.1.8", "1621409a3cb06c750fe845bf954785cffa5fe8f2fca41006008b891877603bf7", [:make, :mix, :rebar3], [], "hexpm", "cd373dc6a028f3e37eca26b073e3a75249513db2f9b0e42520423886801fa7d7"}, "earmark": {:hex, :earmark, "1.4.27", "b413b0379043df51475a9b22ce344e8a58a117516c735b8871e6cdd5ed0f0153", [:mix], [{:earmark_parser, "~> 1.4.26", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "579ebe2eaf4c7e040815a73a268036bcd96e6aab8ad2b1fcd979aaeb1ea47e15"}, "earmark_parser": {:hex, :earmark_parser, "1.4.26", "f4291134583f373c7d8755566122908eb9662df4c4b63caa66a0eabe06569b0a", [:mix], [], "hexpm", "48d460899f8a0c52c5470676611c01f64f3337bad0b26ddab43648428d94aabc"}, "ecto": {:hex, :ecto, "3.6.2", "efdf52acfc4ce29249bab5417415bd50abd62db7b0603b8bab0d7b996548c2bc", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "efad6dfb04e6f986b8a3047822b0f826d9affe8e4ebdd2aeedbfcb14fd48884e"}, @@ -16,6 +21,7 @@ "elixir_make": {:hex, :elixir_make, "0.6.3", "bc07d53221216838d79e03a8019d0839786703129599e9619f4ab74c8c096eac", [:mix], [], "hexpm", "f5cbd651c5678bcaabdbb7857658ee106b12509cd976c2c2fca99688e1daf716"}, "esbuild": {:hex, :esbuild, "0.5.0", "d5bb08ff049d7880ee3609ed5c4b864bd2f46445ea40b16b4acead724fb4c4a3", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}], "hexpm", "f183a0b332d963c4cfaf585477695ea59eef9a6f2204fdd0efa00e099694ffe5"}, "ex_check": {:hex, :ex_check, "0.13.0", "f1389bae8703703b4f4d71c8ed5d117cd4b94b5843a68e6b27e47d4d2c680520", [:mix], [], "hexpm", "93d17de0e6d5bce63da51f2293098fc4408f0cf9350d0179eb71c9c3b6131e49"}, + "exla": {:hex, :exla, "0.4.0", "96665648dfc8daea9ba4cec8cd0ed1314964d8810e9f2a26143c46519e488c77", [:make, :mix], [{:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:nx, "~> 0.4.0", [hex: :nx, repo: "hexpm", optional: false]}, {:xla, "~> 0.3.0", [hex: :xla, repo: "hexpm", optional: false]}], "hexpm", "75ddd86ec890ca3271cf4c39eec020e189aba6a421f48c6e137858a09a2f3ae0"}, "exqlite": {:hex, :exqlite, "0.11.3", "9e02e3b7deca65b404a919c100151a3762b4fab92a884f876ca1a7335fffe80f", [:make, :mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "c49e4ffed36dc5aea2f21f44a7a30cd7c3d59d397c4fa3eebbe1f991f31b9c6e"}, "file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"}, "finch": {:hex, :finch, "0.8.2", "3e3f3f7a333b998a79d6f6d6bba07fad034c3d2b57852820593176f4601d6b19", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: false]}, {:mint, "~> 1.3", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.3.5", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "01b559829dba1a0fe1fd1820a80b192071d28399b45d929c242db34c74203ca1"}, @@ -23,6 +29,7 @@ "gettext": {:hex, :gettext, "0.18.2", "7df3ea191bb56c0309c00a783334b288d08a879f53a7014341284635850a6e55", [:mix], [], "hexpm", "f9f537b13d4fdd30f3039d33cb80144c3aa1f8d9698e47d7bcbcc8df93b1f5c5"}, "git_diff": {:hex, :git_diff, "0.6.2", "07ae323e7ffafa625b209131bf4b44bbadcce70259f9659379fedad4310b38b2", [:mix], [], "hexpm", "b96ab8886f6d63bdde51b748b2682c2996b208c2a58fe4cd18e11f6ce623cdec"}, "hackney": {:hex, :hackney, "1.16.0", "5096ac8e823e3a441477b2d187e30dd3fff1a82991a806b2003845ce72ce2d84", [:rebar3], [{:certifi, "2.5.2", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.1", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.3.0", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.6", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "3bf0bebbd5d3092a3543b783bf065165fa5d3ad4b899b836810e513064134e18"}, + "heroicons": {:hex, :heroicons, "0.2.4", "12824795f25340415bffa9a501b96c89dc856ed732e82acccdab635fee92411b", [:mix], [{:phoenix_html, "~> 2.14 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: false]}], "hexpm", "7d6c51dc8ecaadb37943247da83ec2fbf3f6b479a23f4cf0195bd61ec2268128"}, "html_entities": {:hex, :html_entities, "0.5.2", "9e47e70598da7de2a9ff6af8758399251db6dbb7eebe2b013f2bbd2515895c3c", [:mix], [], "hexpm", "c53ba390403485615623b9531e97696f076ed415e8d8058b1dbaa28181f4fdcc"}, "idna": {:hex, :idna, "6.0.1", "1d038fb2e7668ce41fbf681d2c45902e52b3cb9e9c77b55334353b222c2ee50c", [:rebar3], [{:unicode_util_compat, "0.5.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "a02c8a1c4fd601215bb0b0324c8a6986749f807ce35f25449ec9e69758708122"}, "inflex": {:hex, :inflex, "1.5.0", "e4ff5d900280b2011b24d1ac1c4590986ee5add2ea644c9894e72213cf93ff0b", [:mix], [], "hexpm", "d48609edc5bb7901b95dcc00c1e38f259e8006904865a028954ccfe9336a3384"}, @@ -40,7 +47,10 @@ "nimble_parsec": {:hex, :nimble_parsec, "1.2.3", "244836e6e3f1200c7f30cb56733fd808744eca61fd182f731eac4af635cc6d0b", [:mix], [], "hexpm", "c8d789e39b9131acf7b99291e93dae60ab48ef14a7ee9d58c6964f59efb570b0"}, "nimble_pool": {:hex, :nimble_pool, "0.2.4", "1db8e9f8a53d967d595e0b32a17030cdb6c0dc4a451b8ac787bf601d3f7704c3", [:mix], [], "hexpm", "367e8071e137b787764e6a9992ccb57b276dc2282535f767a07d881951ebeac6"}, "number": {:hex, :number, "1.0.3", "932c8a2d478a181c624138958ca88a78070332191b8061717270d939778c9857", [:mix], [{:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}], "hexpm", "dd397bbc096b2ca965a6a430126cc9cf7b9ef7421130def69bcf572232ca0f18"}, + "nx": {:hex, :nx, "0.4.1", "3cc8e420d0835ab7cac94f253950dee3ff927c68798ae88a3e4ff184a825b042", [:mix], [{:complex, "~> 0.4.2", [hex: :complex, repo: "hexpm", optional: false]}], "hexpm", "0b33fccaf76ebc6e79d53fe1149a70f99838e6505e9e7092e5a0a57b131b27c6"}, + "nx_image": {:hex, :nx_image, "0.1.0", "ae10fa41fa95126f934d6160ef4320f7db583535fb868415f2562fe19969d245", [:mix], [{:nx, "~> 0.4", [hex: :nx, repo: "hexpm", optional: false]}], "hexpm", "60a2928164cdca540b4c180ff25579b97a5f2a650fc890d40db3e1a7798c93ad"}, "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm", "17ef63abde837ad30680ea7f857dd9e7ced9476cdd7b0394432af4bfc241b960"}, + "pgvector": {:hex, :pgvector, "0.1.2", "53ec02fc7cbdc6831e0077149d9fab8821edda2b51e8f136c10b7caf072ad14d", [:mix], [{:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: false]}], "hexpm", "ac9bcee00c8388bcfb97fbc21029d1c827fac5e11e37daab2c87f85856207fd5"}, "phoenix": {:hex, :phoenix, "1.6.12", "f8f8ac077600f84419806dd53114b2e77aedde7a502e74181a7d886355aa0643", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.0", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 1.0", [hex: :phoenix_view, repo: "hexpm", optional: false]}, {:plug, "~> 1.10", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.2", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "2d6cf5583c9c20f7103c40e6014ef802d96553b8e5d6585ad6e627bd5ddb0d12"}, "phoenix_ecto": {:hex, :phoenix_ecto, "4.4.0", "0672ed4e4808b3fbed494dded89958e22fb882de47a97634c0b13e7b0b5f7720", [:mix], [{:ecto, "~> 3.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "09864e558ed31ee00bd48fcc1d4fc58ae9678c9e81649075431e69dbabb43cc1"}, "phoenix_html": {:hex, :phoenix_html, "3.2.0", "1c1219d4b6cb22ac72f12f73dc5fad6c7563104d083f711c3fcd8551a1f4ae11", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "36ec97ba56d25c0136ef1992c37957e4246b649d620958a1f9fa86165f8bc54f"}, @@ -55,7 +65,11 @@ "poison": {:hex, :poison, "3.0.0", "625ebd64d33ae2e65201c2c14d6c85c27cc8b68f2d0dd37828fde9c6920dd131", [:mix], [], "hexpm", "813db7909e9aa20b2f4d42692798beef7a430691b424d0e6b08f7cb11727c721"}, "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm", "dad79704ce5440f3d5a3681c8590b9dc25d1a561e8f5a9c995281012860901e3"}, "postgrex": {:hex, :postgrex, "0.15.9", "46f8fe6f25711aeb861c4d0ae09780facfdf3adbd2fb5594ead61504dd489bda", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "610719103e4cb2223d4ab78f9f0f3e720320eeca6011415ab4137ddef730adee"}, + "progress_bar": {:hex, :progress_bar, "2.0.1", "7b40200112ae533d5adceb80ff75fbe66dc753bca5f6c55c073bfc122d71896d", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}], "hexpm", "2519eb58a2f149a3a094e729378256d8cb6d96a259ec94841bd69fdc71f18f87"}, + "protox": {:hex, :protox, "1.6.10", "41d0b0c5b9190e7d5e6a2b1a03a09257ead6f3d95e6a0cf8b81430b526126908", [:mix], [{:decimal, "~> 1.9 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "9769fca26ae7abfc5cc61308a1e8d9e2400ff89a799599cee7930d21132832d9"}, "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, + "rustler": {:hex, :rustler, "0.26.0", "06a2773d453ee3e9109efda643cf2ae633dedea709e2455ac42b83637c9249bf", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:toml, "~> 0.6", [hex: :toml, repo: "hexpm", optional: false]}], "hexpm", "42961e9d2083d004d5a53e111ad1f0c347efd9a05cb2eb2ffa1d037cdc74db91"}, + "rustler_precompiled": {:hex, :rustler_precompiled, "0.5.5", "a075a92c8e748ce5c4f7b2cf573a072d206a6d8d99c53f627e81d3f2b10616a3", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: false]}, {:rustler, "~> 0.23", [hex: :rustler, repo: "hexpm", optional: true]}], "hexpm", "e8a7f1abfec8d68683bb25d14efc88496f091ef113f7f4c45d39f3606f7223f6"}, "sourceror": {:hex, :sourceror, "0.11.2", "549ce48be666421ac60cfb7f59c8752e0d393baa0b14d06271d3f6a8c1b027ab", [:mix], [], "hexpm", "9ab659118896a36be6eec68ff7b0674cba372fc8e210b1e9dc8cf2b55bb70dfb"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.6", "cf344f5692c82d2cd7554f5ec8fd961548d4fd09e7d22f5b62482e5aeaebd4b0", [:make, :mix, :rebar3], [], "hexpm", "bdb0d2471f453c88ff3908e7686f86f9be327d065cc1ec16fa4540197ea04680"}, "surface": {:hex, :surface, "0.9.1", "6a343564b1d6c17c619ac933cec5680ffe8c68f0cd2d85f780b70f6607750a96", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 0.18.0", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}, {:sourceror, "~> 0.11", [hex: :sourceror, repo: "hexpm", optional: false]}], "hexpm", "131312d35d190739d0e0f1681acb9fce6962d3f81439011a3331bad2976ca372"}, @@ -65,6 +79,11 @@ "telemetry_metrics": {:hex, :telemetry_metrics, "0.6.1", "315d9163a1d4660aedc3fee73f33f1d355dcc76c5c3ab3d59e76e3edf80eef1f", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7be9e0871c41732c233be71e4be11b96e56177bf15dde64a8ac9ce72ac9834c6"}, "telemetry_poller": {:hex, :telemetry_poller, "0.5.1", "21071cc2e536810bac5628b935521ff3e28f0303e770951158c73eaaa01e962a", [:rebar3], [{:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "4cab72069210bc6e7a080cec9afffad1b33370149ed5d379b81c7c5f0c663fd4"}, "tesla": {:hex, :tesla, "1.4.0", "1081bef0124b8bdec1c3d330bbe91956648fb008cf0d3950a369cda466a31a87", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:finch, "~> 0.3", [hex: :finch, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, "~> 1.3", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "bf1374a5569f5fca8e641363b63f7347d680d91388880979a33bc12a6eb3e0aa"}, + "tokenizers": {:hex, :tokenizers, "0.2.0", "3aa9811396680f849803f6a3978a310a653059613592710ce5f883d67ff17a33", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: false]}, {:rustler, ">= 0.0.0", [hex: :rustler, repo: "hexpm", optional: true]}, {:rustler_precompiled, "~> 0.5", [hex: :rustler_precompiled, repo: "hexpm", optional: false]}], "hexpm", "2496fd44cf96bcefc70e75cf7e34126de8b63ccb9ad35967d6d5d8661cbdb6b7"}, + "toml": {:hex, :toml, "0.7.0", "fbcd773caa937d0c7a02c301a1feea25612720ac3fa1ccb8bfd9d30d822911de", [:mix], [], "hexpm", "0690246a2478c1defd100b0c9b89b4ea280a22be9a7b313a8a058a2408a2fa70"}, + "torchx": {:hex, :torchx, "0.4.1", "5aa7f93d7aff85c9f5fbae4c534affa9d16a9ffe9bbbb261cf2dca8ead2f6ab8", [:make, :mix], [{:dll_loader_helper, "~> 0.1.0", [hex: :dll_loader_helper, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:nx, "~> 0.4.1", [hex: :nx, repo: "hexpm", optional: false]}], "hexpm", "3d1ee6b2588cadf2e70e4ea33449bda8e27f315b653f12d1f681070b83854b0e"}, "tzdata": {:hex, :tzdata, "1.0.5", "69f1ee029a49afa04ad77801febaf69385f3d3e3d1e4b56b9469025677b89a28", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "55519aa2a99e5d2095c1e61cc74c9be69688f8ab75c27da724eb8279ff402a5a"}, "unicode_util_compat": {:hex, :unicode_util_compat, "0.5.0", "8516502659002cec19e244ebd90d312183064be95025a319a6c7e89f4bccd65b", [:rebar3], [], "hexpm", "d48d002e15f5cc105a696cf2f1bbb3fc72b4b770a184d8420c8db20da2674b38"}, + "unpickler": {:hex, :unpickler, "0.1.0", "c2262c0819e6985b761e7107546cef96a485f401816be5304a65fdd200d5bd6a", [:mix], [], "hexpm", "e2b3f61e62406187ac52afead8a63bfb4e49394028993f3c4c42712743cab79e"}, + "xla": {:hex, :xla, "0.3.0", "7f40d3d799447bd745e6e61985c922821e355eca37c25754bac299d11f137f2e", [:make, :mix], [{:elixir_make, "~> 0.4", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "0de304f5624ea2f2ae1afdcbd72d33855364be36fed2cc5de8d6d2ec8950926e"}, } diff --git a/memex/priv/repo/migrations/20221214130932_create_vector_extension.exs b/memex/priv/repo/migrations/20221214130932_create_vector_extension.exs new file mode 100644 index 0000000..8fe781f --- /dev/null +++ b/memex/priv/repo/migrations/20221214130932_create_vector_extension.exs @@ -0,0 +1,11 @@ +defmodule Memex.Repo.Migrations.CreateVectorExtension do + use Ecto.Migration + + def up do + execute "CREATE EXTENSION IF NOT EXISTS vector" + end + + def down do + execute "DROP EXTENSION vector" + end +end diff --git a/postgres.Dockerfile b/postgres.Dockerfile new file mode 100644 index 0000000..afcc4f6 --- /dev/null +++ b/postgres.Dockerfile @@ -0,0 +1,15 @@ +FROM postgres:13 + +RUN apt-get update && \ + apt-get install -y --no-install-recommends git ca-certificates build-essential postgresql-server-dev-13 && \ + git clone https://github.com/pgvector/pgvector.git /tmp/pgvector && \ + cd /tmp/pgvector && \ + make clean && \ + make OPTFLAGS="" && \ + make install && \ + mkdir /usr/share/doc/pgvector && \ + cp LICENSE README.md /usr/share/doc/pgvector && \ + rm -r /tmp/pgvector && \ + apt-get remove -y build-essential postgresql-server-dev-13 && \ + apt-get autoremove -y && \ + rm -rf /var/lib/apt/lists/* \ No newline at end of file