diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 897aa1feca..be661c3592 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -180,6 +180,9 @@ jobs: - name: Run triggers example tests run: cargo test --package triggers --features "pg$PG_VER" --no-default-features + - name: Run versioned_so example tests + run: cargo test --package versioned_so --features "pg$PG_VER" --no-default-features + # Attempt to make the cache payload slightly smaller. - name: Clean up built PGX files run: | diff --git a/Cargo.lock b/Cargo.lock index d83e8e8c38..d6b018fe36 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -279,7 +279,7 @@ dependencies = [ [[package]] name = "cargo-pgx" -version = "0.4.3" +version = "0.4.4" dependencies = [ "atty", "cargo_metadata", @@ -300,7 +300,7 @@ dependencies = [ "rayon", "regex", "rttp_client", - "semver 1.0.7", + "semver 1.0.9", "syn", "tracing", "tracing-error", @@ -325,7 +325,7 @@ checksum = "4acbb09d9ee8e23699b9634375c72795d095bf268439da88562cf9b501f181fa" dependencies = [ "camino", "cargo-platform", - "semver 1.0.7", + "semver 1.0.9", "serde", "serde_json", ] @@ -375,9 +375,9 @@ dependencies = [ [[package]] name = "clap" -version = "3.1.10" +version = "3.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3124f3f75ce09e22d1410043e1e24f2ecc44fad3afe4f08408f1f7663d68da2b" +checksum = "85a35a599b11c089a7f49105658d089b8f2cf0882993c17daf6de15285c2c35d" dependencies = [ "atty", "bitflags", @@ -416,9 +416,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.1.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "189ddd3b5d32a70b35e7686054371742a937b0d99128e76dde6340210e966669" +checksum = "a37c35f1112dad5e6e0b1adaff798507497a18fceeb30cceb3bae7d1427b9213" dependencies = [ "os_str_bytes", ] @@ -1037,9 +1037,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.124" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21a41fed9d98f27ab1c6d161da622a4fa35e8a54a8adc24bbf3ddd0ef70b0e50" +checksum = "5916d2ae698f6de9bfb891ad7a8d65c09d232dc58cc4ac433c7da3b2fd84bc2b" [[package]] name = "libflate" @@ -1098,9 +1098,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.16" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6389c490849ff5bc16be905ae24bc913a9c8892e19b2341dbc175e14c341c2b8" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" dependencies = [ "cfg-if", ] @@ -1137,9 +1137,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "memoffset" @@ -1267,9 +1267,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" dependencies = [ "autocfg", ] @@ -1311,18 +1311,30 @@ checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9" [[package]] name = "openssl" -version = "0.10.38" +version = "0.10.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c7ae222234c30df141154f159066c5093ff73b63204dcda7121eb082fc56a95" +checksum = "28f3916d46d9d813a62d7b7d2724d7b14785ac999fb623d990ee4603f9122742" dependencies = [ "bitflags", "cfg-if", "foreign-types", "libc", "once_cell", + "openssl-macros", "openssl-sys", ] +[[package]] +name = "openssl-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "openssl-probe" version = "0.1.5" @@ -1331,9 +1343,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.72" +version = "0.9.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e46109c383602735fa0a2e48dd2b7c892b048e1bf69e5c3b1d804b7d9c203cb" +checksum = "9d5fd19fb3e0a8191c1e34935718976a3e70c112ab9a24af6d7cadccd9d90bc0" dependencies = [ "autocfg", "cc", @@ -1360,36 +1372,34 @@ checksum = "8e22443d1643a904602595ba1cd8f7d896afe56d26712531c5ff73a15b2fbf64" [[package]] name = "owo-colors" -version = "3.3.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e72e30578e0d0993c8ae20823dd9cff2bc5517d2f586a8aef462a581e8a03eb" +checksum = "decf7381921fea4dcb2549c5667eda59b3ec297ab7e2b5fc33eac69d2e7da87b" dependencies = [ "supports-color", ] [[package]] name = "parking_lot" -version = "0.11.2" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +checksum = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58" dependencies = [ - "instant", "lock_api", "parking_lot_core", ] [[package]] name = "parking_lot_core" -version = "0.8.5" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216" +checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929" dependencies = [ "cfg-if", - "instant", "libc", "redox_syscall", "smallvec", - "winapi", + "windows-sys", ] [[package]] @@ -1425,14 +1435,13 @@ dependencies = [ [[package]] name = "pgx" -version = "0.4.3" +version = "0.4.4" dependencies = [ "atomic-traits", "bitflags", "cstr_core", "enum-primitive-derive", "eyre", - "hash32", "heapless", "num-traits", "once_cell", @@ -1452,7 +1461,7 @@ dependencies = [ [[package]] name = "pgx-macros" -version = "0.4.3" +version = "0.4.4" dependencies = [ "pgx-utils", "proc-macro2", @@ -1464,7 +1473,7 @@ dependencies = [ [[package]] name = "pgx-pg-sys" -version = "0.4.3" +version = "0.4.4" dependencies = [ "bindgen", "build-deps", @@ -1484,7 +1493,7 @@ dependencies = [ [[package]] name = "pgx-tests" -version = "0.4.3" +version = "0.4.4" dependencies = [ "eyre", "libc", @@ -1503,7 +1512,7 @@ dependencies = [ [[package]] name = "pgx-utils" -version = "0.4.3" +version = "0.4.4" dependencies = [ "atty", "convert_case", @@ -1551,9 +1560,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e280fbe77cc62c91527259e9442153f4688736748d24660126286329742b4c6c" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" [[package]] name = "pin-utils" @@ -1583,9 +1592,9 @@ dependencies = [ [[package]] name = "postgres" -version = "0.19.2" +version = "0.19.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb76d6535496f633fa799bb872ffb4790e9cbdedda9d35564ca0252f930c0dd5" +checksum = "c8bbcd5f6deb39585a0d9f4ef34c4a41c25b7ad26d23c75d837d78c8e7adc85f" dependencies = [ "bytes", "fallible-iterator", @@ -1597,9 +1606,9 @@ dependencies = [ [[package]] name = "postgres-protocol" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79ec03bce71f18b4a27c4c64c6ba2ddf74686d69b91d8714fb32ead3adaed713" +checksum = "878c6cbf956e03af9aa8204b407b9cbf47c072164800aa918c516cd4b056c50c" dependencies = [ "base64 0.13.0", "byteorder", @@ -1615,9 +1624,9 @@ dependencies = [ [[package]] name = "postgres-types" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04619f94ba0cc80999f4fc7073607cb825bc739a883cb6d20900fc5e009d6b0d" +checksum = "ebd6e8b7189a73169290e89bd24c771071f1012d8fe6f738f5226531f0b03d89" dependencies = [ "bytes", "fallible-iterator", @@ -1632,9 +1641,9 @@ checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" [[package]] name = "prettyplease" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b83ec2d0af5c5c556257ff52c9f98934e243b9fd39604bfb2a9b75ec2e97f18" +checksum = "d9e07e3a46d0771a8a06b5f4441527802830b43e679ba12f44960f48dd4c6803" dependencies = [ "proc-macro2", "syn", @@ -2002,9 +2011,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.7" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d65bd28f48be7196d222d95b9243287f48d27aca604e08497513019ff0502cc4" +checksum = "8cb243bdfdb5936c8dc3c45762a19d12ab4550cdc753bc247637d4ec35a040fd" dependencies = [ "serde", ] @@ -2026,9 +2035,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.136" +version = "1.0.137" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce31e24b01e1e524df96f1c2fdd054405f8d7376249a5110886fb4b658484789" +checksum = "61ea8d54c77f8315140a05f4c7237403bf38b72704d031543aa1d16abbf517d1" dependencies = [ "serde_derive", ] @@ -2057,9 +2066,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.136" +version = "1.0.137" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08597e7152fcd306f41838ed3e37be9eaeed2b61c42e2117266a554fab4662f9" +checksum = "1f26faba0c3959972377d3b2d306ee9f71faee9714294e41bb777f83f88578be" dependencies = [ "proc-macro2", "quote", @@ -2068,9 +2077,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.79" +version = "1.0.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e8d9fa5c3b304765ce1fd9c4c8a3de2c8db365a5b91be52f186efc675681d95" +checksum = "f972498cf015f7c0746cac89ebe1d6ef10c293b94175a243a2d9442c163d9944" dependencies = [ "itoa", "ryu", @@ -2232,9 +2241,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.91" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b683b2b825c8eef438b77c36a06dc262294da3d5a5813fac20da149241dcd44d" +checksum = "7ff7c592601f11445996a06f8ad0c27f094a58857c2f89e97974ab9235b92c52" dependencies = [ "proc-macro2", "quote", @@ -2294,18 +2303,18 @@ checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb" [[package]] name = "thiserror" -version = "1.0.30" +version = "1.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417" +checksum = "bd829fe32373d27f76265620b5309d0340cb8550f523c1dda251d6298069069a" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.30" +version = "1.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b" +checksum = "0396bc89e626244658bef819e22d0cc459e795a5ebe878e6ec336d1674a8d79a" dependencies = [ "proc-macro2", "quote", @@ -2341,9 +2350,9 @@ checksum = "42657b1a6f4d817cda8e7a0ace261fe0cc946cf3a80314390b22cc61ae080792" [[package]] name = "tinyvec" -version = "1.5.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c1c1d5a42b6245520c249549ec267180beaffcc0615401ac8e31853d4b6d8d2" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" dependencies = [ "tinyvec_macros", ] @@ -2356,14 +2365,15 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.17.0" +version = "1.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2af73ac49756f3f7c01172e34a23e5d0216f6c32333757c2c61feb2bbff5a5ee" +checksum = "dce653fb475565de9f6fb0614b28bca8df2c430c0cf84bcd9c843f15de5414cc" dependencies = [ "bytes", "libc", "memchr", "mio", + "once_cell", "pin-project-lite", "socket2", "winapi", @@ -2371,9 +2381,9 @@ dependencies = [ [[package]] name = "tokio-postgres" -version = "0.7.5" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b6c8b33df661b548dcd8f9bf87debb8c56c05657ed291122e1188698c2ece95" +checksum = "19c88a47a23c5d2dc9ecd28fb38fba5fc7e5ddc1fe64488ec145076b0c71c8ae" dependencies = [ "async-trait", "byteorder", @@ -2394,16 +2404,16 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.6.9" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e99e1983e5d376cd8eb4b66604d2e99e79f5bd988c3055891dcd8c9e2604cc0" +checksum = "0edfdeb067411dba2044da6d1cb2df793dd35add7888d73c16e3381ded401764" dependencies = [ "bytes", "futures-core", "futures-sink", - "log", "pin-project-lite", "tokio", + "tracing", ] [[package]] @@ -2429,9 +2439,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e65ce065b4b5c53e73bb28912318cb8c9e9ad3921f1d669eb0e68b4c8143a2b" +checksum = "cc6b8ad3567499f98a1db7a752b07a7c8c7c7c34c332ec00effb2b0027974b7c" dependencies = [ "proc-macro2", "quote", @@ -2460,9 +2470,9 @@ dependencies = [ [[package]] name = "tracing-log" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6923477a48e41c1951f1999ef8bb5a3023eb723ceadafe78ffb65dc366761e3" +checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" dependencies = [ "lazy_static", "log", @@ -2524,9 +2534,9 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a01404663e3db436ed2746d9fefef640d868edae3cceb81c3b8d5732fda678f" +checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" [[package]] name = "unicode-normalization" @@ -2539,9 +2549,9 @@ dependencies = [ [[package]] name = "unicode-xid" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" +checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04" [[package]] name = "url" @@ -2557,9 +2567,9 @@ dependencies = [ [[package]] name = "uuid" -version = "0.8.2" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" +checksum = "8cfcd319456c4d6ea10087ed423473267e1a071f3bc0aa89f80d60997843c6f0" dependencies = [ "getrandom 0.2.6", ] @@ -2588,6 +2598,14 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "versioned_so" +version = "0.0.0" +dependencies = [ + "pgx", + "pgx-tests", +] + [[package]] name = "void" version = "1.0.2" @@ -2663,6 +2681,49 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-sys" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" +dependencies = [ + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" + +[[package]] +name = "windows_i686_gnu" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" + +[[package]] +name = "windows_i686_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" + [[package]] name = "xml-rs" version = "0.8.4" diff --git a/Cargo.toml b/Cargo.toml index 1b58b783e5..d2dfc9e462 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ members = [ "pgx-examples/srf", "pgx-examples/strings", "pgx-examples/triggers", + "pgx-examples/versioned_so", ] [profile.dev.build-override] diff --git a/README.md b/README.md index a737ee52c8..457fecf65d 100644 --- a/README.md +++ b/README.md @@ -206,7 +206,7 @@ my_extension=# SELECT hello_my_extension(); ### 5. Detailed cargo pgx usage -For more details on how to manage pgx extensions see [Managing pgx extensions](./cargo-pgx/README.md.) +For more details on how to manage pgx extensions see [Managing pgx extensions](./cargo-pgx/README.md). ## Upgrading diff --git a/articles/postgresql-aggregates-with-rust.md b/articles/postgresql-aggregates-with-rust.md new file mode 100644 index 0000000000..92a1ce24f5 --- /dev/null +++ b/articles/postgresql-aggregates-with-rust.md @@ -0,0 +1,963 @@ +# PostgreSQL Aggregates with Rust + + +Reaching for something like `SUM(vals)` or `AVG(vals)` is a common habit when using PostgreSQL. These [aggregate functions][postgresql-aggregate-functions] offer users an easy, efficient way to compute results from a set of inputs. + +How do they work? What makes them different than a function? How do we make one? What kinds of other uses exist? + +We'll explore creating some basic ones using SQL, then create an extension that defines aggregates in Rust using [`pgx`][pgx] 0.3.0's new aggregate support. + + + +# Aggregates in the world + +Aggregates have a number of uses, beyond the ones already mentioned (sum and average) we can find slightly less conceptually straightforward aggregates like: + +* JSON/JSONB/XML 'collectors': [`json_agg`][postgresql-aggregate-functions], [`json_object_agg`][postgresql-aggregate-functions], [`xmlagg`][postgresql-aggregate-functions]. +* Bitwise operators: [`bit_and`][postgresql-aggregate-functions], [`bit_xor`][postgresql-aggregate-functions], etc. +* Some [`std::iter::Iterator`][std::iter::Iterator] equivalents: + + [`all`][std::iter::Iterator::all]: [`bool_and`][postgresql-aggregate-functions] + + [`any`][std::iter::Iterator::any]: [`bool_or`][postgresql-aggregate-functions] + + [`collect::>`][std::iter::Iterator::collect]: [`array_agg`][postgresql-aggregate-functions] +* [PostGIS][postgis]'s geospacial [aggregates][postgis-aggregates] such as [`ST_3DExtent`][postgis-aggregates-3d-extent] which produces bounding boxes over geometry sets. +* [PG-Strom][pg-strom]'s [HyperLogLog aggregates][pg-strom-aggregates] such as [`hll_count`][pg-strom-aggregates]. + +So, aggregates can collect items, work like a [`fold`][std::iter::Iterator::fold], or do complex analytical math... how do we make one? + +# What makes an Aggregate? + +Defining an aggregate is via [`CREATE AGGREGATE`][postgresql-create-aggregate] and [`CREATE FUNCTION`][postgresql-create-function], here's a reimplementation of `sum` only for `integer`: + +```sql +CREATE FUNCTION example_sum_state( + state integer, + next integer +) RETURNS integer +LANGUAGE SQL +STRICT +AS $$ + SELECT $1 + $2; +$$; + +CREATE AGGREGATE example_sum(integer) +( + SFUNC = example_sum_state, -- State function + STYPE = integer, -- State type + INITCOND = '0' -- Must be a string or null +); + +SELECT example_sum(value) FROM UNNEST(ARRAY [1, 2, 3]) as value; +-- example_sum +-- ------------- +-- 6 +-- (1 row) +``` + +Conceptually, the aggregate loops over each item in the input and runs the `SFUNC` function on the current state as well as each value. That code is analogous to: + +```rust +fn example_sum(values: Vec) -> isize { + let mut sum = 0; + for value in values { + sum += value; + } + sum +} +``` + +Aggregates are more than just a loop, though. If the aggregate specifies a `combinefunc` PostgreSQL can run different instances of the aggregate over subsets of the data, then combine them later. This is called [*partial aggregation*][postgresql-user-defined-aggregates-partial-aggregates] and enables different worker processes to handle data in parallel. Let's make our `example_sum` aggregate above have a `combinefunc`: + +*(Readers may note we could use `example_sum_state` in this particular case, but not in general, so we're gonna make a new function for demonstration.)* + +```sql +CREATE FUNCTION example_sum_combine( + first integer, + second integer +) RETURNS integer +LANGUAGE SQL +STRICT +AS $$ + SELECT $1 + $2; +$$; + +DROP AGGREGATE example_sum(integer); + +CREATE AGGREGATE example_sum(integer) +( + SFUNC = example_sum_state, + STYPE = integer, + INITCOND = '0', + combinefunc = example_sum_combine +); + +SELECT example_sum(value) FROM generate_series(0, 4000) as value; +-- example_sum +-- ------------- +-- 8002000 +-- (1 row) +``` + +Here's one using `FINALFUNC`, which offers a way to compute some final value from the state: + +```sql +CREATE FUNCTION example_uniq_state( + state text[], + next text +) RETURNS text[] +LANGUAGE SQL +STRICT +AS $$ + SELECT array_append($1, $2); +$$; + +CREATE FUNCTION example_uniq_final( + state text[] +) RETURNS integer +LANGUAGE SQL +STRICT +AS $$ + SELECT count(DISTINCT value) FROM UNNEST(state) as value +$$; + +CREATE AGGREGATE example_uniq(text) +( + SFUNC = example_uniq_state, -- State function + STYPE = text[], -- State type + INITCOND = '{}', -- Must be a string or null + FINALFUNC = example_uniq_final -- Final function +); + +SELECT example_uniq(value) FROM UNNEST(ARRAY ['a', 'a', 'b']) as value; +-- example_uniq +-- -------------- +-- 2 +-- (1 row) +``` + +This is particularly handy as your `STYPE` doesn't need to be the type you return! + +Aggregates can take multiple arguments, too: + +```sql +CREATE FUNCTION example_concat_state( + state text[], + first text, + second text, + third text +) RETURNS text[] +LANGUAGE SQL +STRICT +AS $$ + SELECT array_append($1, concat($2, $3, $4)); +$$; + +CREATE AGGREGATE example_concat(text, text, text) +( + SFUNC = example_concat_state, + STYPE = text[], + INITCOND = '{}' +); + +SELECT example_concat(first, second, third) FROM + UNNEST(ARRAY ['a', 'b', 'c']) as first, + UNNEST(ARRAY ['1', '2', '3']) as second, + UNNEST(ARRAY ['!', '@', '#']) as third; +-- example_concat +-- --------------------------------------------------------------------------------------------------------------- +-- {a1!,a2!,a3!,b1!,b2!,b3!,c1!,c2!,c3!,a1@,a2@,a3@,b1@,b2@,b3@,c1@,c2@,c3@,a1#,a2#,a3#,b1#,b2#,b3#,c1#,c2#,c3#} +-- (1 row) +``` + +See how we see `a1`, `b1`, and `c1`? Multiple arguments might not work as you expect! As you can see, each argument is passed with each other argument. + +```sql +SELECT UNNEST(ARRAY ['a', 'b', 'c']) as first, + UNNEST(ARRAY ['1', '2', '3']) as second, + UNNEST(ARRAY ['!', '@', '#']) as third; +-- first | second | third +-- -------+--------+------- +-- a | 1 | ! +-- b | 2 | @ +-- c | 3 | # +-- (3 rows) +``` + +Aggregates have several more optional fields, such as a `PARALLEL`. Their signatures are documented in the [`CREATE AGGREGATE`][postgresql-create-aggregate] documentation and this article isn't meant to be comprehensive. + +> **Reminder:** You can also create functions with [`pl/pgsql`][postgresql-plpgsql], [`c`][postgresql-xfunc-c], [pl/Python][postgresql-plpython], or even in the experimental [`pl/Rust`][plrust]. + +Extensions can, of course, create aggregates too. Next, let's explore how to do that with Rust using [`pgx`][pgx] 0.3.0's Aggregate support. + +# Familiarizing with `pgx` + +[`pgx`][pgx] is a suite of crates that provide everything required to build, test, and package extensions for PostgreSQL versions 10 through 14 using pure Rust. + +It includes: + +* `cargo-pgx`: A `cargo` plugin that provides commands like `cargo pgx package` and `cargo pgx test`, +* `pgx`: A crate providing macros, high level abstractions (such as SPI), and low level generated bindings for PostgreSQL. +* `pgx-tests`: A crate providing a test framework for running tests inside PostgreSQL. + +**Note:** `pgx` does not currently offer Windows support, but works great in [WSL2][wsl]. + +If a Rust toolchain is not already installed, please follow the instructions on [rustup.rs][rustup-rs]. + +You'll also [need to make sure you have some development libraries][pgx-system-requirements] like `zlib` and `libclang`, as +`cargo pgx init` will, by default, build it's own development PostgreSQL installs. Usually it's possible to +figure out if something is missing from error messages and then discover the required package for the system. + +Install `cargo-pgx` then initialize its development PostgreSQL installations (used for `cargo pgx test` and `cargo pgx run`): + +```bash +$ cargo install cargo-pgx +$ cargo pgx init +# ... +``` + +We can create a new extension with: + +```bash +$ cargo pgx new exploring_aggregates +$ cd exploring_aggregates +``` + +Then run it: + +```bash +$ cargo pgx run +# ... +building extension with features `` +"cargo" "build" + Finished dev [unoptimized + debuginfo] target(s) in 0.06s + +installing extension + Copying control file to `/home/ana/.pgx/13.5/pgx-install/share/postgresql/extension/exploring_aggregates.control` + Copying shared library to `/home/ana/.pgx/13.5/pgx-install/lib/postgresql/exploring_aggregates.so` + Discovering SQL entities + Discovered 1 SQL entities: 0 schemas (0 unique), 1 functions, 0 types, 0 enums, 0 sqls, 0 ords, 0 hashes +running SQL generator +"/home/ana/git/samples/exploring_aggregates/target/debug/sql-generator" "--sql" "/home/ana/.pgx/13.5/pgx-install/share/postgresql/extension/exploring_aggregates--0.0.0.sql" + Copying extension schema file to `/home/ana/.pgx/13.5/pgx-install/share/postgresql/extension/exploring_aggregates--0.0.0.sql` + Finished installing exploring_aggregates + Starting Postgres v13 on port 28813 + Creating database exploring_aggregates +psql (13.5) +Type "help" for help. + +exploring_aggregates=# +``` + +Observing the start of the `src/lib.rs` file, we can see the `pg_module_magic!()` and a function `hello_exploring_aggregates`: + +```rust +use pgx::*; + +pg_module_magic!(); + +#[pg_extern] +fn hello_exploring_aggregates() -> &'static str { + "Hello, exploring_aggregates" +} +``` + +Back on our `psql` prompt, we can load the extension and run the function: + +```sql +CREATE EXTENSION exploring_aggregates; +-- CREATE EXTENSION + +\dx+ exploring_aggregates +-- Objects in extension "exploring_aggregates" +-- Object description +-- --------------------------------------- +-- function hello_exploring_aggregates() +-- (1 row) + +SELECT hello_exploring_aggregates(); +-- hello_exploring_aggregates +-- ----------------------------- +-- Hello, exploring_aggregates +-- (1 row) +``` + +Next, let's run the tests: + +```bash +$ cargo pgx test +"cargo" "test" "--features" " pg_test" + Finished test [unoptimized + debuginfo] target(s) in 0.08s + Running unittests (target/debug/deps/exploring_aggregates-4783beb51375d29c) + +running 1 test +building extension with features ` pg_test` +"cargo" "build" "--features" " pg_test" + Finished dev [unoptimized + debuginfo] target(s) in 0.41s + +installing extension + Copying control file to `/home/ana/.pgx/13.5/pgx-install/share/postgresql/extension/exploring_aggregates.control` + Copying shared library to `/home/ana/.pgx/13.5/pgx-install/lib/postgresql/exploring_aggregates.so` + Discovering SQL entities + Discovered 3 SQL entities: 1 schemas (1 unique), 2 functions, 0 types, 0 enums, 0 sqls, 0 ords, 0 hashes, 0 aggregates +running SQL generator +"/home/ana/git/samples/exploring_aggregates/target/debug/sql-generator" "--sql" "/home/ana/.pgx/13.5/pgx-install/share/postgresql/extension/exploring_aggregates--0.0.0.sql" + Copying extension schema file to `/home/ana/.pgx/13.5/pgx-install/share/postgresql/extension/exploring_aggregates--0.0.0.sql` + Finished installing exploring_aggregates +test tests::pg_test_hello_exploring_aggregates ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 2.44s + +Stopping Postgres + + Running unittests (target/debug/deps/sql_generator-1bb38131b30894e5) + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Doc-tests exploring_aggregates + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s +``` + +We can also inspect the SQL the extension generates: + +```bash +$ cargo pgx schema + Building SQL generator with features `` +"cargo" "build" "--bin" "sql-generator" + Finished dev [unoptimized + debuginfo] target(s) in 0.06s + Discovering SQL entities + Discovered 1 SQL entities: 0 schemas (0 unique), 1 functions, 0 types, 0 enums, 0 sqls, 0 ords, 0 hashes, 0 aggregates +running SQL generator +"/home/ana/git/samples/exploring_aggregates/target/debug/sql-generator" "--sql" "sql/exploring_aggregates-0.0.0.sql" +``` + +This creates `sql/exploring_aggregates-0.0.0.sql`: + +```sql +/* +This file is auto generated by pgx. + +The ordering of items is not stable, it is driven by a dependency graph. +*/ + +-- src/lib.rs:5 +-- exploring_aggregates::hello_exploring_aggregates +CREATE OR REPLACE FUNCTION "hello_exploring_aggregates"() RETURNS text /* &str */ +STRICT +LANGUAGE c /* Rust */ +AS 'MODULE_PATHNAME', 'hello_exploring_aggregates_wrapper'; +``` + +Finally we can create a package for the `pg_config` version installed on the system, this is done in release mode, so it takes a few minutes: + +```bash +$ cargo pgx package +building extension with features `` +"cargo" "build" "--release" + Finished release [optimized] target(s) in 0.07s + +installing extension + Discovering SQL entities + Discovered 1 SQL entities: 0 schemas (0 unique), 1 functions, 0 types, 0 enums, 0 sqls, 0 ords, 0 hashes, 0 aggregates +running SQL generator +"/home/ana/git/samples/exploring_aggregates/target/release/sql-generator" "--sql" "/home/ana/git/samples/exploring_aggregates/target/release/exploring_aggregates-pg13/usr/share/postgresql/13/extension/exploring_aggregates--0.0.0.sql" + Copying extension schema file to `target/release/exploring_aggregates-pg13/usr/share/postgresql/13/extension/exploring_aggregates--0.0.0.sql` + Finished installing exploring_aggregates +``` + +Let's make some aggregates with `pgx` now! + +# Aggregates with [`pgx`][pgx] + +While designing the aggregate support for [`pgx`][pgx] 0.3.0 we wanted to try to make things feel idiomatic and natural from the Rust side, +but it should be flexible enough for any use. + +Aggregates in `pgx` are defined by creating a type (this doesn't necessarily need to be the state type), then using the [`#[pg_aggregate]`][pgx-pg_aggregate] +procedural macro on an [`pgx::Aggregate`][pgx-aggregate-aggregate] implementation for that type. + +The [`pgx::Aggregate`][pgx-aggregate-aggregate] trait has quite a few items (`fn`s, `const`s, `type`s) that you can implement, but the procedural macro can fill in +stubs for all non-essential items. The state type (the implementation target by default) must have a [`#[derive(PostgresType)]`][pgx-postgrestype] declaration, +or be a type PostgreSQL already knows about. + +Here's the simplest aggregate you can make with `pgx`: + +```rust +use pgx::*; +use serde::{Serialize, Deserialize}; + +pg_module_magic!(); + +#[derive(Copy, Clone, Default, Debug, PostgresType, Serialize, Deserialize)] +pub struct DemoSum { + count: i32, +} + +#[pg_aggregate] +impl Aggregate for DemoSum { + const INITIAL_CONDITION: Option<&'static str> = Some(r#"{ "count": 0 }"#); + type Args = i32; + fn state( + mut current: Self::State, + arg: Self::Args, + _fcinfo: pg_sys::FunctionCallInfo + ) -> Self::State { + current.count += arg; + current + } +} +``` + +We can review the generated SQL (generated via `cargo pgx schema`): + +```sql +/* +This file is auto generated by pgx. + +The ordering of items is not stable, it is driven by a dependency graph. +*/ + +-- src/lib.rs:6 +-- exploring_aggregates::DemoSum +CREATE TYPE DemoSum; + +-- src/lib.rs:6 +-- exploring_aggregates::demosum_in +CREATE OR REPLACE FUNCTION "demosum_in"( + "input" cstring /* &cstr_core::CStr */ +) RETURNS DemoSum /* exploring_aggregates::DemoSum */ +IMMUTABLE PARALLEL SAFE STRICT +LANGUAGE c /* Rust */ +AS 'MODULE_PATHNAME', 'demosum_in_wrapper'; + +-- src/lib.rs:6 +-- exploring_aggregates::demosum_out +CREATE OR REPLACE FUNCTION "demosum_out"( + "input" DemoSum /* exploring_aggregates::DemoSum */ +) RETURNS cstring /* &cstr_core::CStr */ +IMMUTABLE PARALLEL SAFE STRICT +LANGUAGE c /* Rust */ +AS 'MODULE_PATHNAME', 'demosum_out_wrapper'; + +-- src/lib.rs:6 +-- exploring_aggregates::DemoSum +CREATE TYPE DemoSum ( + INTERNALLENGTH = variable, + INPUT = demosum_in, /* exploring_aggregates::demosum_in */ + OUTPUT = demosum_out, /* exploring_aggregates::demosum_out */ + STORAGE = extended +); + +-- src/lib.rs:11 +-- exploring_aggregates::demo_sum_state +CREATE OR REPLACE FUNCTION "demo_sum_state"( + "this" DemoSum, /* exploring_aggregates::DemoSum */ + "arg_one" integer /* i32 */ +) RETURNS DemoSum /* exploring_aggregates::DemoSum */ +STRICT +LANGUAGE c /* Rust */ +AS 'MODULE_PATHNAME', 'demo_sum_state_wrapper'; + +-- src/lib.rs:11 +-- exploring_aggregates::DemoSum +CREATE AGGREGATE DemoSum ( + integer /* i32 */ +) +( + SFUNC = "demo_sum_state", /* exploring_aggregates::DemoSum::state */ + STYPE = DemoSum, /* exploring_aggregates::DemoSum */ + INITCOND = '{ "count": 0 }' /* exploring_aggregates::DemoSum::INITIAL_CONDITION */ +); +``` + +We can test it out with `cargo pgx run`: + +```bash +$ cargo pgx run + Stopping Postgres v13 +building extension with features `` +"cargo" "build" + Finished dev [unoptimized + debuginfo] target(s) in 0.06s + +installing extension + Copying control file to `/home/ana/.pgx/13.5/pgx-install/share/postgresql/extension/exploring_aggregates.control` + Copying shared library to `/home/ana/.pgx/13.5/pgx-install/lib/postgresql/exploring_aggregates.so` + Discovering SQL entities + Discovered 5 SQL entities: 0 schemas (0 unique), 3 functions, 1 types, 0 enums, 0 sqls, 0 ords, 0 hashes, 1 aggregates +running SQL generator +"/home/ana/git/samples/exploring_aggregates/target/debug/sql-generator" "--sql" "/home/ana/.pgx/13.5/pgx-install/share/postgresql/extension/exploring_aggregates--0.0.0.sql" + Copying extension schema file to `/home/ana/.pgx/13.5/pgx-install/share/postgresql/extension/exploring_aggregates--0.0.0.sql` + Finished installing exploring_aggregates + Starting Postgres v13 on port 28813 + Re-using existing database exploring_aggregates +psql (13.5) +Type "help" for help. + +exploring_aggregates=# +``` + +Now we're connected via `psql`: + +```sql +CREATE EXTENSION exploring_aggregates; +-- CREATE EXTENSION + +SELECT DemoSum(value) FROM generate_series(0, 4000) as value; +-- demosum +-- ------------------- +-- {"count":8002000} +-- (1 row) +``` + +Pretty cool! + +...But we don't want that silly `{"count": ... }` stuff, just the number! We can resolve this by changing the [`State`][pgx-aggregate-aggregate-state] type, or by adding a [`finalize`][pgx-aggregate-aggregate-finalize] (which maps to `ffunc`) as we saw in the previous section. + +Let's change the [`State`][pgx-aggregate-aggregate-state] this time: + +```rust +#[derive(Copy, Clone, Default, Debug)] +pub struct DemoSum; + +#[pg_aggregate] +impl Aggregate for DemoSum { + const INITIAL_CONDITION: Option<&'static str> = Some(r#"0"#); + type Args = i32; + type State = i32; + + fn state( + mut current: Self::State, + arg: Self::Args, + _fcinfo: pg_sys::FunctionCallInfo, + ) -> Self::State { + current += arg; + current + } +} +``` + +Now when we run it: + +```sql +SELECT DemoSum(value) FROM generate_series(0, 4000) as value; +-- demosum +-- --------- +-- 8002000 +-- (1 row) +``` + +This is a fine reimplementation of `SUM` so far, but as we saw previously we need a [`combine`][pgx-aggregate-aggregate-combine] (mapping to `combinefunc`) to support partial aggregation: + +```rust +#[pg_aggregate] +impl Aggregate for DemoSum { + // ... + fn combine( + mut first: Self::State, + second: Self::State, + _fcinfo: pg_sys::FunctionCallInfo, + ) -> Self::State { + first += second; + first + } +} +``` + +We can also change the name of the generated aggregate, or set the [`PARALLEL`][pgx-aggregate-aggregate-parallel] settings, for example: + +```rust +#[pg_aggregate] +impl Aggregate for DemoSum { + // ... + const NAME: &'static str = "demo_sum"; + const PARALLEL: Option = Some(pgx::aggregate::ParallelOption::Unsafe); + // ... +} +``` + +This generates: + +```sql +-- src/lib.rs:9 +-- exploring_aggregates::DemoSum +CREATE AGGREGATE demo_sum ( + integer /* i32 */ +) +( + SFUNC = "demo_sum_state", /* exploring_aggregates::DemoSum::state */ + STYPE = integer, /* i32 */ + COMBINEFUNC = "demo_sum_combine", /* exploring_aggregates::DemoSum::combine */ + INITCOND = '0', /* exploring_aggregates::DemoSum::INITIAL_CONDITION */ + PARALLEL = UNSAFE /* exploring_aggregates::DemoSum::PARALLEL */ +); +``` + +## Rust state types + +It's possible to use a non-SQL (say, [`HashSet`][std::collections::HashSet]) type as a state by using [`Internal`][pgx::datum::Internal]. + +> When using this strategy, **a `finalize` function must be provided.** + +Here's a unique string counter aggregate that uses a `HashSet`: + +```rust +use pgx::*; +use std::collections::HashSet; + +pg_module_magic!(); + +#[derive(Copy, Clone, Default, Debug)] +pub struct DemoUnique; + +#[pg_aggregate] +impl Aggregate for DemoUnique { + type Args = &'static str; + type State = Internal; + type Finalize = i32; + + fn state( + mut current: Self::State, + arg: Self::Args, + _fcinfo: pg_sys::FunctionCallInfo, + ) -> Self::State { + let inner = unsafe { current.get_or_insert_default::>() }; + + inner.insert(arg.to_string()); + current + } + + fn combine( + mut first: Self::State, + mut second: Self::State, + _fcinfo: pg_sys::FunctionCallInfo + ) -> Self::State { + let first_inner = unsafe { first.get_or_insert_default::>() }; + let second_inner = unsafe { second.get_or_insert_default::>() }; + + let unioned: HashSet<_> = first_inner.union(second_inner).collect(); + Internal::new(unioned) + } + + fn finalize( + mut current: Self::State, + _direct_arg: Self::OrderedSetArgs, + _fcinfo: pg_sys::FunctionCallInfo, + ) -> Self::Finalize { + let inner = unsafe { current.get_or_insert_default::>() }; + + inner.len() as i32 + } +} +``` + +We can test it: + +```sql +SELECT DemoUnique(value) FROM UNNEST(ARRAY ['a', 'a', 'b']) as value; +-- demounique +-- ------------ +-- 2 +-- (1 row) +``` + +Using `Internal` here means that the values it holds get dropped at the end of [` PgMemoryContexts::CurrentMemoryContext`][postgresql-current-memory-contexts], the aggregate context in this case. + + +## Ordered-Set Aggregates + +PostgreSQL also supports what are called [*Ordered-Set Aggregates*][postgresql-ordered-set-aggregate]. Ordered-Set Aggregates can take a **direct argument**, and specify a sort ordering for the inputs. + +> PostgreSQL does *not* order inputs behind the scenes! + +Let's create a simple `percentile_disc` reimplementation to get an idea of how to make one with `pgx`. You'll notice we add [`ORDERED_SET = true`][pgx::aggregate::Aggregate::ORDERED_SET] and set an (optional) [`OrderedSetArgs`][pgx::aggregate::Aggregate::OrderedSetArgs], which determines the direct arguments. + +```rust +#[derive(Copy, Clone, Default, Debug)] +pub struct DemoPercentileDisc; + +#[pg_aggregate] +impl Aggregate for DemoPercentileDisc { + type Args = name!(input, i32); + type State = Internal; + type Finalize = i32; + const ORDERED_SET: bool = true; + type OrderedSetArgs = name!(percentile, f64); + + fn state( + mut current: Self::State, + arg: Self::Args, + _fcinfo: pg_sys::FunctionCallInfo, + ) -> Self::State { + let inner = unsafe { current.get_or_insert_default::>() }; + + inner.push(arg); + current + } + + fn finalize( + mut current: Self::State, + direct_arg: Self::OrderedSetArgs, + _fcinfo: pg_sys::FunctionCallInfo, + ) -> Self::Finalize { + let inner = unsafe { current.get_or_insert_default::>() }; + // This isn't done for us! + inner.sort(); + + let target_index = (inner.len() as f64 * direct_arg).round() as usize; + inner[target_index.saturating_sub(1)] + } +} +``` + +This creates SQL like: + +```sql +-- src/lib.rs:9 +-- exploring_aggregates::DemoPercentileDisc +CREATE AGGREGATE DemoPercentileDisc ( + "percentile" double precision /* f64 */ + ORDER BY + "input" integer /* i32 */ +) +( + SFUNC = "demo_percentile_disc_state", /* exploring_aggregates::DemoPercentileDisc::state */ + STYPE = internal, /* pgx::datum::internal::Internal */ + FINALFUNC = "demo_percentile_disc_finalize" /* exploring_aggregates::DemoPercentileDisc::final */ +); +``` + +We can test it like so: + +```sql +SELECT DemoPercentileDisc(0.5) WITHIN GROUP (ORDER BY income) FROM UNNEST(ARRAY [6000, 70000, 500]) as income; +-- demopercentiledisc +-- -------------------- +-- 6000 +-- (1 row) + +SELECT DemoPercentileDisc(0.05) WITHIN GROUP (ORDER BY income) FROM UNNEST(ARRAY [5, 100000000, 6000, 70000, 500]) as income; +-- demopercentiledisc +-- -------------------- +-- 5 +-- (1 row) +``` + +## Moving-Aggregate mode + +Aggregates can also support [*moving-aggregate mode*][postgresql-moving-aggregate], which can **remove** inputs from the aggregate as well. + +This allows for some optimization if you are using aggregates as window functions. The documentation explains that this is because PostgreSQL doesn't need to recalculate the aggregate each time the frame starting point moves. + +Moving-aggregate mode has it's own [`moving_state`][pgx::aggregate::Aggregate::moving_state] function as well as an [`moving_state_inverse`][pgx::aggregate::Aggregate::moving_state_inverse] function for removing inputs. Because moving-aggregate mode may require some additional tracking on the part of the aggregate, there is also a [`MovingState`][pgx::aggregate::Aggregate::MovingState] associated type as well as a [`moving_state_finalize`][pgx::aggregate::Aggregate::moving_state_finalize] function for any specialized final computation. + +Let's take our sum example above and add moving-aggregate mode support to it: + +```rust +#[derive(Copy, Clone, Default, Debug)] +pub struct DemoSum; + +#[pg_aggregate] +impl Aggregate for DemoSum { + const NAME: &'static str = "demo_sum"; + const PARALLEL: Option = Some(pgx::aggregate::ParallelOption::Unsafe); + const INITIAL_CONDITION: Option<&'static str> = Some(r#"0"#); + const MOVING_INITIAL_CONDITION: Option<&'static str> = Some(r#"0"#); + + type Args = i32; + type State = i32; + type MovingState = i32; + + fn state( + mut current: Self::State, + arg: Self::Args, + _fcinfo: pg_sys::FunctionCallInfo, + ) -> Self::State { + pgx::log!("state({}, {})", current, arg); + current += arg; + current + } + + fn moving_state( + mut current: Self::State, + arg: Self::Args, + _fcinfo: pg_sys::FunctionCallInfo, + ) -> Self::MovingState { + pgx::log!("moving_state({}, {})", current, arg); + current += arg; + current + } + + fn moving_state_inverse( + mut current: Self::State, + arg: Self::Args, + _fcinfo: pg_sys::FunctionCallInfo, + ) -> Self::MovingState { + pgx::log!("moving_state_inverse({}, {})", current, arg); + current -= arg; + current + } + + fn combine( + mut first: Self::State, + second: Self::State, + _fcinfo: pg_sys::FunctionCallInfo, + ) -> Self::State { + pgx::log!("combine({}, {})", first, second); + first += second; + first + } +} +``` + +This generates: + +```sql +-- src/lib.rs:8 +-- exploring_aggregates::DemoSum +CREATE AGGREGATE demo_sum ( + integer /* i32 */ +) +( + SFUNC = "demo_sum_state", /* exploring_aggregates::DemoSum::state */ + STYPE = integer, /* i32 */ + COMBINEFUNC = "demo_sum_combine", /* exploring_aggregates::DemoSum::combine */ + INITCOND = '0', /* exploring_aggregates::DemoSum::INITIAL_CONDITION */ + MSFUNC = "demo_sum_moving_state", /* exploring_aggregates::DemoSum::moving_state */ + MINVFUNC = "demo_sum_moving_state_inverse", /* exploring_aggregates::DemoSum::moving_state_inverse */ + MINITCOND = '0', /* exploring_aggregates::DemoSum::MOVING_INITIAL_CONDITION */ + PARALLEL = UNSAFE, /* exploring_aggregates::DemoSum::PARALLEL */ + MSTYPE = integer /* exploring_aggregates::DemoSum::MovingState = i32 */ +); +``` + +Using it (we'll also turn on logging to see what happens with `SET client_min_messages TO debug;`): + +```sql +SET client_min_messages TO debug; +-- SET + +SELECT demo_sum(value) OVER ( + ROWS CURRENT ROW +) FROM UNNEST(ARRAY [1, 20, 300, 4000]) as value; +-- LOG: moving_state(0, 1) +-- LOG: moving_state(0, 20) +-- LOG: moving_state(0, 300) +-- LOG: moving_state(0, 4000) +-- demo_sum +-- ---------- +-- 1 +-- 20 +-- 300 +-- 4000 +-- (4 rows) +``` + +Inside the `OVER ()` we can use [syntax for window function calls][postgresql-window-function-calls]. + +```sql +SELECT demo_sum(value) OVER ( + ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING +) FROM UNNEST(ARRAY [1, 20, 300, 4000]) as value; +-- LOG: moving_state(0, 1) +-- LOG: moving_state(1, 20) +-- LOG: moving_state(21, 300) +-- LOG: moving_state(321, 4000) +-- demo_sum +-- ---------- +-- 4321 +-- 4321 +-- 4321 +-- 4321 +-- (4 rows) + +SELECT demo_sum(value) OVER ( + ROWS BETWEEN 1 PRECEDING AND CURRENT ROW +) FROM UNNEST(ARRAY [1, 20, 300, 4000]) as value; +-- LOG: moving_state(0, 1) +-- LOG: moving_state(1, 20) +-- LOG: moving_state_inverse(21, 1) +-- LOG: moving_state(20, 300) +-- LOG: moving_state_inverse(320, 20) +-- LOG: moving_state(300, 4000) +-- demo_sum +-- ---------- +-- 1 +-- 21 +-- 320 +-- 4300 +-- (4 rows) + +SELECT demo_sum(value) OVER ( + ORDER BY sorter + ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING +) FROM ( + VALUES (1, 10000), + (2, 1) + ) AS v (sorter, value); +-- LOG: moving_state(0, 10000) +-- LOG: moving_state(10000, 1) +-- LOG: moving_state_inverse(10001, 10000) +-- demo_sum +-- ---------- +-- 10001 +-- 1 +-- (2 rows) +``` + +# Wrapping up + +I had a lot of fun implementing the aggregate support for [`pgx`][pgx], and hope you have just as much fun using it! If you have questions, open up an [issue][pgx-issues]. + +Moving-aggregate mode is pretty new to me, and I'm still learning about it! If you have any good resources I'd love to recieve them from you! + +If you're looking for more materials about aggregates, the TimescaleDB folks wrote about aggregates and how they impacted their hyperfunctions in [this article][timescaledb-article-aggregation]. Also, My pal [Tim McNamara][timclicks] wrote about how to implement harmonic and geometric means as aggregates in [this article][timclicks-article-aggregates]. + +[pgx]: https://github.com/zombodb/pgx +[pgx-issues]: https://github.com/zombodb/pgx/issues +[pgx-aggregate-aggregate]: https://docs.rs/pgx/0.3.0/pgx/aggregate/trait.Aggregate.html +[pgx-aggregate-aggregate-finalize]: https://docs.rs/pgx/0.3.0/pgx/aggregate/trait.Aggregate.html#tymethod.finalize +[pgx-aggregate-aggregate-state]: https://docs.rs/pgx/0.3.0/pgx/aggregate/trait.Aggregate.html#associatedtype.State +[pgx-aggregate-aggregate-combine]: https://docs.rs/pgx/0.3.0/pgx/aggregate/trait.Aggregate.html#tymethod.combine +[pgx-aggregate-aggregate-parallel]: https://docs.rs/pgx/0.3.0/pgx/aggregate/trait.Aggregate.html#associatedconstant.PARALLEL +[pgx::aggregate::Aggregate::ORDERED_SET]: https://docs.rs/pgx/0.3.0/pgx/aggregate/trait.Aggregate.html#associatedconstant.ORDERED_SET +[pgx::aggregate::Aggregate::OrderedSetArgs]: https://docs.rs/pgx/0.3.0/pgx/aggregate/trait.Aggregate.html#associatedtype.OrderedSetArgs +[pgx::aggregate::Aggregate::moving_state]: https://docs.rs/pgx/0.3.0/pgx/aggregate/trait.Aggregate.html#tymethod.moving_state +[pgx::aggregate::Aggregate::moving_state_inverse]: https://docs.rs/pgx/0.3.0/pgx/aggregate/trait.Aggregate.html#tymethod.moving_state_inverse +[pgx::aggregate::Aggregate::MovingState]: https://docs.rs/pgx/0.3.0/pgx/aggregate/trait.Aggregate.html#associatedtype.MovingState +[pgx::aggregate::Aggregate::moving_state_finalize]: https://docs.rs/pgx/0.3.0/pgx/aggregate/trait.Aggregate.html#tymethod.moving_finalize +[pgx::datum::Internal]: https://docs.rs/pgx/0.3.0/pgx/datum/struct.Internal.html +[pgx-pg_aggregate]: https://docs.rs/pgx/0.3.0/pgx/attr.pg_aggregate.html +[pgx-postgrestype]: https://docs.rs/pgx/0.3.0/pgx/derive.PostgresType.html +[pgx-system-requirements]: https://github.com/zombodb/pgx#system-requirements +[plrust]: https://github.com/zombodb/plrust +[rustup-rs]: https://rustup.rs/ +[wsl]: https://docs.microsoft.com/en-us/windows/wsl/install +[std::iter::Iterator]: https://doc.rust-lang.org/std/iter/trait.Iterator.html +[std::iter::Iterator::all]: https://doc.rust-lang.org/std/iter/trait.Iterator.html#method.all +[std::iter::Iterator::any]: https://doc.rust-lang.org/std/iter/trait.Iterator.html#method.any +[std::iter::Iterator::collect]: https://doc.rust-lang.org/std/iter/trait.Iterator.html#method.collect +[std::iter::Iterator::fold]: https://doc.rust-lang.org/std/iter/trait.Iterator.html#method.collect +[std::collections::hashset]: https://doc.rust-lang.org/std/collections/struct.HashSet.html +[postgresql-pl-pgsql]: https://www.postgresql.org/docs/current/plpgsql.html +[postgresql-user-defined-aggregates]: https://www.postgresql.org/docs/current/xaggr.html +[postgresql-user-defined-aggregates-partial-aggregates]: https://www.postgresql.org/docs/current/xaggr.html#XAGGR-PARTIAL-AGGREGATES +[postgresql-aggregate-functions]: https://www.postgresql.org/docs/current/functions-aggregate.html +[postgresql-window-function-calls]: https://www.postgresql.org/docs/current/sql-expressions.html#SYNTAX-WINDOW-FUNCTIONS +[postgresql-create-aggregate]: https://www.postgresql.org/docs/current/sql-createaggregate.html +[postgresql-ordered-set-aggregate]: https://www.postgresql.org/docs/current/xaggr.html#XAGGR-ORDERED-SET-AGGREGATES +[postgresql-moving-aggregate]: https://www.postgresql.org/docs/current/xaggr.html#XAGGR-MOVING-AGGREGATES +[postgresql-create-function]: https://www.postgresql.org/docs/current/sql-createfunction.html +[postgresql-current-memory-contexts]: https://github.com/postgres/postgres/blob/7c1aead6cbe7dcc6c216715fed7a1fb60684c5dc/src/backend/utils/mmgr/README#L72 +[postgresql-xfunc-c]: https://www.postgresql.org/docs/current/xfunc-c.html +[postgresql-plpython]: https://www.postgresql.org/docs/current/plpython.html +[postgresql-plpgsql]: https://www.postgresql.org/docs/current/plpgsql.html +[postgis]: https://postgis.net/ +[postgis-aggregates]: https://postgis.net/docs/PostGIS_Special_Functions_Index.html#PostGIS_Aggregate_Functions +[postgis-aggregates-3d-extent]: https://postgis.net/docs/ST_3DExtent.html +[timescaledb-article-aggregation]: https://blog.timescale.com/blog/how-postgresql-aggregation-works-and-how-it-inspired-our-hyperfunctions-design-2/ +[timclicks]: https://tim.mcnamara.nz/ +[timclicks-article-aggregates]: https://tim.mcnamara.nz/post/177172657187/user-defined-aggregates-in-postgresql +[pg-strom]: https://heterodb.github.io/pg-strom/ +[pg-strom-aggregates]: https://heterodb.github.io/pg-strom/ref_sqlfuncs/#hyperloglog-functions diff --git a/cargo-pgx/Cargo.toml b/cargo-pgx/Cargo.toml index 903bb5d08d..970f885a44 100644 --- a/cargo-pgx/Cargo.toml +++ b/cargo-pgx/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cargo-pgx" -version = "0.4.3" +version = "0.4.4" authors = ["ZomboDB, LLC "] license = "MIT" description = "Cargo subcommand for 'pgx' to make Postgres extension development easy" @@ -17,19 +17,19 @@ edition = "2021" atty = "0.2.14" cargo_metadata = "0.14.2" cargo_toml = "0.11.5" -clap = { version = "3.1.10", features = [ "env", "suggestions", "cargo", "derive" ] } +clap = { version = "3.1.15", features = [ "env", "suggestions", "cargo", "derive" ] } clap-cargo = { version = "0.8.0", features = [ "cargo_metadata" ] } -semver = "1.0.7" -owo-colors = { version = "3.3.0", features = [ "supports-colors" ] } +semver = "1.0.9" +owo-colors = { version = "3.4.0", features = [ "supports-colors" ] } env_proxy = "0.4.1" num_cpus = "1.13.1" -pgx-utils = { path = "../pgx-utils", version = "0.4.3" } +pgx-utils = { path = "../pgx-utils", version = "0.4.4" } proc-macro2 = { version = "1.0.37", features = [ "span-locations" ] } quote = "1.0.18" rayon = "1.5.2" regex = "1.5.5" rttp_client = { version = "0.1.0", features = ["tls-native"] } -syn = { version = "1.0.91", features = [ "extra-traits", "full", "fold", "parsing" ] } +syn = { version = "1.0.92", features = [ "extra-traits", "full", "fold", "parsing" ] } unescape = "0.1.0" fork = "0.1.19" libloading = "0.7.3" diff --git a/cargo-pgx/README.md b/cargo-pgx/README.md index bad7d06b00..1d753eefbd 100644 --- a/cargo-pgx/README.md +++ b/cargo-pgx/README.md @@ -626,7 +626,7 @@ OPTIONS: Print version information ``` -## Inspect you Extension Schema +## Inspect your Extension Schema If you just want to look at the full extension schema that pgx will generate, use `cargo pgx schema`. @@ -686,3 +686,94 @@ OPTIONS: -V, --version Print version information ``` + +## EXPERIMENTAL: Versioned shared-object support + +`pgx` experimentally supports the option to produce a versioned shared library. This allows multiple versions of the +extension to be installed side-by-side, and can enable the deprecation (and removal) of functions between extension +versions. There are some caveats which must be observed when using this functionality. For this reason it is currently +experimental. + +### Activation + +Versioned shared-object support is enabled by removing the `module_pathname` configuration value in the extension's +`.control` file. + +### Concepts + +Postgres has the implicit requirement that C extensions maintain ABI compatibility between versions. The idea behind +this feature is to allow interoperability between two versions of an extension when the new version is not ABI +compatible with the old version. + +The mechanism of operation is to version the name of the shared library file, and to hard-code function definitions to +point to the versioned shared library file. Without versioned shared-object support, the SQL definition of a C function +would look as follows: + +```SQL +CREATE OR REPLACE FUNCTION "hello_extension"() RETURNS text /* &str */ +STRICT +LANGUAGE c /* Rust */ +AS 'MODULE_PATHNAME', 'hello_extension_wrapper'; +``` + +`MODULE_PATHNAME` is replaced by Postgres with the configured value in the `.control` file. For pgx-based extensions, +this is usually set to `$libdir/`. + +When using versioned shared-object support, the same SQL would look as follows: + +```SQL +CREATE OR REPLACE FUNCTION "hello_extension"() RETURNS text /* &str */ +STRICT +LANGUAGE c /* Rust */ +AS '$libdir/extension-0.0.0', 'hello_extension_wrapper'; +``` + +Note that the versioned shared library is hard-coded in the function definition. This corresponds to the +`extension-0.0.0.so` file which `pgx` generates. + +It is important to note that the emitted SQL is version-dependent. This means that all previously-defined C functions +must be redefined to point to the current versioned-so in the version upgrade script. As an example, when updating the +extension version to 0.1.0, the shared object will be named `-0.1.0.so`, and `cargo pgx schema` will +produce the following SQL for the above function: + +```SQL +CREATE OR REPLACE FUNCTION "hello_extension"() RETURNS text /* &str */ +STRICT +LANGUAGE c /* Rust */ +AS '$libdir/extension-0.1.0', 'hello_extension_wrapper'; +``` + +This SQL must be used in the upgrade script from `0.0.0` to `0.1.0` in order to point the `hello_extension` function to +the new shared object. `pgx` _does not_ do any magic to determine in which version a function was introduced or modified +and only place it in the corresponding versioned so file. By extension, you can always expect that the shared library +will contain _all_ functions which are still defined in the extension's source code. + +This feature is not designed to assist in the backwards compatibility of data types. + +### `@MODULE_PATHNAME@` Templating + +In case you are already providing custom SQL definitions for Rust functions, you can use the `@MODULE_PATHNAME@` +template in your custom SQL. This value will be replaced with the path to the actual shared object. + +The following example illustrates how this works: + +```rust +#[pg_extern(sql = r#" + CREATE OR REPLACE FUNCTION tests."overridden_sql_with_fn_name"() RETURNS void + STRICT + LANGUAGE c /* Rust */ + AS '@MODULE_PATHNAME@', '@FUNCTION_NAME@'; +"#)] +fn overridden_sql_with_fn_name() -> bool { + true +} +``` + +### Caveats + +There are some scenarios which are entirely incompatible with this feature, because they rely on some global state in +Postgres, so loading two versions of the shared library will cause trouble. + +These scenarios are: +- when using shared memory +- when using query planner hooks diff --git a/cargo-pgx/src/command/install.rs b/cargo-pgx/src/command/install.rs index dd1a5a3826..3a114e31c8 100644 --- a/cargo-pgx/src/command/install.rs +++ b/cargo-pgx/src/command/install.rs @@ -14,8 +14,8 @@ use crate::{ use cargo_toml::Manifest; use eyre::{eyre, WrapErr}; use owo_colors::OwoColorize; -use pgx_utils::get_target_dir; use pgx_utils::pg_config::PgConfig; +use pgx_utils::{get_target_dir, versioned_so_name}; use std::{ io::BufReader, path::{Path, PathBuf}, @@ -114,6 +114,8 @@ pub(crate) fn install_extension( )); } + let versioned_so = get_property(&package_manifest_path, "module_pathname")?.is_none(); + let build_command_output = build_extension( user_manifest_path.as_ref(), user_package, @@ -152,7 +154,13 @@ pub(crate) fn install_extension( { let mut dest = base_directory.clone(); dest.push(&pkgdir); - dest.push(format!("{}.so", extname)); + let so_name = if versioned_so { + let extver = get_version(&package_manifest_path)?; + versioned_so_name(&extname, &extver) + } else { + extname.clone() + }; + dest.push(format!("{}.so", so_name)); if cfg!(target_os = "macos") { // Remove the existing .so if present. This is a workaround for an @@ -301,7 +309,7 @@ fn get_target_sql_file( let mut dest = base_directory.clone(); dest.push(extdir); - let (_, extname) = crate::command::get::find_control_file(&manifest_path)?; + let (_, extname) = find_control_file(&manifest_path)?; let version = get_version(&manifest_path)?; dest.push(format!("{}--{}.sql", extname, version)); @@ -321,7 +329,7 @@ fn copy_sql_files( skip_build: bool, ) -> eyre::Result<()> { let dest = get_target_sql_file(&package_manifest_path, extdir, base_directory)?; - let (_, extname) = crate::command::get::find_control_file(&package_manifest_path)?; + let (_, extname) = find_control_file(&package_manifest_path)?; crate::command::schema::generate_schema( pg_config, diff --git a/cargo-pgx/src/command/schema.rs b/cargo-pgx/src/command/schema.rs index 2fc7fdc27a..43477a17d9 100644 --- a/cargo-pgx/src/command/schema.rs +++ b/cargo-pgx/src/command/schema.rs @@ -165,6 +165,8 @@ pub(crate) fn generate_schema( )); } + let versioned_so = get_property(&package_manifest_path, "module_pathname")?.is_none(); + let flags = std::env::var("PGX_BUILD_FLAGS").unwrap_or_default(); let mut target_dir_with_profile = pgx_utils::get_target_dir()?; @@ -401,6 +403,8 @@ pub(crate) fn generate_schema( typeid_sql_mapping.clone().into_iter(), source_only_sql_mapping.clone().into_iter(), entities.into_iter(), + package_name.to_string(), + versioned_so, ) .wrap_err("SQL generation error")?; diff --git a/cargo-pgx/src/templates/cargo_toml b/cargo-pgx/src/templates/cargo_toml index 329a7993d2..868d123240 100644 --- a/cargo-pgx/src/templates/cargo_toml +++ b/cargo-pgx/src/templates/cargo_toml @@ -16,10 +16,10 @@ pg14 = ["pgx/pg14", "pgx-tests/pg14" ] pg_test = [] [dependencies] -pgx = "0.4.3" +pgx = "0.4.4" [dev-dependencies] -pgx-tests = "0.4.3" +pgx-tests = "0.4.4" [profile.dev] panic = "unwind" diff --git a/nix/templates/default/Cargo.toml b/nix/templates/default/Cargo.toml index 6ba7ed68cc..2d08c010b8 100644 --- a/nix/templates/default/Cargo.toml +++ b/nix/templates/default/Cargo.toml @@ -16,13 +16,13 @@ pg14 = ["pgx/pg14", "pgx-tests/pg14" ] pg_test = [] [dependencies] -pgx = "0.4.3" -pgx-macros = "0.4.3" -pgx-utils = "0.4.3" +pgx = "0.4.4" +pgx-macros = "0.4.4" +pgx-utils = "0.4.4" [dev-dependencies] -pgx-tests = "0.4.3" +pgx-tests = "0.4.4" tempfile = "3.2.0" once_cell = "1.7.2" diff --git a/pgx-examples/aggregate/Cargo.toml b/pgx-examples/aggregate/Cargo.toml index 23bfaf5615..86c6d36422 100644 --- a/pgx-examples/aggregate/Cargo.toml +++ b/pgx-examples/aggregate/Cargo.toml @@ -17,7 +17,7 @@ pg_test = [] [dependencies] pgx = { path = "../../pgx", default-features = false } -serde = "1.0.136" +serde = "1.0.137" [dev-dependencies] pgx-tests = { path = "../../pgx-tests" } diff --git a/pgx-examples/arrays/Cargo.toml b/pgx-examples/arrays/Cargo.toml index 71649fcde0..1d09e2d543 100644 --- a/pgx-examples/arrays/Cargo.toml +++ b/pgx-examples/arrays/Cargo.toml @@ -17,7 +17,7 @@ pg_test = [] [dependencies] pgx = { path = "../../pgx", default-features = false } -serde = "1.0.136" +serde = "1.0.137" [dev-dependencies] pgx-tests = { path = "../../pgx-tests" } diff --git a/pgx-examples/custom_sql/Cargo.toml b/pgx-examples/custom_sql/Cargo.toml index 7bc66cebdd..0c6ab4acee 100644 --- a/pgx-examples/custom_sql/Cargo.toml +++ b/pgx-examples/custom_sql/Cargo.toml @@ -17,7 +17,7 @@ pg_test = [] [dependencies] pgx = { path = "../../pgx", default-features = false } -serde = "1.0.136" +serde = "1.0.137" [dev-dependencies] pgx-tests = { path = "../../pgx-tests" } diff --git a/pgx-examples/custom_types/Cargo.toml b/pgx-examples/custom_types/Cargo.toml index d9bcce533b..ec798712ac 100644 --- a/pgx-examples/custom_types/Cargo.toml +++ b/pgx-examples/custom_types/Cargo.toml @@ -18,7 +18,7 @@ pg_test = [] [dependencies] pgx = { path = "../../pgx", default-features = false } maplit = "1.0.2" -serde = "1.0.136" +serde = "1.0.137" [dev-dependencies] pgx-tests = { path = "../../pgx-tests" } diff --git a/pgx-examples/nostd/Cargo.toml b/pgx-examples/nostd/Cargo.toml index c025d2591f..23aa1244d0 100644 --- a/pgx-examples/nostd/Cargo.toml +++ b/pgx-examples/nostd/Cargo.toml @@ -17,7 +17,7 @@ pg_test = [] [dependencies] pgx = { path = "../../pgx", default-features = false } -serde = "1.0.136" +serde = "1.0.137" [dev-dependencies] pgx-tests = { path = "../../pgx-tests" } diff --git a/pgx-examples/operators/Cargo.toml b/pgx-examples/operators/Cargo.toml index 2585889c93..81cd955c68 100644 --- a/pgx-examples/operators/Cargo.toml +++ b/pgx-examples/operators/Cargo.toml @@ -18,7 +18,7 @@ pg_test = [] [dependencies] pgx = { path = "../../pgx", default-features = false } maplit = "1.0.2" -serde = "1.0.136" +serde = "1.0.137" [dev-dependencies] pgx-tests = { path = "../../pgx-tests" } diff --git a/pgx-examples/schemas/Cargo.toml b/pgx-examples/schemas/Cargo.toml index 263be2c5ac..70b823e152 100644 --- a/pgx-examples/schemas/Cargo.toml +++ b/pgx-examples/schemas/Cargo.toml @@ -17,7 +17,7 @@ pg_test = [] [dependencies] pgx = { path = "../../pgx", default-features = false } -serde = "1.0.136" +serde = "1.0.137" [dev-dependencies] pgx-tests = { path = "../../pgx-tests" } diff --git a/pgx-examples/shmem/Cargo.toml b/pgx-examples/shmem/Cargo.toml index bfd97112a8..8cf04e0a17 100644 --- a/pgx-examples/shmem/Cargo.toml +++ b/pgx-examples/shmem/Cargo.toml @@ -18,7 +18,7 @@ pg_test = [] [dependencies] heapless = "0.7.10" pgx = { path = "../../pgx", default-features = false } -serde = { version = "1.0.136", features = [ "derive" ] } +serde = { version = "1.0.137", features = [ "derive" ] } [dev-dependencies] pgx-tests = { path = "../../pgx-tests" } diff --git a/pgx-examples/versioned_so/.gitignore b/pgx-examples/versioned_so/.gitignore new file mode 100644 index 0000000000..3906c33241 --- /dev/null +++ b/pgx-examples/versioned_so/.gitignore @@ -0,0 +1,6 @@ +.DS_Store +.idea/ +/target +*.iml +**/*.rs.bk +Cargo.lock diff --git a/pgx-examples/versioned_so/Cargo.toml b/pgx-examples/versioned_so/Cargo.toml new file mode 100644 index 0000000000..0a96ca1d59 --- /dev/null +++ b/pgx-examples/versioned_so/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "versioned_so" +version = "0.0.0" +edition = "2021" + +[lib] +crate-type = ["cdylib"] + +[features] +default = ["pg13"] +pg10 = ["pgx/pg10", "pgx-tests/pg10" ] +pg11 = ["pgx/pg11", "pgx-tests/pg11" ] +pg12 = ["pgx/pg12", "pgx-tests/pg12" ] +pg13 = ["pgx/pg13", "pgx-tests/pg13" ] +pg14 = ["pgx/pg14", "pgx-tests/pg14" ] +pg_test = [] + +[dependencies] +pgx = { path = "../../pgx/", default-features = false } + +[dev-dependencies] +pgx-tests = { path = "../../pgx-tests" } + +# uncomment these if compiling outside of 'pgx' +#[profile.dev] +#panic = "unwind" +# lto = "thin" + +#[profile.release] +#panic = "unwind" +#opt-level = 3 +#lto = "fat" +#codegen-units = 1 diff --git a/pgx-examples/versioned_so/src/lib.rs b/pgx-examples/versioned_so/src/lib.rs new file mode 100644 index 0000000000..a4725e8326 --- /dev/null +++ b/pgx-examples/versioned_so/src/lib.rs @@ -0,0 +1,32 @@ +use pgx::*; + +pg_module_magic!(); + +#[pg_extern] +fn hello_versioned_so() -> &'static str { + "Hello, versioned_so" +} + +#[cfg(any(test, feature = "pg_test"))] +#[pg_schema] +mod tests { + use pgx::*; + + #[pg_test] + fn test_hello_versioned_so() { + assert_eq!("Hello, versioned_so", crate::hello_versioned_so()); + } + +} + +#[cfg(test)] +pub mod pg_test { + pub fn setup(_options: Vec<&str>) { + // perform one-off initialization when the pg_test framework starts + } + + pub fn postgresql_conf_options() -> Vec<&'static str> { + // return any postgresql.conf settings that are required for your tests + vec![] + } +} diff --git a/pgx-examples/versioned_so/versioned_so.control b/pgx-examples/versioned_so/versioned_so.control new file mode 100644 index 0000000000..52014aed9e --- /dev/null +++ b/pgx-examples/versioned_so/versioned_so.control @@ -0,0 +1,6 @@ +comment = 'versioned_so: Created by pgx' +default_version = '@CARGO_VERSION@' +# commenting-out module_pathname results in this extension being built/run/tested in "versioned shared-object mode" +# module_pathname = '$libdir/versioned_so' +relocatable = false +superuser = false diff --git a/pgx-macros/Cargo.toml b/pgx-macros/Cargo.toml index b48fe40744..240a27c805 100644 --- a/pgx-macros/Cargo.toml +++ b/pgx-macros/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pgx-macros" -version = "0.4.3" +version = "0.4.4" authors = ["ZomboDB, LLC "] license = "MIT" description = "Proc Macros for 'pgx'" @@ -18,11 +18,11 @@ proc-macro = true rustc-args = ["--cfg", "docsrs"] [dependencies] -pgx-utils = { path = "../pgx-utils", version = "0.4.3" } +pgx-utils = { path = "../pgx-utils", version = "0.4.4" } proc-macro2 = "1.0.37" quote = "1.0.18" -syn = { version = "1.0.91", features = [ "extra-traits", "full", "fold", "parsing" ] } +syn = { version = "1.0.92", features = [ "extra-traits", "full", "fold", "parsing" ] } unescape = "0.1.0" [dev-dependencies] -serde = { version = "1.0.136", features = ["derive"] } +serde = { version = "1.0.137", features = ["derive"] } diff --git a/pgx-pg-sys/Cargo.toml b/pgx-pg-sys/Cargo.toml index 1f296ce787..6c5769964f 100644 --- a/pgx-pg-sys/Cargo.toml +++ b/pgx-pg-sys/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pgx-pg-sys" -version = "0.4.3" +version = "0.4.4" authors = ["ZomboDB, LLC "] license = "MIT" description = "Generated Rust bindings for Postgres internals, for use with 'pgx'" @@ -29,17 +29,17 @@ rustdoc-args = ["--cfg", "docsrs"] [dependencies] memoffset = "0.6.5" once_cell = "1.10.0" -pgx-macros = { path = "../pgx-macros/", version = "0.4.3" } +pgx-macros = { path = "../pgx-macros/", version = "0.4.4" } [build-dependencies] bindgen = { version = "0.59.2", default-features = false, features = ["runtime"] } build-deps = "0.1.4" -owo-colors = "3.3.0" +owo-colors = "3.4.0" num_cpus = "1.13.1" -pgx-utils = { path = "../pgx-utils/", version = "0.4.3" } +pgx-utils = { path = "../pgx-utils/", version = "0.4.4" } proc-macro2 = "1.0.37" quote = "1.0.18" rayon = "1.5.2" -syn = { version = "1.0.91", features = [ "extra-traits", "full", "fold", "parsing" ] } +syn = { version = "1.0.92", features = [ "extra-traits", "full", "fold", "parsing" ] } eyre = "0.6.8" color-eyre = "0.6.1" diff --git a/pgx-pg-sys/include/pg10.h b/pgx-pg-sys/include/pg10.h index f6a9cd267e..518fc611e6 100644 --- a/pgx-pg-sys/include/pg10.h +++ b/pgx-pg-sys/include/pg10.h @@ -74,6 +74,7 @@ Use of this source code is governed by the MIT license that can be found in the #include "optimizer/restrictinfo.h" #include "optimizer/tlist.h" #include "parser/parse_func.h" +#include "parser/parse_oper.h" #include "parser/parse_type.h" #include "parser/parser.h" #include "parser/parsetree.h" diff --git a/pgx-pg-sys/include/pg11.h b/pgx-pg-sys/include/pg11.h index 9957051513..7d4acccbad 100644 --- a/pgx-pg-sys/include/pg11.h +++ b/pgx-pg-sys/include/pg11.h @@ -72,6 +72,7 @@ Use of this source code is governed by the MIT license that can be found in the #include "optimizer/restrictinfo.h" #include "optimizer/tlist.h" #include "parser/parse_func.h" +#include "parser/parse_oper.h" #include "parser/parse_type.h" #include "parser/parser.h" #include "parser/parsetree.h" diff --git a/pgx-pg-sys/include/pg12.h b/pgx-pg-sys/include/pg12.h index 53fa9ca56a..91a11d26e3 100644 --- a/pgx-pg-sys/include/pg12.h +++ b/pgx-pg-sys/include/pg12.h @@ -72,6 +72,7 @@ Use of this source code is governed by the MIT license that can be found in the #include "optimizer/restrictinfo.h" #include "optimizer/tlist.h" #include "parser/parse_func.h" +#include "parser/parse_oper.h" #include "parser/parse_type.h" #include "parser/parser.h" #include "parser/parsetree.h" diff --git a/pgx-pg-sys/include/pg13.h b/pgx-pg-sys/include/pg13.h index d8e4e52c10..bf920ab77a 100644 --- a/pgx-pg-sys/include/pg13.h +++ b/pgx-pg-sys/include/pg13.h @@ -72,6 +72,7 @@ Use of this source code is governed by the MIT license that can be found in the #include "optimizer/restrictinfo.h" #include "optimizer/tlist.h" #include "parser/parse_func.h" +#include "parser/parse_oper.h" #include "parser/parse_type.h" #include "parser/parser.h" #include "parser/parsetree.h" diff --git a/pgx-pg-sys/include/pg14.h b/pgx-pg-sys/include/pg14.h index d8e4e52c10..bf920ab77a 100644 --- a/pgx-pg-sys/include/pg14.h +++ b/pgx-pg-sys/include/pg14.h @@ -72,6 +72,7 @@ Use of this source code is governed by the MIT license that can be found in the #include "optimizer/restrictinfo.h" #include "optimizer/tlist.h" #include "parser/parse_func.h" +#include "parser/parse_oper.h" #include "parser/parse_type.h" #include "parser/parser.h" #include "parser/parsetree.h" diff --git a/pgx-tests/Cargo.toml b/pgx-tests/Cargo.toml index ef200680a1..afa361f2a0 100644 --- a/pgx-tests/Cargo.toml +++ b/pgx-tests/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pgx-tests" -version = "0.4.3" +version = "0.4.4" authors = ["ZomboDB, LLC "] license = "MIT" description = "Test framework for 'pgx'-based Postgres extensions" @@ -31,16 +31,16 @@ rustc-args = ["--cfg", "docsrs"] rustdoc-args = ["--cfg", "docsrs"] [dependencies] -owo-colors = "3.3.0" +owo-colors = "3.4.0" once_cell = "1.10.0" -libc = "0.2.124" -pgx = { path = "../pgx", default-features = false, version= "0.4.3" } -pgx-macros = { path = "../pgx-macros", version= "0.4.3" } -pgx-utils = { path = "../pgx-utils", version= "0.4.3" } -postgres = "0.19.2" +libc = "0.2.125" +pgx = { path = "../pgx", default-features = false, version= "0.4.4" } +pgx-macros = { path = "../pgx-macros", version= "0.4.4" } +pgx-utils = { path = "../pgx-utils", version= "0.4.4" } +postgres = "0.19.3" regex = "1.5.5" -serde = "1.0.136" -serde_json = "1.0.79" +serde = "1.0.137" +serde_json = "1.0.80" shutdown_hooks = "0.1.0" time = "0.3.9" eyre = "0.6.8" diff --git a/pgx-tests/src/tests/pg_extern_tests.rs b/pgx-tests/src/tests/pg_extern_tests.rs index d1ed30305a..615605dbc8 100644 --- a/pgx-tests/src/tests/pg_extern_tests.rs +++ b/pgx-tests/src/tests/pg_extern_tests.rs @@ -34,12 +34,12 @@ mod tests { assert!(result) } - // Ensures `@FUNCTION_NAME@` is handled. + // Ensures `@MODULE_PATHNAME@` and `@FUNCTION_NAME@` are handled. #[pg_extern(sql = r#" CREATE OR REPLACE FUNCTION tests."overridden_sql_with_fn_name"() RETURNS void STRICT LANGUAGE c /* Rust */ - AS 'MODULE_PATHNAME', '@FUNCTION_NAME@'; + AS '@MODULE_PATHNAME@', '@FUNCTION_NAME@'; "#)] fn overridden_sql_with_fn_name() -> bool { true diff --git a/pgx-utils/Cargo.toml b/pgx-utils/Cargo.toml index f0bb333e76..3a2f06ebea 100644 --- a/pgx-utils/Cargo.toml +++ b/pgx-utils/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pgx-utils" -version = "0.4.3" +version = "0.4.4" authors = ["ZomboDB, LLC "] license = "MIT" description = "Utility functions for 'pgx'" @@ -12,7 +12,7 @@ edition = "2021" [dependencies] atty = "0.2.14" -owo-colors = "3.3.0" +owo-colors = "3.4.0" convert_case = "0.5.0" dirs = "4.0.0" env_proxy = "0.4.1" @@ -20,11 +20,11 @@ proc-macro2 = { version = "1.0.37", features = [ "span-locations" ] } quote = "1.0.18" regex = "1.5.5" rttp_client = "0.1.0" -serde = { version = "1.0.136", features = [ "derive" ] } -serde_derive = "1.0.136" +serde = { version = "1.0.137", features = [ "derive" ] } +serde_derive = "1.0.137" serde-xml-rs = "0.5.1" -serde_json = "1.0.79" -syn = { version = "1.0.91", features = [ "extra-traits", "full", "fold", "parsing" ] } +serde_json = "1.0.80" +syn = { version = "1.0.92", features = [ "extra-traits", "full", "fold", "parsing" ] } syntect = { version = "4.6.0", default-features = false, features = ["default-fancy"] } toml = "0.5.9" unescape = "0.1.0" @@ -34,4 +34,4 @@ tracing = "0.1.34" tracing-error = "0.2.0" tracing-subscriber = { version = "0.3.11", features = [ "env-filter" ] } petgraph = "0.6.0" -prettyplease = "0.1.9" +prettyplease = "0.1.10" diff --git a/pgx-utils/src/lib.rs b/pgx-utils/src/lib.rs index 3a332353c8..c01f0a2efe 100644 --- a/pgx-utils/src/lib.rs +++ b/pgx-utils/src/lib.rs @@ -563,6 +563,10 @@ pub fn anonymonize_lifetimes(value: &mut syn::Type) { } } +pub fn versioned_so_name(extension_name: &str, extension_version: &str) -> String { + format!("{}-{}", extension_name, extension_version) +} + #[cfg(test)] mod tests { use crate::{parse_extern_attributes, ExternArgs}; diff --git a/pgx-utils/src/sql_entity_graph/control_file.rs b/pgx-utils/src/sql_entity_graph/control_file.rs index c854e89a1a..37e7d5ef83 100644 --- a/pgx-utils/src/sql_entity_graph/control_file.rs +++ b/pgx-utils/src/sql_entity_graph/control_file.rs @@ -26,7 +26,7 @@ use tracing_error::SpanTrace; pub struct ControlFile { pub comment: String, pub default_version: String, - pub module_pathname: String, + pub module_pathname: Option, pub relocatable: bool, pub superuser: bool, pub schema: Option, @@ -75,13 +75,7 @@ impl ControlFile { context: SpanTrace::capture(), })? .to_string(), - module_pathname: temp - .get("module_pathname") - .ok_or(ControlFileError::MissingField { - field: "module_pathname", - context: SpanTrace::capture(), - })? - .to_string(), + module_pathname: temp.get("module_pathname").map(|v| v.to_string()), relocatable: temp .get("relocatable") .ok_or(ControlFileError::MissingField { diff --git a/pgx-utils/src/sql_entity_graph/pg_extern/entity/mod.rs b/pgx-utils/src/sql_entity_graph/pg_extern/entity/mod.rs index 854c0fc2d0..c2619e7725 100644 --- a/pgx-utils/src/sql_entity_graph/pg_extern/entity/mod.rs +++ b/pgx-utils/src/sql_entity_graph/pg_extern/entity/mod.rs @@ -105,16 +105,19 @@ impl ToSql for PgExternEntity { extern_attrs.push(ExternArgs::Strict); } + let module_pathname = &context.get_module_pathname(); + let fn_sql = format!("\ CREATE OR REPLACE FUNCTION {schema}\"{name}\"({arguments}) {returns}\n\ {extern_attrs}\ {search_path}\ LANGUAGE c /* Rust */\n\ - AS 'MODULE_PATHNAME', '{unaliased_name}_wrapper';\ + AS '{module_pathname}', '{unaliased_name}_wrapper';\ ", schema = self.schema.map(|schema| format!("{}.", schema)).unwrap_or_else(|| context.schema_prefix_for(&self_index)), name = self.name, unaliased_name = self.unaliased_name, + module_pathname = module_pathname, arguments = if !self.fn_args.is_empty() { let mut args = Vec::new(); for (idx, arg) in self.fn_args.iter().enumerate() { diff --git a/pgx-utils/src/sql_entity_graph/pgx_sql.rs b/pgx-utils/src/sql_entity_graph/pgx_sql.rs index 94302ea01c..1f1452139d 100644 --- a/pgx-utils/src/sql_entity_graph/pgx_sql.rs +++ b/pgx-utils/src/sql_entity_graph/pgx_sql.rs @@ -31,6 +31,7 @@ use crate::sql_entity_graph::{ to_sql::ToSql, SqlGraphEntity, SqlGraphIdentifier, }; +use crate::versioned_so_name; /// A generator for SQL. /// @@ -64,6 +65,8 @@ pub struct PgxSql { pub ords: HashMap, pub hashes: HashMap, pub aggregates: HashMap, + pub extension_name: String, + pub versioned_so: bool, } #[derive(Debug, Clone, Copy, PartialEq, PartialOrd, Eq, Ord)] @@ -79,6 +82,8 @@ impl PgxSql { type_mappings: impl Iterator, source_mappings: impl Iterator, entities: impl Iterator, + extension_name: String, + versioned_so: bool, ) -> eyre::Result { let mut graph = StableGraph::new(); @@ -229,6 +234,8 @@ impl PgxSql { graph_root: root, graph_bootstrap: bootstrap, graph_finalize: finalize, + extension_name: extension_name, + versioned_so, }; this.register_types(); Ok(this) @@ -495,6 +502,16 @@ impl PgxSql { }, ); } + + pub fn get_module_pathname(&self) -> String { + return if self.versioned_so { + let extname = &self.extension_name; + let extver = &self.control.default_version; + format!("$libdir/{}", versioned_so_name(extname, extver)) + } else { + String::from("MODULE_PATHNAME") + }; + } } #[tracing::instrument(level = "error", skip_all)] diff --git a/pgx-utils/src/sql_entity_graph/to_sql/entity.rs b/pgx-utils/src/sql_entity_graph/to_sql/entity.rs index 385a7f2b36..0e5b1d75d8 100644 --- a/pgx-utils/src/sql_entity_graph/to_sql/entity.rs +++ b/pgx-utils/src/sql_entity_graph/to_sql/entity.rs @@ -55,6 +55,10 @@ impl ToSqlConfigEntity { } if let Some(content) = self.content { + let module_pathname = context.get_module_pathname(); + + let content = content.replace("@MODULE_PATHNAME@", &module_pathname); + return Some(Ok(format!( "\n\ {sql_anchor_comment}\n\ @@ -70,14 +74,20 @@ impl ToSqlConfigEntity { .map_err(|e| eyre!(e)) .wrap_err("Failed to run specified `#[pgx(sql = path)] function`"); return match content { - Ok(content) => Some(Ok(format!( - "\n\ + Ok(content) => { + let module_pathname = &context.get_module_pathname(); + + let content = content.replace("@MODULE_PATHNAME@", &module_pathname); + + Some(Ok(format!( + "\n\ {sql_anchor_comment}\n\ {content}\ ", - content = content, - sql_anchor_comment = entity.sql_anchor_comment(), - ))), + content = content, + sql_anchor_comment = entity.sql_anchor_comment(), + ))) + } Err(e) => Some(Err(e)), }; } diff --git a/pgx/Cargo.toml b/pgx/Cargo.toml index 51c496c5c3..8ea0c0e505 100644 --- a/pgx/Cargo.toml +++ b/pgx/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pgx" -version = "0.4.3" +version = "0.4.4" authors = ["ZomboDB, LLC "] license = "MIT" description = "pgx: A Rust framework for creating Postgres extensions" @@ -32,19 +32,18 @@ rustc-args = ["--cfg", "docsrs"] [dependencies] cstr_core = "0.2.5" enum-primitive-derive = "0.2.2" -num-traits = "0.2.14" +num-traits = "0.2.15" seahash = "4.1.0" -pgx-macros = { path = "../pgx-macros/", version = "0.4.3" } -pgx-pg-sys = { path = "../pgx-pg-sys", version = "0.4.3" } -pgx-utils = { path = "../pgx-utils/", version = "0.4.3" } -serde = { version = "1.0.136", features = [ "derive" ] } +pgx-macros = { path = "../pgx-macros/", version = "0.4.4" } +pgx-pg-sys = { path = "../pgx-pg-sys", version = "0.4.4" } +pgx-utils = { path = "../pgx-utils/", version = "0.4.4" } +serde = { version = "1.0.137", features = [ "derive" ] } serde_cbor = "0.11.2" -serde_json = "1.0.79" +serde_json = "1.0.80" time = { version = "0.3.9", features = ["formatting", "parsing", "alloc", "macros"] } atomic-traits = "0.3.0" heapless = "0.7.10" -hash32 = "0.2.1" -uuid = { version = "0.8.2", features = [ "v4" ] } +uuid = { version = "1.0.0", features = [ "v4" ] } once_cell = "1.10.0" bitflags = "1.3.2" eyre = "0.6.8" diff --git a/pgx/src/datum/numeric.rs b/pgx/src/datum/numeric.rs index 4ecec5ae9a..b9f987b6f4 100644 --- a/pgx/src/datum/numeric.rs +++ b/pgx/src/datum/numeric.rs @@ -16,9 +16,15 @@ use serde::{de, Deserialize, Deserializer, Serialize}; use serde_json::Number; use std::fmt; -#[derive(Serialize)] +#[derive(Serialize, Debug)] pub struct Numeric(pub String); +impl std::fmt::Display for Numeric { + fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::result::Result<(), std::fmt::Error> { + fmt.write_fmt(format_args!("{}", self.0)) + } +} + impl<'de> Deserialize<'de> for Numeric { fn deserialize(deserializer: D) -> Result>::Error> where diff --git a/pgx/src/namespace.rs b/pgx/src/namespace.rs index c1f609f097..70e07d2e55 100644 --- a/pgx/src/namespace.rs +++ b/pgx/src/namespace.rs @@ -11,6 +11,7 @@ Use of this source code is governed by the MIT license that can be found in the use crate::list::PgList; use crate::pg_sys; +use pgx_pg_sys::AsPgCStr; /// A helper struct for creating a Postgres `List` of `String`s to qualify an object name pub struct PgQualifiedNameBuilder { @@ -33,9 +34,7 @@ impl PgQualifiedNameBuilder { pub fn push(mut self, value: &str) -> PgQualifiedNameBuilder { unsafe { // SAFETY: the result of pg_sys::makeString is always a valid pointer - self.list.push(pg_sys::makeString( - std::ffi::CString::new(value).unwrap().into_raw(), - )); + self.list.push(pg_sys::makeString(value.as_pg_cstr())); } self } diff --git a/pgx/src/shmem.rs b/pgx/src/shmem.rs index 3c0847f96e..99d4a264e7 100644 --- a/pgx/src/shmem.rs +++ b/pgx/src/shmem.rs @@ -8,6 +8,7 @@ Use of this source code is governed by the MIT license that can be found in the */ use crate::lwlock::*; use crate::{pg_sys, PgAtomic}; +use std::hash::Hash; use uuid::Uuid; /// Custom types that want to participate in shared memory must implement this marker trait @@ -217,7 +218,7 @@ where { } unsafe impl PGXSharedMemory for heapless::Vec {} -unsafe impl PGXSharedMemory +unsafe impl PGXSharedMemory for heapless::IndexMap { }