diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn index 3c71a9b52..51046643f 100644 --- a/.clj-kondo/config.edn +++ b/.clj-kondo/config.edn @@ -26,9 +26,17 @@ {:exclude {puppetlabs.puppetdb.jdbc/call-with-array-converted-query-rows {:namespaces [".*"]} puppetlabs.puppetdb.testutils.services/call-with-puppetdb-instance {:namespaces [".*"]} - puppetlabs.puppetdb.testutils.services/with-puppetdb-instance {:namespaces [".*"]}}} + puppetlabs.puppetdb.testutils.services/with-puppetdb-instance {:namespaces [".*"]} + puppetlabs.trapperkeeper.testutils.logging/atom-logger {:namespaces [".*"]} + puppetlabs.trapperkeeper.testutils.logging/with-log-output {:namespaces [".*"]} + puppetlabs.trapperkeeper.testutils.logging/logs-matching {:namespaces [".*"]}}} :refer-all {:exclude [clojure.test]} :unresolved-symbol {:exclude [(puppetlabs.comidi/GET) - (puppetlabs.comidi/POST)]}}} + (puppetlabs.comidi/POST)]} + :unresolved-var + {:exclude + [digest/sha1 + instaparse.core/transform + com.rpl.specter/ALL]}}} diff --git a/project.clj b/project.clj index 1e5751264..aac40ee3f 100644 --- a/project.clj +++ b/project.clj @@ -1,6 +1,6 @@ -(def pdb-version "8.11.1-SNAPSHOT") +(def pdb-version "8.12.0-SNAPSHOT") -(def clj-parent-version "7.4.0") +(def clj-parent-version "7.6.6") (defn true-in-env? [x] (#{"true" "yes" "1"} (System/getenv x))) @@ -25,15 +25,15 @@ (let [ver (eval '(java.lang.Runtime/version))] {:feature (.feature ver) :interim (.interim ver)})))) -(def i18n-version "0.9.2") +(def i18n-version "1.0.2") (def pdb-dev-deps (concat '[[ring/ring-mock] [timofreiberg/bultitude "0.3.1"] - [puppetlabs/trapperkeeper :classifier "test"] - [puppetlabs/kitchensink :classifier "test"] - [com.puppetlabs/trapperkeeper-webserver-jetty10 :classifier "test"] + [org.openvoxproject/trapperkeeper :classifier "test"] + [org.openvoxproject/kitchensink :classifier "test"] + [org.openvoxproject/trapperkeeper-webserver-jetty10 :classifier "test"] [org.flatland/ordered "1.15.12"] [org.clojure/test.check "1.1.1"] [com.gfredericks/test.chuck "0.2.14"] @@ -42,7 +42,7 @@ [org.yaml/snakeyaml] ;; Only needed for :integration tests - [puppetlabs/trapperkeeper-filesystem-watcher nil]])) + [org.openvoxproject/trapperkeeper-filesystem-watcher nil]])) ;; Don't use lein :clean-targets so that we don't have to repeat ;; ourselves, given that we need to remove some protected files, and @@ -78,17 +78,23 @@ "puppetserver" "vendor"])) +(def pdb-aot-classes + ;; Compile classes first for now: + ;; https://codeberg.org/leiningen/leiningen/issues/99 + '[puppetlabs.puppetdb.jdbc.PDBBytea puppetlabs.puppetdb.jdbc.VecPDBBytea]) + (def pdb-aot-namespaces - (apply vector - #"puppetlabs\.puppetdb\..*" - (->> "resources/puppetlabs/puppetdb/bootstrap.cfg" - clojure.java.io/reader - line-seq - (map clojure.string/trim) - (remove #(re-matches #"#.*" %)) ;; # comments - (remove #(re-matches #"puppetlabs\.puppetdb\.." %)) - (map #(clojure.string/replace % #"(.*)/[^/]+$" "$1")) - (map symbol)))) + (into [] + (concat pdb-aot-classes + [#"puppetlabs\.puppetdb\..*"] + (->> "resources/puppetlabs/puppetdb/bootstrap.cfg" + clojure.java.io/reader + line-seq + (map clojure.string/trim) + (remove #(re-matches #"#.*" %)) ;; # comments + (remove #(re-matches #"puppetlabs\.puppetdb\.." %)) + (map #(clojure.string/replace % #"(.*)/[^/]+$" "$1")) + (map symbol))))) ;; Avoid startup reflection warnings due to ;; https://clojure.atlassian.net/browse/CLJ-2066 @@ -96,13 +102,13 @@ (def pdb-jvm-opts (when (< 8 (:feature pdb-jvm-ver) 17) ["--illegal-access=deny"])) -(defproject puppetlabs/puppetdb pdb-version +(defproject org.openvoxproject/puppetdb pdb-version :description "OpenVox-integrated catalog and fact storage" :license {:name "Apache License, Version 2.0" :url "http://www.apache.org/licenses/LICENSE-2.0.html"} - :url "https://docs.puppetlabs.com/puppetdb/" + :url "https://github.com/openvoxproject/openvoxdb/" :min-lein-version "2.7.1" @@ -128,17 +134,17 @@ [org.clojure/tools.nrepl] ;; Puppet specific - [puppetlabs/comidi] - [puppetlabs/i18n] - [puppetlabs/kitchensink] - [puppetlabs/ssl-utils] - [puppetlabs/stockpile "0.0.4"] - [puppetlabs/structured-logging] - [puppetlabs/trapperkeeper] - [com.puppetlabs/trapperkeeper-webserver-jetty10] - [puppetlabs/trapperkeeper-metrics] - [puppetlabs/trapperkeeper-status] - [puppetlabs/trapperkeeper-authorization] + [org.openvoxproject/comidi] + [org.openvoxproject/i18n] + [org.openvoxproject/kitchensink] + [org.openvoxproject/ssl-utils] + [org.openvoxproject/stockpile "1.0.0"] + [org.openvoxproject/structured-logging "1.0.0"] + [org.openvoxproject/trapperkeeper] + [org.openvoxproject/trapperkeeper-webserver-jetty10] + [org.openvoxproject/trapperkeeper-metrics] + [org.openvoxproject/trapperkeeper-status] + [org.openvoxproject/trapperkeeper-authorization] ;; Various [cheshire] @@ -162,7 +168,7 @@ [com.fasterxml.jackson.core/jackson-databind] ;; Filesystem utilities - [org.apache.commons/commons-lang3] + [org.apache.commons/commons-lang3 "3.20.0"] ;; Database connectivity [com.zaxxer/HikariCP] @@ -177,7 +183,7 @@ :jvm-opts ~pdb-jvm-opts - :deploy-repositories [["clojars" {:url "https://clojars.org/repo" + :deploy-repositories [["releases" {:url "https://clojars.org/repo" :username :env/CLOJARS_USERNAME :password :env/CLOJARS_PASSWORD :sign-releases false}]] @@ -185,7 +191,7 @@ :plugins [[lein-release "1.1.3" :exclusions [org.clojure/clojure]] [lein-cloverage "1.2.4"] [lein-parent "0.3.9"] - [puppetlabs/i18n ~i18n-version]] + [org.openvoxproject/i18n ~i18n-version]] :lein-release {:scm :git :deploy-via :lein-deploy} @@ -216,6 +222,8 @@ ;; "test"]. See the :testutils profile below. :classifiers {:test :testutils} + :aot ~pdb-aot-classes + :profiles {:defaults {:resource-paths ["test-resources"] :dependencies ~pdb-dev-deps :injections [(do @@ -236,12 +244,11 @@ :jvm-opts ~(let [{:keys [feature interim]} pdb-jvm-ver] (conj pdb-jvm-opts (case feature - 8 "-Djava.security.properties==dev-resources/jdk8-fips-security" - 11 "-Djava.security.properties==dev-resources/jdk11on-fips-security" 17 "-Djava.security.properties==dev-resources/jdk11on-fips-security" + 21 "-Djava.security.properties==dev-resources/jdk11on-fips-security" (do) )))}] - :kondo {:dependencies [[clj-kondo "2024.05.24"]]} + :kondo {:dependencies [[clj-kondo "2025.10.23"]]} :ezbake {:dependencies ^:replace [;; NOTE: we need to explicitly pass in `nil` values ;; for the version numbers here in order to correctly ;; inherit the versions from our parent project. @@ -266,7 +273,7 @@ ;; This circular dependency is required because of a bug in ;; ezbake (EZ-35); without it, bootstrap.cfg will not be included ;; in the final package. - [puppetlabs/puppetdb ~pdb-version]] + [org.openvoxproject/puppetdb ~pdb-version]] :name "puppetdb" :plugins [[org.openvoxproject/lein-ezbake ~(or (System/getenv "EZBAKE_VERSION") "2.7.1")]]} :testutils {:source-paths ^:replace ["test"] diff --git a/src/puppetlabs/puppetdb/cli/benchmark.clj b/src/puppetlabs/puppetdb/cli/benchmark.clj index 871c7b7fe..221c5561f 100644 --- a/src/puppetlabs/puppetdb/cli/benchmark.clj +++ b/src/puppetlabs/puppetdb/cli/benchmark.clj @@ -308,7 +308,7 @@ TODO: handle arrays and maps." class) -(defmethod touch-parameter-value String [p] (RandomStringUtils/randomAscii(count p))) +(defmethod touch-parameter-value String [p] (.nextAscii (RandomStringUtils/secure) (count p))) (defmethod touch-parameter-value Number [_] (rand-int 1000000)) (defmethod touch-parameter-value Boolean [p] (not p)) ;; Allow other types to pass through unmutated for now diff --git a/src/puppetlabs/puppetdb/cli/generate.clj b/src/puppetlabs/puppetdb/cli/generate.clj index f00c0e859..11463de35 100644 --- a/src/puppetlabs/puppetdb/cli/generate.clj +++ b/src/puppetlabs/puppetdb/cli/generate.clj @@ -335,7 +335,7 @@ (recur (assoc parameters (parameter-name (rnd/safe-sample-normal 20 5 {:lowerb 5})) - (RandomStringUtils/randomAscii + (.nextAscii (RandomStringUtils/secure) (rnd/safe-sample-normal 50 25 {:upperb (max 50 size)})))) parameters)))) @@ -480,7 +480,7 @@ bsize (rnd/safe-sample-normal avg-blob-size-in-bytes standard-deviation {:lowerb lowerb :upperb upperb}) pname (format "content_blob_%s" (rnd/random-pronouncable-word))] (update-in catalog [:resources (rand-int (count resources)) :parameters] - #(merge % {pname (RandomStringUtils/randomAscii bsize)})))) + #(merge % {pname (.nextAscii (RandomStringUtils/secure) bsize)})))) (defn system-seconds-str "Epoch seconds as a string. Used by default as a version string in Puppet @@ -706,7 +706,7 @@ ([{:keys [file line level message source tags] :or {level (rand-nth ["info" "notice"]) message (rnd/random-sentence-ish) - tags #{level} + tags #{} source "Puppet"}}] (let [final-tags (cset/union (set tags) #{level})] {:file file diff --git a/src/puppetlabs/puppetdb/jdbc.clj b/src/puppetlabs/puppetdb/jdbc.clj index 8a9c9adae..d144581ed 100644 --- a/src/puppetlabs/puppetdb/jdbc.clj +++ b/src/puppetlabs/puppetdb/jdbc.clj @@ -6,6 +6,7 @@ [clojure.string :as str] [clojure.tools.logging :as log] [honey.sql :as hsql] + [next.jdbc.prepare :refer [set-parameter]] [puppetlabs.i18n.core :refer [trs]] [puppetlabs.kitchensink.core :as kitchensink] [puppetlabs.puppetdb.jdbc.internal :refer [limit-result-set!]] @@ -213,7 +214,7 @@ (with-db-transaction [] (with-open [stmt (.prepareStatement ^Connection (:connection *db*) sql)] (doseq [[i param] (map vector (range) params)] - (.setObject stmt (inc i) param)) + (set-parameter param stmt (inc i))) (.setFetchSize stmt (or fetch-size 500)) (with-open [rset (.executeQuery stmt)] (try @@ -245,7 +246,7 @@ (with-db-transaction [] (with-open [stmt (.prepareStatement ^Connection (:connection *db*) sql)] (doseq [[i param] (map vector (range) params)] - (.setObject stmt (inc i) param)) + (set-parameter param stmt (inc i))) (.setFetchSize stmt fetch-size) (let [fix-vals (if as-arrays? #(mapv any-sql-array->vec %) diff --git a/src/puppetlabs/puppetdb/jdbc/PDBBytea.clj b/src/puppetlabs/puppetdb/jdbc/PDBBytea.clj new file mode 100644 index 000000000..02c2b1180 --- /dev/null +++ b/src/puppetlabs/puppetdb/jdbc/PDBBytea.clj @@ -0,0 +1,31 @@ +(ns puppetlabs.puppetdb.jdbc.PDBBytea + "Carrier for bytea parameters, to support clojure.java.jdbc and next.jdbc + protocol extensions. Essentially just a typed wrapper around a byte[]." + (:import + (java.util Arrays)) + (:gen-class + ;;:extends org.postgresql.util.PGobject + :state ^"[B" data + :init init + :constructors {["[B"] []})) + +(def ^:private warn-on-reflection-orig *warn-on-reflection*) +(set! *warn-on-reflection* true) + +(defn -init [data] [[] data]) + +;; Implemented for now because edge-replacement-differential test, for +;; example, uses munge-hash-for-storage to create the test values that +;; it compares for equality -- and who knows what else. Apparently +;; PGobject bytea values, which PDBBytea replaces, compare content for +;; equality, and since we use bytea for hashes, best to be +;; conservative. +(defn -equals [^puppetlabs.puppetdb.jdbc.PDBBytea this x] + (and (instance? puppetlabs.puppetdb.jdbc.PDBBytea x) + (Arrays/equals ^"[B" (.data this) + ^"[B" (.data ^puppetlabs.puppetdb.jdbc.PDBBytea x)))) + +(defn -hashCode [^puppetlabs.puppetdb.jdbc.PDBBytea this] + (Arrays/hashCode ^"[B" (.data this))) + +(set! *warn-on-reflection* warn-on-reflection-orig) diff --git a/src/puppetlabs/puppetdb/jdbc/VecPDBBytea.clj b/src/puppetlabs/puppetdb/jdbc/VecPDBBytea.clj new file mode 100644 index 000000000..256847067 --- /dev/null +++ b/src/puppetlabs/puppetdb/jdbc/VecPDBBytea.clj @@ -0,0 +1,16 @@ +(ns puppetlabs.puppetdb.jdbc.VecPDBBytea + "Carrier for bytea[] parameters, to support clojure.java.jdbc and next.jdbc + protocol extensions. Essentially just a typed wrapper around a byte[][]." + ;; It doesn't look like java.sql.Arrays, which is what this + ;; replaces, define content-based equals(), so we don't need one. + (:gen-class + :state ^"[[B" data + :init init + :constructors {["[[B"] []})) + +(def ^:private warn-on-reflection-orig *warn-on-reflection*) +(set! *warn-on-reflection* true) + +(defn -init [data] [[] data]) + +(set! *warn-on-reflection* warn-on-reflection-orig) diff --git a/src/puppetlabs/puppetdb/random.clj b/src/puppetlabs/puppetdb/random.clj index 6a9c8f5bf..12ae07d81 100644 --- a/src/puppetlabs/puppetdb/random.clj +++ b/src/puppetlabs/puppetdb/random.clj @@ -12,15 +12,15 @@ (defn random-string "Generate a random string of optional length" - ([] (RandomStringUtils/randomAlphabetic (inc (rand-int 10)))) + ([] (.nextAlphabetic (RandomStringUtils/secure) (inc (rand-int 10)))) ([length] - (RandomStringUtils/randomAlphabetic length))) + (.nextAlphabetic (RandomStringUtils/secure) length))) (defn random-string-alpha "Generate a random string of optional length, only lower case alphabet chars" ([] (random-string (inc (rand-int 10)))) ([length] - (.toLowerCase (RandomStringUtils/randomAlphabetic length)))) + (.toLowerCase (.nextAlphabetic (RandomStringUtils/secure) length)))) (defn random-bool "Generate a random boolean" @@ -125,8 +125,8 @@ ([length] (random-pronouncable-word length nil)) ([length sd] (random-pronouncable-word length sd {})) ([length sd {:keys [lowerb upperb] :or {lowerb 1}}] - (let [random-consonant #(RandomStringUtils/random 1 "bcdfghjklmnpqrstvwxz") - random-vowel #(RandomStringUtils/random 1 "aeiouy") + (let [random-consonant #(.next (RandomStringUtils/secure) 1 "bcdfghjklmnpqrstvwxz") + random-vowel #(.next (RandomStringUtils/secure) 1 "aeiouy") bounds (if (nil? upperb) {:lowerb lowerb} {:lowerb lowerb :upperb upperb}) actual-length (if (nil? sd) length (safe-sample-normal length sd bounds))] (->> (for [i (range actual-length)] diff --git a/src/puppetlabs/puppetdb/scf/migrate.clj b/src/puppetlabs/puppetdb/scf/migrate.clj index eca0a77ca..c875938d1 100644 --- a/src/puppetlabs/puppetdb/scf/migrate.clj +++ b/src/puppetlabs/puppetdb/scf/migrate.clj @@ -71,7 +71,7 @@ (java.time LocalDateTime LocalDate ZonedDateTime Instant) (java.time.temporal ChronoUnit) (java.time.format DateTimeFormatter) - (org.postgresql.util PGobject))) + (puppetlabs.puppetdb.jdbc PDBBytea))) (defn init-through-2-3-8 [] @@ -1293,8 +1293,8 @@ " ALTER COLUMN target SET NOT NULL") (str "ALTER TABLE edges_transform" " ALTER COLUMN type SET NOT NULL") - (str "DROP TABLE edges") - (str "ALTER TABLE edges_transform RENAME TO edges") + "DROP TABLE edges" + "ALTER TABLE edges_transform RENAME TO edges" (str "ALTER TABLE ONLY edges ADD CONSTRAINT edges_certname_fkey" " FOREIGN KEY (certname)" " REFERENCES certnames(certname)" @@ -1432,7 +1432,7 @@ from (select unnest(?) as id, unnest(?) as hash) in_data where fact_values.id = in_data.id" [(sutils/array-to-param "bigint" Long ids) - (sutils/array-to-param "bytea" PGobject hashes)]))))) + (sutils/array-to-param "bytea" PDBBytea hashes)]))))) (log/info (trs "[7/8] Indexing fact_values table...")) (jdbc/do-commands @@ -1604,7 +1604,7 @@ _iso-year-week :- s/Str]))] (let [date (partitioning/to-zoned-date-time date) ;; guarantee a ZonedDateTime, so our suffix ends in Z start-of-day (-> date - (.truncatedTo (ChronoUnit/DAYS))) ;; this is a ZonedDateTime + (.truncatedTo ChronoUnit/DAYS)) ;; this is a ZonedDateTime start-of-next-day (-> start-of-day (.plusDays 1)) date-formatter DateTimeFormatter/ISO_OFFSET_DATE_TIME diff --git a/src/puppetlabs/puppetdb/scf/partitioning.clj b/src/puppetlabs/puppetdb/scf/partitioning.clj index 2795ec402..2f99f4249 100644 --- a/src/puppetlabs/puppetdb/scf/partitioning.clj +++ b/src/puppetlabs/puppetdb/scf/partitioning.clj @@ -54,7 +54,7 @@ date :- (s/cond-pre LocalDate LocalDateTime ZonedDateTime Instant java.sql.Timestamp)] (let [date (to-zoned-date-time date) ;; guarantee a ZonedDateTime, so our suffix ends in Z start-of-day (-> date - (.truncatedTo (ChronoUnit/DAYS))) ;; this is a ZonedDateTime + (.truncatedTo ChronoUnit/DAYS)) ;; this is a ZonedDateTime start-of-next-day (-> start-of-day (.plusDays 1)) date-formatter DateTimeFormatter/ISO_OFFSET_DATE_TIME diff --git a/src/puppetlabs/puppetdb/scf/storage.clj b/src/puppetlabs/puppetdb/scf/storage.clj index fd1a44e14..778e278e3 100644 --- a/src/puppetlabs/puppetdb/scf/storage.clj +++ b/src/puppetlabs/puppetdb/scf/storage.clj @@ -58,7 +58,7 @@ (java.time ZoneId ZonedDateTime) (java.util Arrays) (org.joda.time Period) - (org.postgresql.util PGobject))) + (puppetlabs.puppetdb.jdbc PDBBytea))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;; Schemas @@ -475,7 +475,7 @@ (if (seq resource-hashes) (let [resource-array (->> (vec resource-hashes) (map sutils/munge-hash-for-storage) - (sutils/array-to-param "bytea" org.postgresql.util.PGobject)) + (sutils/array-to-param "bytea" PDBBytea)) query (format "SELECT DISTINCT %s as resource FROM resource_params_cache WHERE resource=ANY(?)" (sutils/sql-hash-as-str "resource")) sql-params [query resource-array]] @@ -1039,7 +1039,7 @@ (defn find-package-hashes [package-hashes] (jdbc/call-with-query-rows [(format "SELECT id, %s as package_hash FROM packages WHERE hash = ANY(?)" (sutils/sql-hash-as-str "hash")) - (sutils/array-to-param "bytea" PGobject + (sutils/array-to-param "bytea" PDBBytea (map sutils/munge-hash-for-storage package-hashes))] (fn [rows] (reduce (fn [acc {:keys [id package_hash]}] @@ -1796,13 +1796,12 @@ specified, delete all the events that are older than whichever time is more recent." [{:keys [report-ttl resource-events-ttl incremental? update-lock-status] - :or {resource-events-ttl report-ttl - incremental? false + :or {incremental? false update-lock-status (constantly true)}}] {:pre [(instance? org.joda.time.DateTime report-ttl) - (instance? org.joda.time.DateTime resource-events-ttl) (ifn? update-lock-status)]} - (let [effective-resource-events-ttl + (let [resource-events-ttl (or resource-events-ttl report-ttl) + effective-resource-events-ttl (if (before? report-ttl resource-events-ttl) resource-events-ttl report-ttl)] (if-not (detach-partitions-concurrently?) ;; PG11 diff --git a/src/puppetlabs/puppetdb/scf/storage_utils.clj b/src/puppetlabs/puppetdb/scf/storage_utils.clj index db9b0bf15..ce1efddbd 100644 --- a/src/puppetlabs/puppetdb/scf/storage_utils.clj +++ b/src/puppetlabs/puppetdb/scf/storage_utils.clj @@ -4,6 +4,7 @@ [clojure.java.jdbc :as sql] [clojure.string :as str] [clojure.tools.logging :as log] + [next.jdbc.prepare :refer [SettableParameter]] [puppetlabs.i18n.core :refer [trs tru]] [puppetlabs.kitchensink.core :as kitchensink] [puppetlabs.puppetdb.cheshire :as json] @@ -11,19 +12,28 @@ [puppetlabs.puppetdb.jdbc :as jdbc] [puppetlabs.puppetdb.query.common :refer [bad-query-ex]] [puppetlabs.puppetdb.schema :as pls] + [puppetlabs.puppetdb.utils :refer [byte-array-class]] [schema.core :as s]) (:import - [java.sql Connection] - [java.util UUID] - [org.postgresql.util PGobject])) + (java.sql Connection PreparedStatement) + (java.util HexFormat UUID) + (org.postgresql.util PGobject) + (puppetlabs.puppetdb.jdbc PDBBytea VecPDBBytea))) + +(def ^:private warn-on-reflection-orig *warn-on-reflection*) +(set! *warn-on-reflection* true) ;; SCHEMA (defn array-to-param [col-type java-type values] - (.createArrayOf ^Connection (:connection (jdbc/db)) - col-type - (into-array java-type values))) + ;; We create the VecPDBBytea so we can redirect the parameter to + ;; setArray in the set-parameter specializations. + (if (= java-type PDBBytea) + ;; FIXME: change byte-array-class to byte/1 once we require clojure 1.12+ + (VecPDBBytea. (into-array byte-array-class (map #(.data ^PDBBytea %) values))) + (.createArrayOf ^Connection (:connection (jdbc/db)) + col-type (into-array java-type values)))) (def pg-extension-map "Maps to the table definition in postgres, but only includes some of the @@ -427,9 +437,8 @@ (sql/with-db-connection [_conn db] (sql/execute! db ["vacuum analyze"] {:transaction? false}))) -(defn parse-db-hash - [^PGobject db-hash] - (str/replace (.getValue db-hash) "\\x" "")) +(defn db-hash->hex [^PDBBytea pdbb] + (.formatHex (HexFormat/of) (.data pdbb))) (defn parse-db-uuid [^UUID db-uuid] @@ -454,9 +463,43 @@ (defn bytea-escape [s] (format "\\x%s" s)) -(defn munge-hash-for-storage - [hash] - (str->pgobject "bytea" (bytea-escape hash))) +(extend-protocol sql/ISQLParameter + ;; This became necessary with more recent pgjdbc + ;; versions (42.7.7 (at least) and newer), because it stopped + ;; handling a "bytea" PGobject; it started crashing in + ;; PGBytea/toPGLiteral, which doesn't have a clause to handle the + ;; escaped String that it ends up with, presumably from the PGobject + ;; value. The pgjdbc docs specify .setBytes, so do + ;; that: + ;; https://jdbc.postgresql.org/documentation/binary-data/ + ;; https://clojure-doc.org/articles/ecosystem/java_jdbc/using_sql/ + ;; https://clojure.github.io/java.jdbc/#clojure.java.jdbc/ISQLParameter + PDBBytea + (sql/set-parameter [v ^PreparedStatement s i] + (.setBytes s i (.data v))) + VecPDBBytea + (sql/set-parameter [v ^PreparedStatement s i] + (.setArray s i (.createArrayOf ^Connection (:connection (jdbc/db)) + "bytea" (.data v))))) + +(extend-protocol SettableParameter + PDBBytea + (sql/set-parameter [v ^PreparedStatement s i] (.setBytes s i (.data v))) + VecPDBBytea + (sql/set-parameter [v ^PreparedStatement s i] + (.setArray s i (.createArrayOf ^Connection (:connection (jdbc/db)) + "bytea" (.data v))))) + +(defn munge-hash-for-storage [hash] + ;; Represent hashes as PDBBytea so we can add them to + ;; PreparedStatments via setBytes via the clojure/next jdbc + ;; protocols above. It'd be more efficient if eventually pdb just + ;; used something like PDBBytea or byte[] everywhere, though the + ;; latter would allow mutation and might raise "ownership" + ;; questions. The *most* efficient hash that's not just a byte[] + ;; might be a class with two long members and a short to store the + ;; 20 bytes (i.e. so that the data would be inline in the class). + (PDBBytea. (.parseHex (HexFormat/of) hash))) (defn munge-json-for-storage "Prepare a clojure object for storage depending on db type." @@ -497,3 +540,5 @@ (log/info (trs "Analyzing small tables")) (apply jdbc/do-commands-outside-txn (map #(str "analyze " %) small-tables))) + +(set! *warn-on-reflection* warn-on-reflection-orig) diff --git a/src/puppetlabs/puppetdb/status.clj b/src/puppetlabs/puppetdb/status.clj index 2d160ac51..66c36da69 100644 --- a/src/puppetlabs/puppetdb/status.clj +++ b/src/puppetlabs/puppetdb/status.clj @@ -65,6 +65,6 @@ with the associated `status-callback-fn`" [register-fn status-callback-fn] (register-fn "puppetdb-status" - (get-artifact-version "puppetlabs" "puppetdb") + (get-artifact-version "org.openvoxproject" "puppetdb") 1 status-callback-fn)) diff --git a/src/puppetlabs/puppetdb/threadpool.clj b/src/puppetlabs/puppetdb/threadpool.clj index 4ebfc1f55..5e438c2b4 100644 --- a/src/puppetlabs/puppetdb/threadpool.clj +++ b/src/puppetlabs/puppetdb/threadpool.clj @@ -5,7 +5,7 @@ (:import (java.util.concurrent Semaphore ThreadPoolExecutor TimeUnit SynchronousQueue RejectedExecutionException ExecutorService) - (org.apache.commons.lang3.concurrent BasicThreadFactory$Builder))) + (org.apache.commons.lang3.concurrent BasicThreadFactory))) (def logging-exception-handler "Exception handler that ensures any uncaught exception that occurs @@ -27,7 +27,7 @@ uncaught exceptions are logged as errors. Without this uncaught exceptions get the default which is standard error" [threadpool-name-pattern pool-thread-factory] - (-> (BasicThreadFactory$Builder.) + (-> (BasicThreadFactory/builder) (.wrappedFactory pool-thread-factory) (.namingPattern threadpool-name-pattern) (.uncaughtExceptionHandler logging-exception-handler) diff --git a/test/puppetlabs/puppetdb/command_broadcast_test.clj b/test/puppetlabs/puppetdb/command_broadcast_test.clj index 2f3fad8ac..6ba1d163f 100644 --- a/test/puppetlabs/puppetdb/command_broadcast_test.clj +++ b/test/puppetlabs/puppetdb/command_broadcast_test.clj @@ -97,7 +97,7 @@ port "/metrics/v2/read/" "puppetlabs.puppetdb.storage" - (str ":name=" pg ".store-report-time")) + ":name=" pg ".store-report-time") svc-utils/get-ssl :body :value diff --git a/test/puppetlabs/puppetdb/query_eng/default_reports_test.clj b/test/puppetlabs/puppetdb/query_eng/default_reports_test.clj index db8f20c45..9c3e11649 100644 --- a/test/puppetlabs/puppetdb/query_eng/default_reports_test.clj +++ b/test/puppetlabs/puppetdb/query_eng/default_reports_test.clj @@ -15,11 +15,11 @@ (is (= false (t/mentions-report-type? [op "x" "y"]))) (is (= true (t/mentions-report-type? [op "type" "y"]))))) - (testing (str "unary operator null?") + (testing "unary operator null?" (is (= false (t/mentions-report-type? ["null?" "x" true]))) (is (= true (t/mentions-report-type? ["null?" "type" true])))) - (testing (str "unary operator not") + (testing "unary operator not" (is (= false (t/mentions-report-type? ["not" ["=" "x" "y"]]))) (is (= true (t/mentions-report-type? ["not" ["=" "type" "y"]]))) (is (= true (t/mentions-report-type? ["not" ["null?" "type" true]]))) diff --git a/test/puppetlabs/puppetdb/scf/migrate_partitioning_test.clj b/test/puppetlabs/puppetdb/scf/migrate_partitioning_test.clj index 984cd6ba6..7904a30ff 100644 --- a/test/puppetlabs/puppetdb/scf/migrate_partitioning_test.clj +++ b/test/puppetlabs/puppetdb/scf/migrate_partitioning_test.clj @@ -549,9 +549,9 @@ date-formatter (.withZone (DateTimeFormatter/ofPattern "yyyy-MM-dd HH:mm:ssx") (ZoneId/systemDefault)) start-of-day (.format date-formatter - (.truncatedTo date-of-week (ChronoUnit/DAYS))) + (.truncatedTo date-of-week ChronoUnit/DAYS)) end-of-day (.format date-formatter - (.plusDays (.truncatedTo date-of-week (ChronoUnit/DAYS)) 1)) + (.plusDays (.truncatedTo date-of-week ChronoUnit/DAYS) 1)) pg17-db [17 0] {current-db-version :version} (sutils/db-metadata)] [{:left-only nil @@ -1179,9 +1179,9 @@ date-formatter (.withZone (DateTimeFormatter/ofPattern "yyyy-MM-dd HH:mm:ssx") (ZoneId/systemDefault)) start-of-day (.format date-formatter - (.truncatedTo date-of-week (ChronoUnit/DAYS))) + (.truncatedTo date-of-week ChronoUnit/DAYS)) end-of-day (.format date-formatter - (.plusDays (.truncatedTo date-of-week (ChronoUnit/DAYS)) 1)) + (.plusDays (.truncatedTo date-of-week ChronoUnit/DAYS) 1)) pg17-db [17 0] {current-db-version :version} (sutils/db-metadata)] [{:left-only nil diff --git a/test/puppetlabs/puppetdb/scf/migrate_test.clj b/test/puppetlabs/puppetdb/scf/migrate_test.clj index 2998bf017..81033931c 100644 --- a/test/puppetlabs/puppetdb/scf/migrate_test.clj +++ b/test/puppetlabs/puppetdb/scf/migrate_test.clj @@ -1733,7 +1733,7 @@ [date] (let [date-formatter (.withZone (DateTimeFormatter/ofPattern "yyyy-MM-dd HH:mm:ssx") (ZoneId/systemDefault))] (-> date - (.truncatedTo (ChronoUnit/DAYS)) ;; this is a ZonedDateTime + (.truncatedTo ChronoUnit/DAYS) ;; this is a ZonedDateTime (.format date-formatter)))) (defn resource-events-partition-day-constraint-diff-template diff --git a/test/puppetlabs/puppetdb/scf/storage_test.clj b/test/puppetlabs/puppetdb/scf/storage_test.clj index 8610169a4..6cda275ac 100644 --- a/test/puppetlabs/puppetdb/scf/storage_test.clj +++ b/test/puppetlabs/puppetdb/scf/storage_test.clj @@ -290,7 +290,7 @@ :stable {:hostname "myhost" :kernel "Linux" :uptime_seconds 3600} :volatile {:domain "mynewdomain.com" :fqdn "myhost.mynewdomain.com"}} (-> m - (update :hash sutils/parse-db-hash) + (update :hash sutils/db-hash->hex) (update :paths_hash bytes->hex) (update :stable_hash bytes->hex) (update :stable sutils/parse-db-json)