Skip to content
12 changes: 10 additions & 2 deletions .clj-kondo/config.edn
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,17 @@
{:exclude
{puppetlabs.puppetdb.jdbc/call-with-array-converted-query-rows {:namespaces [".*"]}
puppetlabs.puppetdb.testutils.services/call-with-puppetdb-instance {:namespaces [".*"]}
puppetlabs.puppetdb.testutils.services/with-puppetdb-instance {:namespaces [".*"]}}}
puppetlabs.puppetdb.testutils.services/with-puppetdb-instance {:namespaces [".*"]}
puppetlabs.trapperkeeper.testutils.logging/atom-logger {:namespaces [".*"]}
puppetlabs.trapperkeeper.testutils.logging/with-log-output {:namespaces [".*"]}
puppetlabs.trapperkeeper.testutils.logging/logs-matching {:namespaces [".*"]}}}
:refer-all {:exclude [clojure.test]}
:unresolved-symbol
{:exclude
[(puppetlabs.comidi/GET)
(puppetlabs.comidi/POST)]}}}
(puppetlabs.comidi/POST)]}
:unresolved-var
{:exclude
[digest/sha1
instaparse.core/transform
com.rpl.specter/ALL]}}}
81 changes: 44 additions & 37 deletions project.clj
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
(def pdb-version "8.11.1-SNAPSHOT")
(def pdb-version "8.12.0-SNAPSHOT")

(def clj-parent-version "7.4.0")
(def clj-parent-version "7.6.6")

(defn true-in-env? [x]
(#{"true" "yes" "1"} (System/getenv x)))
Expand All @@ -25,15 +25,15 @@
(let [ver (eval '(java.lang.Runtime/version))]
{:feature (.feature ver) :interim (.interim ver)}))))

(def i18n-version "0.9.2")
(def i18n-version "1.0.2")

(def pdb-dev-deps
(concat
'[[ring/ring-mock]
[timofreiberg/bultitude "0.3.1"]
[puppetlabs/trapperkeeper :classifier "test"]
[puppetlabs/kitchensink :classifier "test"]
[com.puppetlabs/trapperkeeper-webserver-jetty10 :classifier "test"]
[org.openvoxproject/trapperkeeper :classifier "test"]
[org.openvoxproject/kitchensink :classifier "test"]
[org.openvoxproject/trapperkeeper-webserver-jetty10 :classifier "test"]
[org.flatland/ordered "1.15.12"]
[org.clojure/test.check "1.1.1"]
[com.gfredericks/test.chuck "0.2.14"]
Expand All @@ -42,7 +42,7 @@
[org.yaml/snakeyaml]

;; Only needed for :integration tests
[puppetlabs/trapperkeeper-filesystem-watcher nil]]))
[org.openvoxproject/trapperkeeper-filesystem-watcher nil]]))

;; Don't use lein :clean-targets so that we don't have to repeat
;; ourselves, given that we need to remove some protected files, and
Expand Down Expand Up @@ -78,31 +78,37 @@
"puppetserver"
"vendor"]))

(def pdb-aot-classes
;; Compile classes first for now:
;; https://codeberg.org/leiningen/leiningen/issues/99
'[puppetlabs.puppetdb.jdbc.PDBBytea puppetlabs.puppetdb.jdbc.VecPDBBytea])

(def pdb-aot-namespaces
(apply vector
#"puppetlabs\.puppetdb\..*"
(->> "resources/puppetlabs/puppetdb/bootstrap.cfg"
clojure.java.io/reader
line-seq
(map clojure.string/trim)
(remove #(re-matches #"#.*" %)) ;; # comments
(remove #(re-matches #"puppetlabs\.puppetdb\.." %))
(map #(clojure.string/replace % #"(.*)/[^/]+$" "$1"))
(map symbol))))
(into []
(concat pdb-aot-classes
[#"puppetlabs\.puppetdb\..*"]
(->> "resources/puppetlabs/puppetdb/bootstrap.cfg"
clojure.java.io/reader
line-seq
(map clojure.string/trim)
(remove #(re-matches #"#.*" %)) ;; # comments
(remove #(re-matches #"puppetlabs\.puppetdb\.." %))
(map #(clojure.string/replace % #"(.*)/[^/]+$" "$1"))
(map symbol)))))

;; Avoid startup reflection warnings due to
;; https://clojure.atlassian.net/browse/CLJ-2066
;; https://openjdk.java.net/jeps/396
(def pdb-jvm-opts (when (< 8 (:feature pdb-jvm-ver) 17)
["--illegal-access=deny"]))

(defproject puppetlabs/puppetdb pdb-version
(defproject org.openvoxproject/puppetdb pdb-version
:description "OpenVox-integrated catalog and fact storage"

:license {:name "Apache License, Version 2.0"
:url "http://www.apache.org/licenses/LICENSE-2.0.html"}

:url "https://docs.puppetlabs.com/puppetdb/"
:url "https://github.com/openvoxproject/openvoxdb/"

:min-lein-version "2.7.1"

Expand All @@ -128,17 +134,17 @@
[org.clojure/tools.nrepl]

;; Puppet specific
[puppetlabs/comidi]
[puppetlabs/i18n]
[puppetlabs/kitchensink]
[puppetlabs/ssl-utils]
[puppetlabs/stockpile "0.0.4"]
[puppetlabs/structured-logging]
[puppetlabs/trapperkeeper]
[com.puppetlabs/trapperkeeper-webserver-jetty10]
[puppetlabs/trapperkeeper-metrics]
[puppetlabs/trapperkeeper-status]
[puppetlabs/trapperkeeper-authorization]
[org.openvoxproject/comidi]
[org.openvoxproject/i18n]
[org.openvoxproject/kitchensink]
[org.openvoxproject/ssl-utils]
[org.openvoxproject/stockpile "1.0.0"]
[org.openvoxproject/structured-logging "1.0.0"]
[org.openvoxproject/trapperkeeper]
[org.openvoxproject/trapperkeeper-webserver-jetty10]
[org.openvoxproject/trapperkeeper-metrics]
[org.openvoxproject/trapperkeeper-status]
[org.openvoxproject/trapperkeeper-authorization]

;; Various
[cheshire]
Expand All @@ -162,7 +168,7 @@
[com.fasterxml.jackson.core/jackson-databind]

;; Filesystem utilities
[org.apache.commons/commons-lang3]
[org.apache.commons/commons-lang3 "3.20.0"]

;; Database connectivity
[com.zaxxer/HikariCP]
Expand All @@ -177,15 +183,15 @@

:jvm-opts ~pdb-jvm-opts

:deploy-repositories [["clojars" {:url "https://clojars.org/repo"
:deploy-repositories [["releases" {:url "https://clojars.org/repo"
:username :env/CLOJARS_USERNAME
:password :env/CLOJARS_PASSWORD
:sign-releases false}]]

:plugins [[lein-release "1.1.3" :exclusions [org.clojure/clojure]]
[lein-cloverage "1.2.4"]
[lein-parent "0.3.9"]
[puppetlabs/i18n ~i18n-version]]
[org.openvoxproject/i18n ~i18n-version]]

:lein-release {:scm :git
:deploy-via :lein-deploy}
Expand Down Expand Up @@ -216,6 +222,8 @@
;; "test"]. See the :testutils profile below.
:classifiers {:test :testutils}

:aot ~pdb-aot-classes

:profiles {:defaults {:resource-paths ["test-resources"]
:dependencies ~pdb-dev-deps
:injections [(do
Expand All @@ -236,12 +244,11 @@
:jvm-opts ~(let [{:keys [feature interim]} pdb-jvm-ver]
(conj pdb-jvm-opts
(case feature
8 "-Djava.security.properties==dev-resources/jdk8-fips-security"
11 "-Djava.security.properties==dev-resources/jdk11on-fips-security"
17 "-Djava.security.properties==dev-resources/jdk11on-fips-security"
21 "-Djava.security.properties==dev-resources/jdk11on-fips-security"
(do)
)))}]
:kondo {:dependencies [[clj-kondo "2024.05.24"]]}
:kondo {:dependencies [[clj-kondo "2025.10.23"]]}
:ezbake {:dependencies ^:replace [;; NOTE: we need to explicitly pass in `nil` values
;; for the version numbers here in order to correctly
;; inherit the versions from our parent project.
Expand All @@ -266,7 +273,7 @@
;; This circular dependency is required because of a bug in
;; ezbake (EZ-35); without it, bootstrap.cfg will not be included
;; in the final package.
[puppetlabs/puppetdb ~pdb-version]]
[org.openvoxproject/puppetdb ~pdb-version]]
:name "puppetdb"
:plugins [[org.openvoxproject/lein-ezbake ~(or (System/getenv "EZBAKE_VERSION") "2.7.1")]]}
:testutils {:source-paths ^:replace ["test"]
Expand Down
2 changes: 1 addition & 1 deletion src/puppetlabs/puppetdb/cli/benchmark.clj
Original file line number Diff line number Diff line change
Expand Up @@ -308,7 +308,7 @@

TODO: handle arrays and maps."
class)
(defmethod touch-parameter-value String [p] (RandomStringUtils/randomAscii(count p)))
(defmethod touch-parameter-value String [p] (.nextAscii (RandomStringUtils/secure) (count p)))
(defmethod touch-parameter-value Number [_] (rand-int 1000000))
(defmethod touch-parameter-value Boolean [p] (not p))
;; Allow other types to pass through unmutated for now
Expand Down
6 changes: 3 additions & 3 deletions src/puppetlabs/puppetdb/cli/generate.clj
Original file line number Diff line number Diff line change
Expand Up @@ -335,7 +335,7 @@
(recur (assoc parameters
(parameter-name
(rnd/safe-sample-normal 20 5 {:lowerb 5}))
(RandomStringUtils/randomAscii
(.nextAscii (RandomStringUtils/secure)
(rnd/safe-sample-normal 50 25 {:upperb (max 50 size)}))))
parameters))))

Expand Down Expand Up @@ -480,7 +480,7 @@
bsize (rnd/safe-sample-normal avg-blob-size-in-bytes standard-deviation {:lowerb lowerb :upperb upperb})
pname (format "content_blob_%s" (rnd/random-pronouncable-word))]
(update-in catalog [:resources (rand-int (count resources)) :parameters]
#(merge % {pname (RandomStringUtils/randomAscii bsize)}))))
#(merge % {pname (.nextAscii (RandomStringUtils/secure) bsize)}))))

(defn system-seconds-str
"Epoch seconds as a string. Used by default as a version string in Puppet
Expand Down Expand Up @@ -706,7 +706,7 @@
([{:keys [file line level message source tags]
:or {level (rand-nth ["info" "notice"])
message (rnd/random-sentence-ish)
tags #{level}
tags #{}
source "Puppet"}}]
(let [final-tags (cset/union (set tags) #{level})]
{:file file
Expand Down
5 changes: 3 additions & 2 deletions src/puppetlabs/puppetdb/jdbc.clj
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
[clojure.string :as str]
[clojure.tools.logging :as log]
[honey.sql :as hsql]
[next.jdbc.prepare :refer [set-parameter]]
[puppetlabs.i18n.core :refer [trs]]
[puppetlabs.kitchensink.core :as kitchensink]
[puppetlabs.puppetdb.jdbc.internal :refer [limit-result-set!]]
Expand Down Expand Up @@ -213,7 +214,7 @@
(with-db-transaction []
(with-open [stmt (.prepareStatement ^Connection (:connection *db*) sql)]
(doseq [[i param] (map vector (range) params)]
(.setObject stmt (inc i) param))
(set-parameter param stmt (inc i)))
(.setFetchSize stmt (or fetch-size 500))
(with-open [rset (.executeQuery stmt)]
(try
Expand Down Expand Up @@ -245,7 +246,7 @@
(with-db-transaction []
(with-open [stmt (.prepareStatement ^Connection (:connection *db*) sql)]
(doseq [[i param] (map vector (range) params)]
(.setObject stmt (inc i) param))
(set-parameter param stmt (inc i)))
(.setFetchSize stmt fetch-size)
(let [fix-vals (if as-arrays?
#(mapv any-sql-array->vec %)
Expand Down
31 changes: 31 additions & 0 deletions src/puppetlabs/puppetdb/jdbc/PDBBytea.clj
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
(ns puppetlabs.puppetdb.jdbc.PDBBytea
"Carrier for bytea parameters, to support clojure.java.jdbc and next.jdbc
protocol extensions. Essentially just a typed wrapper around a byte[]."
(:import
(java.util Arrays))
(:gen-class
;;:extends org.postgresql.util.PGobject
:state ^"[B" data
:init init
:constructors {["[B"] []}))

(def ^:private warn-on-reflection-orig *warn-on-reflection*)
(set! *warn-on-reflection* true)

(defn -init [data] [[] data])

;; Implemented for now because edge-replacement-differential test, for
;; example, uses munge-hash-for-storage to create the test values that
;; it compares for equality -- and who knows what else. Apparently
;; PGobject bytea values, which PDBBytea replaces, compare content for
;; equality, and since we use bytea for hashes, best to be
;; conservative.
(defn -equals [^puppetlabs.puppetdb.jdbc.PDBBytea this x]
(and (instance? puppetlabs.puppetdb.jdbc.PDBBytea x)
(Arrays/equals ^"[B" (.data this)
^"[B" (.data ^puppetlabs.puppetdb.jdbc.PDBBytea x))))

(defn -hashCode [^puppetlabs.puppetdb.jdbc.PDBBytea this]
(Arrays/hashCode ^"[B" (.data this)))

(set! *warn-on-reflection* warn-on-reflection-orig)
16 changes: 16 additions & 0 deletions src/puppetlabs/puppetdb/jdbc/VecPDBBytea.clj
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
(ns puppetlabs.puppetdb.jdbc.VecPDBBytea
"Carrier for bytea[] parameters, to support clojure.java.jdbc and next.jdbc
protocol extensions. Essentially just a typed wrapper around a byte[][]."
;; It doesn't look like java.sql.Arrays, which is what this
;; replaces, define content-based equals(), so we don't need one.
(:gen-class
:state ^"[[B" data
:init init
:constructors {["[[B"] []}))

(def ^:private warn-on-reflection-orig *warn-on-reflection*)
(set! *warn-on-reflection* true)

(defn -init [data] [[] data])

(set! *warn-on-reflection* warn-on-reflection-orig)
10 changes: 5 additions & 5 deletions src/puppetlabs/puppetdb/random.clj
Original file line number Diff line number Diff line change
Expand Up @@ -12,15 +12,15 @@

(defn random-string
"Generate a random string of optional length"
([] (RandomStringUtils/randomAlphabetic (inc (rand-int 10))))
([] (.nextAlphabetic (RandomStringUtils/secure) (inc (rand-int 10))))
([length]
(RandomStringUtils/randomAlphabetic length)))
(.nextAlphabetic (RandomStringUtils/secure) length)))

(defn random-string-alpha
"Generate a random string of optional length, only lower case alphabet chars"
([] (random-string (inc (rand-int 10))))
([length]
(.toLowerCase (RandomStringUtils/randomAlphabetic length))))
(.toLowerCase (.nextAlphabetic (RandomStringUtils/secure) length))))

(defn random-bool
"Generate a random boolean"
Expand Down Expand Up @@ -125,8 +125,8 @@
([length] (random-pronouncable-word length nil))
([length sd] (random-pronouncable-word length sd {}))
([length sd {:keys [lowerb upperb] :or {lowerb 1}}]
(let [random-consonant #(RandomStringUtils/random 1 "bcdfghjklmnpqrstvwxz")
random-vowel #(RandomStringUtils/random 1 "aeiouy")
(let [random-consonant #(.next (RandomStringUtils/secure) 1 "bcdfghjklmnpqrstvwxz")
random-vowel #(.next (RandomStringUtils/secure) 1 "aeiouy")
bounds (if (nil? upperb) {:lowerb lowerb} {:lowerb lowerb :upperb upperb})
actual-length (if (nil? sd) length (safe-sample-normal length sd bounds))]
(->> (for [i (range actual-length)]
Expand Down
10 changes: 5 additions & 5 deletions src/puppetlabs/puppetdb/scf/migrate.clj
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@
(java.time LocalDateTime LocalDate ZonedDateTime Instant)
(java.time.temporal ChronoUnit)
(java.time.format DateTimeFormatter)
(org.postgresql.util PGobject)))
(puppetlabs.puppetdb.jdbc PDBBytea)))

(defn init-through-2-3-8
[]
Expand Down Expand Up @@ -1293,8 +1293,8 @@
" ALTER COLUMN target SET NOT NULL")
(str "ALTER TABLE edges_transform"
" ALTER COLUMN type SET NOT NULL")
(str "DROP TABLE edges")
(str "ALTER TABLE edges_transform RENAME TO edges")
"DROP TABLE edges"
"ALTER TABLE edges_transform RENAME TO edges"
(str "ALTER TABLE ONLY edges ADD CONSTRAINT edges_certname_fkey"
" FOREIGN KEY (certname)"
" REFERENCES certnames(certname)"
Expand Down Expand Up @@ -1432,7 +1432,7 @@
from (select unnest(?) as id, unnest(?) as hash) in_data
where fact_values.id = in_data.id"
[(sutils/array-to-param "bigint" Long ids)
(sutils/array-to-param "bytea" PGobject hashes)])))))
(sutils/array-to-param "bytea" PDBBytea hashes)])))))

(log/info (trs "[7/8] Indexing fact_values table..."))
(jdbc/do-commands
Expand Down Expand Up @@ -1604,7 +1604,7 @@
_iso-year-week :- s/Str]))]
(let [date (partitioning/to-zoned-date-time date) ;; guarantee a ZonedDateTime, so our suffix ends in Z
start-of-day (-> date
(.truncatedTo (ChronoUnit/DAYS))) ;; this is a ZonedDateTime
(.truncatedTo ChronoUnit/DAYS)) ;; this is a ZonedDateTime
start-of-next-day (-> start-of-day
(.plusDays 1))
date-formatter DateTimeFormatter/ISO_OFFSET_DATE_TIME
Expand Down
2 changes: 1 addition & 1 deletion src/puppetlabs/puppetdb/scf/partitioning.clj
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@
date :- (s/cond-pre LocalDate LocalDateTime ZonedDateTime Instant java.sql.Timestamp)]
(let [date (to-zoned-date-time date) ;; guarantee a ZonedDateTime, so our suffix ends in Z
start-of-day (-> date
(.truncatedTo (ChronoUnit/DAYS))) ;; this is a ZonedDateTime
(.truncatedTo ChronoUnit/DAYS)) ;; this is a ZonedDateTime
start-of-next-day (-> start-of-day
(.plusDays 1))
date-formatter DateTimeFormatter/ISO_OFFSET_DATE_TIME
Expand Down
Loading