Skip to content

Commit 85fa4b5

Browse files
vedangvedang
committed
test(ci): split unit and openrouter integration suites
1 parent 4910c1e commit 85fa4b5

4 files changed

Lines changed: 125 additions & 83 deletions

File tree

‎.github/workflows/integration.yml‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ name: Integration Tests
22

33
on:
44
pull_request:
5-
branches: [ main ]
5+
branches: [ main, master, develop ]
66

77
jobs:
88
integration:
@@ -36,5 +36,6 @@ jobs:
3636
3737
- name: Run integration tests
3838
env:
39-
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
39+
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
40+
OPENROUTER_MODEL: ${{ vars.OPENROUTER_MODEL || 'openai/gpt-4o-mini' }}
4041
run: make test-integration

‎Makefile‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,9 @@ help:
44
@echo "Available targets:"
55
@echo " repl - Start a Clojure REPL"
66
@echo " nrepl - Start an nREPL server on port 7888"
7-
@echo " test - Run tests"
8-
@echo " test-ci - Run tests with documentation reporter"
9-
@echo " test-integration - Run integration tests (requires OPENAI_API_KEY)"
7+
@echo " test - Run unit tests"
8+
@echo " test-ci - Run unit tests in CI"
9+
@echo " test-integration - Run integration tests (requires OPENROUTER_API_KEY)"
1010
@echo " coverage - Run tests with coverage report"
1111
@echo " lint - Run clj-kondo linter"
1212
@echo " compile - Compile and check syntax"
@@ -23,11 +23,14 @@ nrepl:
2323
clojure -M:repl -m nrepl.cmdline --middleware '["cider.nrepl/cider-middleware"]' --port 7888
2424

2525
test:
26-
clojure -M:test -m kaocha.runner
26+
clojure -M:test -m kaocha.runner unit
27+
28+
test-ci:
29+
clojure -M:test -m kaocha.runner unit
2730

2831
test-integration:
29-
@echo "Running integration tests (requires OPENAI_API_KEY)..."
30-
clojure -M:test -m kaocha.runner --focus-meta :integration
32+
@echo "Running integration tests (requires OPENROUTER_API_KEY)..."
33+
clojure -M:test -m kaocha.runner integration
3134

3235
coverage:
3336
clojure -M:test:coverage

‎test/dscloj/integration_test.clj‎

Lines changed: 58 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -2,89 +2,72 @@
22
(:require [clojure.test :refer [deftest is testing use-fixtures]]
33
[dscloj.core :as dscloj]))
44

5-
(def api-key-available?
6-
"Check if OPENAI_API_KEY is set in environment"
7-
(not (nil? (System/getenv "OPENAI_API_KEY"))))
5+
(def openrouter-api-key
6+
"OpenRouter API key for live integration tests."
7+
(System/getenv "OPENROUTER_API_KEY"))
8+
9+
(def openrouter-model
10+
"Cheap, deterministic default model for live integration tests.
11+
CI may override this with OPENROUTER_MODEL."
12+
(or (System/getenv "OPENROUTER_MODEL")
13+
"openai/gpt-4o-mini"))
14+
15+
(defn- api-key-available? []
16+
(boolean (seq openrouter-api-key)))
17+
18+
(defn- ci? []
19+
(= "true" (System/getenv "CI")))
820

921
(defn setup-test-provider
10-
"Fixture to register test provider and skip tests if API key is not available"
22+
"Register OpenRouter provider when OPENROUTER_API_KEY is available.
23+
24+
These tests intentionally keep one live provider smoke. Unit tests already cover
25+
DSCloj parsing/validation in detail; this verifies the generated prompt still
26+
works against a real chat-completions model without making DSCloj responsible
27+
for testing every litellm provider."
1128
[f]
12-
(if api-key-available?
29+
(cond
30+
(api-key-available?)
1331
(do
14-
;; Register test provider
15-
(dscloj/register-provider! :test-openai
16-
{:provider :openai
17-
:model "gpt-3.5-turbo"
18-
:config {:api-key (System/getenv "OPENAI_API_KEY")}})
32+
(dscloj/register-provider! :test-openrouter
33+
{:provider :openrouter
34+
:model openrouter-model
35+
:config {:api-key openrouter-api-key}})
1936
(f))
20-
(println "Skipping integration tests: OPENAI_API_KEY not set")))
37+
38+
(ci?)
39+
(throw (ex-info "OPENROUTER_API_KEY is required in CI for integration tests" {}))
40+
41+
:else
42+
(println "Skipping integration tests: OPENROUTER_API_KEY not set")))
2143

2244
(use-fixtures :once setup-test-provider)
2345

24-
(deftest ^:integration basic-qa-integration-test
25-
(testing "Basic Q&A with OpenAI API"
26-
(when api-key-available?
27-
(let [qa-module {:inputs [{:name :question
28-
:spec :string
29-
:description "A question to answer"}]
30-
:outputs [{:name :answer
31-
:spec :string
32-
:description "The answer to the question"}]
33-
:instructions "Answer the question accurately and concisely."}
34-
;; Use registered provider
35-
result (dscloj/predict :test-openai
36-
qa-module
37-
{:question "What is 2+2? Reply with just the number."}
38-
{:temperature 0.0})]
46+
(deftest ^:integration live-structured-output-integration-test
47+
(testing "Live OpenRouter model can follow DSCloj structured output format"
48+
(when (api-key-available?)
49+
(let [module {:inputs [{:name :question
50+
:spec :string
51+
:description "A deterministic arithmetic question"}]
52+
:outputs [{:name :answer
53+
:spec :string
54+
:description "The answer as text"}
55+
{:name :numeric_answer
56+
:spec :int
57+
:description "The answer as an integer"}
58+
{:name :is_correct
59+
:spec :boolean
60+
:description "Whether the numeric answer is correct"}]
61+
:instructions (str "Answer the arithmetic question. "
62+
"Use 4 for numeric_answer. "
63+
"Use True for is_correct.")}
64+
result (dscloj/predict :test-openrouter
65+
module
66+
{:question "What is 2+2?"}
67+
{:temperature 0.0
68+
:max-tokens 80})]
3969
(is (map? result))
4070
(is (contains? result :answer))
4171
(is (string? (:answer result)))
42-
(is (re-find #"4" (:answer result)))))))
43-
44-
(deftest ^:integration boolean-output-integration-test
45-
(testing "Boolean output parsing with OpenAI API"
46-
(when api-key-available?
47-
(let [validator-module {:inputs [{:name :statement
48-
:spec :string
49-
:description "A statement to verify"}]
50-
:outputs [{:name :is_true
51-
:spec :boolean
52-
:description "Whether the statement is true or false"}]
53-
:instructions "Determine if the statement is true or false."}
54-
result (dscloj/predict :test-openai
55-
validator-module
56-
{:statement "The Earth orbits around the Sun."}
57-
{:temperature 0.0})]
58-
(is (map? result))
59-
(is (contains? result :is_true))
60-
(is (boolean? (:is_true result)))
61-
(is (true? (:is_true result)))))))
62-
63-
(deftest ^:integration multiple-outputs-integration-test
64-
(testing "Multiple outputs with different types"
65-
(when api-key-available?
66-
(let [analyzer-module {:inputs [{:name :text
67-
:spec :string
68-
:description "Text to analyze"}]
69-
:outputs [{:name :word_count
70-
:spec :int
71-
:description "Number of words"}
72-
{:name :has_punctuation
73-
:spec :boolean
74-
:description "Whether text has punctuation"}
75-
{:name :summary
76-
:spec :string
77-
:description "Brief summary"}]
78-
:instructions "Analyze the text and provide word count, check for punctuation, and give a brief summary."}
79-
result (dscloj/predict :test-openai
80-
analyzer-module
81-
{:text "Hello, world! This is a test."}
82-
{:temperature 0.0})]
83-
(is (map? result))
84-
(is (contains? result :word_count))
85-
(is (contains? result :has_punctuation))
86-
(is (contains? result :summary))
87-
(is (number? (:word_count result)))
88-
(is (boolean? (:has_punctuation result)))
89-
(is (string? (:summary result)))
90-
(is (true? (:has_punctuation result)))))))
72+
(is (= 4 (:numeric_answer result)))
73+
(is (true? (:is_correct result)))))))

‎test/dscloj/test_config_test.clj‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
(ns dscloj.test-config-test
2+
(:require [clojure.edn :as edn]
3+
[clojure.string :as str]
4+
[clojure.test :refer [deftest is testing]]))
5+
6+
(defn- read-tests-edn []
7+
(edn/read-string {:readers {'kaocha/v1 identity}}
8+
(slurp "tests.edn")))
9+
10+
(defn- make-target-body
11+
"Return the Makefile body lines for `target` up to the next non-indented target."
12+
[makefile target]
13+
(let [lines (str/split-lines makefile)
14+
target-line (str target ":")]
15+
(->> lines
16+
(drop-while #(not= target-line %))
17+
rest
18+
(take-while #(or (str/blank? %)
19+
(str/starts-with? % "\t")
20+
(str/starts-with? % " ")))
21+
(str/join "\n"))))
22+
23+
(deftest makefile-test-targets-use-explicit-suites
24+
(let [makefile (slurp "Makefile")]
25+
(testing "default and CI test targets run only unit tests"
26+
(is (str/includes? (make-target-body makefile "test")
27+
"clojure -M:test -m kaocha.runner unit"))
28+
(is (str/includes? (make-target-body makefile "test-ci")
29+
"clojure -M:test -m kaocha.runner unit")))
30+
31+
(testing "integration target runs only the integration suite with OpenRouter wording"
32+
(is (str/includes? makefile "test-ci:"))
33+
(is (str/includes? (make-target-body makefile "test-integration")
34+
"clojure -M:test -m kaocha.runner integration"))
35+
(is (str/includes? makefile "OPENROUTER_API_KEY")))))
36+
37+
(deftest kaocha-suites-stay-separated
38+
(let [suites (:tests (read-tests-edn))
39+
by-id (into {} (map (juxt :id identity) suites))]
40+
(is (= [:integration] (get-in by-id [:unit :skip-meta])))
41+
(is (= [:integration] (get-in by-id [:integration :focus-meta])))))
42+
43+
(deftest github-pr-integration-workflow-uses-openrouter
44+
(let [workflow (slurp ".github/workflows/integration.yml")]
45+
(is (str/includes? workflow "pull_request:"))
46+
(is (str/includes? workflow "branches: [ main, master, develop ]"))
47+
(is (str/includes? workflow "OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}"))
48+
(is (str/includes? workflow "OPENROUTER_MODEL:"))
49+
(is (str/includes? workflow "make test-integration"))))
50+
51+
(deftest live-integration-tests-use-openrouter-key
52+
(let [integration-test (slurp "test/dscloj/integration_test.clj")]
53+
(is (str/includes? integration-test "OPENROUTER_API_KEY"))
54+
(is (str/includes? integration-test "OPENROUTER_API_KEY is required in CI"))
55+
(is (not (str/includes? integration-test "OPENAI_API_KEY")))))

0 commit comments

Comments
 (0)