Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 7 additions & 4 deletions .github/workflows/test-integration.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
name: "Test: Integration"

# Java integration test-suite, run against an all-in-one (AIO) server with the
# integration test-data loaded (CN_PERSISTENCE_LOAD_TEST_DATA=true). Replaces the

Check warning on line 4 in .github/workflows/test-integration.yml

View workflow job for this annotation

GitHub Actions / Lint and Validate

4:81 [line-length] line too long (81 > 80 characters)
# legacy Jenkins test runner.
#
# Flow per matrix leg (MYSQL / PGSQL):
Expand All @@ -9,7 +9,7 @@
# -> run DB + AIO + a TLS terminating proxy
# -> extract live secrets from the AIO -> render profiles/<fqdn>/
# -> import the AIO cert into the JDK truststore
# -> full build with tests enabled (mvn install -Dcfg=<fqdn>) across all reactors

Check warning on line 12 in .github/workflows/test-integration.yml

View workflow job for this annotation

GitHub Actions / Lint and Validate

12:81 [line-length] line too long (83 > 80 characters)
# -> publish results (step summary + artifact + PR check).

on:
Expand All @@ -21,17 +21,17 @@
options: ["BOTH", "MYSQL", "PGSQL"]
default: "BOTH"
aio_image:
description: "Prebuilt AIO image to test (leave empty to build from this checkout)"

Check warning on line 24 in .github/workflows/test-integration.yml

View workflow job for this annotation

GitHub Actions / Lint and Validate

24:81 [line-length] line too long (91 > 80 characters)
type: string
default: ""
log_level:
description: "AIO log level (TRACE = detailed FILE logs to debug a failure; INFO = default, lower memory)"

Check warning on line 28 in .github/workflows/test-integration.yml

View workflow job for this annotation

GitHub Actions / Lint and Validate

28:81 [line-length] line too long (114 > 80 characters)
type: choice
options: ["INFO", "TRACE"]
default: "INFO"
# Module selection: leave "all modules" checked to test everything (the default, and what

Check warning on line 32 in .github/workflows/test-integration.yml

View workflow job for this annotation

GitHub Actions / Lint and Validate

32:81 [line-length] line too long (95 > 80 characters)
# PR/nightly always do). Uncheck it and tick specific modules to narrow the suites that run.

Check warning on line 33 in .github/workflows/test-integration.yml

View workflow job for this annotation

GitHub Actions / Lint and Validate

33:81 [line-length] line too long (98 > 80 characters)
# The reactor + AIO are always built in full regardless, so build dependencies are honoured.

Check warning on line 34 in .github/workflows/test-integration.yml

View workflow job for this annotation

GitHub Actions / Lint and Validate

34:81 [line-length] line too long (98 > 80 characters)
all_modules:
description: "Test all modules (uncheck to pick specific modules below)"
type: boolean
Expand Down Expand Up @@ -73,7 +73,7 @@
type: boolean
default: false
script_catalog:
description: "docs/script-catalog (custom scripts load + behavior; PRs run it only when the catalog changes)"
description: "docs/script-catalog (custom scripts load + behavior; nightly always, PRs only when the catalog changes)"

Check warning on line 76 in .github/workflows/test-integration.yml

View workflow job for this annotation

GitHub Actions / Lint and Validate

76:81 [line-length] line too long (126 > 80 characters)
type: boolean
default: false
schedule:
Expand All @@ -100,7 +100,7 @@
contents: read

concurrency:
# Per branch/PR (github.ref), so a newer commit only cancels this ref's own now-stale run.

Check warning on line 103 in .github/workflows/test-integration.yml

View workflow job for this annotation

GitHub Actions / Lint and Validate

103:81 [line-length] line too long (92 > 80 characters)
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

Expand All @@ -111,7 +111,7 @@
matrix: ${{ steps.matrix.outputs.matrix }}
steps:
- name: Harden runner
uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0

Check warning on line 114 in .github/workflows/test-integration.yml

View workflow job for this annotation

GitHub Actions / Lint and Validate

114:81 [line-length] line too long (92 > 80 characters)
with:
egress-policy: audit
- name: Compute persistence matrix
Expand Down Expand Up @@ -164,15 +164,18 @@
# .git is rsync'd to the VM (the maven buildnumber plugin needs it); don't carry the token.
persist-credentials: false

# Script-catalog tests run only on demand: a PR touching the catalog/harness, or a ticked dispatch.
# Script-catalog tests run nightly (so the Zulip report covers them), on a PR touching the
# catalog/harness, or on a dispatch with the box (or "all modules") ticked.
- name: Decide script-catalog tests
id: catalog
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
run: |
run=false
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
run="${{ inputs.script_catalog }}"
if [ "${{ github.event_name }}" = "schedule" ]; then
run=true
elif [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
[ "${{ inputs.script_catalog }}" = "true" ] || [ "${{ inputs.all_modules }}" = "true" ] && run=true
elif [ "${{ github.event_name }}" = "pull_request" ]; then
git fetch -q --no-tags --depth=1 origin "$BASE_SHA"
rc=0
Expand Down
4 changes: 2 additions & 2 deletions automation/ci/script_catalog/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -316,8 +316,8 @@ def basic_lock():
assert login_code(client, lock_acr, lock_uid, password), f"{lock_acr} login with the right password failed"
for _ in range(3):
assert not aio.login(client, lock_acr, lock_uid, "wrong-" + password), "wrong password accepted"
status = (aio.get_user(lock_inum) or {}).get("jansStatus")
assert status == "inactive", f"jansStatus={status!r} after 3 failed logins, expected 'inactive'"
status = (aio.get_user(lock_inum) or {}).get("status")
assert status == "inactive", f"status={status!r} after 3 failed logins, expected 'inactive'"
assert not aio.login(client, lock_acr, lock_uid, password), "locked account still logs in"

behavior(run, "ScriptCatalog.behavior", "introspection_custom_params.py returns public sid", introspection_sid)
Expand Down
3 changes: 3 additions & 0 deletions automation/ci/script_catalog/test_harness.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@
import manifest
import run

# Fixture output (fake FAIL lines) in the CI log reads like real AIO failures.
run.log = lambda *_: None


class CatalogTest(unittest.TestCase):
@classmethod
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@

import java.security.SecureRandom;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.Statement;
import java.util.ArrayList;
Expand Down Expand Up @@ -58,17 +59,12 @@
* Atomic batch insert ({@code persistBatch}) and multi-get ({@code findByDns}) of the SQL backend
* against a real database.
*
* <p>Commit counting uses PostgreSQL statistics: {@code pg_stat_database.xact_commit} (all committed
* transactions of the database, read-only autocommit statements included) and the next transaction id
* {@code txid_snapshot_xmax(txid_current_snapshot())}, which advances once per write transaction.
* PostgreSQL 15+ flushes backend statistics to shared memory at most once per second and an idle
* backend can delay it up to 10 s, so {@code xact_commit} is read after {@link #STATS_SETTLE_MILLIS}.
* The database is shared with other services of the stand, so both counters get commits of other
* sessions. An idle window of the same length is measured first and subtracted; the remaining
* variation of that noise, the pool validation query and the probe itself are covered by
* {@link #WRITE_NOISE_TOLERANCE} and {@link #XACT_NOISE_TOLERANCE}. The write transaction count is the
* exact signal (1 for the batch, one per entry for single persist); {@code xact_commit} also counts
* read-only statements of other sessions, so its bound is looser.</p>
* <p>Commit counting is PostgreSQL only. Rows inserted by one transaction share its {@code xmin}, so the
* distinct {@code xmin} of the inserted rows is the exact transaction count (1 for the batch, one per entry
* for single persist), unaffected by other sessions of the shared stand. Database-wide counters
* ({@code pg_stat_database.xact_commit} and {@code txid_snapshot_xmax(txid_current_snapshot())}) are only
* reported, as they include commits of other sessions; {@code xact_commit} is read after
* {@link #STATS_SETTLE_MILLIS} because PostgreSQL 15+ flushes backend statistics lazily.</p>
*
* @author Yuriy Movchan Date: 11/12/2026
*/
Expand All @@ -81,13 +77,6 @@ public class SqlBatchPersistTest extends BaseOrmTest {
private static final String VERSION_TEST_BASE_DN = "ou=version_test,o=jans";
private static final String BINARY_BASE_DN = "ou=binary,o=jans";

/** Write transactions of other sessions tolerated per measurement (jans-dev: 0-1 per 11 s window) */
private static final int WRITE_NOISE_TOLERANCE = 10;
/**
* Read-only commits of other sessions tolerated per measurement on top of the idle-window baseline
* (jans-dev: 16-34 per 11 s window, so the baseline varies by up to about 20)
*/
private static final int XACT_NOISE_TOLERANCE = 30;
private static final long STATS_SETTLE_MILLIS = 11000L;

private String marker;
Expand Down Expand Up @@ -411,14 +400,8 @@ public void oneCommitPerBatch() throws Exception {
LOG.info(report);
System.out.println("SqlBatchPersistTest.oneCommitPerBatch: " + report);

assertTrue(batchWriteTransactions >= 1 && batchWriteTransactions - noiseWriteTransactions <= 1 + WRITE_NOISE_TOLERANCE, report);
assertTrue(singlesWriteTransactions >= count, report);

long batchXactExcess = batchXactCommits - noiseXactCommits;
long singlesXactExcess = singlesXactCommits - noiseXactCommits;
assertTrue(batchXactCommits >= 1 && batchXactExcess <= 1 + XACT_NOISE_TOLERANCE, report);
assertTrue(singlesXactExcess >= count - XACT_NOISE_TOLERANCE, report);
assertTrue(batchXactExcess * 4 < singlesXactExcess, report);
assertEquals(insertTransactions(sqlEntryManager, batch), new long[] { count, 1 }, "rows, transactions of persistBatch; " + report);
assertEquals(insertTransactions(sqlEntryManager, singles), new long[] { count, count }, "rows, transactions of single persist; " + report);
}

@Test
Expand Down Expand Up @@ -580,6 +563,22 @@ private long[] readCommitCounters(SqlEntryManager sqlEntryManager) throws Except
}
}

/** Rows found and distinct inserting transactions (xmin) of the given sessions */
private long[] insertTransactions(SqlEntryManager sqlEntryManager, List<SessionId> sessions) throws Exception {
String[] ids = sessions.stream().map(SessionId::getId).toArray(String[]::new);
String schema = sqlEntryManager.getOperationService().getConnectionProvider().getSchemaName();
String table = (schema == null ? "" : "\"" + schema + "\".") + "\"jansSessId\"";
try (Connection connection = sqlEntryManager.getOperationService().getConnection();
PreparedStatement statement = connection.prepareStatement(
"SELECT count(*), count(DISTINCT xmin::text) FROM " + table + " WHERE doc_id = ANY(?)")) {
statement.setArray(1, connection.createArrayOf("varchar", ids));
try (ResultSet resultSet = statement.executeQuery()) {
assertTrue(resultSet.next());
return new long[] { resultSet.getLong(1), resultSet.getLong(2) };
}
}
}

private List<SessionId> buildSessions(List<String> ids) {
List<SessionId> result = new ArrayList<>();
for (String id : ids) {
Expand Down
Loading