add engine tests

Deltachaos · Deltachaos · commit c2e90ce85a6d · 2026-02-07T17:29:57.000+01:00
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -7,8 +7,8 @@ on:
     branches: [main, master]
 
 jobs:
-  Test:
-    name: Test
+  test:
+    name: Test (non-engine-consistency)
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -21,3 +21,26 @@ jobs:
 
       - name: Run tests
         run: TEST_IGNORE_SLUGS=engine_consistency_integration lua ./bin/test.lua
+
+  engine-consistency:
+    name: Engine Consistency (${{ matrix.engine }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        engine:
+          - dumbgoblin
+          - fruit21
+          - garbochess
+          - sunfish
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Lua
+        uses: leafo/gh-actions-lua@v9
+        with:
+          luaVersion: "5.4"
+          buildCache: false
+
+      - name: Run engine consistency tests for ${{ matrix.engine }}
+        run: ALLOWED_ENGINES=${{ matrix.engine }} lua ./bin/test.lua --only=engine_consistency_integration
diff --git a/tests/integration/__snapshot__/dumbgoblin_eret_015_-_endgame.txt b/tests/integration/__snapshot__/dumbgoblin_eret_015_-_endgame.txt
@@ -0,0 +1 @@
+h3d7
diff --git a/tests/integration/__snapshot__/dumbgoblin_eret_047_-_relief.txt b/tests/integration/__snapshot__/dumbgoblin_eret_047_-_relief.txt
@@ -0,0 +1 @@
+f1a1
diff --git a/tests/integration/__snapshot__/dumbgoblin_eret_079_-_queen_sacrifice.txt b/tests/integration/__snapshot__/dumbgoblin_eret_079_-_queen_sacrifice.txt
@@ -0,0 +1 @@
+e2e8
diff --git a/tests/integration/__snapshot__/dumbgoblin_eret_111_-_fortress.txt b/tests/integration/__snapshot__/dumbgoblin_eret_111_-_fortress.txt
@@ -0,0 +1 @@
+a2a3
diff --git a/tests/integration/__snapshot__/dumbgoblin_sbd_016.txt b/tests/integration/__snapshot__/dumbgoblin_sbd_016.txt
@@ -0,0 +1 @@
+c7g3
diff --git a/tests/integration/__snapshot__/dumbgoblin_sbd_048.txt b/tests/integration/__snapshot__/dumbgoblin_sbd_048.txt
@@ -0,0 +1 @@
+d1d6
diff --git a/tests/integration/__snapshot__/dumbgoblin_sbd_080.txt b/tests/integration/__snapshot__/dumbgoblin_sbd_080.txt
@@ -0,0 +1 @@
+h4f6
diff --git a/tests/integration/__snapshot__/dumbgoblin_sbd_112.txt b/tests/integration/__snapshot__/dumbgoblin_sbd_112.txt
@@ -0,0 +1 @@
+d4f6
diff --git a/tests/integration/__snapshot__/fruit21_delta_006_-_black_to_move_simplified.txt b/tests/integration/__snapshot__/fruit21_delta_006_-_black_to_move_simplified.txt
@@ -0,0 +1 @@
+e8d7
diff --git a/tests/integration/__snapshot__/garbochess_black_to_move.txt b/tests/integration/__snapshot__/garbochess_black_to_move.txt
@@ -0,0 +1 @@
+d7d5
diff --git a/tests/integration/__snapshot__/garbochess_eret_013_-_center.txt b/tests/integration/__snapshot__/garbochess_eret_013_-_center.txt
@@ -0,0 +1 @@
+c4c5
diff --git a/tests/integration/__snapshot__/garbochess_eret_045_-_initiative.txt b/tests/integration/__snapshot__/garbochess_eret_045_-_initiative.txt
@@ -0,0 +1 @@
+c2d3
diff --git a/tests/integration/__snapshot__/garbochess_eret_077_-_attacking_castle.txt b/tests/integration/__snapshot__/garbochess_eret_077_-_attacking_castle.txt
@@ -0,0 +1 @@
+b2c3
diff --git a/tests/integration/__snapshot__/garbochess_eret_109_-_relief.txt b/tests/integration/__snapshot__/garbochess_eret_109_-_relief.txt
@@ -0,0 +1 @@
+f8g6
diff --git a/tests/integration/__snapshot__/garbochess_sbd_014.txt b/tests/integration/__snapshot__/garbochess_sbd_014.txt
@@ -0,0 +1 @@
+b5d4
diff --git a/tests/integration/__snapshot__/garbochess_sbd_046.txt b/tests/integration/__snapshot__/garbochess_sbd_046.txt
@@ -0,0 +1 @@
+e1g1
diff --git a/tests/integration/__snapshot__/garbochess_sbd_078.txt b/tests/integration/__snapshot__/garbochess_sbd_078.txt
@@ -0,0 +1 @@
+d1c3
diff --git a/tests/integration/__snapshot__/garbochess_sbd_110.txt b/tests/integration/__snapshot__/garbochess_sbd_110.txt
@@ -0,0 +1 @@
+g5e6
diff --git a/tests/integration/engine_consistency_test.lua b/tests/integration/engine_consistency_test.lua
@@ -20,13 +20,6 @@ local projectRoot = (testsDir and testsDir:match("(.*)[/\\]")) or dir
 local sep = package.config:sub(1, 1)
 local snapshotDir = projectRoot .. sep .. "tests" .. sep .. "integration" .. sep .. "__snapshot__"
 
--- All registered engines (id and display name)
-local function getEngineList()
-  local list = Engines:GetEngineList()
-  if not list or #list == 0 then return {} end
-  return list
-end
-
 Test.suite("EngineRunner consistency", "engine_consistency_integration")
 
 -- Sync loopFn: runs all steps in a tight while-loop. No stack growth.
@@ -409,7 +402,7 @@ end
 -- Default opts: engines use ply_limit and/or node_limit as they support (e.g. GarboChess uses ply, Sunfish uses nodes)
 local defaultOpts = { ply_limit = 2, node_limit = 1000 }
 
-local engineList = getEngineList()
+local engineList = Test.getEngineList()
 
 --------------------------------------------------------------------------------
 -- Each engine: all positions
diff --git a/tests/test.lua b/tests/test.lua
@@ -285,4 +285,41 @@ function Test.runAll(dir)
   return Test.summary()
 end
 
+--------------------------------------------------------------------------------
+-- Engine helpers (integration tests)
+--------------------------------------------------------------------------------
+
+--- Get list of all registered engines, optionally filtered by ALLOWED_ENGINES environment variable.
+--- ALLOWED_ENGINES: comma-separated list of engine IDs to include (e.g. "garbochess,dumbgoblin").
+function Test.getEngineList()
+  if not _G.DeltaChess or not _G.DeltaChess.Engines then
+    return {}
+  end
+  local list = _G.DeltaChess.Engines:GetEngineList()
+  if not list or #list == 0 then return {} end
+  
+  -- Filter by ALLOWED_ENGINES if set
+  local allowedEnginesEnv = os.getenv("ALLOWED_ENGINES")
+  if allowedEnginesEnv and allowedEnginesEnv ~= "" then
+    local allowed = {}
+    for engineId in allowedEnginesEnv:gmatch("[^,]+") do
+      local trimmed = engineId:gsub("^%s+", ""):gsub("%s+$", ""):lower()
+      if trimmed ~= "" then
+        allowed[trimmed] = true
+      end
+    end
+    
+    local filtered = {}
+    for _, engine in ipairs(list) do
+      local engineId = (engine.id or ""):lower()
+      if allowed[engineId] then
+        table.insert(filtered, engine)
+      end
+    end
+    return filtered
+  end
+  
+  return list
+end
+
 return Test
