Add Project Code Coverage To CI

.github/workflows/code-coverage-report.yml

@@ -0,0 +1,36 @@
+# On push to master, this workflow builds and publishes the project's coverage 
+# report to: https://plutus.cardano.intersectmbo.org/dev/coverage
+
+name: "📊 Code Coverage Report"
+
+on:
+  push: 
+    branches: 
+      - master 
+  workflow_dispatch:
+  pull_request:
+    branches: 
+      - test-coverage # TODO remove me before merging
+
+jobs:
+  deploy:
+    name: Deploy
+    runs-on: [self-hosted, plutus-ci]
+    permissions:
+      contents: write
+    environment:
+      name: github-pages
+    steps:
+      - name: Checkout
+        uses: actions/checkout@main
+
+      - name: Build Report
+        run: | 
+          nix build --no-warn-dirty --accept-flake-config .#project-coverage-report
+
+      - name: Deploy Report
+        uses: JamesIves/[email protected]
+        with:
+          folder: result/share/hpc/vanilla/html
+          target-folder: dev/coverage
+          single-commit: true

.github/workflows/slack-message-broker.yml

@@ -21,6 +21,7 @@ on:
       - "🌘 Nightly Testsuite"
       - "📝 Papers & Specs"
       - "🏛️ Plinth Template"
+      - "📊 Code Coverage Report"
 
 jobs:
   Send:

cardano-constitution/test/Cardano/Constitution/Validator/Data/GoldenTests.hs

@@ -22,6 +22,7 @@ import Data.Maybe
 import Data.String
 import System.FilePath
 import Test.Tasty
+import Test.Tasty.Extras (ignoreTestWhenHpcEnabled)
 import Test.Tasty.Golden
 
 import Helpers.Guardrail
@@ -67,11 +68,11 @@ test_readable_uplc = testGroup "ReadableUplc" $ M.elems $
 
 tests :: TestTreeWithTestState
 tests = testGroup' "Golden" $ fmap const
-        [ test_cbor
-        , test_budget_large
-        , test_budget_small
+        [ ignoreTestWhenHpcEnabled test_cbor
+        , ignoreTestWhenHpcEnabled test_budget_large
+        , ignoreTestWhenHpcEnabled test_budget_small
         , test_readable_pir
-        , test_readable_uplc
+        , ignoreTestWhenHpcEnabled test_readable_uplc
         ]
 
 -- HELPERS

cardano-constitution/test/Cardano/Constitution/Validator/GoldenTests.hs

@@ -22,6 +22,7 @@ import Data.Maybe
 import Data.String
 import System.FilePath
 import Test.Tasty
+import Test.Tasty.Extras (ignoreTestWhenHpcEnabled)
 import Test.Tasty.Golden
 
 import Helpers.Guardrail
@@ -67,11 +68,11 @@ test_readable_uplc = testGroup "ReadableUplc" $ M.elems $
 
 tests :: TestTreeWithTestState
 tests = testGroup' "Golden" $ fmap const
-        [ test_cbor
-        , test_budget_large
-        , test_budget_small
+        [ ignoreTestWhenHpcEnabled test_cbor
+        , ignoreTestWhenHpcEnabled test_budget_large
+        , ignoreTestWhenHpcEnabled test_budget_small
         , test_readable_pir
-        , test_readable_uplc
+        , ignoreTestWhenHpcEnabled test_readable_uplc
         ]
 
 -- HELPERS

nix/outputs.nix

@@ -64,6 +64,9 @@ let
       [ "devShells" ]; # Won't build on Windows
   };
 
+  project-coverage-report =
+    project.projectVariants.ghc96-coverage.projectCoverageReport;
+
   extra-artifacts =
     { inherit unraveling-recursion-paper; } //
     { inherit metatheory-site; } //
@@ -110,6 +113,7 @@ let
       (project-variants-hydra-jobs) //
       (windows-packages) //
       (packages) //
+      { inherit project-coverage-report; } //
       { devShells = non-profiled-shells; } //
       { required = hydra-required-job; };
     "x86_64-darwin" =
@@ -141,11 +145,13 @@ let
     inherit build-latex-doc;
     inherit build-latex;
     inherit extra-artifacts;
+    inherit windows-packages;
     inherit static-haskell-packages;
     inherit exposed-haskell-packages;
     inherit flattened-ci-jobs;
     inherit nested-ci-jobs;
     inherit metatheory-agda-library;
+    inherit project-coverage-report;
   };
 
 in

nix/project.nix

@@ -19,7 +19,24 @@ let
           enableLibraryProfiling = true;
         }];
         ghc96-coverage.modules = [{
-          doCoverage = true;
+          # NOTE: Enabling coverage for some packages breaks tests due to HPC 
+          # (Haskell Program Coverage) instrumentation. The packages listed 
+          # below represent the largest subset that can be enabled without 
+          # breaking tests.
+          # packages.cardano-constitution.doCoverage = true;
+          # packages.plutus-tx.doCoverage = true;
+          packages.plutus-conformance.doCoverage = true;
+          packages.plutus-core.doCoverage = true;
+          packages.plutus-executables.doCoverage = true;
+          packages.plutus-tx-test-util.doCoverage = true;
+          packages.plutus-ledger-api.doCoverage = true;
+
+          packages.plutus-core.configureFlags = [ "--ghc-option=-D__HPC_ENABLED__" ];
+          # plutus-benchmark
+          # plutus-metatheory
+          # plutus-tx
+          # plutus-tx-plugin
+          # docusaurus-examples
         }];
         ghc810.compiler-nix-name = "ghc810";
         ghc98.compiler-nix-name = "ghc98";

plutus-benchmark/marlowe/test/Spec.hs

@@ -19,6 +19,7 @@ import PlutusLedgerApi.V3 (ExCPU (..), ExMemory (..))
 import System.FilePath ((</>))
 import System.IO (hPutStrLn)
 import Test.Tasty (defaultMain, testGroup)
+import Test.Tasty.Extras (ignoreTestWhenHpcEnabled)
 import UntypedPlutusCore.Size qualified as UPLC
 
 main :: IO ()
@@ -65,19 +66,20 @@ main = withUtf8 $ do
           Lib.measureProgram
           [benchmarkToUPLC Data.rolePayoutValidator bench | bench <- rolePayout]
 
+  let marloweTests = testGroup "Marlowe"
+        [ Lib.goldenUplcMeasurements "budgets" goldenFile actualFile \writeHandle ->
+            for_
+              (semanticsMeasures <> rolePayoutMeasures)
+              \(ExCPU cpu, ExMemory mem, UPLC.Size size) ->
+                hPutStrLn writeHandle $
+                  List.intercalate "\t" [show cpu, show mem, show size]
+        , Lib.goldenUplcMeasurements "data-budgets" goldenFileData actualFileData \writeHandle ->
+            for_
+              (dataSemanticsMeasures <> dataRolePayoutMeasures)
+              \(ExCPU cpu, ExMemory mem, UPLC.Size size) ->
+                hPutStrLn writeHandle $
+                  List.intercalate "\t" [show cpu, show mem, show size]
+        ]
+
   -- Write the measures to the actual file
-  defaultMain
-    $ testGroup "Marlowe"
-    [ Lib.goldenUplcMeasurements "budgets" goldenFile actualFile \writeHandle ->
-        for_
-          (semanticsMeasures <> rolePayoutMeasures)
-          \(ExCPU cpu, ExMemory mem, UPLC.Size size) ->
-            hPutStrLn writeHandle $
-              List.intercalate "\t" [show cpu, show mem, show size]
-    , Lib.goldenUplcMeasurements "data-budgets" goldenFileData actualFileData \writeHandle ->
-        for_
-          (dataSemanticsMeasures <> dataRolePayoutMeasures)
-          \(ExCPU cpu, ExMemory mem, UPLC.Size size) ->
-            hPutStrLn writeHandle $
-              List.intercalate "\t" [show cpu, show mem, show size]
-    ]
+  defaultMain $ ignoreTestWhenHpcEnabled marloweTests

plutus-benchmark/script-contexts/test/V1/Spec.hs

@@ -5,7 +5,7 @@ module V1.Spec (allTests) where
 import Data.Text qualified as Text
 
 import Test.Tasty
-import Test.Tasty.Extras (TestNested, runTestNested, testNestedGhc)
+import Test.Tasty.Extras (TestNested, ignoreTestWhenHpcEnabled, runTestNested, testNestedGhc)
 import Test.Tasty.HUnit
 
 import PlutusBenchmark.Common (Term, compiledCodeToTerm, runTermCek, unsafeRunTermCek)
@@ -47,7 +47,10 @@ testCheckSOPSc1 = testGroup "checkScriptContext1"
         compiledCodeToTerm $ SOP.SC.mkCheckScriptContext1Code (SOP.SC.mkScriptContext 4)
     , testCase "fails on 5" . assertFailed $
         compiledCodeToTerm $ SOP.SC.mkCheckScriptContext1Code (SOP.SC.mkScriptContext 5)
-    , runTestGhcSOP [ Tx.goldenSize "checkScriptContext1" $
+    , ignoreTestWhenHpcEnabled $
+          runTestGhcSOP
+               [
+                    Tx.goldenSize "checkScriptContext1" $
                         SOP.SC.mkCheckScriptContext1Code (SOP.SC.mkScriptContext 1)
                    , Tx.goldenPirReadable "checkScriptContext1" $
                         SOP.SC.mkCheckScriptContext1Code (SOP.SC.mkScriptContext 1)
@@ -59,7 +62,7 @@ testCheckSOPSc1 = testGroup "checkScriptContext1"
                         SOP.SC.mkCheckScriptContext1Code (SOP.SC.mkScriptContext 20)
                    , Tx.goldenEvalCekCatch "checkScriptContext1-20" $
                         [SOP.SC.mkCheckScriptContext1Code (SOP.SC.mkScriptContext 20)]
-          ]
+               ]
     ]
 
 testCheckDataSc1 :: TestTree
@@ -89,19 +92,21 @@ testCheckSOPSc2 = testGroup "checkScriptContext2"
           compiledCodeToTerm $ SOP.SC.mkCheckScriptContext2Code (SOP.SC.mkScriptContext 4)
     , testCase "succeed on 5" . assertSucceeded $
           compiledCodeToTerm $ SOP.SC.mkCheckScriptContext2Code (SOP.SC.mkScriptContext 5)
-    , runTestGhcSOP [ Tx.goldenSize "checkScriptContext2" $
-                        SOP.SC.mkCheckScriptContext2Code (SOP.SC.mkScriptContext 1)
-                   , Tx.goldenPirReadable "checkScriptContext2" $
-                        SOP.SC.mkCheckScriptContext2Code (SOP.SC.mkScriptContext 1)
-                   , Tx.goldenBudget "checkScriptContext2-4" $
-                        SOP.SC.mkCheckScriptContext2Code (SOP.SC.mkScriptContext 4)
-                   , Tx.goldenEvalCekCatch "checkScriptContext2-4" $
-                        [SOP.SC.mkCheckScriptContext2Code (SOP.SC.mkScriptContext 4)]
-                   , Tx.goldenBudget "checkScriptContext2-20" $
-                        SOP.SC.mkCheckScriptContext2Code (SOP.SC.mkScriptContext 20)
-                   , Tx.goldenEvalCekCatch "checkScriptContext2-20" $
-                        [SOP.SC.mkCheckScriptContext2Code (SOP.SC.mkScriptContext 20)]
-                   ]
+    , ignoreTestWhenHpcEnabled $
+          runTestGhcSOP
+               [ Tx.goldenSize "checkScriptContext2" $
+                    SOP.SC.mkCheckScriptContext2Code (SOP.SC.mkScriptContext 1)
+               , Tx.goldenPirReadable "checkScriptContext2" $
+                    SOP.SC.mkCheckScriptContext2Code (SOP.SC.mkScriptContext 1)
+               , Tx.goldenBudget "checkScriptContext2-4" $
+                    SOP.SC.mkCheckScriptContext2Code (SOP.SC.mkScriptContext 4)
+               , Tx.goldenEvalCekCatch "checkScriptContext2-4" $
+                    [SOP.SC.mkCheckScriptContext2Code (SOP.SC.mkScriptContext 4)]
+               , Tx.goldenBudget "checkScriptContext2-20" $
+                    SOP.SC.mkCheckScriptContext2Code (SOP.SC.mkScriptContext 20)
+               , Tx.goldenEvalCekCatch "checkScriptContext2-20" $
+                    [SOP.SC.mkCheckScriptContext2Code (SOP.SC.mkScriptContext 20)]
+               ]
     ]
 
 testCheckDataSc2 :: TestTree

plutus-benchmark/script-contexts/test/V2/Spec.hs

@@ -7,7 +7,7 @@ module V2.Spec (allTests) where
 import Data.Text qualified as Text
 
 import Test.Tasty
-import Test.Tasty.Extras (TestNested, runTestNested, testNestedGhc)
+import Test.Tasty.Extras (TestNested, ignoreTestWhenHpcEnabled, runTestNested, testNestedGhc)
 import Test.Tasty.HUnit
 
 import PlutusBenchmark.Common (Term, compiledCodeToTerm, runTermCek, unsafeRunTermCek)
@@ -51,7 +51,10 @@ testCheckSOPSc1 = testGroup "checkScriptContext1"
         compiledCodeToTerm $ SOP.SC.mkCheckScriptContext1Code (SOP.SC.mkScriptContext 4)
     , testCase "fails on 5" . assertFailed $
         compiledCodeToTerm $ SOP.SC.mkCheckScriptContext1Code (SOP.SC.mkScriptContext 5)
-    , runTestGhcSOP [ Tx.goldenSize "checkScriptContext1" $
+    , ignoreTestWhenHpcEnabled $
+          runTestGhcSOP
+               [
+                    Tx.goldenSize "checkScriptContext1" $
                         SOP.SC.mkCheckScriptContext1Code (SOP.SC.mkScriptContext 1)
                    , Tx.goldenPirReadable "checkScriptContext1" $
                         SOP.SC.mkCheckScriptContext1Code (SOP.SC.mkScriptContext 1)
@@ -63,7 +66,7 @@ testCheckSOPSc1 = testGroup "checkScriptContext1"
                         SOP.SC.mkCheckScriptContext1Code (SOP.SC.mkScriptContext 20)
                    , Tx.goldenEvalCekCatch "checkScriptContext1-20" $
                         [SOP.SC.mkCheckScriptContext1Code (SOP.SC.mkScriptContext 20)]
-          ]
+               ]
     ]
 
 testCheckDataSc1 :: TestTree
@@ -93,7 +96,10 @@ testCheckSOPSc2 = testGroup "checkScriptContext2"
           compiledCodeToTerm $ SOP.SC.mkCheckScriptContext2Code (SOP.SC.mkScriptContext 4)
     , testCase "succeed on 5" . assertSucceeded $
           compiledCodeToTerm $ SOP.SC.mkCheckScriptContext2Code (SOP.SC.mkScriptContext 5)
-    , runTestGhcSOP [ Tx.goldenSize "checkScriptContext2" $
+    , ignoreTestWhenHpcEnabled $
+          runTestGhcSOP
+               [
+                    Tx.goldenSize "checkScriptContext2" $
                         SOP.SC.mkCheckScriptContext2Code (SOP.SC.mkScriptContext 1)
                    , Tx.goldenPirReadable "checkScriptContext2" $
                         SOP.SC.mkCheckScriptContext2Code (SOP.SC.mkScriptContext 1)
@@ -105,7 +111,7 @@ testCheckSOPSc2 = testGroup "checkScriptContext2"
                         SOP.SC.mkCheckScriptContext2Code (SOP.SC.mkScriptContext 20)
                    , Tx.goldenEvalCekCatch "checkScriptContext2-20" $
                         [SOP.SC.mkCheckScriptContext2Code (SOP.SC.mkScriptContext 20)]
-                   ]
+               ]
     ]
 
 testCheckDataSc2 :: TestTree
@@ -157,13 +163,14 @@ testCheckDataScEquality = testGroup "checkScriptContextEquality"
 
 testSOPFwdStakeTrick :: TestTree
 testSOPFwdStakeTrick =
-     runTestGhcSOP
-          [ Tx.goldenSize "sopFwdStakeTrick" testCode
-          , Tx.goldenPirReadable "sopFwdStakeTrick" testAbsCode
-          , Tx.goldenUPlcReadable "sopFwdStakeTrick" testAbsCode
-          , Tx.goldenBudget "sopFwdStakeTrick" testCode
-          , Tx.goldenEvalCekCatch "sopFwdStakeTrick" [testCode]
-          ]
+     ignoreTestWhenHpcEnabled $
+          runTestGhcSOP
+               [ Tx.goldenSize "sopFwdStakeTrick" testCode
+               , Tx.goldenPirReadable "sopFwdStakeTrick" testAbsCode
+               , Tx.goldenUPlcReadable "sopFwdStakeTrick" testAbsCode
+               , Tx.goldenBudget "sopFwdStakeTrick" testCode
+               , Tx.goldenEvalCekCatch "sopFwdStakeTrick" [testCode]
+               ]
   where
      testCredential =
           SOP.SC.mkStakingCredential "someCredential"

plutus-benchmark/script-contexts/test/V3/Spec.hs

@@ -5,7 +5,7 @@ module V3.Spec (allTests) where
 import Data.Text qualified as Text
 
 import Test.Tasty
-import Test.Tasty.Extras (TestNested, runTestNested, testNestedGhc)
+import Test.Tasty.Extras (TestNested, ignoreTestWhenHpcEnabled, runTestNested, testNestedGhc)
 import Test.Tasty.HUnit
 
 import PlutusBenchmark.Common (Term, compiledCodeToTerm, runTermCek, unsafeRunTermCek)
@@ -47,7 +47,10 @@ testCheckSOPSc1 = testGroup "checkScriptContext1"
         compiledCodeToTerm $ SOP.SC.mkCheckScriptContext1Code (SOP.SC.mkScriptContext 4)
     , testCase "fails on 5" . assertFailed $
         compiledCodeToTerm $ SOP.SC.mkCheckScriptContext1Code (SOP.SC.mkScriptContext 5)
-    , runTestGhcSOP [ Tx.goldenSize "checkScriptContext1" $
+    , ignoreTestWhenHpcEnabled $
+          runTestGhcSOP
+               [
+                    Tx.goldenSize "checkScriptContext1" $
                         SOP.SC.mkCheckScriptContext1Code (SOP.SC.mkScriptContext 1)
                    , Tx.goldenPirReadable "checkScriptContext1" $
                         SOP.SC.mkCheckScriptContext1Code (SOP.SC.mkScriptContext 1)
@@ -59,7 +62,7 @@ testCheckSOPSc1 = testGroup "checkScriptContext1"
                         SOP.SC.mkCheckScriptContext1Code (SOP.SC.mkScriptContext 20)
                    , Tx.goldenEvalCekCatch "checkScriptContext1-20" $
                         [SOP.SC.mkCheckScriptContext1Code (SOP.SC.mkScriptContext 20)]
-          ]
+               ]
     ]
 
 testCheckDataSc1 :: TestTree
@@ -89,7 +92,9 @@ testCheckSOPSc2 = testGroup "checkScriptContext2"
           compiledCodeToTerm $ SOP.SC.mkCheckScriptContext2Code (SOP.SC.mkScriptContext 4)
     , testCase "succeed on 5" . assertSucceeded $
           compiledCodeToTerm $ SOP.SC.mkCheckScriptContext2Code (SOP.SC.mkScriptContext 5)
-    , runTestGhcSOP [ Tx.goldenSize "checkScriptContext2" $
+    , ignoreTestWhenHpcEnabled $
+          runTestGhcSOP
+               [ Tx.goldenSize "checkScriptContext2" $
                         SOP.SC.mkCheckScriptContext2Code (SOP.SC.mkScriptContext 1)
                    , Tx.goldenPirReadable "checkScriptContext2" $
                         SOP.SC.mkCheckScriptContext2Code (SOP.SC.mkScriptContext 1)
@@ -101,7 +106,7 @@ testCheckSOPSc2 = testGroup "checkScriptContext2"
                         SOP.SC.mkCheckScriptContext2Code (SOP.SC.mkScriptContext 20)
                    , Tx.goldenEvalCekCatch "checkScriptContext2-20" $
                         [SOP.SC.mkCheckScriptContext2Code (SOP.SC.mkScriptContext 20)]
-                   ]
+               ]
     ]
 
 testCheckDataSc2 :: TestTree
@@ -152,7 +157,7 @@ testCheckDataScEquality = testGroup "checkScriptContextEquality"
     ]
 
 testPurposeIsWellFormed :: TestTree
-testPurposeIsWellFormed = testGroup "purposeIsWellFormed"
+testPurposeIsWellFormed = ignoreTestWhenHpcEnabled $ testGroup "purposeIsWellFormed"
     [ runTestGhcData [  Tx.goldenPirReadable "purposeIsWellFormed" $
                         Data.SC.compiledPurposeIsWellFormed
                    , Tx.goldenSize "purposeIsWellFormed" $