ci.yml

# The main hledger continuous integration tests.
# Code must pass this successfully before it can be merged or pushed to master.
# https://github.com/simonmichael/hledger/settings/branch_protection_rules/17386787
# TRIGGER: Runs on any push to ci branch or any pull request against master.
# ACTION: Builds, tests and saves linux x64 dynamic binaries with stack and the default ghc.

name: ci

on:
  # When manually triggered in github ui, it runs in master.
  workflow_dispatch:
  # When there's a push to the ci branch, it runs in that branch.
  # After it passes, those commits can be merged/pushed to master.
  # (Don't use these branches for pull requests, or it will run twice,
  # https://github.community/t/how-to-trigger-an-action-on-push-or-pull-request-but-not-both/16662/2)
  push:
    branches: [ ci ]
  # When there's a pull request against master, it runs in the pull request's branch.
  # After it passes, that branch can be merged/pushed to master.
  pull_request:
    branches: [ master ]
    # Uncomment to run it only for changes to these paths: (but that could prevent merging)
    # paths:
    #   - '.github/workflows/pushpull.yml'
    #   - 'stack*.yaml'
    #   - 'hledger-lib/**'
    #   - 'hledger/**'
    #   - 'hledger-ui/**'
    #   - 'hledger-web/**'
    #   - 'bin/*.hs'
    #   - 'examples/**'
    # Or to ignore certain paths:
    #   # examples
    #   - '!**.journal'
    #   - '!**.j'
    #   - '!**.ledger'
    #   - '!**.csv'
    #   # docs
    #   - '!**.m4'
    #   - '!**.md'
    #   - '!**.1'
    #   - '!**.5'
    #   - '!**.info'
    #   - '!**.txt'

jobs:
  ci:
    runs-on: ubuntu-24.04
      # arch:  x64
      # image: https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2404-Readme.md

    env:
      stack:     stack
      ghc:       9.10.1
      cachedate: 20250615   # bump this when new caches are needed

      # flag for skipping later steps, declared here to prevent "Context access might be invalid" warnings
      do-all:

    steps:

    - name: Check out
      uses: actions/checkout@v4
      # have to fetch everything for git describe for hledger's --version
      with:
        fetch-depth: 0

    # - name: Print some context for troubleshooting
    #   env:
    #     GITHUB_CONTEXT: ${{ toJson(github) }}
    #   run: |
    #     echo $GITHUB_CONTEXT
    #   # echo "$GITHUB_SHA"
    #   # echo "$GITHUB_REF"
    #   # echo "$GITHUB_HEAD_REF"
    #   # echo "$GITHUB_BASE_REF"
    #   # git log "$GITHUB_BASE_REF"..
    #   # tools/commitlint "$GITHUB_BASE_REF"..


    # EARLY ACTIONS

    - name: Check commit messages
      # keep this step synced in all workflows which do it
      # For a PR, the range will be: master..origin/$GITHUB_HEAD_REF
      # For a push it will be: $BEFORE..
      # For a force push, BEFORE is the previous HEAD, and on github (though not locally) this is an "invalid revision range".
      # 202310: we skip this check when we can't detect the commits, which happens in certain cases
      # related: https://stackoverflow.com/questions/64708371/how-to-run-github-workflow-on-every-commit-of-a-push
      # 202312: ignore this if it fails, it may be not worth the hassle
      env:
        BEFORE: ${{ github.event.before }}
        # NUM:    5
      shell: bash
      run: |
        RANGE=${BEFORE:-origin/master}..${GITHUB_HEAD_REF:-}

        echo "debug: last 10 commits:"
        echo "$(git log --format='%h -%d %s (%an, %ci)' -10)"
        echo "debug: origin/master:"
        echo "$(git log --format='%h -%d %s (%an, %ci)' -1 origin/master)"
        echo "debug: BEFORE=$BEFORE"
        echo "$(git log --format='%h -%d %s (%an, %ci)' -1 $BEFORE)"
        echo "debug: GITHUB_HEAD_REF=$GITHUB_HEAD_REF"
        echo "$(git log --format='%h -%d %s (%an, %ci)' -1 $GITHUB_HEAD_REF)"
        echo "debug: RANGE=$RANGE"
        echo "debug: commits to check:"
        echo "$(git log --format='%h -%d %s (%an, %ci)' --abbrev-commit --date=relative --date-order $RANGE)"

        if git rev-list --quiet $RANGE
        then
          tools/commitlint $RANGE || echo "commit lint failed, ignoring"
        else
          # echo "could not identify commits, checking last $NUM instead:"; tools/commitlint -$NUM
          echo "could not identify commits, not checking them"  # XXX
        fi

    - name: Skip remaining steps if the last commit message begins with ;
      shell: bash
      run: |
        echo "git log -1 --pretty='%s' ${GITHUB_HEAD_REF:+origin/$GITHUB_HEAD_REF} >> $$.gitlog"
        (git log -1 --pretty='%s' ${GITHUB_HEAD_REF:+origin/$GITHUB_HEAD_REF} >> $$.gitlog \
          && (grep -qE '^ *;' $$.gitlog || echo "do-all=true" >> $GITHUB_ENV)) \
          || ( echo "could not identify commit range, continuing CI steps"; echo "do-all=true" >> $GITHUB_ENV )


    - name: Check embedded files
      run: |
        tools/checkembeddedfiles
      if: env.do-all


    # CACHES

    # Remember, caches are immutable - created once and never updated.
    # If a cache is used, a new version of it won't be created.
    # It's probably right to use them for things which don't/needn't change between runs,
    # including:
    # - the package db containing the versions of our deps specified by the stackage snapshot
    #   (or by the build plan found by the first cabal run, if using cabal)
    # - the local bin directory ? what's in there anyway ?
    # - the per-project and per-package .stack-work directories ? these probably change a little on each run.
    #   presumably it's worthwhile to cache their first-run content, even if with every following run
    #   we end up having to rebuild more of that. The alternative is to always rebuild all of it.

    - name: Cache - stack global package db
      id:   stack-global
      uses: actions/cache@v4
      with:
        path: ~/.stack
        # XXX if stack.yaml is a symlink, this fails with
        #  Error: The template is not valid. .github/workflows/push.yml (Line: 103, Col: 14): hashFiles('**.yaml') failed.
        #  Fail to hash files under directory '/home/runner/work/hledger/hledger'
        key: ${{ runner.os }}-x64-stack-global-${{ env.ghc }}-${{ env.cachedate }}-${{ hashFiles('**.yaml') }}
        restore-keys: |
             ${{ runner.os }}-x64-stack-global-${{ env.ghc }}-${{ env.cachedate }}
      if: env.do-all

    - name: Cache - stack-installed programs in ~/.local/bin
      id:   stack-programs
      uses: actions/cache@v4
      with:
        path: ~/.local/bin
        key: ${{ runner.os }}-x64-stack-programs-${{ env.ghc }}-${{ env.cachedate }}-${{ hashFiles('**.yaml') }}
        restore-keys: |
             ${{ runner.os }}-x64-stack-programs-${{ env.ghc }}-${{ env.cachedate }}
      if: env.do-all

    - name: Cache - .stack-work
      uses: actions/cache@v4
      with:
        path: .stack-work
        key: ${{ runner.os }}-x64-stack-work-${{ env.ghc }}-${{ env.cachedate }}-${{ hashFiles('**.yaml') }}
        restore-keys: |
             ${{ runner.os }}-x64-stack-work-${{ env.ghc }}-${{ env.cachedate }}
      if: env.do-all

    - name: Cache - hledger-lib/.stack-work
      uses: actions/cache@v4
      with:
        path: hledger-lib/.stack-work
        key: ${{ runner.os }}-x64-hledger-lib-stack-work-${{ env.ghc }}-${{ env.cachedate }}-${{ hashFiles('hledger-lib/package.yaml') }}
        restore-keys: |
             ${{ runner.os }}-x64-hledger-lib-stack-work-${{ env.ghc }}-${{ env.cachedate }}
      if: env.do-all

    - name: Cache - hledger/.stack-work
      uses: actions/cache@v4
      with:
        path: hledger/.stack-work
        key: ${{ runner.os }}-x64-hledger-stack-work-${{ env.ghc }}-${{ env.cachedate }}-${{ hashFiles('hledger/package.yaml') }}
        restore-keys: |
             ${{ runner.os }}-x64-hledger-stack-work-${{ env.ghc }}-${{ env.cachedate }}
      if: env.do-all

    - name: Cache - hledger-ui/.stack-work
      uses: actions/cache@v4
      with:
        path: hledger-ui/.stack-work
        key: ${{ runner.os }}-x64-hledger-ui-stack-work-${{ env.ghc }}-${{ env.cachedate }}-${{ hashFiles('hledger-ui/package.yaml') }}
        restore-keys: |
             ${{ runner.os }}-x64-hledger-ui-stack-work-${{ env.ghc }}-${{ env.cachedate }}
      if: env.do-all

    - name: Cache - hledger-web/.stack-work
      uses: actions/cache@v4
      with:
        path: hledger-web/.stack-work
        key: ${{ runner.os }}-x64-hledger-web-stack-work-${{ env.ghc }}-${{ env.cachedate }}-${{ hashFiles('hledger-web/package.yaml') }}
        restore-keys: |
             ${{ runner.os }}-x64-hledger-web-stack-work-${{ env.ghc }}-${{ env.cachedate }}
      if: env.do-all

    # Ensure the ghc version we need is installed.
    # The following stack path commands require it, so let's install it first, quietly if possible.
    # Hopefully it will often be in our cached dirs, avoiding reinstall.
    # Don't rely on the preinstalled ghc, it's too variable.
    - name: Ensure ghc is installed
      run: |
        $stack setup --verbosity error
      if: env.do-all

    # - name: Show stack directories
    #   run: |
    #     printf "stack-root: \t";            $stack path --stack-root             # Global Stack root directory
    #     printf "global-config: \t";         $stack path --global-config          # Global Stack configuration file
    #     printf "project-root: \t";          $stack path --project-root           # Project root (derived from stack.yaml file)
    #     printf "config-location: \t";       $stack path --config-location        # Configuration location (where the stack.yaml file is)
    #     printf "bin-path: \t";              $stack path --bin-path               # PATH environment variable
    #     printf "programs: \t";              $stack path --programs               # Install location for GHC and other core tools (see 'stack ls tools' command)
    #     printf "compiler-exe: \t";          $stack path --compiler-exe           # Compiler binary (e.g. ghc)
    #     printf "compiler-bin: \t";          $stack path --compiler-bin           # Directory containing the compiler binary (e.g. ghc)
    #     printf "compiler-tools-bin: \t";    $stack path --compiler-tools-bin     # Directory containing binaries specific to a particular compiler
    #     printf "local-bin: \t";             $stack path --local-bin              # Directory where Stack installs executables (e.g. ~/.local/bin (Unix-like OSs) or %APPDATA%\local\bin (Windows))
    #     printf "extra-include-dirs: \t";    $stack path --extra-include-dirs     # Extra include directories
    #     printf "extra-library-dirs: \t";    $stack path --extra-library-dirs     # Extra library directories
    #     printf "snapshot-pkg-db: \t";       $stack path --snapshot-pkg-db        # Snapshot package database
    #     printf "local-pkg-db: \t";          $stack path --local-pkg-db           # Local project package database
    #     printf "global-pkg-db: \t";         $stack path --global-pkg-db          # Global package database
    #     printf "ghc-package-path: \t";      $stack path --ghc-package-path       # GHC_PACKAGE_PATH environment variable
    #     printf "snapshot-install-root: \t"; $stack path --snapshot-install-root  # Snapshot installation root
    #     printf "local-install-root: \t";    $stack path --local-install-root     # Local project installation root
    #     printf "snapshot-doc-root: \t";     $stack path --snapshot-doc-root      # Snapshot documentation root
    #     printf "local-doc-root: \t";        $stack path --local-doc-root         # Local project documentation root
    #     printf "local-hoogle-root: \t";     $stack path --local-hoogle-root      # Local project documentation root
    #     printf "dist-dir: \t";              $stack path --dist-dir               # Dist work directory, relative to package directory
    #     printf "local-hpc-root: \t";        $stack path --local-hpc-root         # Where HPC reports and tix files are stored
    #   if: env.do-all
    # Example output:
    # stack-root:               /home/runner/.stack
    # global-config:            /home/runner/.stack/config.yaml
    # project-root:             /home/runner/work/hledger/hledger
    # config-location:          /home/runner/work/hledger/hledger/stack.yaml
    # bin-path:                 /home/runner/.stack/snapshots/x86_64-linux-tinfo6/640e58090350012aaeacf3bb95988ba08dbaab0ce8ce8d2cb4b2ea0c5e7dd47b/9.10.1/bin:/home/runner/.stack/compiler-tools/x86_64-linux-tinfo6/ghc-9.10.1/bin:/home/runner/.stack/programs/x86_64-linux/ghc-tinfo6-9.10.1/bin:/snap/bin:/home/runner/.local/bin:/opt/pipx_bin:/home/runner/.cargo/bin:/home/runner/.config/composer/vendor/bin:/usr/local/.ghcup/bin:/home/runner/.dotnet/tools:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin
    # programs:                 /home/runner/.stack/programs/x86_64-linux
    # compiler-exe:             /home/runner/.stack/programs/x86_64-linux/ghc-tinfo6-9.10.1/bin/ghc-9.10.1
    # compiler-bin:             /home/runner/.stack/programs/x86_64-linux/ghc-tinfo6-9.10.1/bin
    # compiler-tools-bin:       /home/runner/.stack/compiler-tools/x86_64-linux-tinfo6/ghc-9.10.1/bin
    # local-bin:                /home/runner/.local/bin
    # extra-include-dirs:       
    # extra-library-dirs:       
    # snapshot-pkg-db:          /home/runner/.stack/snapshots/x86_64-linux-tinfo6/640e58090350012aaeacf3bb95988ba08dbaab0ce8ce8d2cb4b2ea0c5e7dd47b/9.10.1/pkgdb
    # local-pkg-db:             /home/runner/work/hledger/hledger/.stack-work/install/x86_64-linux-tinfo6/640e58090350012aaeacf3bb95988ba08dbaab0ce8ce8d2cb4b2ea0c5e7dd47b/9.10.1/pkgdb
    # global-pkg-db:            /home/runner/.stack/programs/x86_64-linux/ghc-tinfo6-9.10.1/lib/ghc-9.10.1/lib/package.conf.d
    # ghc-package-path:         /home/runner/work/hledger/hledger/.stack-work/install/x86_64-linux-tinfo6/640e58090350012aaeacf3bb95988ba08dbaab0ce8ce8d2cb4b2ea0c5e7dd47b/9.10.1/pkgdb:/home/runner/.stack/snapshots/x86_64-linux-tinfo6/640e58090350012aaeacf3bb95988ba08dbaab0ce8ce8d2cb4b2ea0c5e7dd47b/9.10.1/pkgdb:/home/runner/.stack/programs/x86_64-linux/ghc-tinfo6-9.10.1/lib/ghc-9.10.1/lib/package.conf.d
    # snapshot-install-root:    /home/runner/.stack/snapshots/x86_64-linux-tinfo6/640e58090350012aaeacf3bb95988ba08dbaab0ce8ce8d2cb4b2ea0c5e7dd47b/9.10.1
    # local-install-root:       /home/runner/work/hledger/hledger/.stack-work/install/x86_64-linux-tinfo6/640e58090350012aaeacf3bb95988ba08dbaab0ce8ce8d2cb4b2ea0c5e7dd47b/9.10.1
    # snapshot-doc-root:        /home/runner/.stack/snapshots/x86_64-linux-tinfo6/a475558d986419ca16855615e08d0a95545a229cfbd26fe42e916bee9b2277d3/9.10.1/doc
    # local-doc-root:           /home/runner/work/hledger/hledger/.stack-work/install/x86_64-linux-tinfo6/a475558d986419ca16855615e08d0a95545a229cfbd26fe42e916bee9b2277d3/9.10.1/doc
    # local-hoogle-root:        /home/runner/work/hledger/hledger/.stack-work/hoogle/x86_64-linux-tinfo6/a475558d986419ca16855615e08d0a95545a229cfbd26fe42e916bee9b2277d3/9.10.1
    # dist-dir:                 .stack-work/dist/x86_64-linux-tinfo6/ghc-9.10.1
    # local-hpc-root:           /home/runner/work/hledger/hledger/.stack-work/install/x86_64-linux-tinfo6/640e58090350012aaeacf3bb95988ba08dbaab0ce8ce8d2cb4b2ea0c5e7dd47b/9.10.1/hpc


    # ACTIONS
    # in modular steps for faster & more focussed failures

    # XXX slow, I feel this should happen less often
    # - name: Update package index
    #   run: |
    #     $stack update
    #   if: env.do-all

    - name: Build deps of hledger-lib
      run: |
        $stack build --test --bench hledger-lib --only-dependencies
      if: env.do-all

    - name: Build/test    hledger-lib
      run: |
        $stack install --test --bench hledger-lib --fast --ghc-options=-Werror
      if: env.do-all


    - name: Build deps of hledger
      run: |
        $stack build --test --bench hledger --only-dependencies
      if: env.do-all

    - name: Build/test    hledger
      run: |
        $stack install --test --bench hledger --fast --ghc-options=-Werror
      if: env.do-all


    - name: Build deps of hledger-ui
      run: |
        $stack build --test --bench hledger-ui --only-dependencies
      if: env.do-all

    - name: Build/test    hledger-ui
      run: |
        $stack install --test --bench hledger-ui --fast --ghc-options=-Werror
      if: env.do-all


    - name: Build deps of hledger-web
      run: |
        $stack build --test --bench hledger-web --only-dependencies
      if: env.do-all

    - name: Build/test    hledger-web
      run: |
        $stack install --test --bench hledger-web --fast --ghc-options=-Werror
      if: env.do-all


    - name: Install shelltestrunner
      run: |
        export PATH=~/.local/bin:$PATH
        if [[ ! -x ~/.local/bin/shelltest ]]; then $stack install shelltestrunner-1.10; fi
        shelltest --version
      if: env.do-all

    # Takes ~30s on a 2023 github worker.
    - name: Test functional tests (excluding addons)
      run: |
        export PATH=~/.local/bin:$PATH
        COLUMNS=80 $stack exec -- shelltest --execdir -j16 hledger/test -x /_ -x /addons -x ledger-compat/ledger-baseline -x ledger-compat/ledger-regress -x ledger-compat/ledger-collected
        # XXX run the bin/ func tests corresponding to the GHC version enabled above, only
      if: env.do-all

    # Takes 1m+ on a 2023 github worker.
    # Moved to binaries-mac-arm64 workflow instead;
    # haddock breakage might not be found until release time but it's easy to fix.
    # - name: Test haddock generation
    #   env:
    #     stack: ${{ matrix.plan.stack }}
    #   run: |
    #     printf "haddock version: "; haddock --version
    #     time $stack build --fast --haddock --no-haddock-deps --no-haddock-hyperlink-source --haddock-arguments="--no-print-missing-docs" || echo "HADDOCK FAILED, IGNORING"
    #     # --no-haddock-hyperlink-source is 25% faster
    #     # --no-print-missing-docs is 600% quieter
    #   if: env.do-all

    # ARTIFACTS
    # not much needed, but relatively cheap to save and occasionally useful for troubleshooting ?

    - name: Gather binaries
      id: exes
      run: |
        mkdir tmp
        cd tmp
        cp -P ~/.local/bin/hledger .
        cp -P ~/.local/bin/hledger-ui .
        cp -P ~/.local/bin/hledger-web .
        strip hledger
        strip hledger-ui
        strip hledger-web
      if: env.do-all

    - name: Upload binaries
      uses: actions/upload-artifact@v4
      with:
        name: hledger-linux-x64
        path: |
          tmp/hledger
          tmp/hledger-ui
          tmp/hledger-web
      if: env.do-all



    # SNIPPETS

    # how to set a context variable, and an attempt to make a nice artifact version suffix:
    #   echo "::set-output name=version::$(git branch --show-current | sed 's/-.*//')-$(git rev-parse --short HEAD)"

    # - name: show stuff
    #   run: |
    #     if [[ -e ~/.local/bin ]]; then ls -lFRa ~/.local/bin; fi

    # inspect available context info, per
    # https://docs.github.com/en/free-pro-team@latest/actions/reference/context-and-expression-syntax-for-github-actions.
    # sample output: https://github.com/simonmichael/hledger/runs/1619227104
    # - name: Dump GitHub context
    #   env:
    #     GITHUB_CONTEXT: ${{ toJson(github) }}
    #   run: echo "$GITHUB_CONTEXT"
    # - name: Dump job context
    #   env:
    #     JOB_CONTEXT: ${{ toJson(job) }}
    #   run: echo "$JOB_CONTEXT"
    # - name: Dump steps context
    #   env:
    #     STEPS_CONTEXT: ${{ toJson(steps) }}
    #   run: echo "$STEPS_CONTEXT"
    # - name: Dump runner context
    #   env:
    #     RUNNER_CONTEXT: ${{ toJson(runner) }}
    #   run: echo "$RUNNER_CONTEXT"
    # - name: Dump strategy context
    #   env:
    #     STRATEGY_CONTEXT: ${{ toJson(strategy) }}
    #   run: echo "$STRATEGY_CONTEXT"
    # - name: Dump matrix context
    #   env:
    #     MATRIX_CONTEXT: ${{ toJson(matrix) }}
    #   run: echo "$MATRIX_CONTEXT"