Enable cross-platform support for linux-x86-64, windows-x86-64, macos-x86-64 and macos-arm64

* Provide binaries for Souffle programs using git lfs.
* Add dockerized way to build native binaries (except macOS).
* Extend CI for Windows and macOS
* Update README

Author: m-carrasco

.gitattributes

@@ -1,2 +1,3 @@
 *.sh        eol=lf
-
+souffle/bin/linux-x86-64/ssa-query-linux-x86-64 filter=lfs diff=lfs merge=lfs -text
+souffle/bin/windows-x86-64/ssa-query-windows-x86-64.exe filter=lfs diff=lfs merge=lfs -text

.github/workflows/build.yml

@@ -1,4 +1,4 @@
-name: .NET
+name: .NET (build, test and release if necessary)
 
 on:
   push:
@@ -7,14 +7,69 @@ on:
     branches: [ main ]
 
 jobs:
-  build:
+  build-macos:
+    runs-on: macos-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+        lfs: true
+    - name: Checkout LFS objects
+      run: git lfs checkout
+    - name: Setup Python
+      uses: actions/setup-python@v2.2.2
+    - name: Setup .NET
+      uses: actions/setup-dotnet@v1
+      with:
+        dotnet-version: '6.0.x'
+    - name: Install Souffle
+      run: brew install --HEAD souffle-lang/souffle/souffle
+    - name: Build Souffle program
+      run: |
+        cd $GITHUB_WORKSPACE/souffle && CXX=clang++ ./build-souffle-macos-x86-64-arm64.sh
+    - name: Build dotnet
+      run: |
+        dotnet build
+        dotnet test --verbosity normal
+    - name: Upload ssa-query-macos-x86-64-arm64
+      # only do it when it hits the default branch 
+      # this artifact is consumed by the release job
+      if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
+      uses: actions/upload-artifact@v3
+      with:
+        name: ssa-query-macos-x86-64-arm64
+        path: ${{ github.workspace }}/souffle/bin/macos-x86-64-arm64/ssa-query-macos-x86-64-arm64
+        retention-days: 1
 
-    runs-on: ubuntu-18.04
+  build-windows:
+    runs-on: windows-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+        lfs: true
+    - name: Checkout LFS objects
+      run: git lfs checkout
+    - name: Setup Python
+      uses: actions/setup-python@v2.2.2
+    - name: Setup .NET
+      uses: actions/setup-dotnet@v1
+      with:
+        dotnet-version: '6.0.x'
+    - name: Build dotnet
+      run: |
+        dotnet build
+        dotnet test --verbosity normal
 
+  build-linux:  
+    runs-on: ubuntu-20.04
     steps:
     - uses: actions/checkout@v2
       with:
         fetch-depth: 0
+        lfs: true
+    - name: Checkout LFS objects
+      run: git lfs checkout
     - name: Install GitVersion
       uses: gittools/actions/gitversion/setup@v0.9.7
       with:
@@ -27,18 +82,6 @@ jobs:
           configFilePath: ${{ github.workspace }}/.github/GitVersion.yml
     - name: Setup Python
       uses: actions/setup-python@v2.2.2
-    - name: Setup mono
-      run: |
-        sudo apt install gnupg ca-certificates
-        sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
-        echo "deb https://download.mono-project.com/repo/ubuntu stable-bionic/snapshots/6.12.0.182 main" | sudo tee /etc/apt/sources.list.d/mono-official-stable.list
-        sudo apt update
-        sudo apt-get install -y mono-devel mono-utils
-    - name: Setup llvm-config and FileCheck
-      run: |
-        sudo apt-get install llvm-10 llvm-10-tools
-        sudo ln -s /usr/bin/llvm-config-10 /usr/bin/llvm-config
-        llvm-config --version
     - name: Setup .NET
       uses: actions/setup-dotnet@v1
       with:
@@ -47,35 +90,68 @@ jobs:
       uses: jwlawson/actions-setup-cmake@v1.9
       with:
         cmake-version: '3.21.x'
-    - name: Setup Souffle
+    - name: Setup dependencies
+      run: |
+        sudo chmod +x ci/install-lit.sh && sudo ./ci/install-lit.sh
+        sudo chmod +x ci/install-llvm.sh && sudo ./ci/install-llvm.sh
+        sudo chmod +x ci/install-mono.sh && sudo ./ci/install-mono.sh
+    - name: Build Souffle programs from scratch
       run: |
-        curl -LJO https://github.com/souffle-lang/souffle/releases/download/2.0.2/souffle_2.0.2-1_amd64.deb
-        sudo apt-get update
-        sudo apt-get install mcpp
-        sudo dpkg -i souffle_2.0.2-1_amd64.deb
-    - name: Setup LIT
+        cd $GITHUB_WORKSPACE/souffle && rm -r bin/ && ./build-all-with-docker.sh
+    - name: Build dotnet
       run: |
-        pip install lit
-    - name: Configure
+        dotnet build
+        dotnet test --verbosity normal
+    - name: Configure LIT test cases
       run: |
         mkdir build
         cd build 
         cmake .. 
-    - name: Build
-      run: |
-        cd build
-        make build-souffle
-        make build-dotnet
-    - name: Unit test
-      run: |
-        cd build
-        make check-unit-test
     - name: Integration test
       run: |
         cd build
         lit integration-test -v
+
+  release:
+    if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
+    needs: [build-macos, build-windows, build-linux]
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+        lfs: true
+    - name: Checkout LFS objects
+      run: git lfs checkout
+    - name: Install GitVersion
+      uses: gittools/actions/gitversion/setup@v0.9.7
+      with:
+        versionSpec: '5.x'
+    - name: Calculate version
+      id: gitversion
+      uses: gittools/actions/gitversion/execute@v0.9.7
+      with:
+          useConfigFile: true
+          configFilePath: ${{ github.workspace }}/.github/GitVersion.yml
+    - name: Setup Python
+      uses: actions/setup-python@v2.2.2
+    - name: Setup .NET
+      uses: actions/setup-dotnet@v1
+      with:
+        dotnet-version: '6.0.x'
+    - name: Build Souffle programs from scratch
+      run: |
+        cd $GITHUB_WORKSPACE/souffle && rm -r bin/ && ./build-all-with-docker.sh
+    - name: Fetch macos binary
+      uses: actions/download-artifact@master
+      with:
+        name: ssa-query-macos-x86-64-arm64
+        path: ${{ github.workspace }}/souffle/bin/macos-x86-64-arm64/
+    - name: Build dotnet
+      run: |
+        dotnet build
+        dotnet test --verbosity normal
     - name: Generate nuget packages
-      if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
       run: |
-        SOUFFLE_BIN_DIR=$(pwd)/build/bin/souffle dotnet pack net-ssa.sln -o:build/bin/net-ssa/package --include-symbols --include-source /p:Version=${{ steps.gitversion.outputs.AssemblySemVer }} /p:AssemblyVersion=${{ steps.gitversion.outputs.AssemblySemVer }} /p:InformationalVersion=${{ steps.gitversion.outputs.InformationalVersion }} /p:PackageVersion=${{ steps.gitversion.outputs.AssemblySemVer }}
+        dotnet pack net-ssa.sln -o:build/bin/net-ssa/package --include-symbols --include-source /p:Version=${{ steps.gitversion.outputs.AssemblySemVer }} /p:AssemblyVersion=${{ steps.gitversion.outputs.AssemblySemVer }} /p:InformationalVersion=${{ steps.gitversion.outputs.InformationalVersion }} /p:PackageVersion=${{ steps.gitversion.outputs.AssemblySemVer }}
         dotnet nuget push build/bin/net-ssa/package/net-ssa-lib.${{ steps.gitversion.outputs.NuGetVersionV2 }}.nupkg --api-key ${{ secrets.NUGET_API_KEY }} --source https://api.nuget.org/v3/index.json

.github/workflows/docker-ubuntu.yml

@@ -1,4 +1,4 @@
-name: docker-ubuntu
+name: Docker image
 
 on:
   push:
@@ -12,5 +12,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Checkout LFS objects
+        run: git lfs checkout
       - name: Build the Docker image
+        # On purpose Souffle programs aren't compiled from scratch.
+        # This is helpful if we forget to update the ones shipped in the repository.
         run: cd $GITHUB_WORKSPACE && docker build . --file Dockerfile --tag net-ssa:net-ssa

.gitignore

@@ -1,6 +1,5 @@
 ## Custom entries
 
-souffle/bins/
 build/
 
 ## Ignore Visual Studio temporary files, build results, and

CMakeLists.txt

@@ -3,8 +3,6 @@ project (NET_SSA)
 
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
 
-find_package(Dotnet REQUIRED)
-find_package(Souffle REQUIRED)
 find_package(Python3 REQUIRED COMPONENTS Interpreter)
 find_package(Mono 6.12 REQUIRED COMPONENTS)
 
@@ -17,22 +15,4 @@ find_program(LLVM_CONFIG NAMES llvm-config REQUIRED HINTS /usr/bin/llvm-config)
 message(STATUS "Using llvm-config: ${LLVM_CONFIG}")
 exec_program("${LLVM_CONFIG} --bindir" OUTPUT_VARIABLE LLVM_BINDIR)
 
-find_program(CLANGXX NAMES clang++ HINTS ${LLVM_BINDIR})
-if(NOT CLANGXX)
-    message(FATAL_ERROR "clang++ not found in ${LLVM_BINDIR}.")
-endif()
-
-add_subdirectory(souffle)
-
-set(NET_SSA_BIN_DIR ${CMAKE_CURRENT_BINARY_DIR}/bin/net-ssa)
-add_custom_target(build-dotnet 
-    COMMAND ${CMAKE_COMMAND} -E env SOUFFLE_BIN_DIR=${SOUFFLE_BIN_DIR} ${DOTNET_COMMAND} build -o ${NET_SSA_BIN_DIR}
-    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-    )
-
-add_custom_target(check-unit-test
-    COMMAND ${CMAKE_COMMAND} -E env SOUFFLE_BIN_DIR=${SOUFFLE_BIN_DIR} ${DOTNET_COMMAND} test --verbosity normal
-    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-    )
-
 configure_file("${CMAKE_CURRENT_SOURCE_DIR}/integration-test/lit.site.cfg.py.in" "${CMAKE_CURRENT_BINARY_DIR}/integration-test/lit.site.cfg.py" @ONLY)

Dockerfile

@@ -3,16 +3,19 @@ FROM $REPO:6.0-focal
 
 COPY . /net-ssa
 
-RUN chmod +x /net-ssa/ci/install-souffle.sh && /net-ssa/ci/install-souffle.sh && \
+RUN apt update && \
+    # Installing Souffle is not strictly required but it is handy in case
+    # a new analysis is implemented. This environment should be enough for quick prototyping.
+    chmod +x /net-ssa/ci/install-souffle.sh && /net-ssa/ci/install-souffle.sh && \
     chmod +x /net-ssa/ci/install-cmake.sh && /net-ssa/ci/install-cmake.sh && \
     chmod +x /net-ssa/ci/install-lit.sh && /net-ssa/ci/install-lit.sh && \
     chmod +x /net-ssa/ci/install-llvm.sh && /net-ssa/ci/install-llvm.sh && \
     chmod +x /net-ssa/ci/install-mono.sh && /net-ssa/ci/install-mono.sh
 
-RUN mkdir build && \
-    cd build && \
+RUN cd /net-ssa && \
+    dotnet build && \
+    dotnet test --verbosity normal && \
+    mkdir /build && \
+    cd /build && \
     cmake ../net-ssa && \
-    make build-souffle && \
-    make build-dotnet && \
-    make check-unit-test && \
     lit ./integration-test -vv

Dockerfile-souffle

@@ -0,0 +1,11 @@
+FROM ubuntu:20.04
+
+COPY . /net-ssa
+
+RUN apt-get update && apt-get install -y wget && \
+    chmod +x /net-ssa/ci/install-souffle.sh && /net-ssa/ci/install-souffle.sh && \
+    chmod +x /net-ssa/ci/install-mingw32.sh && /net-ssa/ci/install-mingw32.sh && \
+    cd /net-ssa/souffle && CXX=g++ ./build-souffle-linux-x86-64.sh && \
+    # Copy Souffle's headers for mingw32
+    cp -r /usr/include/souffle/ /usr/x86_64-w64-mingw32/include/souffle/ && \
+    cd /net-ssa/souffle && CXX=x86_64-w64-mingw32-g++-posix ./build-souffle-windows-x86-64.sh

README.md

@@ -1,4 +1,4 @@
-# net-ssa [![.NET](https://github.com/m-carrasco/net-ssa/actions/workflows/build.yml/badge.svg)](https://github.com/m-carrasco/net-ssa/actions/workflows/build.yml) ![Nuget](https://img.shields.io/nuget/v/net-ssa-lib)
+# net-ssa [![.NET (build, test and release if necessary)](https://github.com/m-carrasco/net-ssa/actions/workflows/build.yml/badge.svg?branch=main&event=push)](https://github.com/m-carrasco/net-ssa/actions/workflows/build.yml) [![Docker image](https://github.com/m-carrasco/net-ssa/actions/workflows/docker-ubuntu.yml/badge.svg?branch=main&event=push)](https://github.com/m-carrasco/net-ssa/actions/workflows/docker-ubuntu.yml) ![Nuget](https://img.shields.io/nuget/v/net-ssa-lib)
 
 Microsoft's high-level programming languages, such as C#, are compiled to CIL bytecode. The instruction set architecture of CIL operates on a stack virtual machine with local variables. CIL instruction's operands are implicit because they are elements in the stack. `net-ssa` provides a register-based intermediate representation for CIL where operands become explicit.
 
@@ -14,40 +14,64 @@ If you have any questions or suggestions, feel free to open an issue to discuss
 
 * [Quick setup](#quick-setup)
 * [Build from sources](#build-from-sources)
+* [Build native dependencies](#build-native-dependencies)
 * [Example: net-ssa-cli](#disassembling-with-net-ssa-cli)
 * [Example: net-ssa-lib](#disassembling-with-net-ssa-lib)
 * [Contributing](#contributing)
 * [Acknowledgements](#acknowledgements)
 
 ## Quick setup
+
 It is possible to develop and test `net-ssa` without installing any dependency in your system but [Docker](https://docs.docker.com/get-docker/).
+However, it is adviced to compile the project at least once in the host system. This is mainly for downloading dependencies and correctly setting up any IDE.
 
 1. `git clone git@github.com:m-carrasco/net-ssa.git`
 2. `cd net-ssa`
-3. `docker build -t net-ssa/net-ssa .`
-4. `docker run --name dev -v $(pwd):/net-ssa -ti net-ssa/net-ssa`
+3. `git lfs checkout`
+    * Install [git lfs](https://git-lfs.github.com/)
+4.  `dotnet build && dotnet test`
+    * This is optional, it requires installing dotnet.
+5. `docker build -t net-ssa/net-ssa .`
+6. `docker run --name dev -v $(pwd):/net-ssa -ti net-ssa/net-ssa`
    * This is now an interactive container. `$(pwd)` of the host is shared with the container as `net-ssa` source code folder.
-5. Introduce changes in the source code using your IDE as usual.
-6. Build and test in the container, execute these commands in the container terminal:
+7. Introduce changes in the source code using your IDE as usual.
+8. Build and test in the container, execute these commands in the container terminal:
    * `cd build`
-   * `make build-dotnet`
+   * `(cd /net-ssa && dotnet build)`
    * `lit integration-test/ -vvv`
    * `exit # once you finish working`
-7.  `docker start -i dev # to resume the container terminal`
+9.  `docker start -i dev # to resume the container terminal`
+
 
 ## Build from sources
 
-### Ubuntu 20.04.3
+### Ubuntu 20.04
+
+1. `cd net-ssa`
+2. `git lfs checkout`
+    * Install [git lfs](https://git-lfs.github.com/)
+3. `dotnet build`
+4. `dotnet test`
+5. `mkdir build`
+6. `cd build && cmake ..`
+7. `lit integration-test/ -vvv`
+
+To know the required dependencies for the integration tests (`cmake` and `lit` step), please check the [Dockerfile](https://github.com/m-carrasco/net-ssa/blob/main/Dockerfile).
+The Dockerfile executes the shell scripts under the [ci](https://github.com/m-carrasco/net-ssa/tree/main/ci) folder. You would just need to execute them once in your system.
 
-1. Read [Dockerfile](https://github.com/m-carrasco/net-ssa/blob/main/Dockerfile)
+### Windows and MacOS
 
-### Ubuntu 18.04
+The steps are the same as in Ubuntu. The project building and unit testing is done in the CI. Yet, the integration tests aren't configured.
+Anyway, the dependencies should be the same as in Ubuntu. If you encounter any problem while trying this, please open an issue.
 
-1. Read [./github/workflow/build.yml](https://github.com/m-carrasco/net-ssa/blob/main/.github/workflows/build.yml)
+## Build native dependencies 
 
-### Windows
+`net-ssa` has native dependencies, which are shipped in the project already. You shouldn't need to build them. Usually, this is only required for development. The supported systems are:
+* Linux - x86-64
+* Windows - x86-64
+* MacOS - x86-64 and arm64
 
-It has not been tested yet. However, it should work as long as Souffle can be installed. Souffle can run there using _Windows Subsystem for Linux_.
+In case they must be re-built, [`./net-ssa/souffle/build-all-with-docker.sh`](https://github.com/m-carrasco/net-ssa/blob/main/souffle/build-all-with-docker.sh) is available. The script compiles these dependencies from scratch using the source code in your repository. Under the hood, the script isolates this process using Docker. This only builds the Linux and Windows dependencies. Cross-compilation for MacOS is incredible difficult. If you are a MacOS user, check the CI to figure out the required dependencies and execute `build-souffle-macos-x86-64-arm64.sh`.
 
 ## Examples
 

ci/install-cmake.sh

@@ -1,2 +1,3 @@
 #!/bin/bash
+set -e
 curl -s "https://cmake.org/files/v3.21/cmake-3.21.1-linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local

ci/install-lit.sh

@@ -1,2 +1,3 @@
 #!/bin/bash
+set -e
 apt-get install -y python3-dev pip && pip install lit