ci: add PyInstaller binary build workflow for foryc #9

Workflow file for this run

.github/workflows/build-foryc-binaries.yml at 54fe273

	# Licensed to the Apache Software Foundation (ASF) under one
	# or more contributor license agreements. See the NOTICE file
	# distributed with this work for additional information
	# regarding copyright ownership. The ASF licenses this file
	# to you under the Apache License, Version 2.0 (the
	# "License"); you may not use this file except in compliance
	# with the License. You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing,
	# software distributed under the License is distributed on an
	# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
	# KIND, either express or implied. See the License for the
	# specific language governing permissions and limitations
	# under the License.

	name: Build foryc Binaries

	on:
	workflow_dispatch:

	workflow_call:
	outputs:
	artifacts_ready:
	description: "True if all 5 foryc binaries built and validated."
	value: ${{ jobs.build-complete.outputs.ready }}

	push:
	tags: ["foryc-v*"]

	pull_request:
	paths:
	- "compiler/**"
	- ".github/workflows/build-foryc-binaries.yml"

	permissions:
	contents: read

	env:
	PYTHON_VERSION: "3.11"

	# ─────────────────────────────────────────────────────────────────────────────
	# BUILD JOB
	# Produces one standalone foryc binary per target platform.
	#
	# Runner notes (matched to repo convention in build-native-pr.yml):
	# linux-x86_64 : ubuntu-22.04
	# linux-aarch64 : ubuntu-24.04-arm (native GitHub ARM64 runner)
	# macos-x86_64 : macos-15-intel
	# macos-aarch64 : macos-15 (native Apple Silicon)
	# windows-x86_64: windows-latest (Server 2022)
	#
	# UPX notes:
	# Linux and Windows targets use UPX --best --lzma compression.
	# macOS targets skip UPX entirely — UPX 4.x+ dropped macOS Mach-O support.
	# macOS aarch64 requires ad-hoc codesigning for Apple Silicon.
	#
	# 10 MB constraint:
	# Each binary must remain under 10 MB after UPX compression.
	# This is a hard gate for crates.io embedding in Phase 2.
	# macOS binaries are uncompressed but remain under 10 MB because
	# fory_compiler has zero third-party pip dependencies (pure stdlib only).
	#
	# Windows Defender note:
	# Set-MpPreference -DisableRealtimeMonitoring $true is required in BOTH
	# the build and validate jobs. Add-MpPreference -ExclusionPath is NOT
	# sufficient — it only excludes file system scans, not the memory-level
	# LoadLibrary interception that causes PYI-3104 / ERROR_NOACCESS failures
	# when PyInstaller extracts and maps python311.dll at runtime.
	# ─────────────────────────────────────────────────────────────────────────────
	jobs:
	build:
	name: build / ${{ matrix.target }}
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	include:
	- target: linux-x86_64
	os: ubuntu-22.04
	artifact_name: foryc-linux-x86_64
	binary_path: compiler/dist/foryc
	binary_name: foryc
	use_upx: true
	codesign: false

	- target: linux-aarch64
	os: ubuntu-24.04-arm
	artifact_name: foryc-linux-aarch64
	binary_path: compiler/dist/foryc
	binary_name: foryc
	use_upx: true
	codesign: false

	- target: macos-x86_64
	os: macos-15-intel
	artifact_name: foryc-macos-x86_64
	binary_path: compiler/dist/foryc
	binary_name: foryc
	use_upx: false
	codesign: false

	- target: macos-aarch64
	os: macos-15
	artifact_name: foryc-macos-aarch64
	binary_path: compiler/dist/foryc
	binary_name: foryc
	use_upx: false
	codesign: true

	- target: windows-x86_64
	os: windows-latest
	artifact_name: foryc-windows-x86_64
	binary_path: compiler/dist/foryc.exe
	binary_name: foryc.exe
	use_upx: true
	codesign: false

	steps:
	- name: Checkout
	uses: actions/checkout@v5

	- name: Set up Python ${{ env.PYTHON_VERSION }}
	uses: actions/setup-python@v5
	with:
	python-version: ${{ env.PYTHON_VERSION }}
	cache: "pip"
	cache-dependency-path: \|
	compiler/requirements-dev.txt
	compiler/pyproject.toml

	# ── Install UPX ─────────────────────────────────────────────────────────
	- name: Install UPX (Linux)
	if: runner.os == 'Linux' && matrix.use_upx
	run: \|
	sudo apt-get update -qq
	sudo apt-get install -y upx-ucl
	upx --version

	- name: Install UPX (macOS)
	if: runner.os == 'macOS' && matrix.use_upx
	run: \|
	brew install upx
	upx --version

	- name: Install UPX (Windows)
	if: runner.os == 'Windows' && matrix.use_upx
	shell: pwsh
	run: \|
	choco install upx --yes --no-progress
	upx --version

	# ── Install build dependencies ──────────────────────────────────────────
	- name: Install PyInstaller and fory_compiler
	run: \|
	python -m pip install --upgrade pip
	pip install -r compiler/requirements-dev.txt
	pip install ./compiler

	# ── Verify all hiddenimports are importable ─────────────────────────────
	# PyInstaller silently ignores nonexistent hiddenimports entries.
	# This step catches renamed or removed modules before they silently
	# disappear from the binary.
	- name: Verify all hiddenimports are importable
	working-directory: compiler
	run: \|
	python -c "
	mods = [
	'fory_compiler', 'fory_compiler.cli',
	'fory_compiler.ir', 'fory_compiler.ir.ast',
	'fory_compiler.ir.emitter', 'fory_compiler.ir.validator',
	'fory_compiler.ir.type_id', 'fory_compiler.ir.types',
	'fory_compiler.frontend', 'fory_compiler.frontend.base',
	'fory_compiler.frontend.utils',
	'fory_compiler.frontend.fdl', 'fory_compiler.frontend.fdl.lexer',
	'fory_compiler.frontend.fdl.parser',
	'fory_compiler.frontend.proto', 'fory_compiler.frontend.proto.ast',
	'fory_compiler.frontend.proto.lexer', 'fory_compiler.frontend.proto.parser',
	'fory_compiler.frontend.proto.translator',
	'fory_compiler.frontend.fbs', 'fory_compiler.frontend.fbs.ast',
	'fory_compiler.frontend.fbs.lexer', 'fory_compiler.frontend.fbs.parser',
	'fory_compiler.frontend.fbs.translator',
	'fory_compiler.generators', 'fory_compiler.generators.base',
	'fory_compiler.generators.java', 'fory_compiler.generators.python',
	'fory_compiler.generators.cpp', 'fory_compiler.generators.rust',
	'fory_compiler.generators.go',
	]
	for m in mods:
	__import__(m)
	print(f'OK: {m}')
	print('All hiddenimports verified.')
	"

	# ── Windows Defender (build job) ────────────────────────────────────────
	# Full real-time monitoring disable is required — not just path exclusions.
	# Add-MpPreference -ExclusionPath only covers filesystem scans.
	# The PYI-3104 / LoadLibrary: Invalid access to memory location error
	# is caused by Defender intercepting the memory-mapping of python311.dll
	# during PyInstaller bootloader execution. Only DisableRealtimeMonitoring
	# stops that interception.
	- name: Disable Windows Defender real-time monitoring (PyInstaller workaround)
	if: runner.os == 'Windows'
	shell: pwsh
	run: Set-MpPreference -DisableRealtimeMonitoring $true

	# ── Build ───────────────────────────────────────────────────────────────
	- name: Build standalone binary with PyInstaller
	working-directory: compiler
	run: pyinstaller foryc.spec

	# ── Pre-compression smoke test ──────────────────────────────────────────
	- name: Smoke test (pre-UPX)
	shell: bash
	env:
	BINARY_PATH: ${{ matrix.binary_path }}
	run: \|
	echo "=== Pre-UPX binary size ==="
	python -c "
	import os
	p = os.environ['BINARY_PATH']
	s = os.path.getsize(p)
	print(f'Size: {s:,} bytes ({s/1024/1024:.2f} MB)')
	"
	"$BINARY_PATH" --help

	# ── UPX compression ─────────────────────────────────────────────────────
	- name: Compress with UPX
	if: matrix.use_upx
	shell: bash
	env:
	BINARY_PATH: ${{ matrix.binary_path }}
	run: \|
	upx --best --lzma "$BINARY_PATH"
	echo "=== Post-UPX binary size ==="
	python -c "
	import os
	p = os.environ['BINARY_PATH']
	s = os.path.getsize(p)
	print(f'Size: {s:,} bytes ({s/1024/1024:.2f} MB)')
	"

	# ── macOS aarch64: ad-hoc codesign ───────────────────────────────────────
	# PyInstaller-built Apple Silicon binaries require valid code signatures.
	# UPX is disabled for macOS — this step runs regardless of UPX.
	# --sign - creates an ad-hoc signature; no Apple Developer ID required.
	- name: Ad-hoc codesign binary (macOS aarch64, required for Apple Silicon)
	if: matrix.codesign
	run: \|
	codesign --force --deep --sign - "${{ matrix.binary_path }}"
	codesign --verify --verbose "${{ matrix.binary_path }}"

	# ── Post-compression smoke test ─────────────────────────────────────────
	- name: Smoke test (post-UPX)
	shell: bash
	run: \|
	"${{ matrix.binary_path }}" --help

	# ── crates.io 10 MB size gate ────────────────────────────────────────────
	- name: Assert binary is under 10 MB (crates.io hard limit)
	shell: bash
	env:
	BINARY_PATH: ${{ matrix.binary_path }}
	run: \|
	python -c "
	import os, sys
	path = os.environ['BINARY_PATH']
	size = os.path.getsize(path)
	limit = 10 * 1024 * 1024
	print(f'Final size: {size:,} bytes ({size/1024/1024:.2f} MB)')
	print(f'Limit: {limit:,} bytes (10.00 MB)')
	if size > limit:
	print()
	print('FAIL: Binary exceeds 10 MB crates.io per-crate limit.')
	print('Add exclusions to compiler/foryc.spec or investigate UPX options.')
	sys.exit(1)
	print('PASS')
	"

	# ── Upload artifact ─────────────────────────────────────────────────────
	- name: Upload binary artifact
	uses: actions/upload-artifact@v4
	with:
	name: ${{ matrix.artifact_name }}
	path: ${{ matrix.binary_path }}
	retention-days: 30
	if-no-files-found: error

	# ─────────────────────────────────────────────────────────────────────────────
	# VALIDATE JOB
	# Downloads each binary onto its native runner and runs:
	# 1. --help sanity check
	# Python is NOT set up in this job — binary must be fully self-contained.
	# 2. End-to-end compile of compiler/examples/demo.fdl to Rust output.
	# 3. End-to-end compile of compiler/examples/demo.fdl to Java output.
	# Validates that all 5 language generators are correctly embedded.
	# ─────────────────────────────────────────────────────────────────────────────
	validate:
	name: validate / ${{ matrix.target }}
	needs: build
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	include:
	- target: linux-x86_64
	os: ubuntu-22.04
	artifact_name: foryc-linux-x86_64
	binary_name: foryc

	- target: linux-aarch64
	os: ubuntu-24.04-arm
	artifact_name: foryc-linux-aarch64
	binary_name: foryc

	- target: macos-x86_64
	os: macos-15-intel
	artifact_name: foryc-macos-x86_64
	binary_name: foryc

	- target: macos-aarch64
	os: macos-15
	artifact_name: foryc-macos-aarch64
	binary_name: foryc

	- target: windows-x86_64
	os: windows-latest
	artifact_name: foryc-windows-x86_64
	binary_name: foryc.exe

	steps:
	# Sparse checkout: only compiler/examples/ needed for demo.fdl.
	# Python is deliberately NOT set up — the binary must run standalone.
	- name: Checkout (sparse — compiler/examples only)
	uses: actions/checkout@v5
	with:
	sparse-checkout: compiler/examples
	sparse-checkout-cone-mode: true

	- name: Download artifact
	uses: actions/download-artifact@v4
	with:
	name: ${{ matrix.artifact_name }}
	path: ./artifact

	# ── Windows Defender (validate job) ────────────────────────────────────
	# MUST appear before the binary is executed — not just before the build.
	# PyInstaller onefile extracts python311.dll to RUNNER_TEMP on first run.
	# Defender intercepts the LoadLibrary call at the memory level, causing
	# PYI-3104: Invalid access to memory location.
	# Path exclusions (Add-MpPreference) do NOT fix this — full real-time
	# monitoring disable is required in this job exactly as in the build job.
	- name: Disable Windows Defender real-time monitoring (PyInstaller extraction workaround)
	if: runner.os == 'Windows'
	shell: pwsh
	run: Set-MpPreference -DisableRealtimeMonitoring $true

	- name: Make executable (Unix)
	if: runner.os != 'Windows'
	run: chmod +x ./artifact/${{ matrix.binary_name }}

	# ── Test 1: --help ───────────────────────────────────────────────────────
	- name: Validate --help output
	shell: bash
	run: \|
	OUTPUT=$(./artifact/${{ matrix.binary_name }} --help 2>&1)
	echo "$OUTPUT"
	echo "$OUTPUT" \| grep -qiE "^usage:.*foryc" \|\| {
	echo "ERROR: --help missing expected usage line"
	exit 1
	}

	# ── Test 2: End-to-end FDL → Rust ────────────────────────────────────────
	- name: End-to-end compile demo.fdl to Rust
	shell: bash
	run: \|
	OUT_DIR="${RUNNER_TEMP}/foryc-e2e-rust"
	mkdir -p "${OUT_DIR}"

	echo "=== Compiling compiler/examples/demo.fdl → Rust ==="
	./artifact/${{ matrix.binary_name }} \
	--rust_out "${OUT_DIR}" \
	compiler/examples/demo.fdl

	echo "=== Output files ==="
	ls -la "${OUT_DIR}/"

	RS_COUNT=$(find "${OUT_DIR}" -name "*.rs" -size +0c \| wc -l)
	if [ "${RS_COUNT}" -eq 0 ]; then
	echo "ERROR: no non-empty .rs files generated"
	exit 1
	fi
	echo "PASS: ${RS_COUNT} .rs file(s) generated"

	# ── Test 3: End-to-end FDL → Java ─────────────────────────────────────────
	- name: End-to-end compile demo.fdl to Java
	shell: bash
	run: \|
	JAVA_OUT="${RUNNER_TEMP}/foryc-e2e-java"
	mkdir -p "${JAVA_OUT}"

	echo "=== Compiling compiler/examples/demo.fdl → Java ==="
	./artifact/${{ matrix.binary_name }} \
	--java_out "${JAVA_OUT}" \
	compiler/examples/demo.fdl

	echo "=== Output files ==="
	ls -la "${JAVA_OUT}/"

	JAVA_COUNT=$(find "${JAVA_OUT}" -name "*.java" -size +0c \| wc -l)
	if [ "${JAVA_COUNT}" -eq 0 ]; then
	echo "ERROR: no non-empty .java files generated"
	exit 1
	fi
	echo "PASS: ${JAVA_COUNT} .java file(s) generated"

	# ─────────────────────────────────────────────────────────────────────────────
	# SUMMARY JOB
	# Single required status check for branch protection rules.
	# The Phase 4 release pipeline reads this job's output via workflow_call.
	# ─────────────────────────────────────────────────────────────────────────────
	build-complete:
	name: foryc / all binaries ready
	needs: [build, validate]
	runs-on: ubuntu-latest
	if: always()
	outputs:
	ready: ${{ steps.check.outputs.ready }}
	steps:
	- name: Evaluate results
	id: check
	run: \|
	BUILD="${{ needs.build.result }}"
	VALIDATE="${{ needs.validate.result }}"
	echo "build: ${BUILD}"
	echo "validate: ${VALIDATE}"
	if [[ "${BUILD}" == "success" && "${VALIDATE}" == "success" ]]; then
	echo "ready=true" >> "${GITHUB_OUTPUT}"
	else
	echo "ready=false" >> "${GITHUB_OUTPUT}"
	exit 1
	fi

	# ─────────────────────────────────────────────────────────────────────────────
	# FALLBACK: If ubuntu-24.04-arm runner is unavailable in your fork,
	# replace the linux-aarch64 matrix entry with:
	#
	# - target: linux-aarch64
	# os: ubuntu-22.04
	# artifact_name: foryc-linux-aarch64
	# binary_path: compiler/dist/foryc
	# binary_name: foryc
	# use_upx: true
	# codesign: false
	#
	# Then add these two steps before "Install PyInstaller":
	#
	# - name: Set up QEMU
	# uses: docker/setup-qemu-action@v3
	# with:
	# platforms: arm64
	#
	# - name: Build via Docker (linux-aarch64)
	# if: matrix.target == 'linux-aarch64'
	# uses: addnab/docker-run-action@v3
	# with:
	# image: python:3.11-slim-bookworm
	# options: --platform linux/arm64 -v ${{ github.workspace }}:/ws
	# run: \|
	# apt-get update -qq && apt-get install -y upx-ucl binutils
	# pip install -r /ws/compiler/requirements-dev.txt
	# pip install /ws/compiler
	# cd /ws/compiler && pyinstaller foryc.spec
	#
	# QEMU builds are 10-20x slower than native. Expect 20-30 min per run.
	# ─────────────────────────────────────────────────────────────────────────────

Navigation Menu

Uh oh!

ci: add PyInstaller binary build workflow for foryc #9

Workflow file

ci: add PyInstaller binary build workflow for foryc #9

Uh oh!

Workflow file for this run