diff --git a/.github/actions/push/action.yml b/.github/actions/push/action.yml deleted file mode 100644 index 9ec398fa0..000000000 --- a/.github/actions/push/action.yml +++ /dev/null @@ -1,75 +0,0 @@ -name: 'Push' -inputs: - head_sha: - required: true - artifact_name: - required: true - artifact_image_file: - required: true - artifact_image_name: - required: true - arch: - required: true - image: - required: true - default: ghcr.io/${{ github.repository }}:latest - registry: - required: true - default: ghcr.io - username: - required: true - default: ${{ github.actor }} - password: - required: true - default: ${{ github.token }} -runs: - using: "composite" - steps: - - - uses: actions/download-artifact@v4 - with: - name: ${{ inputs.artifact_name }} - path: /tmp/ - - - shell: bash - run: docker load -i /tmp/${{ inputs.artifact_image_file }} - - - id: image_parts - shell: bash - run: | - IMAGE_TAGLESS=$(echo ${{ inputs.image }} | cut -d':' -f1) - IMAGE_REPO=$(echo $IMAGE_TAGLESS | cut -d'/' -f2,3) - IMAGE_TAG=$(echo ${{ inputs.image }} | cut -d':' -f2) - echo "::set-output name=repo::$IMAGE_REPO" - echo "::set-output name=tag::$IMAGE_TAG" - - - uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 - with: - registry: ${{ inputs.registry }} - username: ${{ inputs.username }} - password: ${{ inputs.password }} - - - shell: bash - run: echo "IMAGE_URL=https://${{ inputs.image }}" >> $GITHUB_ENV - - - if: ${{ inputs.registry == 'docker.io' }} - shell: bash - run: | - echo "IMAGE_URL=https://hub.docker.com/r/${{ steps.image_parts.outputs.repo }}/tags?name=${{ steps.image_parts.outputs.tag }}" >> $GITHUB_ENV - - - shell: bash - run: | - docker push ${{ inputs.image }} - - - uses: actions/github-script@v5 - with: - script: | - github.rest.repos.createCommitStatus({ - owner: context.repo.owner, - repo: context.repo.repo, - sha: '${{ inputs.head_sha }}', - state: 'success', - context: `${{ inputs.image }}`, - target_url: '${{ env.IMAGE_URL }}', - description: 'Available', - }); diff --git a/.github/workflows/action-test.yml b/.github/workflows/action-test.yml index 36f534ffc..992fb1860 100644 --- a/.github/workflows/action-test.yml +++ b/.github/workflows/action-test.yml @@ -7,6 +7,10 @@ on: description: 'Tag to use' type: 'string' default: '' + artifact: + description: 'Artifact to use' + type: 'string' + default: '' jobs: action-setup: @@ -32,7 +36,12 @@ jobs: with: script: | const os = ['ubuntu-latest']; - if (process.env.ACTION_CHANGED === 'true') { + // Only run the additional builds if the action has changed, and this + // is not a test with an artifact image. The additional runs are time + // consuming so only do them if necessary. Don't run them for artifact + // action tests because the test won't be multiplatform since the + // artifact images contain only a single platform. + if (process.env.ACTION_CHANGED === 'true' && '${{ inputs.artifact }}' === '') { os.push('ubuntu-24.04-arm'); os.push('macos-13'); } @@ -50,6 +59,7 @@ jobs: - uses: actions/checkout@v4 - uses: ./ with: + artifact: ${{ inputs.artifact }} tag: ${{ inputs.tag }} - name: "Run basic test making sure RPC and Horizon are available" run: > diff --git a/.github/workflows/build-future.yml b/.github/workflows/build-future.yml deleted file mode 100644 index dfc7bfb0b..000000000 --- a/.github/workflows/build-future.yml +++ /dev/null @@ -1,87 +0,0 @@ -name: Future - -# The `:future` tag points to a build containing unreleased versions of -# software that have been informally released to the futurenet network. - -on: - workflow_call: - secrets: - DOCKERHUB_USERNAME: - required: false - DOCKERHUB_TOKEN: - required: false - inputs: - sha: - description: "Sha to build" - type: "string" - required: true - tag-prefix: - description: "Prefix for the tag name" - type: "string" - default: "" - tag-alias-prefix: - description: 'Prefix for the alias tag name' - type: 'string' - default: '' - -jobs: - amd64: - uses: ./.github/workflows/build.yml - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - with: - sha: ${{ inputs.sha }} - arch: amd64 - tag: ${{ inputs.tag-prefix }}future-amd64 - protocol_version_default: 23 - xdr_ref: v23.0.0 - core_ref: v23.0.1 - horizon_ref: horizon-v23.0.0 - stellar_rpc_ref: v23.0.4 - friendbot_ref: horizon-v23.0.0 - lab_ref: main - test_matrix: | - { - "network": ["local"], - "core": ["core", null], - "horizon": ["horizon", null], - "rpc": ["rpc", null], - "options": [""] - } - - arm64: - uses: ./.github/workflows/build.yml - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - with: - sha: ${{ inputs.sha }} - arch: arm64 - tag: ${{ inputs.tag-prefix }}future-arm64 - protocol_version_default: 23 - xdr_ref: v23.0.0 - core_ref: v23.0.1 - horizon_ref: horizon-v23.0.0 - stellar_rpc_ref: v23.0.4 - friendbot_ref: horizon-v23.0.0 - lab_ref: main - test_matrix: | - { - "network": ["local"], - "core": ["core", null], - "horizon": ["horizon", null], - "rpc": ["rpc", null], - "options": [""] - } - - manifest: - needs: [amd64, arm64] - uses: ./.github/workflows/manifest.yml - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - with: - tag: ${{ inputs.tag-prefix }}future - tag-alias: ${{ inputs.tag-alias-prefix }}future - images: ${{ needs.amd64.outputs.image }} ${{ needs.arm64.outputs.image }} diff --git a/.github/workflows/build-latest.yml b/.github/workflows/build-latest.yml deleted file mode 100644 index 5324338be..000000000 --- a/.github/workflows/build-latest.yml +++ /dev/null @@ -1,95 +0,0 @@ -name: Latest - -# The `:latest` tag should only ever be formally released versions of -# software. That means each ref should only ever be specified as a version -# tag for software that's been released which is not a release candidate. - -on: - workflow_call: - secrets: - DOCKERHUB_USERNAME: - required: false - DOCKERHUB_TOKEN: - required: false - inputs: - sha: - description: 'Sha to build' - type: 'string' - required: true - tag-prefix: - description: 'Prefix for the tag name' - type: 'string' - default: '' - tag-alias-prefix: - description: 'Prefix for the alias tag name' - type: 'string' - default: '' - -jobs: - - amd64: - uses: ./.github/workflows/build.yml - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - with: - sha: ${{ inputs.sha }} - arch: amd64 - tag: ${{ inputs.tag-prefix }}latest-amd64 - protocol_version_default: 23 - xdr_ref: v23.0.0 - core_ref: v23.0.1 - horizon_ref: horizon-v23.0.0 - stellar_rpc_ref: v23.0.4 - friendbot_ref: horizon-v23.0.0 - lab_ref: main - test_matrix: | - { - "network": ["local"], - "core": ["core", null], - "horizon": ["horizon", null], - "rpc": ["rpc", null], - "options": [""], - "include": [ - { "network": "pubnet", "core": "core", "horizon": "horizon", "rpc": "rpc" } - ] - } - - arm64: - uses: ./.github/workflows/build.yml - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - with: - sha: ${{ inputs.sha }} - arch: arm64 - tag: ${{ inputs.tag-prefix }}latest-arm64 - protocol_version_default: 23 - xdr_ref: v23.0.0 - core_ref: v23.0.1 - horizon_ref: horizon-v23.0.0 - stellar_rpc_ref: v23.0.4 - friendbot_ref: horizon-v23.0.0 - lab_ref: main - test_matrix: | - { - "network": ["local"], - "core": ["core", null], - "horizon": ["horizon", null], - "rpc": ["rpc", null], - "options": [""], - "include": [ - { "network": "pubnet", "core": "core", "horizon": "horizon", "rpc": "rpc" } - ] - } - - manifest: - needs: [amd64, arm64] - uses: ./.github/workflows/manifest.yml - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - with: - tag: ${{ inputs.tag-prefix }}latest - tag-alias: ${{ inputs.tag-alias-prefix }}latest - images: ${{ needs.amd64.outputs.image }} ${{ needs.arm64.outputs.image }} diff --git a/.github/workflows/build-start.yml b/.github/workflows/build-start.yml deleted file mode 100644 index 2ebe14f01..000000000 --- a/.github/workflows/build-start.yml +++ /dev/null @@ -1,93 +0,0 @@ -name: Build - -on: - push: - branches: - - main - pull_request: - -# Prevent more than one build of this workflow for a branch to be running at the -# same time, and if multiple are queued, only run the latest, cancelling any -# already running build. The exception being any protected branch, such as -# main, where a build for every commit will run. -concurrency: - group: ${{ github.workflow }}-${{ github.ref_protected == 'true' && github.sha || github.ref }} - cancel-in-progress: true - -jobs: - - complete: - if: always() - needs: [latest, testing, future, latest-action-test, testing-action-test] - runs-on: ubuntu-latest - steps: - - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') - run: exit 1 - - setup: - runs-on: ubuntu-latest - outputs: - tag-prefix: ${{ steps.tag-prefix.outputs.tag-prefix }} - tag-alias-prefix: ${{ steps.tag-prefix.outputs.tag-alias-prefix }} - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 # Get all history for the sha count below. - ref: ${{ github.event.pull_request.head.sha || github.sha }} - - id: tag-prefix - run: | - pr_prefix="${{ github.event_name == 'pull_request' && format('pr{0}-', github.event.pull_request.number) }}" - count="$(git rev-list HEAD --count --first-parent)" - echo "tag-prefix=${pr_prefix}v${count}-" >> $GITHUB_OUTPUT - echo "tag-alias-prefix=${pr_prefix}" >> $GITHUB_OUTPUT - - latest: - needs: [setup] - uses: ./.github/workflows/build-latest.yml - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - with: - sha: ${{ github.event.pull_request.head.sha || github.sha }} - tag-prefix: ${{ needs.setup.outputs.tag-prefix }} - tag-alias-prefix: ${{ needs.setup.outputs.tag-alias-prefix }} - - latest-action-test: - needs: [setup, latest] - if: | - github.event_name == 'push' && github.ref == 'refs/heads/main' - || github.event.pull_request.head.repo.full_name == github.repository - uses: ./.github/workflows/action-test.yml - with: - tag: ${{ needs.setup.outputs.tag-prefix }}latest - - testing: - needs: [setup] - uses: ./.github/workflows/build-testing.yml - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - with: - sha: ${{ github.event.pull_request.head.sha || github.sha }} - tag-prefix: ${{ needs.setup.outputs.tag-prefix }} - tag-alias-prefix: ${{ needs.setup.outputs.tag-alias-prefix }} - - testing-action-test: - needs: [setup, testing] - if: | - github.event_name == 'push' && github.ref == 'refs/heads/main' - || github.event.pull_request.head.repo.full_name == github.repository - uses: ./.github/workflows/action-test.yml - with: - tag: ${{ needs.setup.outputs.tag-prefix }}testing - - future: - needs: [setup] - uses: ./.github/workflows/build-future.yml - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - with: - sha: ${{ github.event.pull_request.head.sha || github.sha }} - tag-prefix: ${{ needs.setup.outputs.tag-prefix }} - tag-alias-prefix: ${{ needs.setup.outputs.tag-alias-prefix }} diff --git a/.github/workflows/build-testing.yml b/.github/workflows/build-testing.yml deleted file mode 100644 index a7f6713fd..000000000 --- a/.github/workflows/build-testing.yml +++ /dev/null @@ -1,98 +0,0 @@ -name: Testing - -# The `:testing` tag should only ever be formally released or release candidate -# versions of software. That means each ref should only ever be specified as a -# version tag for software that's either the latest release candidate or latest -# release. - -on: - workflow_call: - secrets: - DOCKERHUB_USERNAME: - required: false - DOCKERHUB_TOKEN: - required: false - inputs: - sha: - description: 'Sha to build' - type: 'string' - required: true - tag-prefix: - description: 'Prefix for the tag name' - type: 'string' - default: '' - tag-alias-prefix: - description: 'Prefix for the alias tag name' - type: 'string' - default: '' - -jobs: - - amd64: - uses: ./.github/workflows/build.yml - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - with: - sha: ${{ inputs.sha }} - arch: amd64 - tag: ${{ inputs.tag-prefix }}testing-amd64 - protocol_version_default: 23 - xdr_ref: v23.0.0 - core_ref: v23.0.1 - horizon_ref: horizon-v23.0.0 - stellar_rpc_ref: v23.0.4 - friendbot_ref: horizon-v23.0.0 - lab_ref: main - test_matrix: | - { - "network": ["local"], - "core": ["core", null], - "horizon": ["horizon", null], - "rpc": ["rpc", null], - "options": [""], - "include": [ - { "network": "testnet", "core": "core", "horizon": "horizon", "rpc": "rpc" }, - { "network": "pubnet", "core": "core", "horizon": "horizon", "rpc": "rpc" } - ] - } - - arm64: - uses: ./.github/workflows/build.yml - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - with: - sha: ${{ inputs.sha }} - arch: arm64 - tag: ${{ inputs.tag-prefix }}testing-arm64 - protocol_version_default: 23 - xdr_ref: v23.0.0 - core_ref: v23.0.1 - horizon_ref: horizon-v23.0.0 - stellar_rpc_ref: v23.0.4 - friendbot_ref: horizon-v23.0.0 - lab_ref: main - test_matrix: | - { - "network": ["local"], - "core": ["core", null], - "horizon": ["horizon", null], - "rpc": ["rpc", null], - "options": [""], - "include": [ - { "network": "testnet", "core": "core", "horizon": "horizon", "rpc": "rpc" }, - { "network": "pubnet", "core": "core", "horizon": "horizon", "rpc": "rpc" } - ] - } - - manifest: - needs: [amd64, arm64] - uses: ./.github/workflows/manifest.yml - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - with: - tag: ${{ inputs.tag-prefix }}testing - tag-alias: ${{ inputs.tag-alias-prefix }}testing - images: ${{ needs.amd64.outputs.image }} ${{ needs.arm64.outputs.image }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 22bd555cb..5877efa72 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,491 +1,330 @@ +name: Build + +# This workflow builds multiple quickstart images as defined in the images json +# passed to the workflow across the architectures defined in the archs input. +# +# See the images.json file in the repo for how to define the JSON for a set of +# images. +# +# This workflow is intended to be called by third parties to build custom +# quickstart images. See the repository README for an example. +# +# The build process first builds the dependencies (xdr, core, rpc, horizon, +# friendbot, lab). When doing so the dependencies needed as specified by the +# images json are deduplicated so that any software shared by the images is +# only built once. Dependencies are cached and so only rebuilt when needed. +# Dependencies are defined by a tag or branch, but when building those git refs +# are resolved to a sha to ensure stability of the sha throughout the full +# build process. For all dependencies and the final image, amd64 and arm64 +# variants may be built. + on: workflow_call: - secrets: - DOCKERHUB_USERNAME: - required: false - DOCKERHUB_TOKEN: - required: false inputs: - sha: - description: 'Sha to build' - type: 'string' - required: true - arch: - description: 'Architecture to build the image for (amd64, arm64)' - type: 'string' - required: true - tag: - description: 'Tag to use on the image name' - type: 'string' - required: true - protocol_version_default: - description: 'Default protocol version to use on local networks' - type: 'number' - required: true - core_repo: - description: 'Git repo for stellar-core' - type: 'string' - default: 'stellar/stellar-core' - core_ref: - description: 'Git ref for the stellar-core repo' - type: 'string' - required: true - core_configure_flags: - description: 'CONFIGURE_FLAGS used when building stellar-core' - type: 'string' - default: '--disable-tests' - horizon_ref: - description: 'Git ref for the stellar/go repo (horizon)' - type: 'string' - required: true - xdr_ref: - description: 'Git ref for the stellar/rs-stellar-xdr repo' - type: 'string' - required: false - stellar_rpc_ref: - description: 'Git ref for the stellar/stellar-rpc repo (stellar-rpc)' - type: 'string' - required: true - friendbot_ref: - description: 'Git ref for the stellar/go repo (friendbot)' - type: 'string' + repo: + description: "Quickstart repo where quickstart is hosted" + type: "string" + default: "stellar/quickstart" + ref: + description: "Quickstart ref to build should match workflow (sha, branch, tag)" + type: "string" + default: "main" + images: + description: "A custom image.json (a single image from the same format as images.json), if not provided the full images.json is run" + type: "string" required: true - lab_ref: - description: 'Git ref for the stellar/laboratory (lab)' - type: 'string' + archs: + description: 'Architectures to build for as an array (e.g. ["amd64", "arm64"])' + type: "string" required: true - test_matrix: - description: 'JSON matrix for the test job' - type: 'string' - required: true - outputs: - image: - description: 'Image pushed as a result of this build' - value: ${{ jobs.build.outputs.image }} + test: + description: "Whether the image tests should run" + type: "boolean" + default: true + cache_id: + description: "A value insert into cache keys to namespace cache usage, or invalidate it by incrementing" + type: "string" + default: 6 env: - IMAGE: ${{ format('{0}/{1}:{2}', secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io', github.repository, inputs.tag) }} - # Cache ID is a value inserted into cache keys. Whenever changing the build - # in a way that needs to use entirely new fresh builds, increment the number - # by one so that all the keys become new. - CACHE_ID: 5 - ARTIFACT_RETENTION_DAYS_FOR_IMAGE: 7 - ARTIFACT_RETENTION_DAYS_FOR_LOGS: 60 + artifact_retention_days_for_image: 7 + artifact_retention_days_for_logs: 60 jobs: - shas: + complete: + if: always() + name: complete + needs: [setup, load, build, test] runs-on: ubuntu-latest - outputs: - xdr: ${{ steps.xdr.outputs.sha }} - core: ${{ steps.core.outputs.sha }} - rpc: ${{ steps.rpc.outputs.sha }} - horizon: ${{ steps.horizon.outputs.sha }} - friendbot: ${{ steps.friendbot.outputs.sha }} - lab: ${{ steps.lab.outputs.sha }} - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - - id: xdr - run: echo "sha=$(gh api repos/stellar/rs-stellar-xdr/commits/"${{inputs.xdr_ref}}" --jq '.sha')" | tee -a $GITHUB_OUTPUT - - id: core - run: echo "sha=$(gh api repos/${{inputs.core_repo}}/commits/"${{inputs.core_ref}}" --jq '.sha')" | tee -a $GITHUB_OUTPUT - - id: rpc - run: echo "sha=$(gh api repos/stellar/stellar-rpc/commits/"${{inputs.stellar_rpc_ref}}" --jq '.sha')" | tee -a $GITHUB_OUTPUT - - id: horizon - run: echo "sha=$(gh api repos/stellar/go/commits/"${{inputs.horizon_ref}}" --jq '.sha')" | tee -a $GITHUB_OUTPUT - - id: friendbot - run: echo "sha=$(gh api repos/stellar/go/commits/"${{inputs.friendbot_ref}}" --jq '.sha')" | tee -a $GITHUB_OUTPUT - - id: lab - run: echo "sha=$(gh api repos/stellar/laboratory/commits/"${{inputs.lab_ref}}" --jq '.sha')" | tee -a $GITHUB_OUTPUT + - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') + run: exit 1 - load-stellar-core-from-cache: - needs: [shas] + setup: + name: 1 setup runs-on: ubuntu-latest outputs: - cache-hit: ${{ steps.cache.outputs.cache-hit }} - steps: - - id: cache - uses: actions/cache@v3 - with: - path: /tmp/image - key: image-${{ env.CACHE_ID }}-stellar-core-${{ inputs.arch }}-${{ needs.shas.outputs.core }}-${{ inputs.core_configure_flags }} - - name: Upload Stellar-Core Image - if: steps.cache.outputs.cache-hit == 'true' - uses: actions/upload-artifact@v4 - with: - name: image-stellar-core-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/image - retention-days: ${{ env.ARTIFACT_RETENTION_DAYS_FOR_IMAGE }} - - build-stellar-core: - needs: [shas, load-stellar-core-from-cache] - if: ${{ needs.load-stellar-core-from-cache.outputs.cache-hit != 'true' }} - runs-on: ${{ inputs.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} + sha: ${{ steps.sha.outputs.sha }} + images: ${{ steps.images.outputs.images }} + deps: ${{ steps.deps.outputs.deps }} + additional-tests: ${{ steps.tests.outputs.additional-tests }} steps: - - name: Checkout Quickstart for Core docker file - uses: actions/checkout@v3 - with: - ref: ${{ inputs.sha }} - - id: cache - uses: actions/cache@v3 - with: - path: /tmp/image - key: image-${{ env.CACHE_ID }}-stellar-core-${{ inputs.arch }}-${{ needs.shas.outputs.core }}-${{ inputs.core_configure_flags }} - - uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f - - name: Build Stellar-Core Image - run: > - docker buildx build --platform linux/${{ inputs.arch }} - -f Dockerfile.core - -t stellar-core:${{ inputs.arch }} - -o type=docker,dest=/tmp/image - --build-arg REPO="${{ inputs.core_repo }}" - --build-arg REF="${{ needs.shas.outputs.core }}" - --build-arg CONFIGURE_FLAGS='${{ inputs.core_configure_flags }}' . - - name: Upload Stellar-Core Image - uses: actions/upload-artifact@v4 + - uses: actions/checkout@v2 with: - name: image-stellar-core-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/image - retention-days: ${{ env.ARTIFACT_RETENTION_DAYS_FOR_IMAGE }} + repository: ${{ inputs.repo }} + ref: ${{ inputs.ref }} + - name: Sha + id: sha + run: | + echo "sha=$(git rev-parse HEAD)" | tee -a $GITHUB_OUTPUT + - name: Images with Extras + id: images + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + images: ${{ inputs.images }} + run: | + images="$(<<< $images ./.scripts/images-with-extras)" + <<< $images jq + echo "images=$images" >> $GITHUB_OUTPUT + echo "images=$images" >> $GITHUB_ENV + - name: Deps + id: deps + run: | + deps="$(<<< $images ./.scripts/images-deps)" + <<< $deps jq + echo "deps=$deps" >> $GITHUB_OUTPUT + - name: Additional Tests + id: tests + run: | + tests="$(<<< $images ./.scripts/images-additional-tests)" + <<< $tests jq + echo "additional-tests=$tests" >> $GITHUB_OUTPUT - load-stellar-horizon-from-cache: - needs: [shas] + load: + needs: [setup] + strategy: + matrix: + dep: ${{ fromJSON(needs.setup.outputs.deps) }} + arch: ${{ fromJSON(inputs.archs) }} + fail-fast: false + name: 2 load (${{ matrix.dep.name }}, ${{ matrix.dep.ref }}, ${{ matrix.arch }}, ${{ matrix.dep.options && toJSON(matrix.dep.options) || '-' }}) runs-on: ubuntu-latest - outputs: - cache-hit: ${{ steps.cache.outputs.cache-hit }} - steps: - - id: cache - uses: actions/cache@v3 - with: - path: /tmp/image - key: image-${{ env.CACHE_ID }}-stellar-horizon-${{ inputs.arch }}-${{ needs.shas.outputs.horizon }} - - name: Upload Stellar-Horizon Image - if: steps.cache.outputs.cache-hit == 'true' - uses: actions/upload-artifact@v4 - with: - name: image-stellar-horizon-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/image - retention-days: ${{ env.ARTIFACT_RETENTION_DAYS_FOR_IMAGE }} - - build-stellar-horizon: - needs: [shas, load-stellar-horizon-from-cache] - if: ${{ needs.load-stellar-horizon-from-cache.outputs.cache-hit != 'true' }} - runs-on: ${{ inputs.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} + env: + dep_json: ${{ toJSON(matrix.dep) }} + image_filename: image-${{ matrix.dep.name }}-${{ matrix.dep.id }}-${{ matrix.arch }}.tar + json_filename: image-${{ matrix.dep.name }}-${{ matrix.dep.id }}-${{ matrix.arch }}.json + missing_filename: missing-${{ matrix.dep.name }}-${{ matrix.dep.id }}-${{ matrix.arch }}.json steps: - - id: cache - uses: actions/cache@v3 - with: - path: /tmp/image - key: image-${{ env.CACHE_ID }}-stellar-horizon-${{ inputs.arch }}-${{ needs.shas.outputs.horizon }} - - name: Checkout Quickstart for Horizon docker file - uses: actions/checkout@v3 - with: - ref: ${{ inputs.sha }} - - name: Setup buildx - uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f - - name: Build Stellar-Horizon Image + - name: Create Dep Details JSON (with arch) run: > - docker buildx build --platform linux/${{ inputs.arch }} - -f Dockerfile.horizon - -t stellar-horizon:${{ inputs.arch }} -o type=docker,dest=/tmp/image - --build-arg REF="${{ needs.shas.outputs.horizon }}" . - - name: Upload Stellar-Horizon Image + echo "${dep_json}" + | jq --arg arch ${{ matrix.arch }} '.arch = $arch' + | tee /tmp/${{ env.json_filename }} + - name: Upload Dep Details JSON uses: actions/upload-artifact@v4 with: - name: image-stellar-horizon-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/image - retention-days: ${{ env.ARTIFACT_RETENTION_DAYS_FOR_IMAGE }} - - load-stellar-friendbot-from-cache: - needs: [shas] - runs-on: ubuntu-latest - outputs: - cache-hit: ${{ steps.cache.outputs.cache-hit }} - steps: - - id: cache - uses: actions/cache@v3 + name: ${{ env.json_filename }} + path: /tmp/${{ env.json_filename }} + retention-days: ${{ env.artifact_retention_days_for_image }} + - name: Find Image in Cache + id: cache + uses: actions/cache/restore@v3 with: - path: /tmp/image - key: image-${{ env.CACHE_ID }}-stellar-friendbot-${{ inputs.arch }}-${{ needs.shas.outputs.friendbot }} - - name: Upload Stellar-Friendbot Image + key: ${{ inputs.cache_id }}-${{ env.image_filename }} + path: /tmp/${{ env.image_filename }} + - name: Upload Image to Artifacts if: steps.cache.outputs.cache-hit == 'true' uses: actions/upload-artifact@v4 with: - name: image-stellar-friendbot-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/image - retention-days: ${{ env.ARTIFACT_RETENTION_DAYS_FOR_IMAGE }} - - build-stellar-friendbot: - needs: [shas, load-stellar-friendbot-from-cache] - if: ${{ needs.load-stellar-friendbot-from-cache.outputs.cache-hit != 'true' }} - runs-on: ${{ inputs.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} - steps: - - id: cache - uses: actions/cache@v3 - with: - path: /tmp/image - key: image-${{ env.CACHE_ID }}-stellar-friendbot-${{ inputs.arch }}-${{ needs.shas.outputs.friendbot }} - - name: Checkout Quickstart for Friendbot docker file - uses: actions/checkout@v3 - with: - ref: ${{ inputs.sha }} - - name: Setup buildx - uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f - - name: Build Stellar-Friendbot Image - run: > - docker buildx build --platform linux/${{ inputs.arch }} - -f Dockerfile.friendbot - -t stellar-friendbot:${{ inputs.arch }} - -o type=docker,dest=/tmp/image - --build-arg REF="${{ needs.shas.outputs.friendbot }}" . - - name: Upload Stellar-Friendbot Image + name: ${{ env.image_filename }} + path: /tmp/${{ env.image_filename }} + retention-days: ${{ env.artifact_retention_days_for_image }} + - name: Upload Dep Details as Missing Marker Due to Cache Miss + if: steps.cache.outputs.cache-hit != 'true' uses: actions/upload-artifact@v4 with: - name: image-stellar-friendbot-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/image - retention-days: ${{ env.ARTIFACT_RETENTION_DAYS_FOR_IMAGE }} + name: ${{ env.missing_filename }} + path: /tmp/${{ env.json_filename }} + retention-days: ${{ env.artifact_retention_days_for_image }} - load-stellar-rpc-from-cache: - needs: [shas] + prepare: + needs: [load] + name: 3 prepare runs-on: ubuntu-latest outputs: - cache-hit: ${{ steps.cache.outputs.cache-hit }} + deps-to-build: ${{ steps.deps-to-build.outputs.deps }} steps: - - id: cache - uses: actions/cache@v3 - with: - path: /tmp/image - key: image-${{ env.CACHE_ID }}-stellar-rpc-${{ inputs.arch }}-${{ needs.shas.outputs.rpc }} - - name: Upload Stellar-Core Image - if: steps.cache.outputs.cache-hit == 'true' - uses: actions/upload-artifact@v4 + - name: Download Missing Markers + uses: actions/download-artifact@v4 with: - name: image-stellar-rpc-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/image - retention-days: ${{ env.ARTIFACT_RETENTION_DAYS_FOR_IMAGE }} + pattern: missing-* + merge-multiple: true + path: /tmp/missing + - name: Collect Deps-to-Build from Missing Markers + id: deps-to-build + run: | + deps="$(find /tmp/missing -name "*.json" -exec cat {} \; | jq -c -s '.')" + echo "deps=$deps" | tee -a $GITHUB_OUTPUT - build-stellar-rpc: - needs: [shas, load-stellar-rpc-from-cache] - if: ${{ needs.load-stellar-rpc-from-cache.outputs.cache-hit != 'true' }} - runs-on: ${{ inputs.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} + build-dep: + needs: [setup, prepare] + if: needs.prepare.outputs.deps-to-build != '[]' + strategy: + matrix: + dep: ${{ fromJSON(needs.prepare.outputs.deps-to-build) }} + fail-fast: false + name: 4 build (${{ matrix.dep.name }}, ${{ matrix.dep.ref }}, ${{ matrix.dep.arch }}, ${{ matrix.dep.options && toJSON(matrix.dep.options) || '-' }}) + runs-on: ${{ matrix.dep.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} + env: + image_filename: image-${{ matrix.dep.name }}-${{ matrix.dep.id }}-${{ matrix.dep.arch }}.tar steps: - - id: cache - uses: actions/cache@v3 - with: - path: /tmp/image - key: image-${{ env.CACHE_ID }}-stellar-rpc-${{ inputs.arch }}-${{ needs.shas.outputs.rpc }} - - name: Checkout Quickstart for RPC docker file - uses: actions/checkout@v3 + - uses: actions/checkout@v3 with: - ref: ${{ inputs.sha }} + repository: ${{ inputs.repo }} + ref: ${{ needs.setup.outputs.sha }} - uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f - - name: Build Stellar-rpc Image - run: > - docker buildx build --platform linux/${{ inputs.arch }} - -f Dockerfile.rpc - -t stellar-rpc:${{ inputs.arch }} - -o type=docker,dest=/tmp/image - --build-arg REF="${{ needs.shas.outputs.rpc }}" . - - name: Upload Stellar-rpc Image - uses: actions/upload-artifact@v4 - with: - name: image-stellar-rpc-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/image - retention-days: ${{ env.ARTIFACT_RETENTION_DAYS_FOR_IMAGE }} - - load-stellar-lab-from-cache: - needs: [shas] - runs-on: ubuntu-latest - outputs: - cache-hit: ${{ steps.cache.outputs.cache-hit }} - steps: - - id: cache - uses: actions/cache@v3 - with: - path: /tmp/image - key: image-${{ env.CACHE_ID }}-stellar-lab-${{ inputs.arch }}-${{ needs.shas.outputs.lab }} - - name: Upload Stellar-Lab Image - if: steps.cache.outputs.cache-hit == 'true' - uses: actions/upload-artifact@v4 - with: - name: image-stellar-lab-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/image - retention-days: ${{ env.ARTIFACT_RETENTION_DAYS_FOR_IMAGE }} - - build-stellar-lab: - needs: [shas, load-stellar-lab-from-cache] - if: ${{ needs.load-stellar-lab-from-cache.outputs.cache-hit != 'true' }} - runs-on: ${{ inputs.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} - steps: - - id: cache - uses: actions/cache@v3 - with: - path: /tmp/image - key: image-${{ env.CACHE_ID }}-stellar-lab-${{ inputs.arch }}-${{ needs.shas.outputs.lab }} - - name: Checkout Quickstart for Horizon docker file - uses: actions/checkout@v3 - with: - ref: ${{ inputs.sha }} - - name: Setup buildx - uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f - - name: Build Stellar-lab Image + - name: Build Image + env: + options_json: ${{ toJSON(matrix.dep.options) }} run: > - docker buildx build --platform linux/${{ inputs.arch }} - -f Dockerfile.lab - -t stellar-lab:${{ inputs.arch }} -o type=docker,dest=/tmp/image - --build-arg NEXT_PUBLIC_COMMIT_HASH=${{ needs.shas.outputs.lab }} . - - name: Upload Stellar-lab Image - uses: actions/upload-artifact@v4 - with: - name: image-stellar-lab-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/image - retention-days: ${{ env.ARTIFACT_RETENTION_DAYS_FOR_IMAGE }} - - load-rs-stellar-xdr-from-cache: - needs: [shas] - runs-on: ubuntu-latest - outputs: - cache-hit: ${{ steps.cache.outputs.cache-hit }} - steps: - - id: cache - uses: actions/cache@v3 - with: - path: /tmp/image - key: image-${{ env.CACHE_ID }}-rs-stellar-xdr-${{ inputs.arch }}-${{ needs.shas.outputs.xdr }} - - name: Upload Stellar-Core Image - if: steps.cache.outputs.cache-hit == 'true' - uses: actions/upload-artifact@v4 - with: - name: image-rs-stellar-xdr-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/image - retention-days: ${{ env.ARTIFACT_RETENTION_DAYS_FOR_IMAGE }} - - build-rs-stellar-xdr: - needs: [shas, load-rs-stellar-xdr-from-cache] - if: ${{ needs.load-rs-stellar-xdr-from-cache.outputs.cache-hit != 'true' }} - runs-on: ${{ inputs.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} - steps: - - name: Checkout Quickstart for Horizon docker file - uses: actions/checkout@v3 - with: - ref: ${{ inputs.sha }} - - id: cache - uses: actions/cache@v3 + docker buildx build + --platform linux/${{ matrix.dep.arch }} + -f Dockerfile.${{ matrix.dep.name }} + -t stellar-${{ matrix.dep.name }}:${{ matrix.dep.arch }} + -o type=docker,dest=/tmp/${image_filename} + --build-arg REPO="${{ matrix.dep.repo }}" + --build-arg REF="${{ matrix.dep.sha }}" + --build-arg OPTIONS="${options_json}" + . + - name: Upload Image to Cache + uses: actions/cache/save@v3 + id: cache with: - path: /tmp/image - key: image-${{ env.CACHE_ID }}-rs-stellar-xdr-${{ inputs.arch }}-${{ needs.shas.outputs.xdr }} - - uses: docker/setup-buildx-action@5146db6c4d81fbfd508899f851bbb3883a96ff9f - - name: Build Stellar-Rs-Xdr Image - run: > - docker buildx build --platform linux/${{ inputs.arch }} - -f Dockerfile.xdr - -t stellar-rs-xdr:${{ inputs.arch }} - -o type=docker,dest=/tmp/image - --build-arg REPO=stellar/rs-stellar-xdr - --build-arg REF="${{ needs.shas.outputs.xdr }}" . - - name: Upload Stellar-Rs-Xdr Image + key: ${{ inputs.cache_id }}-${{ env.image_filename }} + path: /tmp/${{ env.image_filename }} + - name: Upload Image to Artifacts uses: actions/upload-artifact@v4 with: - name: image-rs-stellar-xdr-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/image - retention-days: ${{ env.ARTIFACT_RETENTION_DAYS_FOR_IMAGE }} + name: ${{ env.image_filename }} + path: /tmp/${{ env.image_filename }} + retention-days: ${{ env.artifact_retention_days_for_image }} build: - needs: [build-stellar-core, build-stellar-horizon, build-rs-stellar-xdr, build-stellar-friendbot, build-stellar-rpc, build-stellar-lab] + needs: [setup, load, build-dep] if: always() && !failure() && !cancelled() - outputs: - image: ${{ steps.image.outputs.name }} - runs-on: ${{ inputs.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} + strategy: + matrix: + image: ${{ fromJSON(needs.setup.outputs.images) }} + arch: ${{ fromJSON(inputs.archs) }} + fail-fast: false + name: 5 build (quickstart, ${{ matrix.image.tag }}, ${{ matrix.arch }}) + runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} + env: + image_json: ${{ toJSON(matrix.image) }} steps: - uses: actions/checkout@v3 with: - ref: ${{ inputs.sha }} - - name: Download Stellar XDR + repository: ${{ inputs.repo }} + ref: ${{ needs.setup.outputs.sha }} + - name: Collect Dep IDs + id: ids + run: + echo "$(<<< $image_json jq -r '.deps[] | "\(.name)=\(.id)"')" | tee -a $GITHUB_OUTPUT + - name: Write Image Config + run: | + echo "$image_json" > .image.json + - name: Download Image XDR uses: actions/download-artifact@v4 with: - name: image-rs-stellar-xdr-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/stellar-xdr - - name: Download Stellar-Core Image + pattern: image-xdr-${{ steps.ids.outputs.xdr }}-${{ matrix.arch }}.* + merge-multiple: true + path: /tmp/images + - name: Download Image Core uses: actions/download-artifact@v4 with: - name: image-stellar-core-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/stellar-core - - name: Download Stellar-Horizon Image + pattern: image-core-${{ steps.ids.outputs.core }}-${{ matrix.arch }}.* + merge-multiple: true + path: /tmp/images + - name: Download Image RPC uses: actions/download-artifact@v4 with: - name: image-stellar-horizon-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/stellar-horizon - - name: Download Stellar-Friendbot Image + pattern: image-rpc-${{ steps.ids.outputs.rpc }}-${{ matrix.arch }}.* + merge-multiple: true + path: /tmp/images + - name: Download Image Horizon uses: actions/download-artifact@v4 with: - name: image-stellar-friendbot-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/stellar-friendbot - - name: Download Stellar-Lab Image + pattern: image-horizon-${{ steps.ids.outputs.horizon }}-${{ matrix.arch }}.* + merge-multiple: true + path: /tmp/images + - name: Download Image Friendbot uses: actions/download-artifact@v4 with: - name: image-stellar-lab-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/stellar-lab - - name: Download Stellar-rpc Image + pattern: image-friendbot-${{ steps.ids.outputs.friendbot }}-${{ matrix.arch }}.* + merge-multiple: true + path: /tmp/images + - name: Download Image Lab uses: actions/download-artifact@v4 with: - name: image-stellar-rpc-${{ inputs.tag }}-${{ inputs.arch }} - path: /tmp/stellar-rpc - - name: Load Stellar-Core Image - run: docker load -i /tmp/stellar-core/image - - name: Load Stellar-Horizon Image - run: docker load -i /tmp/stellar-horizon/image - - name: Load Stellar-Friendbot Image - run: docker load -i /tmp/stellar-friendbot/image - - name: Load Stellar-Lab Image - run: docker load -i /tmp/stellar-lab/image - - name: Load Stellar-rpc Image - run: docker load -i /tmp/stellar-rpc/image - - name: Load Stellar-Rs-Xdr Image - run: docker load -i /tmp/stellar-xdr/image - - id: image - name: Image Name - run: echo "name=$IMAGE" >> $GITHUB_OUTPUT + pattern: image-lab-${{ steps.ids.outputs.lab }}-${{ matrix.arch }}.* + merge-multiple: true + path: /tmp/images + - name: Load Image into Docker + run: | + ls -lah /tmp/images/ + for image in /tmp/images/*.tar; do + echo Loading image $image + < "${image/%.tar/.json}" jq + docker load -i $image + done - name: Pull Base Image - run: docker pull --platform linux/${{ inputs.arch }} ubuntu:22.04 + run: docker pull --platform linux/${{ matrix.arch }} ubuntu:22.04 # Docker buildx cannot be used to build the dev quickstart image because # buildx does not yet support importing existing images, like the core and # horizon images above, into a buildx builder's cache. Buildx would be # preferred because it can output a smaller image file faster than docker # save can. Once buildx supports it we can update. # https://github.com/docker/buildx/issues/847 - - name: Build Quickstart Image + - name: Build Image run: > docker build - --platform linux/${{ inputs.arch }} + --platform linux/${{ matrix.arch }} -f Dockerfile - -t $IMAGE - --label org.opencontainers.image.revision="${{ inputs.sha }}" + -t quickstart:${{ matrix.image.tag }}-${{ matrix.arch }} + --label org.opencontainers.image.revision="${{ needs.setup.outputs.sha }}" + --build-arg REVISION="${{ needs.setup.outputs.sha }}" + --build-arg PROTOCOL_VERSION_DEFAULT="${{ matrix.image.config.protocol_version_default }}" + --build-arg XDR_IMAGE_REF=stellar-xdr:${{ matrix.arch }} + --build-arg CORE_IMAGE_REF=stellar-core:${{ matrix.arch }} + --build-arg RPC_IMAGE_REF=stellar-rpc:${{ matrix.arch }} + --build-arg HORIZON_IMAGE_REF=stellar-horizon:${{ matrix.arch }} + --build-arg FRIENDBOT_IMAGE_REF=stellar-friendbot:${{ matrix.arch }} + --build-arg LAB_IMAGE_REF=stellar-lab:${{ matrix.arch }} . - --build-arg REVISION="${{ inputs.sha }}" - --build-arg PROTOCOL_VERSION_DEFAULT="${{ inputs.protocol_version_default }}" - --build-arg STELLAR_XDR_IMAGE_REF=stellar-rs-xdr:${{ inputs.arch }} - --build-arg STELLAR_CORE_IMAGE_REF=stellar-core:${{ inputs.arch }} - --build-arg HORIZON_IMAGE_REF=stellar-horizon:${{ inputs.arch }} - --build-arg FRIENDBOT_IMAGE_REF=stellar-friendbot:${{ inputs.arch }} - --build-arg STELLAR_RPC_IMAGE_REF=stellar-rpc:${{ inputs.arch }} - --build-arg LAB_IMAGE_REF=stellar-lab:${{ inputs.arch }} - name: Save Quickstart Image - run: docker save $IMAGE -o /tmp/image - - name: Upload Quickstart Image + run: docker save quickstart -o /tmp/image + - name: Upload Quickstart Image to Artifacts uses: actions/upload-artifact@v4 with: - name: image-${{ inputs.tag }}-${{ inputs.arch }} + name: image-quickstart-${{ matrix.image.tag }}-${{ matrix.arch }}.tar path: /tmp/image - retention-days: ${{ env.ARTIFACT_RETENTION_DAYS_FOR_IMAGE }} + retention-days: ${{ env.artifact_retention_days_for_image }} test: - needs: build - if: always() && !failure() && !cancelled() + needs: [setup, build] + if: always() && !failure() && !cancelled() && (github.event_name != 'workflow_call' || inputs.test == true) strategy: - matrix: ${{ fromJSON(inputs.test_matrix) }} + matrix: + tag: ${{ fromJSON(needs.setup.outputs.images).*.tag }} + arch: ${{ fromJSON(inputs.archs) }} + network: ["local"] + enable: ["core","rpc","core,rpc,horizon"] + options: [""] + include: ${{ fromJSON(needs.setup.outputs.additional-tests) }} fail-fast: false - runs-on: ${{ inputs.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} + name: 6 test (${{ matrix.tag }}, ${{ matrix.arch }}, ${{ matrix.network }}, ${{ matrix.enable }} ${{ matrix.options || '' }}) + runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} steps: - name: Free up disk space if: matrix.network == 'pubnet' @@ -497,32 +336,29 @@ jobs: df -h - uses: actions/checkout@v2 with: - ref: ${{ inputs.sha }} + repository: ${{ inputs.repo }} + ref: ${{ needs.setup.outputs.sha }} - name: Download Quickstart Image uses: actions/download-artifact@v4 with: - name: image-${{ inputs.tag }}-${{ inputs.arch }} + name: image-quickstart-${{ matrix.tag }}-${{ matrix.arch }}.tar path: /tmp/ - name: Load Quickstart Image run: docker load -i /tmp/image - - if: inputs.arch == 'arm64' - uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 - with: - platforms: arm64 - name: Prepare Logs Directory run: mkdir -p logs - name: Run Quickstart Image run: > docker run - --platform linux/${{ inputs.arch }} + --platform linux/${{ matrix.arch }} -d -p "8000:8000" -p "11626:11626" --name stellar - $IMAGE + quickstart:${{ matrix.tag }}-${{ matrix.arch }} --${{ matrix.network }} - --enable ${{ matrix.core }},${{ matrix.horizon }},${{ matrix.rpc }} + --enable ${{ matrix.enable }} ${{ matrix.options }} - name: Set up Go uses: actions/setup-go@v2 @@ -531,28 +367,28 @@ jobs: - name: Sleep until supervisor is up run: sleep 10 - name: Run core test - if: ${{ matrix.core }} + if: ${{ contains(matrix.enable, 'core') }} run: | docker logs stellar -f & echo "supervisorctl tail -f stellar-core" | docker exec -i stellar sh & go run tests/test_core.go curl http://localhost:11626/info - name: Run horizon up test - if: ${{ matrix.horizon }} + if: ${{ contains(matrix.enable, 'horizon') }} run: | docker logs stellar -f & echo "supervisorctl tail -f horizon" | docker exec -i stellar sh & go run tests/test_horizon_up.go curl http://localhost:8000 - name: Run horizon core up test - if: ${{ matrix.horizon && matrix.network != 'pubnet' }} + if: ${{ contains(matrix.enable, 'horizon') && matrix.network != 'pubnet' }} run: | docker logs stellar -f & echo "supervisorctl tail -f horizon" | docker exec -i stellar sh & go run tests/test_horizon_core_up.go curl http://localhost:8000 - name: Run horizon ingesting test - if: ${{ matrix.horizon && matrix.network != 'pubnet' }} + if: ${{ contains(matrix.enable, 'horizon') && matrix.network != 'pubnet' }} run: | docker logs stellar -f & echo "supervisorctl tail -f stellar-core" | docker exec -i stellar sh & @@ -560,20 +396,20 @@ jobs: go run tests/test_horizon_ingesting.go curl http://localhost:8000 - name: Run friendbot test - if: ${{ matrix.horizon && matrix.network == 'local' }} + if: ${{ contains(matrix.enable, 'horizon') && matrix.network == 'local' }} run: | docker logs stellar -f & echo "supervisorctl tail -f friendbot" | docker exec -i stellar sh & echo "supervisorctl tail -f horizon" | docker exec -i stellar sh & go run tests/test_friendbot.go - name: Run stellar rpc up test - if: ${{ matrix.rpc }} + if: ${{ contains(matrix.enable, 'rpc') }} run: | docker logs stellar -f & echo "supervisorctl tail -f stellar-rpc" | docker exec -i stellar sh & go run tests/test_stellar_rpc_up.go - name: Run stellar rpc healthy test - if: ${{ matrix.rpc && matrix.network != 'pubnet' }} + if: ${{ contains(matrix.enable, 'rpc') && matrix.network != 'pubnet' }} run: | docker logs stellar -f & echo "supervisorctl tail -f stellar-rpc" | docker exec -i stellar sh & @@ -585,54 +421,6 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: logs-${{ inputs.tag }}-${{ inputs.arch }}-test-${{ strategy.job-index }} + name: logs-${{ matrix.tag }}-${{ matrix.arch }}-test-${{ strategy.job-index }} path: logs - retention-days: ${{ env.ARTIFACT_RETENTION_DAYS_FOR_LOGS }} - - push-pr: - # Push image to registry after build for pull requests from a local branch. - if: always() && !failure() && !cancelled() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository - needs: build - permissions: - packages: write - statuses: write - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - ref: ${{ inputs.sha }} - - id: push - uses: ./.github/actions/push - with: - head_sha: ${{ inputs.sha }} - artifact_name: image-${{ inputs.tag }}-${{ inputs.arch }} - artifact_image_file: image - arch: ${{ inputs.arch }} - image: ${{ env.IMAGE }} - registry: ${{ secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io' }} - username: ${{ secrets.DOCKERHUB_USERNAME || github.actor }} - password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} - - push-release: - # Push image to registry after test for main. - if: always() && !failure() && !cancelled() && github.event_name == 'push' && github.ref == 'refs/heads/main' - needs: [build, test] - permissions: - packages: write - statuses: write - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - ref: ${{ inputs.sha }} - - id: push - uses: ./.github/actions/push - with: - head_sha: ${{ inputs.sha }} - artifact_name: image-${{ inputs.tag }}-${{ inputs.arch }} - artifact_image_file: image - arch: ${{ inputs.arch }} - image: ${{ env.IMAGE }} - registry: ${{ secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io' }} - username: ${{ secrets.DOCKERHUB_USERNAME || github.actor }} - password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} + retention-days: ${{ env.artifact_retention_days_for_logs }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..324b2b1b3 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,124 @@ +name: CI + +# This workflow builds all the images defined in images.json with the specified +# dependencies, and runs a set of locally defined tests. +# +# Each image defined in images.json specifies the events the image should be +# rebuilt and pushed. Most images will do so for push and pull_request, but can +# also be specified to build on a schedule. +# +# The tests in this repo are designed to make sure that when quickstart starts +# up it is able to either sync with an existing network, or start a new network +# that is progressing. +# +# This workflow also tests that the quickstart action functions. + +on: + push: + branches: + - main + pull_request: + schedule: + - cron: '0 0 * * *' + +# Prevent more than one build of this workflow for a branch to be running at the +# same time, and if multiple are queued, only run the latest, cancelling any +# already running build. The exception being any protected branch, such as +# main, where a build for every commit will run. +concurrency: + group: ${{ github.workflow }}-${{ github.ref_protected == 'true' && github.sha || github.ref }} + cancel-in-progress: true + +jobs: + + complete: + if: always() + name: complete + needs: [build, action-using-artifact, push, action-using-registry] + runs-on: ubuntu-latest + steps: + - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') + run: exit 1 + + setup: + name: 1 setup + runs-on: ubuntu-latest + outputs: + tag-prefix: ${{ steps.tag-prefix.outputs.tag-prefix }} + tag-alias-prefix: ${{ steps.tag-prefix.outputs.tag-alias-prefix }} + images: ${{ steps.images.outputs.images }} + tags: ${{ steps.tags.outputs.tags }} + archs: ${{ steps.archs.outputs.archs }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 # Get all history for the sha count below. + ref: ${{ github.event.pull_request.head.sha || github.sha }} + - name: Tag Prefix + id: tag-prefix + run: | + pr_prefix="${{ github.event_name == 'pull_request' && format('pr{0}-', github.event.pull_request.number) || '' }}" + commit_count="$(git rev-list HEAD --count --first-parent)" + build_number="${{ github.run_number }}.${{ github.run_attempt }}" + echo "tag-prefix=${pr_prefix}v${commit_count}-b${build_number}-" | tee -a $GITHUB_OUTPUT + echo "tag-alias-prefix=${pr_prefix}" | tee -a $GITHUB_OUTPUT + - name: Images + id: images + run: | + images="$(> $GITHUB_ENV + echo "images=$images" >> $GITHUB_OUTPUT + - name: Tags + id: tags + run: | + tags="$(<<< $images jq -c '[.[].tag]')" + <<< $tags jq + echo "tags=$tags" >> $GITHUB_OUTPUT + - name: Architectures + id: archs + run: | + archs='["amd64","arm64"]' + <<< $archs jq + echo "archs=$archs" >> $GITHUB_OUTPUT + + build: + name: 2 build + needs: setup + uses: ./.github/workflows/build.yml + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + images: ${{ needs.setup.outputs.images }} + archs: ${{ needs.setup.outputs.archs }} + + action-using-artifact: + needs: [setup, build] + name: 3 test action artifact + uses: ./.github/workflows/action-test.yml + with: + artifact: image-quickstart-latest-amd64.tar + tag: latest-amd64 + + push: + name: 4 push + needs: [setup, build] + uses: ./.github/workflows/push.yml + with: + sha: ${{ github.event.pull_request.head.sha || github.sha }} + tags: ${{ needs.setup.outputs.tags }} + archs: ${{ needs.setup.outputs.archs }} + tag-prefix: ${{ needs.setup.outputs.tag-prefix }} + tag-alias-prefix: ${{ needs.setup.outputs.tag-alias-prefix }} + registry_repo: ${{ github.repository }} + secrets: + registry: ${{ secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io' }} + registry_username: ${{ secrets.DOCKERHUB_USERNAME || github.actor }} + registry_password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} + + action-using-registry: + needs: [setup, push] + name: 5 test action registry + uses: ./.github/workflows/action-test.yml + with: + tag: ${{ needs.setup.outputs.tag-prefix }}latest diff --git a/.github/workflows/manifest.yml b/.github/workflows/manifest.yml deleted file mode 100644 index 752ae89a1..000000000 --- a/.github/workflows/manifest.yml +++ /dev/null @@ -1,90 +0,0 @@ -on: - workflow_call: - secrets: - DOCKERHUB_USERNAME: - required: false - DOCKERHUB_TOKEN: - required: false - inputs: - tag: - description: 'Tag to use as the manifest list image name' - type: 'string' - required: true - tag-alias: - description: 'Tag to alias to the tag of the manifest, e.g. "latest"' - type: 'string' - required: true - images: - description: 'Space separated list of images to include in the manifest list' - type: 'string' - required: true - outputs: - image: - description: 'Image pushed as a result of this build' - value: ${{ jobs.build.outputs.image }} - -env: - HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} - IMAGE: ${{ format('{0}/{1}:{2}', secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io', github.repository, inputs.tag) }} - IMAGE_ALIAS: ${{ format('{0}/{1}:{2}', secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io', github.repository, inputs.tag-alias) }} - REGISTRY: ${{ secrets.DOCKERHUB_TOKEN && 'docker.io' || 'ghcr.io' }} - -jobs: - - input-validations: - runs-on: ubuntu-latest - steps: - - name: Check tag and tag-alias format has pr prefix for pr builds - if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main') }} - run: | - if ! [[ "${{ inputs.tag }}" =~ ^pr[0-9]+- ]]; then - echo "Error: 'tag' input must start with 'prN-'" - exit 1 - fi - if ! [[ "${{ inputs.tag-alias }}" =~ ^pr[0-9]+- ]]; then - echo "Error: 'tag-alias' input must start with 'prN-'" - exit 1 - fi - - push: - needs: input-validations - if: ${{ github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) }} - permissions: - packages: write - statuses: write - runs-on: ubuntu-latest - steps: - - id: image_parts - run: | - IMAGE_TAGLESS=$(echo ${{ env.IMAGE }} | cut -d':' -f1) - IMAGE_REPO=$(echo $IMAGE_TAGLESS | cut -d'/' -f2,3) - IMAGE_TAG=$(echo ${{ env.IMAGE }} | cut -d':' -f2) - echo "::set-output name=repo::$IMAGE_REPO" - echo "::set-output name=tag::$IMAGE_TAG" - - uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 - with: - registry: ${{ env.REGISTRY }} - username: ${{ secrets.DOCKERHUB_USERNAME || github.actor }} - password: ${{ secrets.DOCKERHUB_TOKEN || github.token }} - - run: echo "IMAGE_URL=https://${{ env.IMAGE }}" >> $GITHUB_ENV - - if: ${{ env.REGISTRY == 'docker.io' }} - run: | - echo "IMAGE_URL=https://hub.docker.com/r/${{ steps.image_parts.outputs.repo }}/tags?name=${{ steps.image_parts.outputs.tag }}" >> $GITHUB_ENV - - run: | - docker manifest create ${{ env.IMAGE }} ${{ inputs.images }} - - run: | - docker manifest push ${{ env.IMAGE }} - - run: | - docker buildx imagetools create -t ${{ env.IMAGE_ALIAS }} ${{ env.IMAGE }} - - uses: actions/github-script@v5 - with: - script: | - github.rest.repos.createCommitStatus({ - owner: context.repo.owner, - repo: context.repo.repo, - sha: '${{ env.HEAD_SHA }}', - state: 'success', - context: `${{ env.IMAGE }}`, - target_url: '${{ env.IMAGE_URL }}', - description: 'Available', - }); diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml new file mode 100644 index 000000000..d6cf9be3e --- /dev/null +++ b/.github/workflows/push.yml @@ -0,0 +1,188 @@ +name: Push + +# This workflow pushes multiple quickstart images that have been built with the +# build.yml. This workflow and the build.yml workflow are coupled, and this +# workflow makes assumptions about what artifact names the images hae been +# saved under. This workflow is not intended to be called by third parties. + +on: + workflow_call: + inputs: + sha: + description: 'Sha to connect push status notifications to' + type: "string" + required: true + tags: + description: 'Tags to push (e.g. ["latest", "testing", ...])' + type: "string" + required: true + archs: + description: 'Architectures to push for as an array (e.g. ["amd64", "arm64"])' + type: "string" + required: true + tag-prefix: + description: "Tag prefix for the image when pushed (e.g. pr877-v21-)" + type: "string" + default: '' + tag-alias-prefix: + description: "A second tag prefix for the image when pushed that'll be pushed in addition as an alias (e.g. pr877-)" + type: "string" + default: '' + registry_repo: + description: "Repo at the registry to push to" + type: "string" + secrets: + registry: + description: "Registry to push to" + registry_username: + description: "Username to auth with the registry" + registry_password: + description: "Password to auth with the registry" + +jobs: + + complete: + if: always() + name: complete + needs: [push, push-manifest, push-alias] + runs-on: ubuntu-latest + steps: + - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') + run: exit 1 + + push: + strategy: + matrix: + tag: ${{ fromJSON(inputs.tags) }} + arch: ${{ fromJSON(inputs.archs) }} + fail-fast: false + name: 1 push (${{ matrix.tag }}, ${{ matrix.arch }}) + permissions: + packages: write + statuses: write + runs-on: ubuntu-latest + steps: + - name: Create Tag + id: tag + run: echo "tag=${{ inputs.tag-prefix }}${{ matrix.tag }}-${{ matrix.arch }}" | tee -a $GITHUB_OUTPUT + - name: Download Image from Artifacts + uses: actions/download-artifact@v4 + with: + name: image-quickstart-${{ matrix.tag }}-${{ matrix.arch }}.tar + path: /tmp/ + - name: Load Image + run: docker load -i /tmp/image + - name: Tag Image + run: > + docker tag + quickstart:${{ matrix.tag }}-${{ matrix.arch }} + ${{ secrets.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag }} + - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + with: + registry: ${{ secrets.registry }} + username: ${{ secrets.registry_username }} + password: ${{ secrets.registry_password }} + - name: Push Image + run: > + docker push + ${{ secrets.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag }} + - name: Post Status with Image Name + uses: actions/github-script@v5 + with: + script: | + github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: '${{ inputs.sha }}', + state: 'success', + context: `${{ secrets.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag }}`, + description: 'Available', + }); + + push-manifest: + needs: push + strategy: + matrix: + tag: ${{ fromJSON(inputs.tags) }} + fail-fast: false + name: 2 push manifest (${{ matrix.tag }}) + permissions: + packages: write + statuses: write + runs-on: ubuntu-latest + steps: + - name: Create Tag + id: tag + run: | + echo "tag=${{ inputs.tag-prefix }}${{ matrix.tag }}" | tee -a $GITHUB_OUTPUT + - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + with: + registry: ${{ secrets.registry }} + username: ${{ secrets.registry_username }} + password: ${{ secrets.registry_password }} + - name: Prepare Image List + id: images + env: + archs: ${{ inputs.archs }} + run: | + images="$(<<< $archs jq -r 'map("${{ secrets.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag }}-\(.)") | join(" ")')" + echo "images=$images" | tee -a $GITHUB_OUTPUT + - name: Create Manifest + run: > + docker manifest create + ${{ secrets.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag }} + ${{ steps.images.outputs.images }} + - name: Push Manifest + run: > + docker manifest push + ${{ secrets.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag }} + - uses: actions/github-script@v5 + with: + script: | + github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: '${{ inputs.sha }}', + state: 'success', + context: '${{ secrets.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag }}', + description: 'Available', + }); + + push-alias: + needs: push-manifest + strategy: + matrix: + tag: ${{ fromJSON(inputs.tags) }} + fail-fast: false + name: 3 push alias (${{ matrix.tag }}) + permissions: + packages: write + statuses: write + runs-on: ubuntu-latest + steps: + - name: Create Tag + id: tag + run: | + echo "tag=${{ inputs.tag-prefix }}${{ matrix.tag }}" | tee -a $GITHUB_OUTPUT + echo "tag-alias=${{ inputs.tag-alias-prefix }}${{ matrix.tag }}" | tee -a $GITHUB_OUTPUT + - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + with: + registry: ${{ secrets.registry }} + username: ${{ secrets.registry_username }} + password: ${{ secrets.registry_password }} + - name: Push Alias + run: > + docker buildx imagetools create -t + ${{ secrets.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag-alias }} + ${{ secrets.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag }} + - uses: actions/github-script@v5 + with: + script: | + github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: '${{ inputs.sha }}', + state: 'success', + context: '${{ secrets.registry }}/${{ inputs.registry_repo }}:${{ steps.tag.outputs.tag-alias }}', + description: 'Available', + }); diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..4070854d4 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/.image.json diff --git a/.scripts/images-additional-tests b/.scripts/images-additional-tests new file mode 100755 index 000000000..eabfd80a3 --- /dev/null +++ b/.scripts/images-additional-tests @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 + +import json +import sys + +# Accepts as stdin a JSON object in the format of images.json. Outputs an array +# of all the additional test cases defined in the images merged together. +# Usage: < images.json ./.scripts/images-additional-tests + +images = json.load(sys.stdin) +tests = [] +for image in images: + tag = image['tag'] + for test in image['additional-tests']: + tests.append({'tag': tag, **test}) + +print(json.dumps(tests)) diff --git a/.scripts/images-deps b/.scripts/images-deps new file mode 100755 index 000000000..8d1d5e776 --- /dev/null +++ b/.scripts/images-deps @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -e +set -u +set -o pipefail + +# Accepts as stdin a JSON object in the format of images.json. Outputs an array +# of all dependencies that need to be built across all the images. +# Usage: < images.json ./.scripts/images-deps + +jq -c '[ .[] | .deps[] ] | unique' diff --git a/.scripts/images-with-extras b/.scripts/images-with-extras new file mode 100755 index 000000000..9b9ed3b8b --- /dev/null +++ b/.scripts/images-with-extras @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 + +import json +import sys +import subprocess +import hashlib + +# Accepts as stdin a JSON object in the format of images.json. +# And adds some calculatble elements used during the build. +# +# 1. Resolves any 'ref' values in the JSON to a revision sha. +# 2. Hashes the entire dep details and injects the hash as an id into the deps +# details. The id can be used to uniquely identify a dep configuration. +# +# Outputs the original JSON modified. +# +# Usage: < images.json ./.scripts/images-with-extras + +images = json.load(sys.stdin) + +cache = {} + +for image in images: + tag = image["tag"] + for dep in image["deps"]: + name = dep["name"] + repo = dep["repo"] + ref = dep["ref"] + print(f"{tag} {name} {repo} {ref} ...", file=sys.stderr) + key = (name, repo, ref) + if key in cache: + sha = cache[key] + else: + sha = subprocess.run( + ["gh", "api", f"repos/{repo}/commits/{ref}", "--jq", ".sha"], + capture_output=True, + text=True, + check=True + ).stdout.strip() + cache[key] = sha + print(f" • revision sha = {sha}", file=sys.stderr) + dep["sha"] = sha + + dep_str = json.dumps(dep, separators=(',', ':')) + id = hashlib.sha256(dep_str.encode()).hexdigest() + dep["id"] = id + print(f" • id = {id}", file=sys.stderr) + +print(json.dumps(images, separators=(',', ':'))) diff --git a/Dockerfile b/Dockerfile index 1474ca979..395176cfe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,21 +1,21 @@ -ARG STELLAR_XDR_IMAGE_REF -ARG STELLAR_CORE_IMAGE_REF +ARG XDR_IMAGE_REF +ARG CORE_IMAGE_REF ARG HORIZON_IMAGE_REF ARG FRIENDBOT_IMAGE_REF -ARG STELLAR_RPC_IMAGE_REF +ARG RPC_IMAGE_REF ARG LAB_IMAGE_REF -FROM $STELLAR_XDR_IMAGE_REF AS stellar-xdr -FROM $STELLAR_CORE_IMAGE_REF AS stellar-core +FROM $XDR_IMAGE_REF AS xdr +FROM $CORE_IMAGE_REF AS core FROM $HORIZON_IMAGE_REF AS horizon FROM $FRIENDBOT_IMAGE_REF AS friendbot -FROM $STELLAR_RPC_IMAGE_REF AS stellar-rpc +FROM $RPC_IMAGE_REF AS rpc FROM $LAB_IMAGE_REF AS lab FROM ubuntu:22.04 ARG REVISION -ENV REVISION $REVISION +ENV REVISION=$REVISION EXPOSE 5432 EXPOSE 6060 @@ -29,11 +29,11 @@ EXPOSE 11626 ADD dependencies / RUN /dependencies -COPY --from=stellar-xdr /stellar-xdr /usr/local/bin/stellar-xdr -COPY --from=stellar-core /stellar-core /usr/bin/stellar-core +COPY --from=xdr /stellar-xdr /usr/local/bin/stellar-xdr +COPY --from=core /stellar-core /usr/bin/stellar-core COPY --from=horizon /horizon /usr/bin/stellar-horizon COPY --from=friendbot /friendbot /usr/local/bin/friendbot -COPY --from=stellar-rpc /stellar-rpc /usr/bin/stellar-rpc +COPY --from=rpc /stellar-rpc /usr/bin/stellar-rpc COPY --from=lab /lab /opt/stellar/lab COPY --from=lab /node /usr/bin/ @@ -42,6 +42,8 @@ RUN adduser --system --group --quiet --home /var/lib/stellar --disabled-password RUN ["mkdir", "-p", "/opt/stellar"] RUN ["touch", "/opt/stellar/.docker-ephemeral"] +ADD .image.json /image.json + RUN ["rm", "-fr", "/etc/supervisor"] RUN ["ln", "-sT", "/opt/stellar/supervisor/etc", "/etc/supervisor"] @@ -57,8 +59,9 @@ ADD futurenet /opt/stellar-default/futurenet ADD start / RUN ["chmod", "+x", "start"] + ARG PROTOCOL_VERSION_DEFAULT RUN test -n "$PROTOCOL_VERSION_DEFAULT" || (echo "Image build arg PROTOCOL_VERSION_DEFAULT required and not set" && false) -ENV PROTOCOL_VERSION_DEFAULT $PROTOCOL_VERSION_DEFAULT +ENV PROTOCOL_VERSION_DEFAULT=$PROTOCOL_VERSION_DEFAULT ENTRYPOINT ["/start"] diff --git a/Dockerfile.core b/Dockerfile.core index 491a1ff83..1e1f053de 100644 --- a/Dockerfile.core +++ b/Dockerfile.core @@ -6,10 +6,13 @@ RUN apt-get update && \ git build-essential pkg-config autoconf automake libtool \ bison flex sed perl libpq-dev parallel libunwind-dev \ clang-12 libc++abi-12-dev libc++-12-dev \ - postgresql curl + postgresql curl jq ARG REPO ARG REF +ARG OPTIONS +RUN echo "$OPTIONS" | jq -r '.configure_flags // ""' > /tmp/arg_configure_flags + WORKDIR /wd RUN git clone https://github.com/${REPO} /wd RUN git fetch origin ${REF} @@ -22,14 +25,13 @@ ARG CC=clang-12 ARG CXX=clang++-12 ARG CFLAGS='-O3 -g1 -fno-omit-frame-pointer' ARG CXXFLAGS='-O3 -g1 -fno-omit-frame-pointer -stdlib=libc++' -ARG CONFIGURE_FLAGS='' RUN sysctl vm.mmap_rnd_bits=28 RUN ./autogen.sh RUN ./install-rust.sh ENV PATH "/root/.cargo/bin:$PATH" -RUN ./configure CC="${CC}" CXX="${CXX}" CFLAGS="${CFLAGS}" CXXFLAGS="${CXXFLAGS}" ${CONFIGURE_FLAGS} +RUN sh -c './configure CC="${CC}" CXX="${CXX}" CFLAGS="${CFLAGS}" CXXFLAGS="${CXXFLAGS}" $( $@ + +# Extract configuration from selected image +PROTOCOL_VERSION_DEFAULT = $(shell < $(IMAGE_JSON) jq -r '.config.protocol_version_default') +XDR_REPO = $(shell < $(IMAGE_JSON) jq -r '.deps[] | select(.name == "xdr") | .repo') +XDR_SHA = $(shell < $(IMAGE_JSON) jq -r '.deps[] | select(.name == "xdr") | .sha') +CORE_REPO = $(shell < $(IMAGE_JSON) jq -r '.deps[] | select(.name == "core") | .repo') +CORE_SHA = $(shell < $(IMAGE_JSON) jq -r '.deps[] | select(.name == "core") | .sha') +CORE_OPTIONS = $(shell < $(IMAGE_JSON) jq -c '.deps[] | select(.name == "core") | .options // {}') +RPC_REPO = $(shell < $(IMAGE_JSON) jq -r '.deps[] | select(.name == "rpc") | .repo') +RPC_SHA = $(shell < $(IMAGE_JSON) jq -r '.deps[] | select(.name == "rpc") | .sha') +HORIZON_REPO = $(shell < $(IMAGE_JSON) jq -r '.deps[] | select(.name == "horizon") | .repo') +HORIZON_SHA = $(shell < $(IMAGE_JSON) jq -r '.deps[] | select(.name == "horizon") | .sha') +FRIENDBOT_REPO = $(shell < $(IMAGE_JSON) jq -r '.deps[] | select(.name == "friendbot") | .repo') +FRIENDBOT_SHA = $(shell < $(IMAGE_JSON) jq -r '.deps[] | select(.name == "friendbot") | .sha') +LAB_REPO = $(shell < $(IMAGE_JSON) jq -r '.deps[] | select(.name == "lab") | .repo') +LAB_SHA = $(shell < $(IMAGE_JSON) jq -r '.deps[] | select(.name == "lab") | .sha') run: - docker run --rm --name stellar -p 8000:8000 stellar/quickstart:$(TAG) --local --enable-stellar-rpc + docker run --rm --name stellar -p 8000:8000 stellar/quickstart:$(TAG) --local logs: docker exec stellar /bin/sh -c 'tail -F /var/log/supervisor/*' @@ -22,61 +33,34 @@ logs: console: docker exec -it stellar /bin/bash -build-latest: - $(MAKE) build TAG=latest \ - PROTOCOL_VERSION_DEFAULT=23 \ - XDR_REF=v23.0.0 \ - CORE_REF=v23.0.1 \ - HORIZON_REF=horizon-v23.0.0 \ - STELLAR_RPC_REF=v23.0.4 \ - FRIENDBOT_REF=horizon-v23.0.0 - -build-testing: - $(MAKE) build TAG=testing \ - PROTOCOL_VERSION_DEFAULT=23 \ - XDR_REF=v23.0.0 \ - CORE_REF=v23.0.1 \ - HORIZON_REF=horizon-v23.0.0 \ - STELLAR_RPC_REF=v23.0.4 \ - FRIENDBOT_REF=horizon-v23.0.0 - -build-future: - $(MAKE) build TAG=future \ - PROTOCOL_VERSION_DEFAULT=23 \ - XDR_REF=v23.0.0 \ - CORE_REF=v23.0.1 \ - HORIZON_REF=horizon-v23.0.0 \ - STELLAR_RPC_REF=v23.0.4 \ - FRIENDBOT_REF=horizon-v23.0.0 - -build: - $(MAKE) -j 4 build-deps +build: $(IMAGE_JSON) + $(MAKE) build-deps docker build -t stellar/quickstart:$(TAG) -f Dockerfile . \ --build-arg REVISION=$(REVISION) \ --build-arg PROTOCOL_VERSION_DEFAULT=$(PROTOCOL_VERSION_DEFAULT) \ - --build-arg STELLAR_XDR_IMAGE_REF=stellar-xdr:$(XDR_REF) \ - --build-arg STELLAR_CORE_IMAGE_REF=stellar-core:$(CORE_REF) \ - --build-arg HORIZON_IMAGE_REF=stellar-horizon:$(HORIZON_REF) \ - --build-arg FRIENDBOT_IMAGE_REF=stellar-friendbot:$(FRIENDBOT_REF) \ - --build-arg STELLAR_RPC_IMAGE_REF=stellar-rpc:$(STELLAR_RPC_REF) \ - --build-arg LAB_IMAGE_REF=stellar-lab:$(LAB_REF) + --build-arg XDR_IMAGE_REF=stellar-xdr:$(XDR_SHA) \ + --build-arg CORE_IMAGE_REF=stellar-core:$(CORE_SHA) \ + --build-arg RPC_IMAGE_REF=stellar-rpc:$(RPC_SHA) \ + --build-arg HORIZON_IMAGE_REF=stellar-horizon:$(HORIZON_SHA) \ + --build-arg FRIENDBOT_IMAGE_REF=stellar-friendbot:$(FRIENDBOT_SHA) \ + --build-arg LAB_IMAGE_REF=stellar-lab:$(LAB_SHA) -build-deps: build-deps-xdr build-deps-core build-deps-horizon build-deps-friendbot build-deps-stellar-rpc build-deps-lab +build-deps: build-deps-xdr build-deps-rpc build-deps-horizon build-deps-friendbot build-deps-lab build-deps-core -build-deps-xdr: - docker build -t stellar-xdr:$(XDR_REF) -f Dockerfile.xdr . --build-arg REPO="$(XDR_REPO)" --build-arg REF="$(XDR_REF)" +build-deps-xdr: $(IMAGE_JSON) + docker build -t stellar-xdr:$(XDR_SHA) -f Dockerfile.xdr . --build-arg REPO="$(XDR_REPO)" --build-arg REF="$(XDR_SHA)" -build-deps-core: - docker build -t stellar-core:$(CORE_REF) -f Dockerfile.core . --build-arg REPO="$(CORE_REPO)" --build-arg REF="$(CORE_REF)" --build-arg CONFIGURE_FLAGS="$(CORE_CONFIGURE_FLAGS)" +build-deps-core: $(IMAGE_JSON) + docker build -t stellar-core:$(CORE_SHA) -f Dockerfile.core . --build-arg REPO="$(CORE_REPO)" --build-arg REF="$(CORE_SHA)" --build-arg OPTIONS='$(CORE_OPTIONS)' -build-deps-horizon: - docker build -t stellar-horizon:$(HORIZON_REF) -f Dockerfile.horizon . --build-arg REF="$(HORIZON_REF)" +build-deps-rpc: $(IMAGE_JSON) + docker build -t stellar-rpc:$(RPC_SHA) -f Dockerfile.rpc . --build-arg=REPO="$(RPC_REPO)" --build-arg REF="$(RPC_SHA)" -build-deps-friendbot: - docker build -t stellar-friendbot:$(FRIENDBOT_REF) -f Dockerfile.friendbot . --build-arg REF="$(FRIENDBOT_REF)" +build-deps-horizon: $(IMAGE_JSON) + docker build -t stellar-horizon:$(HORIZON_SHA) -f Dockerfile.horizon . --build-arg REPO="$(HORIZON_REPO)" --build-arg REF="$(HORIZON_SHA)" -build-deps-stellar-rpc: - docker build -t stellar-rpc:$(STELLAR_RPC_REF) -f Dockerfile.rpc . --build-arg REF="$(STELLAR_RPC_REF)" +build-deps-friendbot: $(IMAGE_JSON) + docker build -t stellar-friendbot:$(FRIENDBOT_SHA) -f Dockerfile.friendbot . --build-arg REPO="$(FRIENDBOT_REPO)" --build-arg REF="$(FRIENDBOT_SHA)" -build-deps-lab: - docker build -t stellar-lab:$(LAB_REF) -f Dockerfile.lab . --build-arg NEXT_PUBLIC_COMMIT_HASH=$(LAB_REF) +build-deps-lab: $(IMAGE_JSON) + docker build -t stellar-lab:$(LAB_SHA) -f Dockerfile.lab . --build-arg REPO="$(LAB_REPO)" --build-arg NEXT_PUBLIC_COMMIT_HASH=$(LAB_SHA) diff --git a/README.md b/README.md index 4e608762e..b358111e4 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Stellar Quickstart is the fastest way to spin up a complete Stellar blockchain d > uses: stellar/quickstart@main > ``` > -> See [Using in GitHub Actions] for more configuration options. +> See [Using in GitHub Actions] for more configuration options and how to build and run a custom configuration of quickstart. [`stellar-cli`]: https://github.com/stellar/stellar-cli @@ -217,6 +217,46 @@ jobs: # - Friendbot: http://localhost:8000/friendbot ``` +#### Custom Builds + +The quickstart image can also be built with custom software, such as custom versions of core, rpc, horizon, and so on. Use a workflow as follows to build a custom quickstart image and then run it using the action. + +```yaml +on: [push, pull_request] + +jobs: + build-custom: + uses: stellar/quickstart/.github/workflows/build.yml@main + with: + images: | + [ + { + "tag": "custom", + "config": { + "protocol_version_default": 23 + }, + "deps": [ + { "name": "xdr", "repo": "stellar/rs-stellar-xdr", "ref": "v23.0.0" }, + { "name": "core", "repo": "stellar/stellar-core", "ref": "v23.0.1", "options": { "configure_flags": "--disable-tests" } }, + { "name": "rpc", "repo": "stellar/stellar-rpc", "ref": "v23.0.1" }, + { "name": "horizon", "repo": "stellar/go", "ref": "horizon-v23.0.0" }, + { "name": "friendbot", "repo": "stellar/go", "ref": "horizon-v23.0.0" }, + { "name": "lab", "repo": "stellar/laboratory", "ref": "main" } + ], + "additional-tests": [] + } + ] + archs: '["amd64"]' + use-custom: + needs: build-custom + runs-on: ubuntu-latest + steps: + - uses: stellar/quickstart@main + with: + artifact: image-quickstart-custom-amd64.tar + tag: custom-amd64 +``` + ### Deploy to Digital Ocean @@ -243,30 +283,11 @@ _Disclaimer_: The DigitalOcean server is publicly accessible on the Internet. Do ### Building Custom Images To build a quickstart image with custom or specific versions of stellar-core, -horizon, etc, use the `Makefile`. The following parameters can be specified to -customize the version of each component, and for stellar-core the features it is -built with. - -- `TAG`: The docker tag to assign to the build. Default `dev`. -- `CORE_REF`: The git reference of stellar-core to build. -- `CORE_CONFIGURE_FLAGS`: The `CONFIGURE_FLAGS` to configure the stellar-core - build with. Typically include `--disable-tests`, and to enable the next protocol - version that is still in development, add - `--enable-next-protocol-version-unsafe-for-production`. -- `HORIZON_REF`: The git reference of stellar-horizon to build. -- `FRIENDBOT_REF`: The git reference of stellar-friendbot to build. -- `STELLAR_RPC_REF`: The git reference of stellar-rpc to build. - -For example: +horizon, etc, use the `Makefile`. Edit the `images.json` file, adding a new +image then build that image specifying its tag name: ``` -make build \ - TAG=future \ - CORE_REF=... \ - CORE_CONFIGURE_FLAGS=... \ - HORIZON_REF=... \ - FRIENDBOT_REF=... \ - STELLAR_RPC_REF=... +make build TAG=mytag ``` ### Background vs. Interactive containers diff --git a/action.yml b/action.yml index c3e7bdbcd..d7b7c2e4d 100644 --- a/action.yml +++ b/action.yml @@ -5,10 +5,13 @@ inputs: description: "Image tag of quickstart image to use" required: true default: "latest" + artifact: + description: "Artifact to collect image from" + type: string + default: "" image: description: "Image for the quickstart image to use" - required: true - default: "docker.io/stellar/quickstart" + default: "" enable: description: "Services to enable" default: "core,horizon,rpc" @@ -35,6 +38,17 @@ runs: steps: - name: Set up Docker uses: docker/setup-docker-action@b60f85385d03ac8acfca6d9996982511d8620a19 # v4 + + - if: inputs.artifact + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.artifact }} + path: /tmp/ + + - if: inputs.artifact + run: docker load -i /tmp/image + shell: bash + - run: > docker run -d --name stellar -p 8000:8000 @@ -53,8 +67,9 @@ runs: --health-interval ${{ inputs.health_interval }}s --health-timeout ${{ inputs.health_timeout }}s --health-retries ${{ inputs.health_retries }} - ${{ inputs.image }}:${{ inputs.tag }} + ${{ inputs.image || (inputs.artifact && 'quickstart' || 'docker.io/stellar/quickstart') }}:${{ inputs.tag }} shell: bash + - name: "Wait for container to be healthy" run: | i=0 diff --git a/images.json b/images.json new file mode 100644 index 000000000..dbd0250ae --- /dev/null +++ b/images.json @@ -0,0 +1,71 @@ +[ + { + "tag": "latest", + "events": ["pull_request", "push"], + "config": { + "protocol_version_default": 23 + }, + "deps": [ + { "name": "xdr", "repo": "stellar/rs-stellar-xdr", "ref": "v23.0.0" }, + { "name": "core", "repo": "stellar/stellar-core", "ref": "v23.0.1", "options": { "configure_flags": "--disable-tests" } }, + { "name": "rpc", "repo": "stellar/stellar-rpc", "ref": "v23.0.1" }, + { "name": "horizon", "repo": "stellar/go", "ref": "horizon-v23.0.0" }, + { "name": "friendbot", "repo": "stellar/go", "ref": "horizon-v23.0.0" }, + { "name": "lab", "repo": "stellar/laboratory", "ref": "main" } + ], + "additional-tests": [ + { "arch": "amd64", "network": "pubnet", "enable": "core,rpc,horizon" } + ] + }, + { + "tag": "testing", + "events": ["pull_request", "push"], + "config": { + "protocol_version_default": 23 + }, + "deps": [ + { "name": "xdr", "repo": "stellar/rs-stellar-xdr", "ref": "v23.0.0" }, + { "name": "core", "repo": "stellar/stellar-core", "ref": "v23.0.1", "options": { "configure_flags": "--disable-tests" } }, + { "name": "rpc", "repo": "stellar/stellar-rpc", "ref": "v23.0.1" }, + { "name": "horizon", "repo": "stellar/go", "ref": "horizon-v23.0.0" }, + { "name": "friendbot", "repo": "stellar/go", "ref": "horizon-v23.0.0" }, + { "name": "lab", "repo": "stellar/laboratory", "ref": "main" } + ], + "additional-tests": [ + { "arch": "amd64", "network": "testnet", "enable": "core,rpc,horizon" }, + { "arch": "amd64", "network": "pubnet", "enable": "core,rpc,horizon" } + ] + }, + { + "tag": "future", + "events": ["pull_request", "push"], + "config": { + "protocol_version_default": 23 + }, + "deps": [ + { "name": "xdr", "repo": "stellar/rs-stellar-xdr", "ref": "v23.0.0" }, + { "name": "core", "repo": "stellar/stellar-core", "ref": "v23.0.1", "options": { "configure_flags": "--disable-tests" } }, + { "name": "rpc", "repo": "stellar/stellar-rpc", "ref": "v23.0.1" }, + { "name": "horizon", "repo": "stellar/go", "ref": "horizon-v23.0.0" }, + { "name": "friendbot", "repo": "stellar/go", "ref": "horizon-v23.0.0" }, + { "name": "lab", "repo": "stellar/laboratory", "ref": "main" } + ], + "additional-tests": [] + }, + { + "tag": "nightly", + "events": ["push", "schedule"], + "config": { + "protocol_version_default": 23 + }, + "deps": [ + { "name": "xdr", "repo": "stellar/rs-stellar-xdr", "ref": "main" }, + { "name": "core", "repo": "stellar/stellar-core", "ref": "master", "options": { "configure_flags": "--disable-tests" } }, + { "name": "rpc", "repo": "stellar/stellar-rpc", "ref": "main" }, + { "name": "horizon", "repo": "stellar/go", "ref": "master" }, + { "name": "friendbot", "repo": "stellar/go", "ref": "master" }, + { "name": "lab", "repo": "stellar/laboratory", "ref": "main" } + ], + "additional-tests": [] + } +] diff --git a/start b/start index edc6c9bec..f3a85e433 100755 --- a/start +++ b/start @@ -80,12 +80,18 @@ function start() { echo "versions:" echo " quickstart: $REVISION" - echo " stellar-core:" + echo " xdr:" + echo "$(stellar-xdr version | sed 's/^/ /')" + echo " core:" echo "$(stellar-core version 2>/dev/null | sed 's/^/ /')" + echo " rpc:" + echo "$(stellar-rpc version | sed 's/^/ /')" echo " horizon:" echo "$(stellar-horizon version | sed 's/^/ /')" - echo " stellar-rpc:" - echo "$(stellar-rpc version | sed 's/^/ /')" + echo " friendbot:" + echo " $(< /image.json jq -r '.deps[] | select(.name == "friendbot") | "\(.ref) (\(.sha))"')" + echo " lab:" + echo " $(< /image.json jq -r '.deps[] | select(.name == "lab") | "\(.ref) (\(.sha))"')" echo "mode: $STELLAR_MODE" echo "network: $NETWORK"