diff --git a/news/239.feature b/news/239.feature new file mode 100644 index 00000000..4ff91882 --- /dev/null +++ b/news/239.feature @@ -0,0 +1 @@ +GitLab CI/CD support @erral diff --git a/templates/projects/monorepo/cookiecutter.json b/templates/projects/monorepo/cookiecutter.json index aefe55f6..57539d9d 100644 --- a/templates/projects/monorepo/cookiecutter.json +++ b/templates/projects/monorepo/cookiecutter.json @@ -17,6 +17,7 @@ "devops_cache": ["1", "0"], "devops_ansible": ["1", "0"], "devops_gha_deploy": ["1", "0"], + "devops_gitlab_deploy": ["1", "0"], "initialize_documentation": ["1", "0"], "__project_slug": "{{ cookiecutter.project_slug }}", "__repository_url": "https://github.com/{{ cookiecutter.github_organization }}/{{ cookiecutter.__project_slug }}", @@ -116,6 +117,11 @@ "1": "Yes", "0": "No" }, + "devops_gitlab_deploy": { + "__prompt__": "Add GitLab Action to Deploy this project?", + "1": "Yes", + "0": "No" + }, "initialize_documentation": { "__prompt__": "Would you like to add a documentation scaffold to your project?", "1": "Yes", diff --git a/templates/projects/monorepo/hooks/post_gen_project.py b/templates/projects/monorepo/hooks/post_gen_project.py index b063b801..e0e716da 100644 --- a/templates/projects/monorepo/hooks/post_gen_project.py +++ b/templates/projects/monorepo/hooks/post_gen_project.py @@ -42,6 +42,10 @@ "devops/.env_gha", "devops/README-GHA.md", ], + "devops-gitlab": [ + ".gitlab-ci.yml", + "devops/README-GITLAB.md", + ], "docs-0": [ ".github/workflows/docs.yml", ".github/workflows/rtd-pr-preview.yml", @@ -62,6 +66,11 @@ def handle_devops_gha_deploy(context: OrderedDict, output_dir: Path): files.remove_files(output_dir, POST_GEN_TO_REMOVE["devops-gha"]) +def handle_devops_gitlab_deploy(context: OrderedDict, output_dir: Path): + """Clean up gitlab deploy.""" + files.remove_files(output_dir, POST_GEN_TO_REMOVE["gitlab"]) + + def handle_docs_cleanup(context: OrderedDict, output_dir: Path): """Clean up GitHub Actions deploy.""" answer = context.get("initialize_documentation") @@ -84,6 +93,31 @@ def handle_git_initialization(context: OrderedDict, output_dir: Path): git.initialize_repository(output_dir) +def handle_container_registry_gitlab(context: OrderedDict, output_dir: Path): + """ show a warning when using GitLab as container registry""" + + msg = """ + [bold blue]{{ cookiecutter.title }}[/bold blue] + + You have selected GitLab as your container registry. + + This template uses `registry.gitlab.com` as the default registry + for your containers. + + If you are using your own GitLab instance, please go the the relevant + Makefiles and change the container registry address, replacing + `registry.gitlab.com` with the URL of your GitLab instance's registry. + + Sorry for the convenience, + The Plone Community. + """ + console.panel( + title="GitLab container registry information", + subtitle="", + msg=msg, + url="https://plone.org/", + ) + def generate_addons_backend(context, output_dir): """Run Plone Addon generator.""" output_dir = output_dir @@ -221,6 +255,19 @@ def main(): context.get("devops_gha_deploy") ), # {{ cookiecutter.devops_gha_deploy }} ], + [ + handle_devops_gitlab_deploy, + "Remove GitLab Actions deployment files", + not int( + context.get("devops_gitlab_deploy") + ), # {{ cookiecutter.devops_gitlab_deploy }} + ], + [ + handle_container_registry_gitlab, + "Remove GitLab Actions deployment files", + context.get("container_registry", "") == "gitlab" + # {{ cookiecutter.container_registry }} + ], [ handle_docs_setup, "Organize documentation files", diff --git a/templates/projects/monorepo/{{ cookiecutter.__folder_name }}/.gitlab-ci.yml b/templates/projects/monorepo/{{ cookiecutter.__folder_name }}/.gitlab-ci.yml new file mode 100644 index 00000000..0d58407d --- /dev/null +++ b/templates/projects/monorepo/{{ cookiecutter.__folder_name }}/.gitlab-ci.yml @@ -0,0 +1,175 @@ +default: + image: docker:24.0.5 + services: + - docker:24.0.5-dind + before_script: + - apk add --no-cache make bash python3 ncurses openssh-client-common + - echo "$CI_REGISTRY_PASSWORD" | docker login $CI_REGISTRY -u $CI_REGISTRY_USER --password-stdin + +stages: + - check + - test + - build + - deploy + +variables: + # Use TLS https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#tls-enabled + # DOCKER_HOST: tcp://docker:2376 + DOCKER_TLS_CERTDIR: "/certs" + CONTAINER_BACKEND_RELEASE_IMAGE: $CI_REGISTRY_IMAGE/backend + CONTAINER_FRONTEND_RELEASE_IMAGE: $CI_REGISTRY_IMAGE/frontend + {%- if cookiecutter.devops_cache == '1' %} + CONTAINER_VARNISH_RELEASE_IMAGE: $CI_REGISTRY_IMAGE/varnish + {%- endif %} + +check-backend: + stage: check + before_script: + - apk add --no-cache make bash python3 ncurses openssh-client-common git + script: + - cd backend + - make check + + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + +check-frontend: + image: node:20-alpine + stage: check + before_script: + - apk add --no-cache make bash python3 ncurses openssh-client-common git + - corepack enable + script: + - cd frontend + - make install + - make lint + - make ci-i18n + + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + + +test-backend: + stage: test + image: python:3.11-bookworm + before_script: + - apt update + - apt install build-essential python3 python3-dev git -y + script: + - cd backend + - make install + - make test + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + +test-frontend: + image: node:20-alpine + stage: test + before_script: + - apk add --no-cache make bash python3 ncurses openssh-client-common git + - corepack enable + + script: + - cd frontend + - make install + - make ci-test + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + +build-backend: + stage: build + before_script: + - apk add --no-cache make bash python3 ncurses openssh-client-common + script: + - set -e # Stop if the execution of any command fails + - cd backend + - make build-image + - echo "$CI_REGISTRY_PASSWORD" | docker login $CI_REGISTRY -u $CI_REGISTRY_USER --password-stdin + - docker push $CONTAINER_BACKEND_RELEASE_IMAGE:latest + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + +build-frontend: + stage: build + before_script: + - apk add --no-cache make bash python3 ncurses openssh-client-common + + script: + - set -e # Stop if the execution of any command fails + - cd frontend + - make build-image + - echo "$CI_REGISTRY_PASSWORD" | docker login $CI_REGISTRY -u $CI_REGISTRY_USER --password-stdin + - docker push $CONTAINER_FRONTEND_RELEASE_IMAGE:latest + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + +{%- if cookiecutter.devops_cache == '1' %} +build-varnish: + stage: build + before_script: + - apk add --no-cache make bash python3 ncurses openssh-client-common + script: + - cd devops/varnish + - docker build . -t $CONTAINER_VARNISH_RELEASE_IMAGE:latest + - echo "$CI_REGISTRY_PASSWORD" | docker login $CI_REGISTRY -u $CI_REGISTRY_USER --password-stdin + - docker push $CONTAINER_VARNISH_RELEASE_IMAGE:latest + + rules: + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' +{%- endif %} + +deploy: + stage: deploy + rules: + - if: $CI_COMMIT_REF_NAME == "main" + script: + - mkdir -p ~/.ssh/ + - touch ~/.ssh/known_hosts + - echo "$SSH_KNOWN_HOSTS" >> ~/.ssh/known_hosts + - chmod 644 ~/.ssh/known_hosts + - cd devops + - eval `ssh-agent -s` + - echo "${DEPLOY_SSH_PRIVATE_KEY}" | tr -d '\r' | ssh-add - > /dev/null # add ssh key + - export TERM=dumb + - touch .env + - 'echo "DEPLOY_ENV: ${DEPLOY_ENV}" >> .env' + - 'echo "DEPLOY_HOST: ${DEPLOY_HOST}" >> .env' + - 'echo "DEPLOY_PORT: ${DEPLOY_PORT}" >> .env' + - 'echo "DEPLOY_USER: ${DEPLOY_USER}" >> .env' + - 'echo "DOCKER_CONFIG: ${DOCKER_CONFIG}" >> .env' + - 'echo "STACK_NAME: ${STACK_NAME}" >> .env' + - 'echo "STACK_PARAM: ${CI_COMMIT_TAG}" >> .env' + - make docker-setup + - make stack-deploy + environment: production +# deploy-with-auxiliary-docker-image: +# stage: deploy +# variables: +# REGISTRY: ${CI_REGISTRY} +# USERNAME: ${CI_REGISTRY_USER} +# PASSWORD: ${CI_REGISTRY_PASSWORD} +# REMOTE_HOST: ${DEPLOY_HOST} +# REMOTE_PORT: ${DEPLOY_PORT} +# REMOTE_USER: ${DEPLOY_USER} +# REMOTE_PRIVATE_KEY: "${DEPLOY_SSH_PRIVATE_KEY}" +# STACK_FILE: devops/stacks/${DEPLOY_HOST}.yml +# STACK_NAME: ${STACK_NAME} +# DEPLOY_IMAGE: ghcr.io/kitconcept/docker-stack-deploy:latest +# script: +# - docker pull ${DEPLOY_IMAGE} +# - docker run --rm +# -v "$(pwd)":/github/workspace +# -v /var/run/docker.sock:/var/run/docker.sock +# -e REGISTRY=${REGISTRY} +# -e USERNAME=${USERNAME} +# -e PASSWORD=${PASSWORD} +# -e REMOTE_HOST=${REMOTE_HOST} +# -e REMOTE_PORT=${REMOTE_PORT} +# -e REMOTE_USER=${REMOTE_USER} +# -e REMOTE_PRIVATE_KEY="${REMOTE_PRIVATE_KEY}" +# -e STACK_FILE=${STACK_FILE} +# -e STACK_NAME=${STACK_NAME} +# ${DEPLOY_IMAGE} +# only: +# - main +# environment: production \ No newline at end of file diff --git a/templates/projects/monorepo/{{ cookiecutter.__folder_name }}/devops/README-GITLAB.md b/templates/projects/monorepo/{{ cookiecutter.__folder_name }}/devops/README-GITLAB.md index 570ffe45..f018a4ef 100644 --- a/templates/projects/monorepo/{{ cookiecutter.__folder_name }}/devops/README-GITLAB.md +++ b/templates/projects/monorepo/{{ cookiecutter.__folder_name }}/devops/README-GITLAB.md @@ -4,6 +4,54 @@ Welcome to the DevOps pipelines guide for deploying your project using GitLab pi This README provides step-by-step instructions to set up your GitLab repository and initiate manual deployment pipelines. Follow each step carefully to correctly configure your environment and secrets. +## Runner setup 🛠️ + +If you use your own runners, you'll need to configure them to use the "Docker-in-Docker" method to enable Docker commands for your CI/CD jobs. +You can read about it in the [GitLab Documentation](https://docs.gitlab.com/ci/docker/using_docker_build/#use-docker-in-docker). + +A GitLab runner configuration like this works out of the box. +Perhaps this is not the most secure setup, but it works. + +[Register a runner](https://docs.gitlab.com/runner/register/) in one of your servers. +Then modify its configuration file `/etc/gitlab-runner/config.toml` as needed. +The following is an example configuration file. + +```toml +concurrent = 1 +check_interval = 0 +connection_max_age = "15m0s" +shutdown_timeout = 0 + +[session_server] + session_timeout = 1800 + +[[runners]] + name = "runner1" + url = "https://gitlab.com" + id = + token = "" + token_obtained_at = 2024-09-17T08:51:36Z + token_expires_at = 0001-01-01T00:00:00Z + executor = "docker" + [runners.custom_build_dir] + [runners.cache] + MaxUploadedArchiveSize = 0 + [runners.cache.s3] + [runners.cache.gcs] + [runners.cache.azure] + [runners.docker] + tls_verify = false + image = "docker:24.0.5" + privileged = true + disable_entrypoint_overwrite = false + oom_kill_disable = false + disable_cache = false + volumes = ["/var/run/docker.sock:/var/run/docker.sock", "/certs/client", "/cache"] + shm_size = 0 + network_mtu = 0 +``` + + ## Repository setup 🛠️ See [Get started with GitLab CI/CD](https://docs.gitlab.com/ee/ci/). @@ -17,6 +65,8 @@ See [Get started with GitLab CI/CD](https://docs.gitlab.com/ee/ci/). 4. Expand the `Variables`. 5. Add all variables and their corresponding values, copying them from the `.env` file. This file is not commited into the repository, and that's why we need to add them here. + > [!NOTE] + > Do not mark any of the variables as _protected_. ### Step 2: Add deployment's host SSH key as a known host @@ -35,20 +85,18 @@ See [Use SSH keys to communicate with GitLab](https://docs.gitlab.com/ee/user/ss 3. Copy the private key as a `Variable` like in the previous step. Name the variable `DEPLOY_SSH_PRIVATE_KEY`. -*Note:* instead of creating a new key, you can use the one that this template creates for you when you run `make server-setup`. +> [!NOTE] +> Instead of creating a new key, you can use the one that this template creates for you when you run `make server-setup`. Check the section "Server Setup" in the file `README.md` in the same folder as this file for details. Be aware that the key generated by this template will not be added to your git repository because it is explicitly excluded by the `.gitignore` file. -## Automatic deployment 🚀 - -The deployment is executed automatically on each commit to `main` branch. +## Development and deployment 🚀 -You may want to adapt your development workflow to that, and use a `develop` branch to develop and create merge requests for deployments. +Development should be done in a separate branch, such as `develop`. -## Manual deployment 🚀 +When opening a merge request from your development branch to `main`, CI/CD runs the checks and tests, and, if they pass, then builds the Docker images. -If you want to do manual deployments, you can enable the `when: manual` option of the `.gitlab-ci.yml` file. -By default it is commented. +When merging the merge request, the project will be deployed. ## Deployment with an auxiliary docker image diff --git a/templates/sub/project_settings/{{ cookiecutter.__folder_name }}/backend/Makefile b/templates/sub/project_settings/{{ cookiecutter.__folder_name }}/backend/Makefile index f8f595b5..e56fc0a5 100644 --- a/templates/sub/project_settings/{{ cookiecutter.__folder_name }}/backend/Makefile +++ b/templates/sub/project_settings/{{ cookiecutter.__folder_name }}/backend/Makefile @@ -16,6 +16,7 @@ RESET=`tput sgr0` YELLOW=`tput setaf 3` IMAGE_NAME_PREFIX={{ cookiecutter.__container_image_prefix }} +IMAGE_NAME_SEPARATOR={%- if cookiecutter.container_registry == 'gitlab' %}/{%- else %}-{%- endif %} IMAGE_TAG=latest # Python checks @@ -134,7 +135,7 @@ test-coverage: $(VENV_FOLDER) ## run tests with coverage # Build Docker images .PHONY: build-image build-image: ## Build Docker Images - @docker build . -t $(IMAGE_NAME_PREFIX)-backend:$(IMAGE_TAG) -f Dockerfile --build-arg PLONE_VERSION=$(PLONE_VERSION) + @docker build . -t $(IMAGE_NAME_PREFIX)$(IMAGE_NAME_SEPARATOR)backend:$(IMAGE_TAG) -f Dockerfile --build-arg PLONE_VERSION=$(PLONE_VERSION) # Acceptance tests .PHONY: acceptance-backend-start @@ -143,7 +144,7 @@ acceptance-backend-start: ## Start backend acceptance server .PHONY: acceptance-image-build acceptance-image-build: ## Build Docker Images - @docker build . -t $(IMAGE_NAME_PREFIX)-backend-acceptance:$(IMAGE_TAG) -f Dockerfile.acceptance --build-arg PLONE_VERSION=$(PLONE_VERSION) + @docker build . -t $(IMAGE_NAME_PREFIX)$(IMAGE_NAME_SEPARATOR)backend-acceptance:$(IMAGE_TAG) -f Dockerfile.acceptance --build-arg PLONE_VERSION=$(PLONE_VERSION) ## Add bobtemplates features (check bobtemplates.plone's documentation to get the list of available features) add: $(VENV_FOLDER) diff --git a/templates/sub/project_settings/{{ cookiecutter.__folder_name }}/frontend/Makefile b/templates/sub/project_settings/{{ cookiecutter.__folder_name }}/frontend/Makefile index fcd588d8..c040483e 100644 --- a/templates/sub/project_settings/{{ cookiecutter.__folder_name }}/frontend/Makefile +++ b/templates/sub/project_settings/{{ cookiecutter.__folder_name }}/frontend/Makefile @@ -24,7 +24,11 @@ DOCKER_IMAGE=plone/server-dev:${PLONE_VERSION} DOCKER_IMAGE_ACCEPTANCE=plone/server-acceptance:${PLONE_VERSION} ADDON_NAME='{{ cookiecutter.__npm_package_name }}' +{%- if cookiecutter.container_registry == 'gitlab' %} +IMAGE_NAME={{cookiecutter.__container_image_prefix}}/frontend +{%- else %} IMAGE_NAME={{cookiecutter.__container_image_prefix}}-frontend +{%- endif %} IMAGE_TAG=latest VOLTO_VERSION = $(shell cat ./mrs.developer.json | python -c "import sys, json; print(json.load(sys.stdin)['core']['tag'])")