Improvements and cleanup

Author: rimi-itk

.github/workflows/pr.yaml

@@ -8,7 +8,7 @@ jobs:
       fail-fast: false
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           fetch-depth: 2
 
@@ -23,10 +23,10 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        php-versions: [ '8.2' ]
+        php-versions: [ '8.1', '8.2', '8.3' ]
         dependency-version: [ prefer-stable ]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Setup PHP, with composer and extensions
         uses: shivammathur/setup-php@v2
         with:
@@ -55,10 +55,10 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        php-versions: [ '8.2' ]
+        php-versions: [ '8.1', '8.2', '8.3' ]
         dependency-version: [ prefer-stable ]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Setup PHP, with composer and extensions
         uses: shivammathur/setup-php@v2
         with:
@@ -83,40 +83,26 @@ jobs:
         run: |
           composer coding-standards-check
 
-  markdownlint:
+  markdown-coding-standards:
+    name: Markdown coding standards
     runs-on: ubuntu-latest
-    name: markdownlint
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
-      - name: Get yarn cache directory path
-        id: yarn-cache-dir-path
-        run: echo "::set-output name=dir::$(yarn cache dir)"
-      - name: Cache yarn packages
-        uses: actions/cache@v2
-        id: yarn-cache
-        with:
-          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-yarn-
-      - name: Yarn install
-        uses: actions/setup-node@v2
-        with:
-          node-version: '18'
-      - run: yarn install
-      - name: markdownlint
-        run: yarn coding-standards-check/markdownlint
+        uses: actions/checkout@v4
+
+      - name: Coding standards
+        run: |
+          docker run --rm --volume "$PWD":/md peterdavehello/markdownlint markdownlint --ignore vendor --ignore LICENSE.md '**/*.md'
 
   code-analysis:
     name: PHP - Code analysis
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        php-versions: [ '8.2' ]
+        php-versions: [ '8.1', '8.2', '8.3' ]
         dependency-version: [ prefer-stable ]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Setup PHP, with composer and extensions
         uses: shivammathur/setup-php@v2
         with:
@@ -140,4 +126,4 @@ jobs:
           composer install --no-interaction --no-progress --no-scripts
       - name: code-analysis
         run: |
-          composer code-analysis
+          composer code-analysis

.markdownlint.jsonc

@@ -0,0 +1,13 @@
+{
+  "default": true,
+  // https://github.com/DavidAnson/markdownlint/blob/main/doc/md013.md
+  "line-length": {
+    "line_length": 120,
+    "code_blocks": false,
+    "tables": false
+  },
+  // https://github.com/DavidAnson/markdownlint/blob/main/doc/md024.md
+  "no-duplicate-heading": {
+    "siblings_only": true
+  }
+}

.markdownlintrc

@@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+* [PR-19](https://github.com/itk-dev/serviceplatformen_organisation_api/pull/19)
+  Multiple databases
+
 ## [1.0.3] 2024-02-29
 
 * Restore to default monolog config.

CHANGELOG.md

@@ -2,7 +2,7 @@
 
 ## About the project
 
-Sets up an API with data from [Serviceplatformen Organisation](https://digitaliseringskataloget.dk/integration/sf1500).
+Sets up an API with data from [Serviceplatformen Organisation][sf1500].
 
 ### Built with
 
@@ -119,14 +119,68 @@ for help run
 docker compose exec phpfpm bin/console organisation:fetch:data --help
 ```
 
-**To avoid issues with memory leaks during development add the
-`--no-debug` flag to the fetch data command.** You may also want to
+__To avoid issues with memory leaks during development add the
+`--no-debug` flag to the fetch data command.__ You may also want to
 add the verbose flag to see progress.
 
 ```sh
 docker compose exec phpfpm bin/console --no-debug organisation:fetch:data -vvv
 ```
 
+## Multi-database setup
+
+Fetching all data from [SF1500] can be unstable, so for production we use a multi-database setup; one database is
+*current* and the other is used to fetch data. If all data is successfully fetched into ‘the other“ database, it will
+become ‘the current“ database.
+
+Data is fetched by running `bin/fetch-data`, but first a little configuration must be made:
+
+```sh
+# Copy the example config
+cp bin/fetch-data.config.example bin/fetch-data.config
+```
+
+Edit `bin/fetch-data.config` to match your; the most important (and only required) config variable is `database_urls`.
+
+When the configuration is in place, run
+
+``` sh
+bin/fetch-data
+```
+
+to fetch all data into the first database, i.e. the first entry in `database_urls`. If the command succeeds, the first
+database will become the current database. If you run the command again, data will be fetched into the second database,
+and if the fetching succeeds, the second database will become the current database.
+
+For testing that the switching of databases actually work, set
+
+```sh
+# bin/fetch-data.config
+max=10
+```
+
+in `bin/fetch-data.config` to fetch at most 10 items of each type.
+
+To test that switching databases does not accur in case of an error during fetching, you can define a bogus database URL
+before running `bin/fetch-data`:
+
+```sh
+# bin/fetch-data.config
+database_urls=(
+    'mysql://db:[email protected]:3306/db'
+)
+```
+
+If nedd be, you can define which [compose command](https://docs.docker.com/compose/compose-application-model/#cli) (the
+default being `docker compose`) is actually used to run the compose setup:
+
+```sh
+# bin/fetch-data.config
+compose=my-custom-compose
+```
+
+See [`bin/fetch-data.config.example`](bin/fetch-data.config.example) for details on configuration variables.
+
 ## API
 
 See
@@ -209,8 +263,8 @@ we decided to adhere to in this project.
 * Markdown files (markdownlint standard rules)
 
    ```sh
-   docker compose run --rm node yarn install
-   docker compose run --rm node yarn coding-standards-check
+   docker run --rm --volume "$PWD:/md" peterdavehello/markdownlint markdownlint --ignore LICENSE.md --ignore vendor/ '**/*.md' --fix
+   docker run --rm --volume "$PWD:/md" peterdavehello/markdownlint markdownlint --ignore LICENSE.md --ignore vendor/ '**/*.md'
    ```
 
 * Shell scripts
@@ -244,3 +298,5 @@ psalm
 ```sh
 docker compose exec phpfpm composer code-analysis
 ```
+
+[sf1500]: https://digitaliseringskataloget.dk/integration/sf1500

README.md

@@ -20,7 +20,7 @@ source "$config_file"
 
 if [ -z "${database_urls:-}" ]; then
     cat <<EOF >&2
-Missing
+Missing or empty
 
 database_urls=(…)
 
@@ -31,13 +31,6 @@ EOF
     exit 1
 fi
 
-# https://www.php.net/manual/en/reserved.constants.php#constant.php-int-max
-max=${max:-9223372036854775807}
-page_size=${page_size:-1000}
-
-echo max: "$max"
-echo page_size: "$page_size"
-
 function compose() {
     if [[ -n "${compose_script:-}" ]]; then
        "$compose_script" "$@"
@@ -64,12 +57,22 @@ echo current_index: "$current_index"
 echo next_database_url: "$next_database_url"
 echo next_index: "$next_index"
 
+# https://www.php.net/manual/en/reserved.constants.php#constant.php-int-max
+max=${max:-9223372036854775807}
+page_size=${page_size:-1000}
+
+echo max: "$max"
+echo page_size: "$page_size"
+
+# Fetch data into "next" database
+# Cf.
+compose exec --env DATABASE_URL="$next_database_url" phpfpm bin/console doctrine:migrations:migrate --no-interaction first
 compose exec --env DATABASE_URL="$next_database_url" phpfpm bin/console doctrine:migrations:migrate --no-interaction
 compose exec --env DATABASE_URL="$next_database_url" phpfpm bin/console --no-debug organisation:fetch:data --max="$max" --page-size="$page_size" -vvv
 
-echo "DATABASE_URL='$next_database_url'" | tee >| .env.local.database
+echo "Data fetched into $next_database_url"
 
+# Switch to next database.
+echo "DATABASE_URL='$next_database_url'" | tee >| .env.local.database
 # Re-up to read in new database URL.
 compose up --detach
-
-echo 'SUCCESS!'

bin/fetch-data

@@ -1,9 +1,13 @@
 # Database URLs
-# database_urls=(
-# )
+database_urls=(
+    # Databases defined in ../docker-compose.override.yml
+    'mysql://db:db@database-0:3306/db?serverVersion=10.11.2-MariaDB&charset=utf8mb4'
+    'mysql://db:db@database-1:3306/db?serverVersion=10.11.2-MariaDB&charset=utf8mb4'
+)
 
-# Optional compose script path
-# compose_script=
+# Optional path to compose script. The path is relative to the directory of this file.
+# compose_script=docker compose
 
+# Options for bin/console organisation:fetch:data
 # max=10
 # page_size=5

bin/fetch-data.config.example

@@ -4,12 +4,6 @@ services:
     # which may already have a nginx container.
     container_name: organisation_api
 
-  node:
-    image: node:18
-    volumes:
-      - .:/app:delegated
-    working_dir: /app
-
   phpfpm:
     # https://docs.docker.com/compose/compose-file/05-services/#env_file
     env_file: