Feature/SK-1688 | Rework Client

.ci/tests/examples/print_logs.sh

@@ -22,11 +22,23 @@ if [ "$service" == "api-server" ]; then
 fi
 
 if [ "$service" == "combiner" ]; then
-    echo "Reducer logs"
+    echo "Combiner logs"
     docker logs "$(basename $PWD)-combiner-1"
     exit 0
 fi
 
+if [ "$service" == "controller" ]; then
+    echo "Controller logs"
+    docker logs "$(basename $PWD)-controller-1"
+    exit 0
+fi
+
+if [ "$service" == "hooks" ]; then
+    echo "Hooks logs"
+    docker logs "hook"
+    exit 0
+fi
+
 if [ "$service" == "client" ]; then
     echo "Client 0 logs"
     if [ "$example" == "mnist-keras" ]; then

.ci/tests/examples/run.sh

@@ -28,7 +28,7 @@ else
     docker compose \
         -f ../../docker-compose.yaml \
         -f docker-compose.override.yaml \
-        up -d --build combiner api-server mongo minio client1   
+        up -d --build combiner controller api-server hooks mongo minio client1   
 fi
 
 # add server functions to python path to import server functions code
@@ -40,6 +40,9 @@ python ../../.ci/tests/examples/wait_for.py reducer
 >&2 echo "Wait for combiners to connect"
 python ../../.ci/tests/examples/wait_for.py combiners
 
+>&2 echo "Wait for controller to connect"
+python ../../.ci/tests/examples/wait_for.py controller
+
 >&2 echo "Upload compute package"
 python ../../.ci/tests/examples/api_test.py set_package --path package.tgz --helper "$helper" --name test
 
@@ -61,20 +64,6 @@ fi
 >&2 echo "Checking rounds success"
 python ../../.ci/tests/examples/wait_for.py rounds
 
->&2 echo "Test client connection with dowloaded settings"
-# Get config
-python ../../.ci/tests/examples/api_test.py get_client_config --output ../../client.yaml
-
-# Redeploy clients with config
-docker compose \
-    -f ../../docker-compose.yaml \
-    -f docker-compose.override.yaml \
-    -f ../../.ci/tests/examples/compose-client-settings.override.yaml \
-    up -d
-
->&2 echo "Wait for clients to reconnect"
-python ../../.ci/tests/examples/wait_for.py clients
-
 >&2 echo "Test API GET requests"
 python ../../.ci/tests/examples/api_test.py test_api_get_methods
 

.ci/tests/examples/wait_for.py

@@ -63,6 +63,20 @@ def _test_nodes(n_nodes, node_type, reducer_host='localhost', reducer_port='8092
         _eprint(f'Request exception enconuntered: {e}.')
         return False
 
+def _test_controller(reducer_host='localhost', reducer_port='8092'):
+    try:
+        response = requests.get(
+            f'http://{reducer_host}:{reducer_port}/get_controller_status', verify=False)
+
+        if response.status_code == 200:
+            data = json.loads(response.content)
+            _eprint(f'Controller is running: {data}')
+            return True
+
+    except Exception as e:
+        _eprint(f'Request exception encountered: {e}.')
+        return False
+
 
 def rounds(n_rounds=3):
     assert (_retry(_test_rounds, n_rounds=n_rounds))
@@ -79,6 +93,9 @@ def combiners(n_combiners=1):
 def reducer():
     assert (_retry(_test_nodes, n_nodes=1, node_type='reducer'))
 
+def controller():
+    assert (_retry(_test_controller))
+
 
 if __name__ == '__main__':
     fire.Fire()

.github/workflows/build-containers.yaml

@@ -1,22 +1,26 @@
-name: "build containers"
+name: build containers
 
 on:
   workflow_dispatch:
+    inputs:
+      custom_tag:
+        description: "Extra image tag to add (e.g. torchimage)"
+        required: false
+        default: ""
+      extra_pip_packages:
+        description: "Space-separated pip packages to add (e.g. numpy==1.26.4 uvicorn)"
+        required: false
+        default: ""
   push:
-    branches:
-      - master
-      - develop
+    branches: [ master, develop ]
   pull_request:
-    branches:
-      - develop
-      - master
+    branches: [ develop, master ]
   release:
     types: [published]
 
 jobs:
   build-containers:
     runs-on: ubuntu-latest
-
     permissions:
       packages: write
       contents: read
@@ -43,14 +47,14 @@ jobs:
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
             type=sha
+            type=raw,value=${{ inputs.custom_tag }},enable=${{ inputs.custom_tag != '' }}
 
       - name: Log in to GitHub Container Registry
         uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-
 
       - name: Build and push
         uses: docker/build-push-action@v6
@@ -60,3 +64,5 @@ jobs:
           tags: ${{ steps.meta1.outputs.tags }}
           labels: ${{ steps.meta1.outputs.labels }}
           file: Dockerfile
+          build-args: |
+            EXTRA_PIP_PACKAGES=${{ inputs.extra_pip_packages }}

.github/workflows/code-checks.yaml

@@ -26,7 +26,9 @@ jobs:
           --exclude-dir='flower-client'
           --exclude='tests.py'
           --exclude='controller_cmd.py'
+          --exclude='api_server_cmd.py'
           --exclude='fedn_pb2_grpc.py'
+          --exclude='fedn_pb2.pyi'
           --exclude='combiner_cmd.py'
           --exclude='run_cmd.py'
           --exclude='README.rst'

.github/workflows/integration-tests.yaml

@@ -15,6 +15,7 @@ on:
       - '.github/**'
     branches:
       - "**"
+  workflow_dispatch:
 
 jobs:
   integration-tests:
@@ -50,7 +51,15 @@ jobs:
       - name: print logs combiner
         if: failure()
         run: .ci/tests/examples/print_logs.sh combiner ${{ matrix.to_test }}
+
+      - name: print logs controller
+        if: failure()
+        run: .ci/tests/examples/print_logs.sh controller ${{ matrix.to_test }}
 
+      - name: print logs hooks
+        if: failure()
+        run: .ci/tests/examples/print_logs.sh hooks ${{ matrix.to_test }} 
+
       - name: print logs client
         if: failure()
         run: .ci/tests/examples/print_logs.sh client ${{ matrix.to_test }}
@@ -62,3 +71,5 @@ jobs:
       - name: print logs minio
         if: failure()
         run: .ci/tests/examples/print_logs.sh minio ${{ matrix.to_test }}
+
+

.gitignore

@@ -175,8 +175,8 @@ config/settings-combiner.yaml
 config/extra-hosts-client.yaml
 config/extra-hosts-reducer.yaml
 config/settings-client.yaml
-config/settings-reducer.yaml
-config/settings-combiner.yaml
+config/settings-api-server.yaml
+config/settings-controller.yaml
 config/settings-hooks.yaml
 
 ./tmp/*

.readthedocs.yaml

@@ -7,11 +7,16 @@ build:
   jobs:
     pre_build:
      - sphinx-apidoc --ext-autodoc --module-first -o docs fedn ./*tests* ./fedn/cli* ./fedn/common* ./fedn/network/api/v1* ./fedn/network/grpc/fedn_pb2.py ./fedn/network/grpc/fedn_pb2_grpc.py  ./fedn/network/api/server.py ./fedn/network/controller/controlbase.py
+
 sphinx:
    configuration: docs/conf.py
+   fail_on_warning: false
 
 python:
   install:
     - method: pip
       path: .
     - requirements: docs/requirements.txt
+
+# Ensure documentation is publicly accessible
+# Make sure this is not set to private in RTD settings

Dockerfile

@@ -5,7 +5,7 @@ FROM $BASE_IMG AS builder
 ARG GRPC_HEALTH_PROBE_VERSION=""
 ARG REQUIREMENTS=""
 
-ARG INSTALL_TORCH=0
+ARG EXTRA_PIP_PACKAGES=""
 
 WORKDIR /build
 
@@ -36,9 +36,12 @@ RUN python -m venv /venv \
     fi \
     && rm -rf /build/requirements.txt
 
-# only install torch when asked
-RUN if [ "$INSTALL_TORCH" = "1" ]; then /venv/bin/pip install torch; fi
-
+RUN if [ -n "$EXTRA_PIP_PACKAGES" ]; then \
+    echo "Installing extra pip packages: $EXTRA_PIP_PACKAGES" && \
+    /venv/bin/pip install --no-cache-dir $EXTRA_PIP_PACKAGES; \
+    else \
+    echo "No EXTRA_PIP_PACKAGES provided"; \
+    fi
 
 # Install grpc health probe
 RUN if [ ! -z "$GRPC_HEALTH_PROBE_VERSION" ]; then \
@@ -79,4 +82,3 @@ RUN set -ex \
 USER appuser
 
 ENTRYPOINT [ "/venv/bin/fedn" ]
-

Dockerfile.dev

@@ -12,7 +12,8 @@ COPY . /app
 COPY config/settings-client.yaml.template /app/config/settings-client.yaml
 COPY config/settings-combiner.yaml.template /app/config/settings-combiner.yaml
 COPY config/settings-hooks.yaml.template /app/config/settings-hooks.yaml
-COPY config/settings-reducer.yaml.template /app/config/settings-reducer.yaml
+COPY config/settings-api-server.yaml.template /app/config/settings-api-server.yaml
+COPY config/settings-controller.yaml.template /app/config/settings-controller.yaml
 COPY $REQUIREMENTS /app/config/requirements.txt
 
 # Install developer tools (needed for psutil)

config/reducer-settings.override.yaml

@@ -6,4 +6,4 @@ services:
   reducer:
     volumes:
       - ${HOST_REPO_DIR:-.}:/app
-      - ${HOST_REPO_DIR:-.}/config/settings-reducer.yaml:/app/config/settings-reducer.yaml
+      - ${HOST_REPO_DIR:-.}/config/settings-api-server.yaml:/app/config/settings-api-server.yaml

config/settings-reducer.yaml.template → config/settings-api-server.yaml.template

@@ -1,9 +1,13 @@
 network_id: fedn-network
-controller:
+api:
   host: api-server
   port: 8092
   debug: True
 
+controller:
+  host: controller
+  port: 12090
+
 statestore:
   # Available DB types are MongoDB, PostgreSQL, SQLite
   type: MongoDB

config/settings-controller.yaml.local.template

@@ -1,5 +1,5 @@
 network_id: fedn-network
-controller:
+api:
   host: localhost
   port: 8092
   debug: True

config/settings-controller.yaml.template

@@ -0,0 +1,35 @@
+network_id: fedn-network
+
+api:
+  host: api-server
+  port: 8092
+
+controller:
+  host: controller
+  port: 12090
+  debug: True
+
+statestore:
+  # Available DB types are MongoDB, PostgreSQL, SQLite
+  type: MongoDB
+  mongo_config:
+    username: fedn_admin
+    password: password
+    host: mongo
+    port: 6534
+  postgres_config:
+    username: fedn_admin
+    password: password
+    host: fedn_postgres
+    port: 5432
+
+storage:
+  storage_type: BOTO3
+  storage_config:
+    storage_endpoint_url: http://minio:9000
+    storage_access_key: fedn_admin
+    storage_secret_key: password
+    storage_bucket: fedn-models
+    context_bucket: fedn-context
+    storage_secure_mode: False
+    storage_verify_ssl: False

docker-compose.dev.yaml

@@ -67,8 +67,8 @@ services:
       - USER=test
       - PROJECT=project
       - FLASK_DEBUG=1
-      - STATESTORE_CONFIG=/app/config/settings-reducer.yaml.template
-      - MODELSTORAGE_CONFIG=/app/config/settings-reducer.yaml.template
+      - STATESTORE_CONFIG=/app/config/settings-api-server.yaml.template
+      - MODELSTORAGE_CONFIG=/app/config/settings-api-server.yaml.template
       - FEDN_COMPUTE_PACKAGE_DIR=/app
       - TMPDIR=/app/tmp
     build:
@@ -84,11 +84,41 @@ services:
       - mongo
       - fedn_postgres
     command:
-      - controller
+      - api-server
       - start
     ports:
       - 8092:8092
 
+  controller:
+    environment:
+      - PYTHONUNBUFFERED=0
+      - GET_HOSTS_FROM=dns
+      - STATESTORE_CONFIG=/app/config/settings-controller.yaml.template
+      - MODELSTORAGE_CONFIG=/app/config/settings-controller.yaml.template
+      - TMPDIR=/app/tmp
+    build:
+      context: .
+      args:
+        BASE_IMG: ${BASE_IMG:-python:3.12-slim}
+    working_dir: /app
+    volumes:
+      - ${HOST_REPO_DIR:-.}/fedn:/app/fedn
+    healthcheck:
+      test: [ "CMD", "/app/grpc_health_probe", "-addr=localhost:12090" ]
+      interval: 20s
+      timeout: 10s
+      retries: 5
+    depends_on:
+      - minio
+      - mongo
+    command:
+      - controller
+      - start
+      - --init
+      - config/settings-controller.yaml.template
+    ports:
+      - 12090:12090
+
   # Combiner
   combiner:
     environment: