diff --git a/.dockerignore b/.dockerignore index d17d343a4..22e802da8 100644 --- a/.dockerignore +++ b/.dockerignore @@ -11,4 +11,5 @@ examples/* **/*.npz **/data **/*.tgz -dist \ No newline at end of file +dist +Dockerfile \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 348beb747..6e226d1e2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,77 +1,36 @@ -# Stage 1: Builder -ARG BASE_IMG=python:3.12-slim -FROM $BASE_IMG AS builder +# ARG BASE_IMG=python:3.11-slim +ARG RUNTIME_IMG=gcr.io/distroless/python3 +# FROM $BASE_IMG AS builder +FROM python:3.11-slim AS builder ARG GRPC_HEALTH_PROBE_VERSION="" -ARG REQUIREMENTS="" WORKDIR /build -# Temporarily add the Debian Testing repository to install zlib1g 1:1.3.dfsg+really1.3.1-1+b1 (fixed CVE-2023-45853) -# Both zlib1g and zlib1g-dev are installed in the builder stage. -RUN echo "deb http://deb.debian.org/debian testing main" > /etc/apt/sources.list.d/testing.list \ - && apt-get update \ - && apt-get install -y --no-install-recommends -t testing zlib1g=1:1.3.dfsg+really1.3.1-1+b1 zlib1g-dev=1:1.3.dfsg+really1.3.1-1+b1 \ - && rm -rf /etc/apt/sources.list.d/testing.list \ - && apt-get clean \ +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3-dev gcc wget zlib1g-dev \ && rm -rf /var/lib/apt/lists/* -# Install build dependencies -RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends python3-dev gcc wget \ - && rm -rf /var/lib/apt/lists/* - -# Add FEDn and default configs -COPY . /build -COPY $REQUIREMENTS /build/requirements.txt - -# Install dependencies -RUN python -m venv /venv \ - && /venv/bin/pip install --upgrade pip \ - && /venv/bin/pip install --no-cache-dir 'setuptools>=65' \ - && /venv/bin/pip install --no-cache-dir . \ - && if [[ ! -z "$REQUIREMENTS" ]]; then \ - /venv/bin/pip install --no-cache-dir -r /build/requirements.txt; \ - fi \ - && rm -rf /build/requirements.txt - - # Install grpc health probe RUN if [ ! -z "$GRPC_HEALTH_PROBE_VERSION" ]; then \ wget -qO /build/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ chmod +x /build/grpc_health_probe; \ fi -# Stage 2: Runtime -FROM $BASE_IMG +COPY . /build -WORKDIR /app +RUN mkdir /python-dist \ + && pip install --upgrade pip \ + && pip install --prefix=/python-dist --no-cache-dir 'setuptools>=65' . -# Copy application and venv from the builder stage -COPY --from=builder /venv /venv -COPY --from=builder /build /app +# Stage 2: Distroless Runtime -# Use a non-root user -RUN set -ex \ - # Create a non-root user - && addgroup --system --gid 1001 appgroup \ - && adduser --system --uid 1001 --gid 1001 --no-create-home appuser \ - # Creare application specific tmp directory, set ENV TMPDIR to /app/tmp - && mkdir -p /app/tmp \ - && chown -R appuser:appgroup /venv /app \ - # Temporarily add the Debian Testing repository to install zlib1g 1:1.3.dfsg+really1.3.1-1+b1 (fixed CVE-2023-45853) - && echo "deb http://deb.debian.org/debian testing main" > /etc/apt/sources.list.d/testing.list \ - && apt-get update \ - && apt-get install -y --no-install-recommends -t testing zlib1g=1:1.3.dfsg+really1.3.1-1+b1 \ - && rm -rf /etc/apt/sources.list.d/testing.list \ - # Update package index and upgrade all installed packages - && apt-get update \ - && apt-get upgrade -y \ - # Clean up - && apt-get autoremove -y \ - && apt-get clean -y \ - && rm -rf /var/lib/apt/lists/* +FROM $RUNTIME_IMG + +COPY --from=builder /python-dist /python-dist +COPY --from=builder /build /app -USER appuser +ENV PYTHONPATH=/python-dist/lib/python3.11/site-packages -ENTRYPOINT [ "/venv/bin/fedn" ] +ENTRYPOINT ["python3", "/python-dist/bin/fedn"] diff --git a/docker-compose.yaml b/docker-compose.yaml index 598aad0cb..1700f624e 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -109,7 +109,7 @@ services: retries: 5 depends_on: - api-server - - hooks + # - hooks # Hooks hooks: container_name: hook @@ -124,9 +124,9 @@ services: working_dir: /app volumes: - ${HOST_REPO_DIR:-.}/fedn:/app/fedn - entrypoint: [ "sh", "-c" ] command: - - "/venv/bin/pip install --no-cache-dir -e . && /venv/bin/fedn hooks start" + - hooks + - start ports: - 12081:12081 healthcheck: @@ -143,7 +143,8 @@ services: build: context: . args: - BASE_IMG: ${BASE_IMG:-python:3.10-slim} + BASE_IMG: ${BASE_IMG:-python:3.11-slim} + RUNTIME_IMG: ${RUNTIME_IMG:-python:3.11-slim} working_dir: /app volumes: - ${HOST_REPO_DIR:-.}/fedn:/app/fedn diff --git a/fedn/network/api/gunicorn_app.py b/fedn/network/api/gunicorn_app.py index 8c5cb0c30..d249f716c 100644 --- a/fedn/network/api/gunicorn_app.py +++ b/fedn/network/api/gunicorn_app.py @@ -1,4 +1,6 @@ from gunicorn.app.base import BaseApplication + + class GunicornApp(BaseApplication): def __init__(self, app, options=None): self.options = options or {} @@ -14,10 +16,11 @@ def load_config(self): def load(self): return self.application -def run_gunicorn(app, host,port,workers=4): +def run_gunicorn(app, host, port, workers=4): bind_address = f"{host}:{port}" options = { "bind": bind_address, # Specify the bind address and port here "workers": workers, + "preload_app": True } GunicornApp(app, options).run() diff --git a/fedn/network/api/server.py b/fedn/network/api/server.py index 834797aab..b4eab0373 100644 --- a/fedn/network/api/server.py +++ b/fedn/network/api/server.py @@ -487,7 +487,7 @@ def start_server_api(): if debug: app.run(debug=debug, port=port, host=host) else: - workers = os.cpu_count() + workers = config.get("num_workers", 2) gunicorn_app.run_gunicorn(app, host, port, workers) diff --git a/fedn/network/storage/s3/saasrepository.py b/fedn/network/storage/s3/saasrepository.py index 9a734a831..8bb32bee2 100644 --- a/fedn/network/storage/s3/saasrepository.py +++ b/fedn/network/storage/s3/saasrepository.py @@ -153,16 +153,5 @@ def delete_artifact(self, instance_name: str, bucket: str) -> None: logger.error(f"Could not delete artifact: {instance_name}. Error: {err}") def create_bucket(self, bucket_name: str) -> None: - """Create a new bucket. If bucket exists, do nothing. - - :param bucket_name: The name of the bucket - :type bucket_name: str - """ - logger.info(f"Creating bucket: {bucket_name}") - - try: - if not self.client.bucket_exists(bucket_name): - self.client.make_bucket(bucket_name) - except InvalidResponseError as err: - logger.error(f"Failed to create bucket: {bucket_name}. Error: {err}") - raise + # No need to create any buckets in the SaaS. + pass