WIP Uses one Compose service per component

jbaptperez · jbaptperez · commit 4f1abfbdf54a · 2025-11-11T23:03:01.000+01:00
diff --git a/contrib/docker/dev/.env.template b/contrib/docker/dev/.env.template
@@ -11,6 +11,9 @@ TIMESKETCH_CONF_DIR="/etc/timesketch"
 TIMESKETCH_SECRET_KEY="L4np0jV3yAdAFdbVzWRMaBqiFMV8FKYd+Je1WKE40o8="
 TIMESKETCH_USER="dev"
 TIMESKETCH_PASSWORD="dev"
+TIMESKETCH_USER_NAME="timesketch"
+TIMESKETCH_USER_UID="1000"
+TIMESKETCH_USER_GID="1000"
 
 POSTGRES_USER="timesketch"
 POSTGRES_PASSWORD="password"
diff --git a/contrib/docker/dev/compose.yaml b/contrib/docker/dev/compose.yaml
@@ -10,13 +10,105 @@ volumes:
   prometheus-data:
 
 services:
-  timesketch:
-    image: timesketch-timesketch/dev:latest
+  setup:
+    image: timesketch-setup:latest
     build:
       context: ../../..
       dockerfile: contrib/docker/dev/timesketch/Dockerfile
+      target: setup
       args:
         BASE_IMAGE: "${TIMESKETCH_BASE_IMAGE:?}"
+        TIMESKETCH_USER_NAME: "${TIMESKETCH_USER_NAME:?}"
+        TIMESKETCH_USER_UID: "${TIMESKETCH_USER_UID:?}"
+        TIMESKETCH_USER_GID: "${TIMESKETCH_USER_GID:?}"
+        TIMESKETCH_CONF_DIR: "${TIMESKETCH_CONF_DIR:?}"
+        GIFT_PPA_TRACK: "${GIFT_PPA_TRACK:?}"
+        GIFT_PPA_URL: "${GIFT_PPA_URL:?}"
+        NODE_VERSION: "${NODE_VERSION:?}"
+        NODE_PPA_URL: "${NODE_PPA_URL:?}"
+        NODE_NPMRC: "${NODE_NPMRC?}"
+        YARN_YARNRC: "${YARN_YARNRC?}"
+        PYTHON_PIP_CONF: "${PYTHON_PIP_CONF?}"
+    command: timesketch
+    env_file:
+      - timesketch/timesketch.env
+    volumes:
+      - "../../../:/usr/local/src/timesketch/"
+      - "./timesketch/timesketch.conf:${TIMESKETCH_CONF_DIR:?}/timesketch.conf:ro"
+      - "./timesketch/sigma_rules.txt:${TIMESKETCH_CONF_DIR:?}/sigma_rules.txt:ro"
+      - "../../../data/sigma_config.yaml:${TIMESKETCH_CONF_DIR:?}/sigma_config.yaml:ro"
+      - "../../../data/sigma:${TIMESKETCH_CONF_DIR:?}/sigma:ro"
+    depends_on:
+      postgresql:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    networks:
+      - timesketch-dev
+
+  celery-worker:
+    image: timesketch-celery-worker:latest
+    build:
+      context: ../../..
+      dockerfile: contrib/docker/dev/timesketch/Dockerfile
+      target: celery-worker
+      args:
+        BASE_IMAGE: "${TIMESKETCH_BASE_IMAGE:?}"
+        TIMESKETCH_USER_NAME: "${TIMESKETCH_USER_NAME:?}"
+        TIMESKETCH_USER_UID: "${TIMESKETCH_USER_UID:?}"
+        TIMESKETCH_USER_GID: "${TIMESKETCH_USER_GID:?}"
+        TIMESKETCH_CONF_DIR: "${TIMESKETCH_CONF_DIR:?}"
+        GIFT_PPA_TRACK: "${GIFT_PPA_TRACK:?}"
+        GIFT_PPA_URL: "${GIFT_PPA_URL:?}"
+        NODE_VERSION: "${NODE_VERSION:?}"
+        NODE_PPA_URL: "${NODE_PPA_URL:?}"
+        NODE_NPMRC: "${NODE_NPMRC?}"
+        YARN_YARNRC: "${YARN_YARNRC?}"
+        PYTHON_PIP_CONF: "${PYTHON_PIP_CONF?}"
+    command: timesketch
+    env_file:
+      - timesketch/timesketch.env
+    volumes:
+      - "../../../:/usr/local/src/timesketch/"
+      - "./timesketch/timesketch.conf:${TIMESKETCH_CONF_DIR:?}/timesketch.conf:ro"
+      - "../../../data/regex_features.yaml:${TIMESKETCH_CONF_DIR:?}/regex_features.yaml:ro"
+      - "../../../data/winevt_features.yaml:${TIMESKETCH_CONF_DIR:?}/winevt_features.yaml:ro"
+      - "../../../data/tags.yaml:${TIMESKETCH_CONF_DIR:?}/tags.yaml:ro"
+      - "../../../data/intelligence_tag_metadata.yaml:${TIMESKETCH_CONF_DIR:?}/intelligence_tag_metadata.yaml:ro"
+      - "../../../data/plaso.mappings:${TIMESKETCH_CONF_DIR:?}/plaso.mappings:ro"
+      - "../../../data/generic.mappings:${TIMESKETCH_CONF_DIR:?}/generic.mappings:ro"
+      - "../../../data/ontology.yaml:${TIMESKETCH_CONF_DIR:?}/ontology.yaml:ro"
+      - "../../../data/data_finder.yaml:${TIMESKETCH_CONF_DIR:?}/data_finder.yaml:ro"
+      - "../../../data/bigquery_matcher.yaml:${TIMESKETCH_CONF_DIR:?}/bigquery_matcher.yaml:ro"
+      - "../../../data/sigma_config.yaml:${TIMESKETCH_CONF_DIR:?}/sigma_config.yaml:ro"
+      - "../../../data/sigma:${TIMESKETCH_CONF_DIR:?}/sigma:ro"
+      - "../../../data/dfiq:${TIMESKETCH_CONF_DIR:?}/dfiq:ro"
+      - "../../../data/context_links.yaml:${TIMESKETCH_CONF_DIR:?}/context_links.yaml:ro"
+      - "../../../data/plaso_formatters.yaml:${TIMESKETCH_CONF_DIR:?}/plaso_formatters.yaml:ro"
+      - "../../../data/nl2q:${TIMESKETCH_CONF_DIR:?}/nl2q:ro"
+      - "../../../data/llm_summarize:${TIMESKETCH_CONF_DIR:?}/llm_summarize:ro"
+    depends_on:
+      postgresql:
+        condition: service_healthy
+      opensearch:
+        condition: service_started
+      redis:
+        condition: service_healthy
+    networks:
+      - timesketch-dev
+
+  gunicorn:
+    image: timesketch-gunicorn:latest
+    build:
+      context: ../../..
+      dockerfile: contrib/docker/dev/timesketch/Dockerfile
+      target: gunicorn
+      args:
+        BASE_IMAGE: "${TIMESKETCH_BASE_IMAGE:?}"
+        TIMESKETCH_USER_NAME: "${TIMESKETCH_USER_NAME:?}"
+        TIMESKETCH_USER_UID: "${TIMESKETCH_USER_UID:?}"
+        TIMESKETCH_USER_GID: "${TIMESKETCH_USER_GID:?}"
+        TIMESKETCH_CONF_DIR: "${TIMESKETCH_CONF_DIR:?}"
         GIFT_PPA_TRACK: "${GIFT_PPA_TRACK:?}"
         GIFT_PPA_URL: "${GIFT_PPA_URL:?}"
         NODE_VERSION: "${NODE_VERSION:?}"
@@ -29,9 +121,6 @@ services:
       - name: gunicorn
         published: "5000"
         target: 5000
-      - name: vite
-        published: "5001"
-        target: 5001
       - name: metrics
         published: "8080"
         target: 8080
@@ -40,7 +129,6 @@ services:
     volumes:
       - "../../../:/usr/local/src/timesketch/"
       - "./timesketch/timesketch.conf:${TIMESKETCH_CONF_DIR:?}/timesketch.conf:ro"
-      - "./timesketch/sigma_rules.txt:${TIMESKETCH_CONF_DIR:?}/sigma_rules.txt:ro"
       - "../../../data/regex_features.yaml:${TIMESKETCH_CONF_DIR:?}/regex_features.yaml:ro"
       - "../../../data/winevt_features.yaml:${TIMESKETCH_CONF_DIR:?}/winevt_features.yaml:ro"
       - "../../../data/tags.yaml:${TIMESKETCH_CONF_DIR:?}/tags.yaml:ro"
@@ -61,9 +149,49 @@ services:
       postgresql:
         condition: service_healthy
       opensearch:
+        condition: service_started
+      redis:
         condition: service_healthy
+    networks:
+      - timesketch-dev
+
+  vue-cli-service:
+    image: timesketch-vue-cli-service:latest
+    build:
+      context: ../../..
+      dockerfile: contrib/docker/dev/timesketch/Dockerfile
+      target: vue-cli-service
+      args:
+        BASE_IMAGE: "${TIMESKETCH_BASE_IMAGE:?}"
+        TIMESKETCH_USER_NAME: "${TIMESKETCH_USER_NAME:?}"
+        TIMESKETCH_USER_UID: "${TIMESKETCH_USER_UID:?}"
+        TIMESKETCH_USER_GID: "${TIMESKETCH_USER_GID:?}"
+        TIMESKETCH_CONF_DIR: "${TIMESKETCH_CONF_DIR:?}"
+        GIFT_PPA_TRACK: "${GIFT_PPA_TRACK:?}"
+        GIFT_PPA_URL: "${GIFT_PPA_URL:?}"
+        NODE_VERSION: "${NODE_VERSION:?}"
+        NODE_PPA_URL: "${NODE_PPA_URL:?}"
+        NODE_NPMRC: "${NODE_NPMRC?}"
+        YARN_YARNRC: "${YARN_YARNRC?}"
+        PYTHON_PIP_CONF: "${PYTHON_PIP_CONF?}"
+    command: timesketch
+    ports:
+      - name: vue-cli-service
+        published: "5001"
+        target: 5001
+    env_file:
+      - timesketch/timesketch.env
+    volumes:
+      - "../../../:/usr/local/src/timesketch/"
+    depends_on:
+      postgresql:
+        condition: service_healthy
+      opensearch:
+        condition: service_started
       redis:
         condition: service_healthy
+      gunicorn:
+        condition: service_healthy
     networks:
       - timesketch-dev
 
@@ -129,7 +257,8 @@ services:
 
     command: --config.file=/etc/prometheus/prometheus.yml
     depends_on:
-      - timesketch
+      gunicorn:
+        condition: service_healthy
     networks:
       - timesketch-dev
 
@@ -149,6 +278,6 @@ services:
       - "/tmp/:/usr/local/src/picadata/"
     depends_on:
       opensearch:
-        condition: service_healthy
+        condition: service_started
     networks:
       - timesketch-dev
diff --git a/contrib/docker/dev/timesketch/Dockerfile b/contrib/docker/dev/timesketch/Dockerfile
@@ -1,12 +1,33 @@
 # Use the official Docker Hub Ubuntu base image
 ARG BASE_IMAGE="ubuntu:24.04"
-FROM $BASE_IMAGE
+FROM ${BASE_IMAGE} AS common
 
 # Prevent needing to configure debian packages, stopping the setup of
 # the docker container.
 RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
 
-RUN apt-get update && apt-get install -y --no-install-recommends \
+ARG TIMESKETCH_USER_NAME="timesketch"
+ARG TIMESKETCH_USER_UID="1000"
+ARG TIMESKETCH_USER_GID="1000"
+ARG TIMESKETCH_CONF_DIR="/etc/timesketch"
+RUN if ! id -u "${TIMESKETCH_USER_UID}" &>/dev/null; then \
+    echo "Creating user ${USER_NAME} (${TIMESKETCH_USER_UID}:${TIMESKETCH_USER_GID})..."; \
+      if getent group "${TIMESKETCH_USER_GID}" >/dev/null; then \
+        echo "Group with GID ${TIMESKETCH_USER_GID} already exists."; \
+      else \
+        echo "Creating group ${TIMESKETCH_USER_NAME} with GID ${TIMESKETCH_USER_GID}..."; \
+        groupadd -g "${TIMESKETCH_USER_GID}" "${TIMESKETCH_USER_NAME}"; \
+      fi; \
+      useradd -m -u "${TIMESKETCH_USER_UID}" -g "${TIMESKETCH_USER_GID}" -s /bin/bash "${TIMESKETCH_USER_NAME}"; \
+  else \
+    echo "User with UID ${TIMESKETCH_USER_UID} already exists."; \
+  fi \
+  && for d in "${TIMESKETCH_CONF_DIR}" "/usr/local/src/sigma"; do \
+    mkdir -p "${d}" \
+    && chown "${TIMESKETCH_USER_UID}:${TIMESKETCH_USER_GID}" "${d}"; \
+    done \
+  && apt-get update \
+  && apt-get install -y --no-install-recommends \
     software-properties-common \
     apt-transport-https \
     apt-utils \
@@ -61,38 +82,75 @@ RUN set -eux \
 ARG NODE_NPMRC=""
 RUN if [ -n "${NODE_NPMRC}" ]; then \
     env echo -e "${NODE_NPMRC}" > ~/.npmrc; \
+    cp ~/.npmrc ~${TIMESKETCH_USER_NAME}/; \
+    chown "${TIMESKETCH_USER_UID}:${TIMESKETCH_USER_GID}" ~${TIMESKETCH_USER_NAME}/.npmrc; \
   fi
 
 ARG YARN_YARNRC=""
 RUN if [ -n "${YARN_YARNRC}" ]; then \
     env echo -e "${YARN_YARNRC}" > ~/.yarnrc; \
+    cp ~/.yarnrc ~${TIMESKETCH_USER_NAME}/; \
+    chown "${TIMESKETCH_USER_UID}:${TIMESKETCH_USER_GID}" ~${TIMESKETCH_USER_NAME}/.yarnrc; \
   fi
 
 ARG PYTHON_PIP_CONF=""
 RUN if [ -n "${PYTHON_PIP_CONF}" ]; then \
-    mkdir -p ~/.config/pip; \
+    mkdir -p ~/.config/pip ~${TIMESKETCH_USER_NAME}/.config/pip; \
     env echo -e "${PYTHON_PIP_CONF}" > ~/.config/pip/pip.conf; \
+    cp ~/.config/pip/pip.conf ~${TIMESKETCH_USER_NAME}/.config/pip/pip.conf; \
+    chown -R "${TIMESKETCH_USER_UID}:${TIMESKETCH_USER_GID}" ~${TIMESKETCH_USER_NAME}/.config; \
   fi
 
 # Install Yarn for frontend development
 RUN npm install --global yarn
 
+USER "${TIMESKETCH_USER_NAME}"
+
 # Install dependencies for Timesketch in a virtual environment
-COPY ["requirements.txt", "/timesketch-requirements.txt"]
-COPY ["test_requirements.txt", "/timesketch-test-requirements.txt"]
-RUN python3 -m venv --upgrade-deps --system-site-packages /opt/venv \
-    && . /opt/venv/bin/activate \
-    && pip install --no-cache-dir \
-        -r /timesketch-requirements.txt \
-        -r /timesketch-test-requirements.txt \
-        psycopg2-binary
+COPY --chown="${TIMESKETCH_USER_UID}:${TIMESKETCH_USER_GID}" ["api_client", "/usr/local/src/timesketch/api_client/"]
+COPY --chown="${TIMESKETCH_USER_UID}:${TIMESKETCH_USER_GID}" ["cli_client", "/usr/local/src/timesketch/cli_client/"]
+COPY --chown="${TIMESKETCH_USER_UID}:${TIMESKETCH_USER_GID}" ["end_to_end_tests", "/usr/local/src/timesketch/end_to_end_tests/"]
+COPY --chown="${TIMESKETCH_USER_UID}:${TIMESKETCH_USER_GID}" ["importer_client", "/usr/local/src/timesketch/importer_client/"]
+COPY --chown="${TIMESKETCH_USER_UID}:${TIMESKETCH_USER_GID}" ["timesketch", "/usr/local/src/timesketch/timesketch/"]
+COPY --chown="${TIMESKETCH_USER_UID}:${TIMESKETCH_USER_GID}" ["tests", "/usr/local/src/timesketch/tests/"]
+COPY --chown="${TIMESKETCH_USER_UID}:${TIMESKETCH_USER_GID}" [ \
+  "requirements.txt", \
+  "setup.py", \
+  "test_requirements.txt", \
+  "/usr/local/src/timesketch/" \
+]
+
+RUN python3 -m venv --upgrade-deps --system-site-packages "${HOME}/venv" \
+  && . "${HOME}/venv/bin/activate" \
+  && pip install --no-cache-dir \
+    -r /usr/local/src/timesketch/requirements.txt \
+    -r /usr/local/src/timesketch/test_requirements.txt \
+  && pip install -e /usr/local/src/timesketch
 
 # Update the PATH to include the virtual environment
-ENV PATH="/opt/venv/bin:${PATH}"
+ENV PATH="/home/${TIMESKETCH_USER_NAME}/venv/bin:${PATH}"
+ENV TIMESKETCH_CONF_DIR="${TIMESKETCH_CONF_DIR}"
+
+FROM common AS setup
+
+COPY --chown=root:root --chmod=755 ["contrib/docker/dev/timesketch/setup-docker-entrypoint.sh", "/usr/local/bin/docker-entrypoint.sh"]
+ENTRYPOINT ["docker-entrypoint.sh"]
+
+FROM common AS celery-worker
+
+COPY --chown=root:root --chmod=755 ["contrib/docker/dev/timesketch/celery-worker-docker-entrypoint.sh", "/usr/local/bin/docker-entrypoint.sh"]
+ENTRYPOINT ["docker-entrypoint.sh"]
+
+FROM common AS gunicorn
+
+COPY --chown=root:root --chmod=755 ["contrib/docker/dev/timesketch/gunicorn-docker-entrypoint.sh", "/usr/local/bin/docker-entrypoint.sh"]
+
+HEALTHCHECK --interval=10s --timeout=5s --start-period=15s --start-interval=2s --retries=1 \
+    CMD ["curl", "-f", "-s", "http://localhost:5000/"]
+
+ENTRYPOINT ["docker-entrypoint.sh"]
 
-# Copy the entrypoint script into the container
-COPY ["contrib/docker/dev/timesketch/docker-entrypoint.sh", "/"]
-RUN chmod a+x /docker-entrypoint.sh
+FROM common AS vue-cli-service
 
-# Load the entrypoint script to be run later
-ENTRYPOINT ["/docker-entrypoint.sh"]
+COPY --chown=root:root --chmod=755 ["contrib/docker/dev/timesketch/vue-cli-service-docker-entrypoint.sh", "/usr/local/bin/docker-entrypoint.sh"]
+ENTRYPOINT ["docker-entrypoint.sh"]
diff --git a/contrib/docker/dev/timesketch/celery-worker-docker-entrypoint.sh b/contrib/docker/dev/timesketch/celery-worker-docker-entrypoint.sh
@@ -0,0 +1,6 @@
+#!/usr/bin/env sh
+
+exec celery \
+    -A timesketch.lib.tasks \
+    worker \
+    --loglevel debug
diff --git a/contrib/docker/dev/timesketch/gunicorn-docker-entrypoint.sh b/contrib/docker/dev/timesketch/gunicorn-docker-entrypoint.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/env sh
+
+exec gunicorn \
+    --reload \
+    -b 0.0.0.0:5000 \
+    --log-file - \
+    --timeout 600 \
+    -c /usr/local/src/timesketch/data/gunicorn_config.py \
+    timesketch.wsgi:application
diff --git a/contrib/docker/dev/timesketch/setup-docker-entrypoint.sh b/contrib/docker/dev/timesketch/setup-docker-entrypoint.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Add web user
+tsctl create-user --password "${TIMESKETCH_PASSWORD}" "${TIMESKETCH_USER}"
+
+# Add Sigma rules
+git clone --depth 1 https://github.com/SigmaHQ/sigma /usr/local/src/sigma
+
+# for each line in sigma_rules.txt execute the command
+while IFS= read -r line; do
+  if [ -f "${line}" ]; then
+    tsctl import-sigma-rules "${line}" &
+  else
+    echo "Skipping non existing Sigma rule: ${line}"
+  fi
+done < "${TIMESKETCH_CONF_DIR}/sigma_rules.txt"
+wait
diff --git a/contrib/docker/dev/timesketch/timesketch.env b/contrib/docker/dev/timesketch/timesketch.env
@@ -1,6 +1,5 @@
 TIMESKETCH_USER="${TIMESKETCH_USER}"
 TIMESKETCH_PASSWORD="${TIMESKETCH_PASSWORD}"
-TIMESKETCH_CONF_DIR="${TIMESKETCH_CONF_DIR}"
 
 SECRET_KEY="${TIMESKETCH_SECRET_KEY}"
 POSTGRES_USER="${POSTGRES_USER}"
diff --git a/contrib/docker/dev/timesketch/vue-cli-service-docker-entrypoint.sh b/contrib/docker/dev/timesketch/vue-cli-service-docker-entrypoint.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -e
+
+cd /usr/local/src/timesketch/timesketch/frontend-ng
+
+yarn install
+exec yarn run serve
