Skip to content

Commit 3d15cbe

Browse files
jbaptperezjbaptperez
committed
WIP Uses one Compose service per component
1 parent b23e29c commit 3d15cbe

File tree

9 files changed

+300
-54
lines changed

9 files changed

+300
-54
lines changed

‎contrib/docker/dev/.env.template‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,9 @@ TIMESKETCH_CONF_DIR="/etc/timesketch"
1111
TIMESKETCH_SECRET_KEY="L4np0jV3yAdAFdbVzWRMaBqiFMV8FKYd+Je1WKE40o8="
1212
TIMESKETCH_USER="dev"
1313
TIMESKETCH_PASSWORD="dev"
14+
TIMESKETCH_USER_NAME="timesketch"
15+
TIMESKETCH_USER_UID="1000"
16+
TIMESKETCH_USER_GID="1000"
1417

1518
POSTGRES_USER="timesketch"
1619
POSTGRES_PASSWORD="password"

‎contrib/docker/dev/compose.yaml‎

Lines changed: 132 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -4,19 +4,111 @@ networks:
44
timesketch-dev:
55

66
volumes:
7+
setup-data:
78
opensearch-data:
89
postgresql-data:
910
redis-data:
1011
prometheus-data:
1112

1213
services:
13-
timesketch:
14-
image: timesketch-timesketch/dev:latest
14+
setup:
15+
image: timesketch-setup:latest
1516
build:
1617
context: ../../..
1718
dockerfile: contrib/docker/dev/timesketch/Dockerfile
19+
target: setup
1820
args:
1921
BASE_IMAGE: "${TIMESKETCH_BASE_IMAGE:?}"
22+
TIMESKETCH_USER_NAME: "${TIMESKETCH_USER_NAME:?}"
23+
TIMESKETCH_USER_UID: "${TIMESKETCH_USER_UID:?}"
24+
TIMESKETCH_USER_GID: "${TIMESKETCH_USER_GID:?}"
25+
TIMESKETCH_CONF_DIR: "${TIMESKETCH_CONF_DIR:?}"
26+
GIFT_PPA_TRACK: "${GIFT_PPA_TRACK:?}"
27+
GIFT_PPA_URL: "${GIFT_PPA_URL:?}"
28+
NODE_VERSION: "${NODE_VERSION:?}"
29+
NODE_PPA_URL: "${NODE_PPA_URL:?}"
30+
NODE_NPMRC: "${NODE_NPMRC?}"
31+
YARN_YARNRC: "${YARN_YARNRC?}"
32+
PYTHON_PIP_CONF: "${PYTHON_PIP_CONF?}"
33+
command: timesketch
34+
env_file:
35+
- timesketch/timesketch.env
36+
volumes:
37+
- "setup-data:/usr/local/src/sigma"
38+
- "../../../:/usr/local/src/timesketch"
39+
- "./timesketch/timesketch.conf:${TIMESKETCH_CONF_DIR:?}/timesketch.conf:ro"
40+
- "./timesketch/sigma_rules.txt:${TIMESKETCH_CONF_DIR:?}/sigma_rules.txt:ro"
41+
- "../../../data/sigma_config.yaml:${TIMESKETCH_CONF_DIR:?}/sigma_config.yaml:ro"
42+
- "../../../data/sigma:${TIMESKETCH_CONF_DIR:?}/sigma:ro"
43+
depends_on:
44+
postgresql:
45+
condition: service_healthy
46+
redis:
47+
condition: service_healthy
48+
networks:
49+
- timesketch-dev
50+
51+
celery-worker:
52+
image: timesketch-celery-worker:latest
53+
build:
54+
context: ../../..
55+
dockerfile: contrib/docker/dev/timesketch/Dockerfile
56+
target: celery-worker
57+
args:
58+
BASE_IMAGE: "${TIMESKETCH_BASE_IMAGE:?}"
59+
TIMESKETCH_USER_NAME: "${TIMESKETCH_USER_NAME:?}"
60+
TIMESKETCH_USER_UID: "${TIMESKETCH_USER_UID:?}"
61+
TIMESKETCH_USER_GID: "${TIMESKETCH_USER_GID:?}"
62+
TIMESKETCH_CONF_DIR: "${TIMESKETCH_CONF_DIR:?}"
63+
GIFT_PPA_TRACK: "${GIFT_PPA_TRACK:?}"
64+
GIFT_PPA_URL: "${GIFT_PPA_URL:?}"
65+
NODE_VERSION: "${NODE_VERSION:?}"
66+
NODE_PPA_URL: "${NODE_PPA_URL:?}"
67+
NODE_NPMRC: "${NODE_NPMRC?}"
68+
YARN_YARNRC: "${YARN_YARNRC?}"
69+
PYTHON_PIP_CONF: "${PYTHON_PIP_CONF?}"
70+
command: timesketch
71+
env_file:
72+
- timesketch/timesketch.env
73+
volumes:
74+
- "../../../:/usr/local/src/timesketch/"
75+
- "./timesketch/timesketch.conf:${TIMESKETCH_CONF_DIR:?}/timesketch.conf:ro"
76+
- "../../../data/regex_features.yaml:${TIMESKETCH_CONF_DIR:?}/regex_features.yaml:ro"
77+
- "../../../data/winevt_features.yaml:${TIMESKETCH_CONF_DIR:?}/winevt_features.yaml:ro"
78+
- "../../../data/tags.yaml:${TIMESKETCH_CONF_DIR:?}/tags.yaml:ro"
79+
- "../../../data/intelligence_tag_metadata.yaml:${TIMESKETCH_CONF_DIR:?}/intelligence_tag_metadata.yaml:ro"
80+
- "../../../data/plaso.mappings:${TIMESKETCH_CONF_DIR:?}/plaso.mappings:ro"
81+
- "../../../data/generic.mappings:${TIMESKETCH_CONF_DIR:?}/generic.mappings:ro"
82+
- "../../../data/ontology.yaml:${TIMESKETCH_CONF_DIR:?}/ontology.yaml:ro"
83+
- "../../../data/data_finder.yaml:${TIMESKETCH_CONF_DIR:?}/data_finder.yaml:ro"
84+
- "../../../data/bigquery_matcher.yaml:${TIMESKETCH_CONF_DIR:?}/bigquery_matcher.yaml:ro"
85+
- "../../../data/sigma_config.yaml:${TIMESKETCH_CONF_DIR:?}/sigma_config.yaml:ro"
86+
- "../../../data/sigma:${TIMESKETCH_CONF_DIR:?}/sigma:ro"
87+
- "../../../data/dfiq:${TIMESKETCH_CONF_DIR:?}/dfiq:ro"
88+
- "../../../data/context_links.yaml:${TIMESKETCH_CONF_DIR:?}/context_links.yaml:ro"
89+
- "../../../data/plaso_formatters.yaml:${TIMESKETCH_CONF_DIR:?}/plaso_formatters.yaml:ro"
90+
- "../../../data/nl2q:${TIMESKETCH_CONF_DIR:?}/nl2q:ro"
91+
- "../../../data/llm_summarize:${TIMESKETCH_CONF_DIR:?}/llm_summarize:ro"
92+
depends_on:
93+
setup:
94+
condition: service_completed_successfully
95+
opensearch:
96+
condition: service_started
97+
networks:
98+
- timesketch-dev
99+
100+
gunicorn:
101+
image: timesketch-gunicorn:latest
102+
build:
103+
context: ../../..
104+
dockerfile: contrib/docker/dev/timesketch/Dockerfile
105+
target: gunicorn
106+
args:
107+
BASE_IMAGE: "${TIMESKETCH_BASE_IMAGE:?}"
108+
TIMESKETCH_USER_NAME: "${TIMESKETCH_USER_NAME:?}"
109+
TIMESKETCH_USER_UID: "${TIMESKETCH_USER_UID:?}"
110+
TIMESKETCH_USER_GID: "${TIMESKETCH_USER_GID:?}"
111+
TIMESKETCH_CONF_DIR: "${TIMESKETCH_CONF_DIR:?}"
20112
GIFT_PPA_TRACK: "${GIFT_PPA_TRACK:?}"
21113
GIFT_PPA_URL: "${GIFT_PPA_URL:?}"
22114
NODE_VERSION: "${NODE_VERSION:?}"
@@ -29,9 +121,6 @@ services:
29121
- name: gunicorn
30122
published: "5000"
31123
target: 5000
32-
- name: vite
33-
published: "5001"
34-
target: 5001
35124
- name: metrics
36125
published: "8080"
37126
target: 8080
@@ -40,7 +129,6 @@ services:
40129
volumes:
41130
- "../../../:/usr/local/src/timesketch/"
42131
- "./timesketch/timesketch.conf:${TIMESKETCH_CONF_DIR:?}/timesketch.conf:ro"
43-
- "./timesketch/sigma_rules.txt:${TIMESKETCH_CONF_DIR:?}/sigma_rules.txt:ro"
44132
- "../../../data/regex_features.yaml:${TIMESKETCH_CONF_DIR:?}/regex_features.yaml:ro"
45133
- "../../../data/winevt_features.yaml:${TIMESKETCH_CONF_DIR:?}/winevt_features.yaml:ro"
46134
- "../../../data/tags.yaml:${TIMESKETCH_CONF_DIR:?}/tags.yaml:ro"
@@ -58,11 +146,41 @@ services:
58146
- "../../../data/nl2q:${TIMESKETCH_CONF_DIR:?}/nl2q:ro"
59147
- "../../../data/llm_summarize:${TIMESKETCH_CONF_DIR:?}/llm_summarize:ro"
60148
depends_on:
61-
postgresql:
62-
condition: service_healthy
63-
opensearch:
64-
condition: service_healthy
65-
redis:
149+
setup:
150+
condition: service_completed_successfully
151+
networks:
152+
- timesketch-dev
153+
154+
vue-cli-service:
155+
image: timesketch-vue-cli-service:latest
156+
build:
157+
context: ../../..
158+
dockerfile: contrib/docker/dev/timesketch/Dockerfile
159+
target: vue-cli-service
160+
args:
161+
BASE_IMAGE: "${TIMESKETCH_BASE_IMAGE:?}"
162+
TIMESKETCH_USER_NAME: "${TIMESKETCH_USER_NAME:?}"
163+
TIMESKETCH_USER_UID: "${TIMESKETCH_USER_UID:?}"
164+
TIMESKETCH_USER_GID: "${TIMESKETCH_USER_GID:?}"
165+
TIMESKETCH_CONF_DIR: "${TIMESKETCH_CONF_DIR:?}"
166+
GIFT_PPA_TRACK: "${GIFT_PPA_TRACK:?}"
167+
GIFT_PPA_URL: "${GIFT_PPA_URL:?}"
168+
NODE_VERSION: "${NODE_VERSION:?}"
169+
NODE_PPA_URL: "${NODE_PPA_URL:?}"
170+
NODE_NPMRC: "${NODE_NPMRC?}"
171+
YARN_YARNRC: "${YARN_YARNRC?}"
172+
PYTHON_PIP_CONF: "${PYTHON_PIP_CONF?}"
173+
command: timesketch
174+
ports:
175+
- name: vue-cli-service
176+
published: "5001"
177+
target: 5001
178+
env_file:
179+
- timesketch/timesketch.env
180+
volumes:
181+
- "../../../:/usr/local/src/timesketch/"
182+
depends_on:
183+
gunicorn:
66184
condition: service_healthy
67185
networks:
68186
- timesketch-dev
@@ -129,7 +247,8 @@ services:
129247

130248
command: --config.file=/etc/prometheus/prometheus.yml
131249
depends_on:
132-
- timesketch
250+
gunicorn:
251+
condition: service_healthy
133252
networks:
134253
- timesketch-dev
135254

@@ -149,6 +268,6 @@ services:
149268
- "/tmp/:/usr/local/src/picadata/"
150269
depends_on:
151270
opensearch:
152-
condition: service_healthy
271+
condition: service_started
153272
networks:
154273
- timesketch-dev

‎contrib/docker/dev/postgresql/postgresql.env‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
1-
POSTGRES_USER="${POSTGRES_USER}"
2-
POSTGRES_PASSWORD="${POSTGRES_PASSWORD}"
3-
POSTGRES_DB="${POSTGRES_DB}"
1+
POSTGRES_USER="${POSTGRES_USER?-}"
2+
POSTGRES_PASSWORD="${POSTGRES_PASSWORD?-}"
3+
POSTGRES_DB="${POSTGRES_DB?-}"

‎contrib/docker/dev/timesketch/Dockerfile‎

Lines changed: 103 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,32 @@
11
# Use the official Docker Hub Ubuntu base image
22
ARG BASE_IMAGE="ubuntu:24.04"
3-
FROM $BASE_IMAGE
3+
FROM ${BASE_IMAGE} AS common
44

5-
# Prevent needing to configure debian packages, stopping the setup of
6-
# the docker container.
7-
RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
5+
USER root
86

9-
RUN apt-get update && apt-get install -y --no-install-recommends \
7+
ARG TIMESKETCH_USER_NAME="timesketch"
8+
ARG TIMESKETCH_USER_UID="1000"
9+
ARG TIMESKETCH_USER_GID="1000"
10+
ARG TIMESKETCH_CONF_DIR="/etc/timesketch"
11+
RUN if ! id -u "${TIMESKETCH_USER_UID}" &>/dev/null; then \
12+
echo "Creating user ${USER_NAME} (${TIMESKETCH_USER_UID}:${TIMESKETCH_USER_GID})..."; \
13+
if getent group "${TIMESKETCH_USER_GID}" >/dev/null; then \
14+
echo "Group with GID ${TIMESKETCH_USER_GID} already exists."; \
15+
else \
16+
echo "Creating group ${TIMESKETCH_USER_NAME} with GID ${TIMESKETCH_USER_GID}..."; \
17+
groupadd -g "${TIMESKETCH_USER_GID}" "${TIMESKETCH_USER_NAME}"; \
18+
fi; \
19+
useradd -m -u "${TIMESKETCH_USER_UID}" -g "${TIMESKETCH_USER_GID}" -s /bin/bash "${TIMESKETCH_USER_NAME}"; \
20+
else \
21+
echo "User with UID ${TIMESKETCH_USER_UID} already exists."; \
22+
fi \
23+
&& for d in "${TIMESKETCH_CONF_DIR}" "/usr/local/src/sigma"; do \
24+
mkdir -p "${d}" \
25+
&& chown "${TIMESKETCH_USER_UID}:${TIMESKETCH_USER_GID}" "${d}"; \
26+
done \
27+
&& echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections \
28+
&& apt-get update \
29+
&& apt-get install -y --no-install-recommends \
1030
software-properties-common \
1131
apt-transport-https \
1232
apt-utils \
@@ -26,6 +46,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
2646
python3-venv \
2747
&& rm -rf /var/lib/apt/lists/*
2848

49+
FROM common AS common-python
50+
2951
# Install Plaso
3052
ARG GIFT_PPA_TRACK="stable"
3153
ARG GIFT_PPA_URL="http://ppa.launchpad.net/gift/${GIFT_PPA_TRACK}/ubuntu"
@@ -39,12 +61,70 @@ RUN set -eux \
3961
&& apt-get install -y --no-install-recommends \
4062
plaso-tools \
4163
&& apt-get clean \
42-
&& rm -rf /var/lib/apt/lists/* ~/.gnupg
64+
&& rm -rf /var/lib/apt/lists/* /root/.gnupg
4365

4466
# Fix for broken PPA dependency in Ubuntu 24.04: Plaso needs the 'events'
4567
# library for its opensearch output module.
4668
RUN pip3 install --break-system-packages events
4769

70+
ARG PYTHON_PIP_CONF=""
71+
RUN if [ -n "${PYTHON_PIP_CONF}" ]; then \
72+
mkdir -p /root/.config/pip /home/${TIMESKETCH_USER_NAME}/.config/pip; \
73+
env echo -e "${PYTHON_PIP_CONF}" > /root/.config/pip/pip.conf; \
74+
cp /root/.config/pip/pip.conf /home/${TIMESKETCH_USER_NAME}/.config/pip/pip.conf; \
75+
chown -R "${TIMESKETCH_USER_UID}:${TIMESKETCH_USER_GID}" /home/${TIMESKETCH_USER_NAME}/.config; \
76+
fi
77+
78+
USER "${TIMESKETCH_USER_NAME}"
79+
80+
# Install dependencies for Timesketch in a virtual environment
81+
COPY --chown="${TIMESKETCH_USER_UID}:${TIMESKETCH_USER_GID}" ["api_client", "/usr/local/src/timesketch/api_client/"]
82+
COPY --chown="${TIMESKETCH_USER_UID}:${TIMESKETCH_USER_GID}" ["cli_client", "/usr/local/src/timesketch/cli_client/"]
83+
COPY --chown="${TIMESKETCH_USER_UID}:${TIMESKETCH_USER_GID}" ["end_to_end_tests", "/usr/local/src/timesketch/end_to_end_tests/"]
84+
COPY --chown="${TIMESKETCH_USER_UID}:${TIMESKETCH_USER_GID}" ["importer_client", "/usr/local/src/timesketch/importer_client/"]
85+
COPY --chown="${TIMESKETCH_USER_UID}:${TIMESKETCH_USER_GID}" ["timesketch", "/usr/local/src/timesketch/timesketch/"]
86+
COPY --chown="${TIMESKETCH_USER_UID}:${TIMESKETCH_USER_GID}" ["tests", "/usr/local/src/timesketch/tests/"]
87+
COPY --chown="${TIMESKETCH_USER_UID}:${TIMESKETCH_USER_GID}" [ \
88+
"requirements.txt", \
89+
"setup.py", \
90+
"test_requirements.txt", \
91+
"/usr/local/src/timesketch/" \
92+
]
93+
94+
RUN python3 -m venv --upgrade-deps --system-site-packages "${HOME}/venv" \
95+
&& . "${HOME}/venv/bin/activate" \
96+
&& pip install --no-cache-dir \
97+
-r /usr/local/src/timesketch/requirements.txt \
98+
-r /usr/local/src/timesketch/test_requirements.txt \
99+
&& pip install -e /usr/local/src/timesketch
100+
101+
# Update the PATH to include the virtual environment
102+
ENV PATH="/home/${TIMESKETCH_USER_NAME}/venv/bin:${PATH}"
103+
ENV TIMESKETCH_CONF_DIR="${TIMESKETCH_CONF_DIR}"
104+
105+
FROM common-python AS setup
106+
107+
COPY --chown=root:root --chmod=755 ["contrib/docker/dev/timesketch/setup-docker-entrypoint.sh", "/usr/local/bin/docker-entrypoint.sh"]
108+
ENTRYPOINT ["docker-entrypoint.sh"]
109+
110+
FROM common-python AS celery-worker
111+
112+
COPY --chown=root:root --chmod=755 ["contrib/docker/dev/timesketch/celery-worker-docker-entrypoint.sh", "/usr/local/bin/docker-entrypoint.sh"]
113+
ENTRYPOINT ["docker-entrypoint.sh"]
114+
115+
FROM common-python AS gunicorn
116+
117+
COPY --chown=root:root --chmod=755 ["contrib/docker/dev/timesketch/gunicorn-docker-entrypoint.sh", "/usr/local/bin/docker-entrypoint.sh"]
118+
119+
HEALTHCHECK --interval=10s --timeout=5s --start-period=15s --start-interval=2s --retries=1 \
120+
CMD ["curl", "-f", "-s", "http://localhost:5000/"]
121+
122+
ENTRYPOINT ["docker-entrypoint.sh"]
123+
124+
FROM common AS vue-cli-service
125+
126+
USER root
127+
48128
# Install NodeJS for frontend development
49129
ARG NODE_VERSION="20.x"
50130
ARG NODE_PPA_URL="https://deb.nodesource.com/node_${NODE_VERSION}"
@@ -56,43 +136,35 @@ RUN set -eux \
56136
&& apt-get install -y --no-install-recommends \
57137
nodejs \
58138
&& apt-get clean \
59-
&& rm -rf /var/lib/apt/lists/* ~/.gnupg
139+
&& rm -rf /var/lib/apt/lists/* /root/.gnupg
60140

141+
ARG TIMESKETCH_USER_NAME="timesketch"
142+
ARG TIMESKETCH_USER_UID="1000"
143+
ARG TIMESKETCH_USER_GID="1000"
61144
ARG NODE_NPMRC=""
62145
RUN if [ -n "${NODE_NPMRC}" ]; then \
63-
env echo -e "${NODE_NPMRC}" > ~/.npmrc; \
146+
env echo -e "${NODE_NPMRC}" > /root/.npmrc; \
147+
cp /root/.npmrc /home/${TIMESKETCH_USER_NAME}/.npmrc; \
148+
chown "${TIMESKETCH_USER_UID}:${TIMESKETCH_USER_GID}" /home/${TIMESKETCH_USER_NAME}/.npmrc; \
64149
fi
65150

66151
ARG YARN_YARNRC=""
67152
RUN if [ -n "${YARN_YARNRC}" ]; then \
68-
env echo -e "${YARN_YARNRC}" > ~/.yarnrc; \
69-
fi
70-
71-
ARG PYTHON_PIP_CONF=""
72-
RUN if [ -n "${PYTHON_PIP_CONF}" ]; then \
73-
mkdir -p ~/.config/pip; \
74-
env echo -e "${PYTHON_PIP_CONF}" > ~/.config/pip/pip.conf; \
153+
env echo -e "${YARN_YARNRC}" > /root/.yarnrc; \
154+
cp /root/.yarnrc /home/${TIMESKETCH_USER_NAME}/.yarnrc; \
155+
chown "${TIMESKETCH_USER_UID}:${TIMESKETCH_USER_GID}" /home/${TIMESKETCH_USER_NAME}/.yarnrc; \
75156
fi
76157

77158
# Install Yarn for frontend development
78159
RUN npm install --global yarn
79160

80-
# Install dependencies for Timesketch in a virtual environment
81-
COPY ["requirements.txt", "/timesketch-requirements.txt"]
82-
COPY ["test_requirements.txt", "/timesketch-test-requirements.txt"]
83-
RUN python3 -m venv --upgrade-deps --system-site-packages /opt/venv \
84-
&& . /opt/venv/bin/activate \
85-
&& pip install --no-cache-dir \
86-
-r /timesketch-requirements.txt \
87-
-r /timesketch-test-requirements.txt \
88-
psycopg2-binary
161+
USER "${TIMESKETCH_USER_NAME}"
89162

90-
# Update the PATH to include the virtual environment
91-
ENV PATH="/opt/venv/bin:${PATH}"
163+
COPY --chown="${TIMESKETCH_USER_UID}:${TIMESKETCH_USER_GID}" ["timesketch", "/usr/local/src/timesketch/timesketch/"]
92164

93-
# Copy the entrypoint script into the container
94-
COPY ["contrib/docker/dev/timesketch/docker-entrypoint.sh", "/"]
95-
RUN chmod a+x /docker-entrypoint.sh
165+
RUN if ! yarn --cwd=/usr/local/src/timesketch/timesketch/frontend-ng install; then \
166+
yarn --cwd=/usr/local/src/timesketch/timesketch/frontend-ng install --no-lockfile; \
167+
fi
96168

97-
# Load the entrypoint script to be run later
98-
ENTRYPOINT ["/docker-entrypoint.sh"]
169+
COPY --chown=root:root --chmod=755 ["contrib/docker/dev/timesketch/vue-cli-service-docker-entrypoint.sh", "/usr/local/bin/docker-entrypoint.sh"]
170+
ENTRYPOINT ["docker-entrypoint.sh"]

0 commit comments

Comments
 (0)