Make testssl parallel

Author: antibagr

.env

@@ -1,9 +1,10 @@
 ENVIRONMENT=development
 DEBUG=true
+TEST_SSL_WORKDIR=/home/testssl_user
 TEST_SSL_CONTAINER_NAME=testssl.sh
 TEST_SSL_OUTPUT_FILE=output.json
-TEST_SSL_INPUT_FILE=input.txt
-TEST_SSL_DATA_DIR=/data
+TEST_SSL_INPUT_FILE=/data/input.txt
+TEST_SSL_COMMANDS_FILE=commands.txt
 CLICKHOUSE_HOST=clickhouse
 CLICKHOUSE_PORT=8123
 CLICKHOUSE_USER=default

Makefile

@@ -2,7 +2,7 @@ SOURCES = app tests
 
 .DEFAULT_GOAL := help
 
-DOCKER_COMPOSE_FILE = contrib/docker-compose.yml
+DOCKER_COMPOSE_FILE = deploy/docker-compose.clickhouse.yaml
 DOCKER_COMPOSE_PROJECT_NAME = app
 
 help: ## Display this help screen
@@ -33,7 +33,7 @@ run: ## Run the development Django server
 .PHONY: run
 
 compose-up: ## Run the development Django server with docker-compose
-	COMPOSE_PROJECT_NAME=${DOCKER_COMPOSE_PROJECT_NAME} docker-compose -f ${DOCKER_COMPOSE_FILE} --env-file .env up --build --remove-orphans --force-recreate
+	docker-compose --project-name ${DOCKER_COMPOSE_PROJECT_NAME} --env-file .env -f ${DOCKER_COMPOSE_FILE} up --force-recreate --build --remove-orphans
 .PHONY: compose-up
 
 compose-down: ## Stop the development Django server with docker-compose

app/lib/testssl/__init__.py

@@ -0,0 +1,7 @@
+from .container import TestSSLContainer
+from .parser import TestSSLJsonParser
+
+__all__ = [
+    "TestSSLContainer",
+    "TestSSLJsonParser",
+]

app/lib/docker.py app/lib/testssl/container.py

@@ -18,11 +18,15 @@ def __init__(
         *,
         client: docker.DockerClient,
         container_name: str,
-        output_path: pathlib.Path,
+        workdir: pathlib.Path,
+        output_file_name: pathlib.Path,
+        commands_file_name: pathlib.Path,
     ) -> None:
         self._client = client
         self._container_name = container_name
-        self._output_path = output_path
+        self._workdir = workdir
+        self._output_file_name = output_file_name
+        self._commands_file_name = commands_file_name
 
     @property
     def container(self) -> Container:
@@ -34,20 +38,32 @@ def stop(self) -> None:
     def wait_for_complete(self) -> None:
         try:
             logger.info("Waiting for container to complete")
-            self.container.wait()
+            _, output = self.container.exec_run(
+                f"parallel --jobs 16 --bar -a {self._commands_file_name.as_posix()}",
+                tty=True,
+                stream=True,
+                user="testssl_user",
+                workdir=self._workdir.as_posix(),
+            )
+            for line in output:
+                logger.info(line.decode("utf-8").strip())
+            logger.info("Finished executing commands")
         except requests.exceptions.ReadTimeout:
             logger.error("Container timeout")
             self.container.kill()
 
     def get_json(self) -> list[TestSSLRecord]:
-        tar_gz, _ = self.container.get_archive(self._output_path, encode_stream=True)
+        logger.info(self._workdir / self._output_file_name.name)
+        tar_gz, _ = self.container.get_archive(
+            self._workdir / self._output_file_name.name, encode_stream=True
+        )
 
         with tarfile.open(
             fileobj=io.BytesIO(b"".join(tar_gz)),
             mode="r",
         ) as tar:
             try:
-                if _file := tar.extractfile(self._output_path.name):
+                if _file := tar.extractfile(self._output_file_name.name):
                     json_bytes = _file.read()
                 else:
                     raise KeyError

app/lib/test_ssl_parser.py app/lib/testssl/parser.py

@@ -5,8 +5,7 @@
 from loguru import logger
 
 from app.dto.entities.fqdn import FQDN
-from app.lib.docker import TestSSLContainer
-from app.lib.test_ssl_parser import TestSSLJsonParser
+from app.lib.testssl import TestSSLContainer, TestSSLJsonParser
 
 
 @t.final

app/repository/provider.py

@@ -5,8 +5,7 @@
 import docker
 from loguru import logger
 
-from app.lib.docker import TestSSLContainer
-from app.lib.test_ssl_parser import TestSSLJsonParser
+from app.lib.testssl import TestSSLContainer, TestSSLJsonParser
 from app.repository.db import DB
 from app.repository.provider import TestSSLDataProviderRepository
 from app.services.ssl_checker import SSLCheckerService
@@ -20,22 +19,24 @@
     user=settings.CLICKHOUSE_USER,
     password=settings.CLICKHOUSE_PASSWORD,
     secure=False,
-    connect_timeout=15,
+    connect_timeout=settings.CLICKHOUSE_CONNECT_TIMEOUT,
 )
 
 test_ssl_parser = TestSSLJsonParser()
 test_ssl_container = TestSSLContainer(
     client=docker_client,
     container_name=settings.TEST_SSL_CONTAINER_NAME,
-    output_path=settings.TEST_SSL_DATA_DIR / settings.TEST_SSL_OUTPUT_FILE,
+    workdir=settings.TEST_SSL_WORKDIR,
+    output_file_name=settings.TEST_SSL_OUTPUT_FILE,
+    commands_file_name=settings.TEST_SSL_COMMANDS_FILE,
 )
 
 # Repository Layer
 db = DB(client=clickhouse_client, table_name=settings.CLICKHOUSE_TABLE_NAME)
 data_provider_repo = TestSSLDataProviderRepository(
     container=test_ssl_container,
     json_parser=test_ssl_parser,
-    input_path=settings.TEST_SSL_DATA_DIR / settings.TEST_SSL_INPUT_FILE,
+    input_path=settings.TEST_SSL_INPUT_FILE,
 )
 
 

app/services/service.py

@@ -19,13 +19,15 @@ class Settings(BaseSettings):
     TEST_SSL_CONTAINER_NAME: str
     TEST_SSL_OUTPUT_FILE: pathlib.Path
     TEST_SSL_INPUT_FILE: pathlib.Path
-    TEST_SSL_DATA_DIR: pathlib.Path
+    TEST_SSL_COMMANDS_FILE: pathlib.Path
+    TEST_SSL_WORKDIR: pathlib.Path
 
     CLICKHOUSE_HOST: str
     CLICKHOUSE_PORT: int
     CLICKHOUSE_USER: str
     CLICKHOUSE_PASSWORD: str
     CLICKHOUSE_TABLE_NAME: str
+    CLICKHOUSE_CONNECT_TIMEOUT: int = 15
 
     @property
     def is_production(self) -> bool:

app/settings.py

@@ -3,24 +3,39 @@ version: "3.9"
 services:
   web:
     container_name: web
-    extends:
-      file: common.yaml
-      service: web
+    build:
+      context: ..
+      dockerfile: deploy/web/Dockerfile.win
+    env_file:
+      - ../.env
+    volumes:
+      - ../data:/data
+      - /var/run/docker.sock:/var/run/docker.sock
     depends_on:
       - clickhouse
+      - testssl.sh
     networks:
       - clickhouse-network
 
   testssl.sh:
     container_name: testssl.sh
-    extends:
-      file: common.yaml
-      service: testssl.sh
+    build:
+      context: ..
+      dockerfile: deploy/testssl.Dockerfile
+    environment:
+      - MASS_TESTING_MODE=parallel
+      - TEST_SSL_INPUT_FILE=./data/input.txt
+    env_file:
+      - ../.env
+    deploy:
+      resources:
+        reservations:
+          cpus: "4"
+          memory: 4G
 
   clickhouse:
     container_name: clickhouse
     image: clickhouse/clickhouse-server
-    restart: always
     environment:
       - CLICKHOUSE_DB=${CLICKHOUSE_DB}
       - CLICKHOUSE_USER=${CLICKHOUSE_USER}
@@ -29,6 +44,8 @@ services:
     ports:
       - ${CLICKHOUSE_PORT}:8123
       - 9000:9000
+    env_file:
+      - ../.env
     networks:
       - clickhouse-network
     volumes:

deploy/common.yaml

@@ -0,0 +1,28 @@
+# Dockerfile for running testssl.sh in parallel
+FROM ubuntu:latest
+
+# Create user
+RUN groupadd -r testssl_user && useradd -r -g testssl_user testssl_user
+
+# Update and install necessary packages
+RUN apt-get update && apt-get install -y bash wget parallel bsdmainutils dnsutils
+RUN wget -O testssl.tar.gz https://github.com/drwetter/testssl.sh/archive/master.tar.gz && \
+    mkdir /opt/testssl && \
+    tar xf testssl.tar.gz --strip-components=1 -C /opt/testssl && \
+    ln -s /opt/testssl/testssl.sh /usr/local/bin/testssl.sh && \
+    rm -rf testssl.tar.gz && \
+    mkdir -p ~/.parallel && touch ~/.parallel/will-cite && \
+    chown -R testssl_user:testssl_user /opt/testssl /usr/local/bin/testssl.sh
+
+# Switch to the non-root user
+USER testssl_user
+WORKDIR /home/testssl_user
+
+# Copy necessary files
+COPY --chown=testssl_user:testssl_user ./scripts/get_cmd_lines.sh ./get_cmd_lines.sh
+COPY --chown=testssl_user:testssl_user ./data/input.txt ./input.txt
+
+# Generate parallel commands as the non-root user
+RUN ./get_cmd_lines.sh ./input.txt ./commands.txt
+
+ENTRYPOINT [ "tail", "-f", "/dev/null" ]

deploy/data/output.json

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Check if the input file exists
+if [ ! -f "$1" ]; then
+    echo "Input file not found: $1"
+    exit 1
+fi
+
+# Remove output file if it already exists
+if [ -f "$2" ]; then
+    rm "$2"
+fi
+
+while IFS= read -r line || [ -n "$line" ]; do
+    host=$(echo "$line" | cut -d: -f1)
+    port=$(echo "$line" | cut -d: -f2)
+    cmd="MAX_PARALLEL=100 testssl.sh --jsonfile output.json --mode parallel --quiet --protocols --server-defaults --append $host:$port"
+
+    echo "$cmd" >>"$2"
+done <"$1"
+
+chmod +x "$2"
+echo "commands generated in $2"

deploy/docker-compose.mongo.yaml

@@ -0,0 +1 @@
+

deploy/docker-compose.clickhouse.yaml deploy/docker-compose.yaml

@@ -2,6 +2,14 @@
 
 from app.main import main
 
+TEST_OUTPUT_JSON = """
+{
+    "data": {
+        "hello": "world"
+    }
+}
+"""
+
 
 def test_main() -> None:
     main()