Skip to content

Commit 29e6c8a

Browse files
awtknsawtkns
authored
🐛 Improve ssl DevX (#1117)
* 🐛 Improve SSL devx
1 parent db76d1d commit 29e6c8a

File tree

7 files changed

+165
-116
lines changed

7 files changed

+165
-116
lines changed

‎cli/src/envGenerator.js

+107-108
Original file line numberDiff line numberDiff line change
@@ -3,141 +3,140 @@ import fs from "fs";
33
import chalk from "chalk";
44

55
export const generateEnv = (envValues) => {
6-
let isDockerCompose = envValues.runOption === "docker-compose";
7-
let dbPort = isDockerCompose ? 3307 : 3306;
8-
let platformUrl = isDockerCompose
9-
? "http://host.docker.internal:8000"
10-
: "http://localhost:8000";
11-
12-
const envDefinition = getEnvDefinition(
13-
envValues,
14-
isDockerCompose,
15-
dbPort,
16-
platformUrl,
17-
);
18-
19-
const envFileContent = generateEnvFileContent(envDefinition);
20-
saveEnvFile(envFileContent);
6+
let isDockerCompose = envValues.runOption === "docker-compose";
7+
let dbPort = isDockerCompose ? 3307 : 3306;
8+
let platformUrl = isDockerCompose
9+
? "http://host.docker.internal:8000"
10+
: "http://localhost:8000";
11+
12+
const envDefinition = getEnvDefinition(
13+
envValues,
14+
isDockerCompose,
15+
dbPort,
16+
platformUrl
17+
);
18+
19+
const envFileContent = generateEnvFileContent(envDefinition);
20+
saveEnvFile(envFileContent);
2121
};
2222

2323
const getEnvDefinition = (envValues, isDockerCompose, dbPort, platformUrl) => {
24-
return {
25-
"Deployment Environment": {
26-
NODE_ENV: "development",
27-
NEXT_PUBLIC_VERCEL_ENV: "${NODE_ENV}",
28-
},
29-
NextJS: {
30-
NEXT_PUBLIC_BACKEND_URL: "http://localhost:8000",
31-
NEXT_PUBLIC_MAX_LOOPS: 100,
32-
},
33-
"Next Auth config": {
34-
NEXTAUTH_SECRET: generateAuthSecret(),
35-
NEXTAUTH_URL: "http://localhost:3000",
36-
},
37-
"Auth providers (Use if you want to get out of development mode sign-in)": {
38-
GOOGLE_CLIENT_ID: "***",
39-
GOOGLE_CLIENT_SECRET: "***",
40-
GITHUB_CLIENT_ID: "***",
41-
GITHUB_CLIENT_SECRET: "***",
42-
DISCORD_CLIENT_SECRET: "***",
43-
DISCORD_CLIENT_ID: "***",
44-
},
45-
Backend: {
46-
REWORKD_PLATFORM_ENVIRONMENT: "${NODE_ENV}",
47-
REWORKD_PLATFORM_FF_MOCK_MODE_ENABLED: false,
48-
REWORKD_PLATFORM_MAX_LOOPS: "${NEXT_PUBLIC_MAX_LOOPS}",
49-
REWORKD_PLATFORM_OPENAI_API_KEY:
50-
envValues.OpenAIApiKey || '"<change me>"',
51-
REWORKD_PLATFORM_FRONTEND_URL: "http://localhost:3000",
52-
REWORKD_PLATFORM_RELOAD: true,
53-
REWORKD_PLATFORM_OPENAI_API_BASE: "https://api.openai.com/v1",
54-
REWORKD_PLATFORM_SERP_API_KEY: envValues.serpApiKey || '""',
55-
REWORKD_PLATFORM_REPLICATE_API_KEY: envValues.replicateApiKey || '""',
56-
},
57-
"Database (Backend)": {
58-
REWORKD_PLATFORM_DATABASE_USER: "reworkd_platform",
59-
REWORKD_PLATFORM_DATABASE_PASSWORD: "reworkd_platform",
60-
REWORKD_PLATFORM_DATABASE_HOST: "db",
61-
REWORKD_PLATFORM_DATABASE_PORT: dbPort,
62-
REWORKD_PLATFORM_DATABASE_NAME: "reworkd_platform",
63-
REWORKD_PLATFORM_DATABASE_URL:
64-
"mysql://${REWORKD_PLATFORM_DATABASE_USER}:${REWORKD_PLATFORM_DATABASE_PASSWORD}@${REWORKD_PLATFORM_DATABASE_HOST}:${REWORKD_PLATFORM_DATABASE_PORT}/${REWORKD_PLATFORM_DATABASE_NAME}",
65-
REWORKD_PLATFORM_DB_CA_PATH: isDockerCompose ? "/etc/ssl/certs/ca-certificates.crt" : "",
66-
},
67-
"Database (Frontend)": {
68-
DATABASE_USER: "reworkd_platform",
69-
DATABASE_PASSWORD: "reworkd_platform",
70-
DATABASE_HOST: "db",
71-
DATABASE_PORT: dbPort,
72-
DATABASE_NAME: "reworkd_platform",
73-
DATABASE_URL:
74-
"mysql://${DATABASE_USER}:${DATABASE_PASSWORD}@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_NAME}",
75-
},
76-
};
24+
return {
25+
"Deployment Environment": {
26+
NODE_ENV: "development",
27+
NEXT_PUBLIC_VERCEL_ENV: "${NODE_ENV}",
28+
},
29+
NextJS: {
30+
NEXT_PUBLIC_BACKEND_URL: "http://localhost:8000",
31+
NEXT_PUBLIC_MAX_LOOPS: 100,
32+
},
33+
"Next Auth config": {
34+
NEXTAUTH_SECRET: generateAuthSecret(),
35+
NEXTAUTH_URL: "http://localhost:3000",
36+
},
37+
"Auth providers (Use if you want to get out of development mode sign-in)": {
38+
GOOGLE_CLIENT_ID: "***",
39+
GOOGLE_CLIENT_SECRET: "***",
40+
GITHUB_CLIENT_ID: "***",
41+
GITHUB_CLIENT_SECRET: "***",
42+
DISCORD_CLIENT_SECRET: "***",
43+
DISCORD_CLIENT_ID: "***",
44+
},
45+
Backend: {
46+
REWORKD_PLATFORM_ENVIRONMENT: "${NODE_ENV}",
47+
REWORKD_PLATFORM_FF_MOCK_MODE_ENABLED: false,
48+
REWORKD_PLATFORM_MAX_LOOPS: "${NEXT_PUBLIC_MAX_LOOPS}",
49+
REWORKD_PLATFORM_OPENAI_API_KEY:
50+
envValues.OpenAIApiKey || '"<change me>"',
51+
REWORKD_PLATFORM_FRONTEND_URL: "http://localhost:3000",
52+
REWORKD_PLATFORM_RELOAD: true,
53+
REWORKD_PLATFORM_OPENAI_API_BASE: "https://api.openai.com/v1",
54+
REWORKD_PLATFORM_SERP_API_KEY: envValues.serpApiKey || '""',
55+
REWORKD_PLATFORM_REPLICATE_API_KEY: envValues.replicateApiKey || '""',
56+
},
57+
"Database (Backend)": {
58+
REWORKD_PLATFORM_DATABASE_USER: "reworkd_platform",
59+
REWORKD_PLATFORM_DATABASE_PASSWORD: "reworkd_platform",
60+
REWORKD_PLATFORM_DATABASE_HOST: "db",
61+
REWORKD_PLATFORM_DATABASE_PORT: dbPort,
62+
REWORKD_PLATFORM_DATABASE_NAME: "reworkd_platform",
63+
REWORKD_PLATFORM_DATABASE_URL:
64+
"mysql://${REWORKD_PLATFORM_DATABASE_USER}:${REWORKD_PLATFORM_DATABASE_PASSWORD}@${REWORKD_PLATFORM_DATABASE_HOST}:${REWORKD_PLATFORM_DATABASE_PORT}/${REWORKD_PLATFORM_DATABASE_NAME}",
65+
},
66+
"Database (Frontend)": {
67+
DATABASE_USER: "reworkd_platform",
68+
DATABASE_PASSWORD: "reworkd_platform",
69+
DATABASE_HOST: "db",
70+
DATABASE_PORT: dbPort,
71+
DATABASE_NAME: "reworkd_platform",
72+
DATABASE_URL:
73+
"mysql://${DATABASE_USER}:${DATABASE_PASSWORD}@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_NAME}",
74+
},
75+
};
7776
};
7877

7978
const generateEnvFileContent = (config) => {
80-
let configFile = "";
81-
82-
Object.entries(config).forEach(([section, variables]) => {
83-
configFile += `# ${section}:\n`;
84-
Object.entries(variables).forEach(([key, value]) => {
85-
configFile += `${key}=${value}\n`;
86-
});
87-
configFile += "\n";
79+
let configFile = "";
80+
81+
Object.entries(config).forEach(([section, variables]) => {
82+
configFile += `# ${section}:\n`;
83+
Object.entries(variables).forEach(([key, value]) => {
84+
configFile += `${key}=${value}\n`;
8885
});
86+
configFile += "\n";
87+
});
8988

90-
return configFile.trim();
89+
return configFile.trim();
9190
};
9291

9392
const generateAuthSecret = () => {
94-
const length = 32;
95-
const buffer = crypto.randomBytes(length);
96-
return buffer.toString("base64");
93+
const length = 32;
94+
const buffer = crypto.randomBytes(length);
95+
return buffer.toString("base64");
9796
};
9897

9998
const ENV_PATH = "../next/.env";
10099
const BACKEND_ENV_PATH = "../platform/.env";
101100

102101
export const doesEnvFileExist = () => {
103-
return fs.existsSync(ENV_PATH);
102+
return fs.existsSync(ENV_PATH);
104103
};
105104

106105
// Read the existing env file, test if it is missing any keys or contains any extra keys
107106
export const testEnvFile = () => {
108-
const data = fs.readFileSync(ENV_PATH, "utf8");
107+
const data = fs.readFileSync(ENV_PATH, "utf8");
109108

110-
// Make a fake definition to compare the keys of
111-
const envDefinition = getEnvDefinition({}, "", "", "", "");
109+
// Make a fake definition to compare the keys of
110+
const envDefinition = getEnvDefinition({}, "", "", "", "");
112111

113-
const lines = data
114-
.split("\n")
115-
.filter((line) => !line.startsWith("#") && line.trim() !== "");
116-
const envKeysFromFile = lines.map((line) => line.split("=")[0]);
112+
const lines = data
113+
.split("\n")
114+
.filter((line) => !line.startsWith("#") && line.trim() !== "");
115+
const envKeysFromFile = lines.map((line) => line.split("=")[0]);
117116

118-
const envKeysFromDef = Object.entries(envDefinition).flatMap(
119-
([section, entries]) => Object.keys(entries)
120-
);
117+
const envKeysFromDef = Object.entries(envDefinition).flatMap(
118+
([section, entries]) => Object.keys(entries)
119+
);
121120

122-
const missingFromFile = envKeysFromDef.filter(
123-
(key) => !envKeysFromFile.includes(key)
124-
);
121+
const missingFromFile = envKeysFromDef.filter(
122+
(key) => !envKeysFromFile.includes(key)
123+
);
124+
125+
if (missingFromFile.length > 0) {
126+
let errorMessage = "\nYour ./next/.env is missing the following keys:\n";
127+
missingFromFile.forEach((key) => {
128+
errorMessage += chalk.whiteBright(`- ❌ ${key}\n`);
129+
});
130+
errorMessage += "\n";
125131

126-
if(missingFromFile.length > 0) {
127-
let errorMessage = "\nYour ./next/.env is missing the following keys:\n";
128-
missingFromFile.forEach((key) => {
129-
errorMessage += chalk.whiteBright(`- ❌ ${key}\n`);
130-
});
131-
errorMessage += "\n";
132-
133-
errorMessage += chalk.red(
134-
"We recommend deleting your .env file(s) and restarting this script."
135-
);
136-
throw new Error(errorMessage);
137-
}
132+
errorMessage += chalk.red(
133+
"We recommend deleting your .env file(s) and restarting this script."
134+
);
135+
throw new Error(errorMessage);
136+
}
138137
};
139138

140139
export const saveEnvFile = (envFileContent) => {
141-
fs.writeFileSync(ENV_PATH, envFileContent);
142-
fs.writeFileSync(BACKEND_ENV_PATH, envFileContent);
140+
fs.writeFileSync(ENV_PATH, envFileContent);
141+
fs.writeFileSync(BACKEND_ENV_PATH, envFileContent);
143142
};

‎platform/reworkd_platform/db/utils.py

+4-3
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
1-
import ssl
1+
from ssl import CERT_REQUIRED
22

33
from sqlalchemy import text
44
from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
55

6+
from reworkd_platform.services.ssl import get_ssl_context
67
from reworkd_platform.settings import settings
78

89

@@ -18,8 +19,8 @@ def create_engine() -> AsyncEngine:
1819
echo=settings.db_echo,
1920
)
2021

21-
ssl_context = ssl.create_default_context(cafile=settings.db_ca_path)
22-
ssl_context.verify_mode = ssl.CERT_REQUIRED
22+
ssl_context = get_ssl_context(settings)
23+
ssl_context.verify_mode = CERT_REQUIRED
2324
connect_args = {"ssl": ssl_context}
2425

2526
return create_async_engine(

‎platform/reworkd_platform/services/kafka/consumers/base.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
import asyncio
22
import json
3-
import ssl
43
from abc import ABC, abstractmethod
54
from typing import Any, Protocol
65

76
from aiokafka import AIOKafkaConsumer, ConsumerRecord
87
from loguru import logger
98

9+
from reworkd_platform.services.ssl import get_ssl_context
1010
from reworkd_platform.settings import Settings
1111

1212

@@ -38,7 +38,7 @@ def __init__(
3838
security_protocol="SASL_SSL",
3939
sasl_plain_username=settings.kafka_username,
4040
sasl_plain_password=settings.kafka_password,
41-
ssl_context=ssl.create_default_context(cafile=settings.db_ca_path),
41+
ssl_context=get_ssl_context(settings),
4242
enable_auto_commit=True,
4343
auto_offset_reset="earliest",
4444
value_deserializer=deserializer.deserialize,

‎platform/reworkd_platform/services/kafka/producers/base.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
1-
from ssl import create_default_context
21
from typing import Literal
32

43
from aiokafka import AIOKafkaProducer
54
from loguru import logger
65
from pydantic import BaseModel
76

7+
from reworkd_platform.services.ssl import get_ssl_context
88
from reworkd_platform.settings import Settings
99

1010
TOPICS = Literal["workflow_task"]
@@ -19,7 +19,7 @@ def __init__(self, settings: Settings):
1919
security_protocol="SASL_SSL",
2020
sasl_plain_username=settings.kafka_username,
2121
sasl_plain_password=settings.kafka_password,
22-
ssl_context=create_default_context(cafile=settings.db_ca_path),
22+
ssl_context=get_ssl_context(settings),
2323
)
2424

2525
self._headers = (

‎platform/reworkd_platform/services/ssl.py

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
from ssl import create_default_context, SSLContext
2+
from typing import List, Optional
3+
4+
from reworkd_platform.settings import Settings
5+
6+
MACOS_CERT_PATH = "/etc/ssl/cert.pem"
7+
DOCKER_CERT_PATH = "/etc/ssl/certs/ca-certificates.crt"
8+
9+
10+
def get_ssl_context(
11+
settings: Settings, paths: Optional[List[str]] = None
12+
) -> SSLContext:
13+
if settings.db_ca_path:
14+
return create_default_context(cafile=settings.db_ca_path)
15+
16+
for path in paths or [MACOS_CERT_PATH, DOCKER_CERT_PATH]:
17+
try:
18+
return create_default_context(cafile=path)
19+
except FileNotFoundError:
20+
continue
21+
22+
raise ValueError(
23+
"No CA certificates found for your OS. To fix this, please run change "
24+
"db_ca_path in your settings.py to point to a valid CA certificate file."
25+
)

‎platform/reworkd_platform/settings.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ class Settings(BaseSettings):
7272
db_pass: str = "reworkd_platform"
7373
db_base: str = "reworkd_platform"
7474
db_echo: bool = False
75-
db_ca_path: str = "/etc/ssl/cert.pem"
75+
db_ca_path: Optional[str] = None
7676

7777
# Variables for Weaviate db.
7878
vector_db_url: Optional[str] = None

‎platform/reworkd_platform/tests/test_ssl.py

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
import pytest
2+
3+
from reworkd_platform.services.ssl import (
4+
get_ssl_context,
5+
)
6+
from reworkd_platform.settings import Settings
7+
8+
9+
def test_get_ssl_context():
10+
get_ssl_context(Settings())
11+
12+
13+
def test_get_ssl_context_raise():
14+
settings = Settings()
15+
16+
with pytest.raises(ValueError):
17+
get_ssl_context(settings, paths=["/test/cert.pem"])
18+
19+
20+
def test_get_ssl_context_specified_raise():
21+
settings = Settings(db_ca_path="/test/cert.pem")
22+
23+
with pytest.raises(FileNotFoundError):
24+
get_ssl_context(settings)

0 commit comments

Comments
 (0)