Skip to content

Commit 4801dea

Browse files
committed
feat: get multi actors working e2e on docker compose
1 parent 919290e commit 4801dea

File tree

14 files changed

+331
-116
lines changed

14 files changed

+331
-116
lines changed

docker/dev-full/docker-compose.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -356,6 +356,8 @@ services:
356356
volumes:
357357
- vector-server-data:/var/lib/vector
358358
- ./vector-server:/etc/vector
359+
# environment:
360+
# - VECTOR_LOG=debug
359361
networks:
360362
- rivet-network
361363

docker/dev-full/vector-client/vector.yaml

Lines changed: 6 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -21,15 +21,10 @@ sources:
2121
include:
2222
- /var/lib/rivet-client/log
2323

24-
pegboard_v8_isolate_runner:
25-
type: file
26-
include:
27-
- /var/lib/rivet-client/runner/log
28-
2924
pegboard_container_runners:
3025
type: file
3126
include:
32-
- /var/lib/rivet-client/actors/*/log
27+
- /var/lib/rivet-client/runners/*/log
3328

3429
transforms:
3530
filter_metrics:
@@ -46,7 +41,7 @@ transforms:
4641
.tags.server_id = "fc67e54e-5d6a-4726-ab23-77b0e54f068f"
4742
.tags.datacenter_id = "f288913c-735d-4188-bf9b-2fcf6eac7b9c"
4843
.tags.cluster_id = "unknown"
49-
.tags.pool_type = "pegboard_isolate"
44+
.tags.pool_type = "pegboard"
5045
.tags.public_ip = "127.0.0.1"
5146
5247
pegboard_manager_add_meta:
@@ -60,36 +55,22 @@ transforms:
6055
.server_id = "fc67e54e-5d6a-4726-ab23-77b0e54f068f"
6156
.datacenter_id = "f288913c-735d-4188-bf9b-2fcf6eac7b9c"
6257
.tags.cluster_id = "unknown"
63-
.pool_type = "pegboard_isolate"
64-
.public_ip = "127.0.0.1"
65-
66-
pegboard_v8_isolate_runner_add_meta:
67-
type: remap
68-
inputs:
69-
- pegboard_v8_isolate_runner
70-
source: |
71-
.source = "pegboard_v8_isolate_runner"
72-
73-
.client_id = "fc67e54e-5d6a-4726-ab23-77b0e54f068f"
74-
.server_id = "fc67e54e-5d6a-4726-ab23-77b0e54f068f"
75-
.datacenter_id = "f288913c-735d-4188-bf9b-2fcf6eac7b9c"
76-
.tags.cluster_id = "unknown"
77-
.pool_type = "pegboard_isolate"
58+
.pool_type = "pegboard"
7859
.public_ip = "127.0.0.1"
79-
60+
8061
pegboard_container_runner_add_meta:
8162
type: remap
8263
inputs:
8364
- pegboard_container_runners
8465
source: |
8566
.source = "pegboard_container_runner"
86-
.actor_id = parse_regex!(.file, r'/etc/pegboard/actors/(?P<actor_id>[0-9a-fA-F-]+)/log').actor_id
67+
.runner_id = parse_regex!(.file, r'/etc/pegboard/runners/(?P<runner_id>[0-9a-fA-F-]+)/log').runner_id
8768
8869
.client_id = "fc67e54e-5d6a-4726-ab23-77b0e54f068f"
8970
.server_id = "fc67e54e-5d6a-4726-ab23-77b0e54f068f"
9071
.datacenter_id = "f288913c-735d-4188-bf9b-2fcf6eac7b9c"
9172
.cluster_id = "unknown"
92-
.pool_type = "pegboard_isolate"
73+
.pool_type = "pegboard"
9374
.public_ip = "127.0.0.1"
9475
9576
sinks:
@@ -99,7 +80,6 @@ sinks:
9980
- metrics_add_meta
10081
- dynamic_events_http
10182
- pegboard_manager_add_meta
102-
- pegboard_v8_isolate_runner_add_meta
10383
- pegboard_container_runner_add_meta
10484
address: vector-server:6000
10585
healthcheck:

docker/dev-full/vector-server/vector.yaml

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,15 @@ transforms:
3434
condition:
3535
type: vrl
3636
source: .source == "actors"
37+
38+
runners:
39+
type: filter
40+
inputs:
41+
- vector
42+
- tcp_json
43+
condition:
44+
type: vrl
45+
source: .source == "runners"
3746

3847
job_run:
3948
type: filter
@@ -118,6 +127,21 @@ sinks:
118127
password: vector
119128
batch:
120129
timeout_secs: 1.0
130+
131+
clickhouse_runner_logs:
132+
type: clickhouse
133+
inputs:
134+
- runners
135+
compression: gzip
136+
database: db_pegboard_runner_log
137+
endpoint: http://clickhouse:8123
138+
table: runner_logs
139+
auth:
140+
strategy: basic
141+
user: vector
142+
password: vector
143+
batch:
144+
timeout_secs: 1.0
121145

122146
clickhouse_job_run_logs:
123147
type: clickhouse

docker/http-debug/Dockerfile

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
FROM python:3.11-slim
2+
3+
WORKDIR /app
4+
5+
# Install required packages
6+
RUN pip install flask
7+
8+
# Create the debug server
9+
COPY debug_server.py .
10+
11+
# Expose port 8080
12+
EXPOSE 8080
13+
14+
# Run the server
15+
CMD ["python", "debug_server.py"]

docker/http-debug/debug_server.py

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
from flask import Flask, request, jsonify
2+
import gzip
3+
import zlib
4+
from datetime import datetime
5+
6+
app = Flask(__name__)
7+
8+
def log_request_details():
9+
"""Log all request details in a formatted way"""
10+
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
11+
12+
print("=" * 80)
13+
print(f"[{timestamp}] NEW REQUEST")
14+
print("=" * 80)
15+
16+
# Method and URL
17+
print(f"Method: {request.method}")
18+
print(f"URL: {request.url}")
19+
print(f"Path: {request.path}")
20+
print(f"Query String: {request.query_string.decode('utf-8')}")
21+
22+
# Headers
23+
print("\n--- HEADERS ---")
24+
for header_name, header_value in request.headers:
25+
print(f"{header_name}: {header_value}")
26+
27+
# Query Parameters
28+
if request.args:
29+
print("\n--- QUERY PARAMETERS ---")
30+
for key, value in request.args.items():
31+
print(f"{key}: {value}")
32+
33+
# Form Data
34+
if request.form:
35+
print("\n--- FORM DATA ---")
36+
for key, value in request.form.items():
37+
print(f"{key}: {value}")
38+
39+
# Files
40+
if request.files:
41+
print("\n--- FILES ---")
42+
for key, file in request.files.items():
43+
print(f"{key}: {file.filename} (Content-Type: {file.content_type})")
44+
45+
# Raw Body with decompression support
46+
try:
47+
raw_body = request.get_data()
48+
content_encoding = request.headers.get('Content-Encoding', '').lower()
49+
50+
print("\n--- REQUEST BODY ---")
51+
52+
if not raw_body:
53+
print("(empty)")
54+
else:
55+
# Handle compressed content
56+
decompressed_body = None
57+
58+
if content_encoding == 'gzip':
59+
try:
60+
decompressed_body = gzip.decompress(raw_body).decode('utf-8')
61+
print("(Content was gzip-compressed, showing decompressed version)")
62+
except Exception as e:
63+
print(f"Failed to decompress gzip content: {e}")
64+
elif content_encoding == 'deflate':
65+
try:
66+
decompressed_body = zlib.decompress(raw_body).decode('utf-8')
67+
print("(Content was deflate-compressed, showing decompressed version)")
68+
except Exception as e:
69+
print(f"Failed to decompress deflate content: {e}")
70+
elif content_encoding in ['br', 'brotli']:
71+
print("(Content is brotli-compressed - brotli decompression not available)")
72+
print("Raw compressed data (first 200 bytes):")
73+
print(repr(raw_body[:200]))
74+
elif content_encoding == 'zstd':
75+
print("(Content is zstd-compressed - zstd decompression not available)")
76+
print("Raw compressed data (first 200 bytes):")
77+
print(repr(raw_body[:200]))
78+
else:
79+
# No compression or unknown compression
80+
try:
81+
decompressed_body = raw_body.decode('utf-8')
82+
except UnicodeDecodeError:
83+
print("(Binary content - showing first 200 bytes as hex)")
84+
print(raw_body[:200].hex())
85+
86+
# Display the decompressed content
87+
if decompressed_body:
88+
print(decompressed_body)
89+
90+
# Also show raw length info
91+
print(f"\nRaw body length: {len(raw_body)} bytes")
92+
print(f"Decompressed length: {len(decompressed_body)} bytes")
93+
94+
except Exception as e:
95+
print(f"\n--- REQUEST BODY ---")
96+
print(f"Error reading body: {e}")
97+
98+
print("=" * 80)
99+
print()
100+
101+
# Catch all routes for any HTTP method
102+
@app.route('/', defaults={'path': ''}, methods=['GET', 'POST', 'PUT', 'DELETE', 'PATCH', 'HEAD', 'OPTIONS'])
103+
@app.route('/<path:path>', methods=['GET', 'POST', 'PUT', 'DELETE', 'PATCH', 'HEAD', 'OPTIONS'])
104+
def debug_endpoint(path):
105+
log_request_details()
106+
107+
# Return a simple response
108+
response_data = {
109+
"message": "Request received and logged",
110+
"method": request.method,
111+
"path": f"/{path}" if path else "/",
112+
"timestamp": datetime.now().isoformat()
113+
}
114+
115+
return jsonify(response_data), 200
116+
117+
if __name__ == '__main__':
118+
print("Starting HTTP Debug Server...")
119+
print("All incoming requests will be logged to console")
120+
print("Server listening on port 8080")
121+
print("=" * 80)
122+
app.run(host='0.0.0.0', port=8080, debug=False)

0 commit comments

Comments
 (0)