Skip to content

Commit 0fe4979

Browse files
NathanFlurryNathanFlurry
committed
feat(toolchain): remote build method
1 parent 1511dc4 commit 0fe4979

File tree

32 files changed

+6792
-186
lines changed

32 files changed

+6792
-186
lines changed

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎cloud/packages/ci-manager/Dockerfile‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
# Build stage - includes dev dependencies
2+
FROM node:22-alpine AS builder
3+
ENV NODE_ENV=development
4+
5+
WORKDIR /app
6+
RUN npm i -g corepack && corepack enable
7+
COPY package.json yarn.lock ./
8+
RUN yarn install --immutable
9+
COPY . .
10+
RUN yarn build
11+
12+
# Production stage - only prod dependencies
13+
FROM node:22-alpine AS release
14+
ENV NODE_ENV=production
15+
16+
RUN adduser -s /bin/sh -D rivet
17+
WORKDIR /app
18+
RUN apk add --no-cache skopeo umoci && npm i -g corepack && corepack enable
19+
COPY package.json yarn.lock ./
20+
RUN yarn config set nodeLinker node-modules && yarn install --immutable
21+
COPY --from=builder /app/dist ./dist
22+
RUN chown -R rivet:rivet /app
23+
USER rivet
24+
EXPOSE 3000
25+
CMD ["node", "dist/index.js"]

‎cloud/packages/ci-manager/package.json‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
{
2+
"name": "ci-manager",
3+
"private": true,
4+
"peerDependencies": {
5+
"typescript": "^5"
6+
},
7+
"scripts": {
8+
"prepare-builds": "tsx scripts/prepare-builds.ts",
9+
"check-types": "tsc --noEmit",
10+
"test": "vitest",
11+
"build": "tsc"
12+
},
13+
"dependencies": {
14+
"@rivet-gg/api": "^25.4.2",
15+
"hono": "^4.7.11",
16+
"nanoevents": "^9.1.0",
17+
"zod": "^3.25.56"
18+
},
19+
"devDependencies": {
20+
"@hono/node-server": "^1.14.4",
21+
"@types/eventsource": "^1.1.15",
22+
"@types/node": "^22.15.30",
23+
"eventsource": "^4.0.0",
24+
"get-port": "^7.1.0",
25+
"tar": "^7.4.3",
26+
"tsx": "^4.19.4",
27+
"typescript": "^5.7.2",
28+
"vitest": "^3.2.2",
29+
"zx": "^8.1.9"
30+
},
31+
"packageManager": "[email protected]"
32+
}

‎cloud/packages/ci-manager/src/build-store.ts‎

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
import { BuildInfo, BuildEvent, Status } from "./types";
2+
import { randomUUID } from "crypto";
3+
import { mkdir, rm } from "fs/promises";
4+
import { join, dirname } from "path";
5+
import { createNanoEvents } from "nanoevents";
6+
7+
export class BuildStore {
8+
private builds = new Map<string, BuildInfo>();
9+
private tempDir: string;
10+
public emitter = createNanoEvents<{
11+
"build-event": (buildId: string, event: BuildEvent) => void;
12+
"status-change": (buildId: string, status: Status) => void;
13+
}>();
14+
15+
constructor(tempDir: string = "/tmp/ci-builds") {
16+
this.tempDir = tempDir;
17+
}
18+
19+
async init() {
20+
await mkdir(this.tempDir, { recursive: true });
21+
}
22+
23+
createBuild(buildName: string, dockerfilePath: string, environmentId: string): string {
24+
const id = randomUUID();
25+
const contextPath = join(this.tempDir, id, "context.tar.gz");
26+
const outputPath = join(this.tempDir, id, "output.tar.gz");
27+
28+
const build: BuildInfo = {
29+
id,
30+
status: { type: "starting", data: {} },
31+
buildName,
32+
dockerfilePath,
33+
environmentId,
34+
contextPath,
35+
outputPath,
36+
events: [],
37+
createdAt: new Date(),
38+
};
39+
40+
// Set up 10-minute cleanup timeout
41+
build.cleanupTimeout = setTimeout(
42+
() => {
43+
this.cleanupBuild(id, "timeout");
44+
},
45+
10 * 60 * 1000,
46+
); // 10 minutes
47+
48+
this.builds.set(id, build);
49+
return id;
50+
}
51+
52+
getBuild(id: string): BuildInfo | undefined {
53+
return this.builds.get(id);
54+
}
55+
56+
updateStatus(id: string, status: Status) {
57+
const build = this.builds.get(id);
58+
if (
59+
build &&
60+
build.status.type !== "success" &&
61+
build.status.type !== "failure"
62+
) {
63+
build.status = status;
64+
const event = { type: "status", data: status } as BuildEvent;
65+
build.events.push(event);
66+
this.emitter.emit("build-event", id, event);
67+
this.emitter.emit("status-change", id, status);
68+
console.log(`[${id}] status: ${JSON.stringify(status)}`);
69+
}
70+
}
71+
72+
addLog(id: string, line: string) {
73+
console.log(`[${id}] ${line}`);
74+
const build = this.builds.get(id);
75+
if (build) {
76+
const event: BuildEvent = { type: "log", data: { line } };
77+
build.events.push(event);
78+
this.emitter.emit("build-event", id, event);
79+
}
80+
}
81+
82+
setContainerProcess(id: string, process: any) {
83+
const build = this.builds.get(id);
84+
if (build) {
85+
build.containerProcess = process;
86+
}
87+
}
88+
89+
getContextPath(id: string): string | undefined {
90+
return this.builds.get(id)?.contextPath;
91+
}
92+
93+
getOutputPath(id: string): string | undefined {
94+
return this.builds.get(id)?.outputPath;
95+
}
96+
97+
markDownloaded(id: string) {
98+
const build = this.builds.get(id);
99+
if (build) {
100+
build.downloadedAt = new Date();
101+
// Trigger cleanup after download
102+
setTimeout(() => {
103+
this.cleanupBuild(id, "downloaded");
104+
}, 1000); // Small delay to ensure download is complete
105+
}
106+
}
107+
108+
private async cleanupBuild(id: string, reason: "timeout" | "downloaded") {
109+
const build = this.builds.get(id);
110+
if (!build) return;
111+
112+
console.log(`Cleaning up build ${id} (reason: ${reason})`);
113+
114+
try {
115+
// Clear the timeout if it exists
116+
if (build.cleanupTimeout) {
117+
clearTimeout(build.cleanupTimeout);
118+
}
119+
120+
// Remove build directory and all files
121+
if (build.contextPath) {
122+
const buildDir = dirname(build.contextPath);
123+
try {
124+
await rm(buildDir, { recursive: true, force: true });
125+
console.log(`Removed build directory: ${buildDir}`);
126+
} catch (error) {
127+
console.warn(
128+
`Failed to remove build directory ${buildDir}:`,
129+
error,
130+
);
131+
}
132+
}
133+
134+
// Remove from memory
135+
this.builds.delete(id);
136+
console.log(`Build ${id} cleaned up successfully`);
137+
} catch (error) {
138+
console.error(`Error cleaning up build ${id}:`, error);
139+
}
140+
}
141+
142+
// Manual cleanup method for testing or admin use
143+
async manualCleanup(id: string) {
144+
await this.cleanupBuild(id, "downloaded");
145+
}
146+
}

‎cloud/packages/ci-manager/src/executors/docker.ts‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
import { spawn } from "node:child_process";
2+
import { BuildStore } from "../build-store";
3+
4+
export async function runDockerBuild(
5+
buildStore: BuildStore,
6+
serverUrl: string,
7+
buildId: string,
8+
): Promise<void> {
9+
const build = buildStore.getBuild(buildId);
10+
if (!build) {
11+
throw new Error(`Build ${buildId} not found`);
12+
}
13+
14+
const contextUrl = `${serverUrl}/builds/${buildId}/kaniko/context.tar.gz`;
15+
const outputUrl = `${serverUrl}/builds/${buildId}/kaniko/output.tar.gz`;
16+
17+
const kanikoArgs = [
18+
"run",
19+
"--rm",
20+
"--network=host",
21+
"-e",
22+
`CONTEXT_URL=${contextUrl}`,
23+
"-e",
24+
`OUTPUT_URL=${outputUrl}`,
25+
"-e",
26+
`DESTINATION=${buildId}:latest`,
27+
"-e",
28+
`DOCKERFILE_PATH=${build.dockerfilePath}`,
29+
"ci-runner",
30+
];
31+
32+
buildStore.addLog(
33+
buildId,
34+
`Starting kaniko with args: docker ${kanikoArgs.join(" ")}`,
35+
);
36+
37+
return new Promise<void>((resolve, reject) => {
38+
const dockerProcess = spawn("docker", kanikoArgs, {
39+
stdio: ["pipe", "pipe", "pipe"],
40+
});
41+
42+
buildStore.setContainerProcess(buildId, dockerProcess);
43+
44+
dockerProcess.stdout?.on("data", (data) => {
45+
const lines = data
46+
.toString()
47+
.split("\n")
48+
.filter((line: string) => line.trim());
49+
lines.forEach((line: string) => {
50+
buildStore.addLog(buildId, `[kaniko] ${line}`);
51+
});
52+
});
53+
54+
dockerProcess.stderr?.on("data", (data) => {
55+
const lines = data
56+
.toString()
57+
.split("\n")
58+
.filter((line: string) => line.trim());
59+
lines.forEach((line: string) => {
60+
buildStore.addLog(buildId, `[kaniko-error] ${line}`);
61+
});
62+
});
63+
64+
dockerProcess.on("close", (code) => {
65+
buildStore.addLog(buildId, `Docker process closed with exit code: ${code}`);
66+
buildStore.updateStatus(buildId, { type: "finishing", data: {} });
67+
68+
if (code === 0) {
69+
resolve();
70+
} else {
71+
buildStore.updateStatus(buildId, {
72+
type: "failure",
73+
data: { reason: `Container exited with code ${code}` },
74+
});
75+
reject(new Error(`Container exited with code ${code}`));
76+
}
77+
});
78+
79+
dockerProcess.on("spawn", () => {
80+
buildStore.addLog(buildId, "Docker process spawned successfully");
81+
});
82+
83+
dockerProcess.on("error", (error) => {
84+
buildStore.addLog(buildId, `Docker process error: ${error.message}`);
85+
buildStore.updateStatus(buildId, {
86+
type: "failure",
87+
data: { reason: `Failed to start kaniko: ${error.message}` },
88+
});
89+
reject(error);
90+
});
91+
});
92+
}
93+
94+

0 commit comments

Comments
 (0)