BM-1948: prover cluster update

bento/crates/api/src/lib.rs

@@ -169,7 +169,7 @@ impl IntoResponse for AppError {
 #[clap(author, version, about, long_about = None)]
 pub struct Args {
     /// Bind address for REST api
-    #[clap(long, default_value = "0.0.0.0:8080")]
+    #[clap(long, default_value = "0.0.0.0:8081")]
     bind_addr: String,
 
     /// SQL DB Connection pool connections

compose.yml

@@ -253,21 +253,20 @@ services:
       PROVER_RPC_URL: ${PROVER_RPC_URL:-${RPC_URL}}
     entrypoint: /app/broker --db-url 'sqlite:///db/broker.db' --config-file /app/broker.toml --bento-api-url http://localhost:8081
 
-# # Example second broker with different configuration
-# broker2:
-#   <<: *broker-common
-#   volumes:
-#     - type: bind
-#       source: ./broker2.toml
-#       target: /app/broker.toml
-#     - broker2-data:/db/
-#   environment:
-#     <<: *broker-environment
-#     # Note: use a different variable if you want to use different private keys in each broker
-#     PRIVATE_KEY: ${PROVER_PRIVATE_KEY:-${PRIVATE_KEY}}
-#     RPC_URL: ${RPC_URL_2}
-#   entrypoint: /app/broker --db-url 'sqlite:///db/broker2.db' --config-file /app/broker.toml --bento-api-url http://localhost:8081
-
+  # # Example second broker with different configuration
+  # broker2:
+  #   <<: *broker-common
+  #   volumes:
+  #     - type: bind
+  #       source: ./broker2.toml
+  #       target: /app/broker.toml
+  #     - broker2-data:/db/
+  #   environment:
+  #     <<: *broker-environment
+  #     # Note: use a different variable if you want to use different private keys in each broker
+  #     PRIVATE_KEY: ${PROVER_PRIVATE_KEY:-${PRIVATE_KEY}}
+  #     RPC_URL: ${RPC_URL_2}
+  #   entrypoint: /app/broker --db-url 'sqlite:///db/broker2.db' --config-file /app/broker.toml --bento-api-url http://localhost:8081
 
   miner:
     build:

infra/prover-cluster/Pulumi.prod-nightly-8453.yaml

@@ -35,3 +35,19 @@ config:
     secure: v1:9FrKfdSIsieo1bxD:S8s/9tyJN0U4V2r4PT81CQYWHgR9tbA=
   prover-cluster:apiKey:
     secure: v1:cQTlzOGOQeKDgN3q:xi9tY9JD1CKOjVZhxbnSczW3xiHs4dg+Ioi11z1lYiLkzuITbQQaqljkXtujYP/U1hirI5CLD7OzpOQ=
+  prover-cluster:mcyclePrice: "0.00000001"
+  prover-cluster:peakProveKhz: 10000
+  prover-cluster:minDeadline: 0
+  prover-cluster:lookbackBlocks: 300
+  prover-cluster:maxCollateral: "200"
+  prover-cluster:maxFileSize: "0"
+  prover-cluster:maxMcycleLimit: "0"
+  prover-cluster:maxConcurrentProofs: 1
+  prover-cluster:balanceWarnThreshold: "0"
+  prover-cluster:balanceErrorThreshold: "0"
+  prover-cluster:collateralBalanceWarnThreshold: "0"
+  prover-cluster:collateralBalanceErrorThreshold: "0"
+  prover-cluster:denyRequestorAddresses: ""
+  prover-cluster:maxFetchRetries: 3
+  prover-cluster:allowClientAddresses: ""
+  prover-cluster:lockinPriorityGas: "300"

infra/prover-cluster/Pulumi.prod-release-8453.yaml

@@ -35,3 +35,20 @@ config:
     secure: v1:9FrKfdSIsieo1bxD:S8s/9tyJN0U4V2r4PT81CQYWHgR9tbA=
   prover-cluster:apiKey:
     secure: v1:s3dh8LCcaq1tXtT2:Rbsvh1ktdKrD9WkEVc403al+3xRDFbbWp1FSKAlqsFy9t5mHBhTzXSF6hcTn8dzFsO/1+mNUDCwQktQ=
+  prover-cluster:allowListRequestorAddresses: '["0x323ec32ef13716bfdc3e0b2a96d7bd7cbcb9d57b", "0x2D611BE1e2E49C7b639A88b507b532DaE35e492b", "0xb59bb74fa0c1611eF6A4989a92C0d3ca6942fC0c", "0x1578934C9B006B9568D884C421704d996D402B78" ]'
+  prover-cluster:mcyclePrice: "0"
+  prover-cluster:peakProveKhz: 10000
+  prover-cluster:minDeadline: 0
+  prover-cluster:lookbackBlocks: 300
+  prover-cluster:maxCollateral: "200"
+  prover-cluster:maxFileSize: "0"
+  prover-cluster:maxMcycleLimit: "0"
+  prover-cluster:maxConcurrentProofs: 1
+  prover-cluster:balanceWarnThreshold: "0"
+  prover-cluster:balanceErrorThreshold: "0"
+  prover-cluster:collateralBalanceWarnThreshold: "0"
+  prover-cluster:collateralBalanceErrorThreshold: "0"
+  prover-cluster:denyRequestorAddresses: ""
+  prover-cluster:maxFetchRetries: 3
+  prover-cluster:allowClientAddresses: ""
+  prover-cluster:lockinPriorityGas: "300"

infra/prover-cluster/components/AutoScalingGroupComponent.ts

@@ -1,7 +1,7 @@
 import * as pulumi from "@pulumi/pulumi";
 import * as aws from "@pulumi/aws";
 import * as crypto from "crypto";
-import {BaseComponent, BaseComponentConfig} from "./BaseComponent";
+import { BaseComponent, BaseComponentConfig } from "./BaseComponent";
 
 export interface AutoScalingGroupConfig extends BaseComponentConfig {
     launchTemplateId: pulumi.Output<string>;
@@ -46,7 +46,7 @@ export class AutoScalingGroupComponent extends BaseComponent {
                 },
                 {
                     key: "Name",
-                    value: this.generateTagName(`${config.componentType}-asg`),
+                    value: this.generateName(`${config.componentType}`),
                     propagateAtLaunch: false,
                 },
                 {

infra/prover-cluster/components/DataServicesComponent.ts

@@ -0,0 +1,195 @@
+import * as pulumi from "@pulumi/pulumi";
+import * as aws from "@pulumi/aws";
+import { BaseComponent, BaseComponentConfig } from "./BaseComponent";
+
+export interface DataServicesComponentConfig extends BaseComponentConfig {
+    taskDBName: string;
+    taskDBUsername: string;
+    taskDBPassword: string;
+    securityGroupId: pulumi.Output<string>;
+    rdsInstanceClass?: string;
+    redisNodeType?: string;
+}
+
+export class DataServicesComponent extends BaseComponent {
+    public readonly rdsInstance: aws.rds.Instance;
+    public readonly rdsEndpoint: pulumi.Output<string>;
+    public readonly redisCluster: aws.elasticache.ReplicationGroup;
+    public readonly redisEndpoint: pulumi.Output<string>;
+    public readonly s3Bucket: aws.s3.Bucket;
+    public readonly s3BucketName: pulumi.Output<string>;
+    public readonly dbSubnetGroup: aws.rds.SubnetGroup;
+    public readonly rdsSecurityGroup: aws.ec2.SecurityGroup;
+    public readonly redisSecurityGroup: aws.ec2.SecurityGroup;
+    public readonly redisSubnetGroup: aws.elasticache.SubnetGroup;
+
+    constructor(config: DataServicesComponentConfig) {
+        super(config, "boundless-bento");
+
+        // Create DB subnet group
+        this.dbSubnetGroup = new aws.rds.SubnetGroup(`${config.stackName}-db-subnet-group`, {
+            name: this.generateName("db-subnet-group"),
+            subnetIds: config.privateSubnetIds,
+            tags: {
+                Environment: config.environment,
+                Stack: config.stackName,
+                Component: "data-services",
+            },
+        });
+
+        // Create RDS security group
+        this.rdsSecurityGroup = new aws.ec2.SecurityGroup(`${config.stackName}-rds-sg`, {
+            name: this.generateName("rds-sg"),
+            vpcId: config.vpcId,
+            description: "Security group for RDS PostgreSQL",
+            ingress: [{
+                protocol: "tcp",
+                fromPort: 5432,
+                toPort: 5432,
+                securityGroups: [config.securityGroupId],
+                description: "PostgreSQL access from cluster instances",
+            }],
+            egress: [{
+                protocol: "-1",
+                fromPort: 0,
+                toPort: 0,
+                cidrBlocks: ["0.0.0.0/0"],
+                description: "All outbound traffic",
+            }],
+            tags: {
+                Environment: config.environment,
+                Stack: config.stackName,
+                Component: "rds",
+            },
+        });
+
+        // Create Redis subnet group
+        // ElastiCache subnet group names must be unique across account and max 40 chars
+        // Use format: bb-redis-{stackName} (truncated to fit 40 chars)
+        const prefix = "bb-redis-";
+        const maxStackNameLength = 40 - prefix.length;
+        const truncatedStackName = config.stackName.length > maxStackNameLength
+            ? config.stackName.substring(0, maxStackNameLength)
+            : config.stackName;
+        const redisSubnetGroupName = `${prefix}${truncatedStackName}`;
+        this.redisSubnetGroup = new aws.elasticache.SubnetGroup(`${config.stackName}-redis-subnet-group`, {
+            name: redisSubnetGroupName,
+            subnetIds: config.privateSubnetIds,
+            tags: {
+                Environment: config.environment,
+                Stack: config.stackName,
+                Component: "data-services",
+            },
+        });
+
+        // Create Redis security group
+        this.redisSecurityGroup = new aws.ec2.SecurityGroup(`${config.stackName}-redis`, {
+            name: config.stackName,
+            vpcId: config.vpcId,
+            description: "Security group for ElastiCache Redis",
+            ingress: [{
+                protocol: "tcp",
+                fromPort: 6379,
+                toPort: 6379,
+                securityGroups: [config.securityGroupId],
+                description: "Redis access from cluster instances",
+            }],
+            egress: [{
+                protocol: "-1",
+                fromPort: 0,
+                toPort: 0,
+                cidrBlocks: ["0.0.0.0/0"],
+                description: "All outbound traffic",
+            }],
+            tags: {
+                Environment: config.environment,
+                Stack: config.stackName,
+                Component: "redis",
+            },
+        });
+
+        // Create RDS PostgreSQL instance
+        this.rdsInstance = new aws.rds.Instance(`${config.stackName}`, {
+            identifier: this.generateName("postgres"),
+            engine: "postgres",
+            engineVersion: "17.4",
+            instanceClass: config.rdsInstanceClass || "db.t4g.micro",
+            allocatedStorage: 20,
+            maxAllocatedStorage: 100,
+            storageType: "gp3",
+            storageEncrypted: true,
+            dbName: config.taskDBName,
+            username: config.taskDBUsername,
+            password: config.taskDBPassword,
+            port: 5432,
+            publiclyAccessible: false,
+            dbSubnetGroupName: this.dbSubnetGroup.name,
+            vpcSecurityGroupIds: [this.rdsSecurityGroup.id],
+            skipFinalSnapshot: true,
+            backupRetentionPeriod: 7,
+            tags: {
+                Environment: config.environment,
+                Stack: config.stackName,
+                Component: "postgres",
+            },
+        });
+
+        // RDS endpoint is just the hostname, need to append port
+        this.rdsEndpoint = pulumi.interpolate`${this.rdsInstance.endpoint}:${this.rdsInstance.port}`;
+
+        // Create ElastiCache Redis replication group
+        // ElastiCache replication group IDs must be 1-40 characters
+        // Use a shorter name: stackName-redis (max 40 chars)
+        const redisGroupId = config.stackName.length > 35
+            ? `${config.stackName.substring(0, 34)}-redis`
+            : `${config.stackName}-redis`;
+        this.redisCluster = new aws.elasticache.ReplicationGroup(`${config.stackName}`, {
+            replicationGroupId: redisGroupId,
+            description: `Redis cluster for ${config.stackName}`,
+            engine: "redis",
+            engineVersion: "7.1",
+            nodeType: config.redisNodeType || "cache.t4g.micro",
+            port: 6379,
+            parameterGroupName: "default.redis7",
+            numCacheClusters: 1,
+            subnetGroupName: this.redisSubnetGroup.name,
+            securityGroupIds: [this.redisSecurityGroup.id],
+            atRestEncryptionEnabled: true,
+            transitEncryptionEnabled: false,
+            automaticFailoverEnabled: false,
+            tags: {
+                Name: config.stackName,
+                Environment: config.environment,
+                Stack: config.stackName,
+                Component: "redis",
+            },
+        });
+
+        // Redis endpoint is just the hostname, need to append port
+        this.redisEndpoint = pulumi.interpolate`${this.redisCluster.primaryEndpointAddress}:${this.redisCluster.port}`;
+
+        // Create S3 bucket for workflow storage
+        const bucketName = this.generateName("bento-storage");
+        this.s3Bucket = new aws.s3.Bucket(`${config.stackName}-storage`, {
+            bucket: bucketName,
+            forceDestroy: false, // Prevent accidental deletion
+            tags: {
+                Environment: config.environment,
+                Stack: config.stackName,
+                Component: "s3-storage",
+            },
+        });
+
+        // Block public access
+        new aws.s3.BucketPublicAccessBlock(`${config.stackName}-storage-pab`, {
+            bucket: this.s3Bucket.id,
+            blockPublicAcls: true,
+            blockPublicPolicy: true,
+            ignorePublicAcls: true,
+            restrictPublicBuckets: true,
+        });
+
+        this.s3BucketName = this.s3Bucket.id;
+    }
+}
+