Merge pull request #18 from guardrails-ai/get-guard-steps

Get Guard Step

.dockerignore

@@ -1,2 +1,9 @@
-.venv
+.venv*
 diagrams
+.github
+buildspecs
+tests
+docker*
+Docker*
+*.sh
+ingestion-service-spec.yml

.github/workflows/pr_qa.yml

@@ -18,7 +18,7 @@ jobs:
       - name: Quality Checks
         run: |
           python -m pip install --upgrade pip;
-          pip install -r requirements.txt;
+          make build;
           make qa;
           
           ##

.github/workflows/publish_image.yml

@@ -0,0 +1,38 @@
+name: Publish Image To ER
+on:
+  push:
+    branches: 
+      - 'main'
+  workflow_dispatch:
+
+jobs:
+  build:
+    name: Build Image
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-1
+
+      # - name: Login to Amazon ECR
+      #   id: login-ecr
+      #   uses: aws-actions/amazon-ecr-login@v1
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build, tag, and push image to Amazon ECR
+        env:
+          IMAGE_NAME: api
+          ECR_REPO_NAME: guardrails-validation-service-test
+        run: |
+          bash buildspecs/build.sh

.gitignore

@@ -1,5 +1,5 @@
 **/__pycache__
-.venv
+.venv*
 postgres
 venv
 guardrails-sdk
@@ -16,3 +16,7 @@ coverage.json
 .ropeproject
 install*.sh
 guardrails-custom-validators
+guardrails-api-client
+guardrails
+*.env
+opentelemetry-lambda-layer

Dockerfile.dev

@@ -1,5 +1,4 @@
-# FROM python:3.6
-FROM public.ecr.aws/docker/library/python:3.11-slim
+FROM public.ecr.aws/docker/library/python:3.11.5-slim
 
 # Create app directory
 WORKDIR /app
@@ -12,6 +11,10 @@ RUN python3 -m venv /opt/venv
 # Enable venv
 ENV PATH="/opt/venv/bin:$PATH"
 
+# Install gcc
+RUN apt-get update
+RUN apt-get install -y gcc
+
 COPY ./guardrails-sdk ./guardrails-sdk
 
 COPY ./guard-rails-api-client ./guard-rails-api-client
@@ -21,16 +24,15 @@ COPY ./guardrails-custom-validators ./guardrails-custom-validators
 # Copy the requirements file
 COPY requirements.txt .
 
-# RUN apt-get update
-# RUN apt-get install build-essential -y
-# RUN apt-get install python3-dev -y
-# RUN apt-get install libevent-dev -y
 RUN pip install ./guardrails-sdk
 RUN pip install ./guard-rails-api-client
 
 # Install app dependencies
 RUN pip install -r requirements.txt
 
+# Download punkt data
+RUN python -m nltk.downloader -d /opt/nltk_data punkt
+
 # Freeze dependencies
 RUN pip freeze > requirements.txt
 

Dockerfile.prod

@@ -0,0 +1,45 @@
+FROM public.ecr.aws/docker/library/python:3.11.5-slim
+
+COPY --from=public.ecr.aws/awsguru/aws-lambda-adapter:0.7.1 /lambda-adapter /opt/extensions/lambda-adapter
+COPY ./opentelemetry-lambda-layer /opt
+
+# Create app directory
+WORKDIR /app
+
+# check the version
+RUN python3 --version
+# start the virtual environment
+RUN python3 -m venv /opt/venv
+
+# Enable venv
+ENV PATH="/opt/venv/bin:$PATH"
+
+# Install git and curl
+RUN apt-get update
+RUN apt-get install -y git curl gcc
+
+# Copy pre-built api client
+COPY ./guard-rails-api-client ./guard-rails-api-client
+
+# Copy the requirements file
+COPY requirements.txt .
+
+RUN pip install ./guard-rails-api-client
+RUN pip install git+https://github.com/guardrails-ai/guardrails-internal.git@telemetry
+RUN curl https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem -o ./global-bundle.pem
+
+# Install app dependencies
+RUN pip install -r requirements.txt
+
+# Download punkt data
+RUN python -m nltk.downloader -d /opt/nltk_data punkt
+
+# Freeze dependencies
+RUN pip freeze > requirements.txt
+
+# Copy the whole folder inside the Image filesystem
+COPY . .
+
+EXPOSE 8000
+
+CMD opentelemetry-instrument gunicorn --bind 0.0.0.0:8000 --timeout=5 --threads=10 "app:create_app()"

Makefile

@@ -1,6 +1,12 @@
 build-sdk:
 	bash build-sdk.sh
 
+build:
+	pip install -r requirements.txt;
+	pip install git+https://github.com/guardrails-ai/guardrails-internal.git@telemetry;
+	make build-sdk;
+	pip install ./guard-rails-api-client
+
 dev:
 	bash ./dev.sh
 

app.py

@@ -5,6 +5,7 @@
 app = Flask(__name__)
 CORS(app)
 
+
 def create_app():
     api_doc(
         app,
@@ -14,7 +15,7 @@ def create_app():
     )
 
     from src.clients.postgres_client import PostgresClient
-    
+
     pg_client = PostgresClient()
     pg_client.initialize(app)
 
@@ -24,4 +25,4 @@ def create_app():
     app.register_blueprint(root_bp)
     app.register_blueprint(guards_bp)
 
-    return app
+    return app

buildspecs/build.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+callerIdentity=$(aws sts get-caller-identity);
+accountId=$(jq -r .Account <<< $callerIdentity);
+imageName="${IMAGE_NAME:-validation-service}";
+repoName="${ECR_REPO_NAME:-guardrails-validation-service-test}";
+commitSha=$(git rev-parse HEAD);
+region="${AWS_DEFAULT_REGION:-us-east-1}";
+defaultEcrEndpoint="${accountId}.dkr.ecr.${region}.amazonaws.com";
+ecrEndpoint="${ECR_ENDPOINT:-$defaultEcrEndpoint}"
+ecrImageUrl="${ecrEndpoint}/${repoName}";
+
+# Setup unpublished api client
+echo "Building api client..."
+bash build-sdk.sh
+
+# Building OTEL Collector extension
+if [ -d "opentelemetry-lambda-layer" ]; then
+  rm -rf opentelemetry-lambda-layer
+fi
+
+curl $(aws lambda get-layer-version-by-arn --arn arn:aws:lambda:us-east-1:184161586896:layer:opentelemetry-collector-arm64-0_2_0:1 --query 'Content.Location' --output text) --output otel-collector.zip
+unzip otel-collector.zip -d ./opentelemetry-lambda-layer
+rm otel-collector.zip
+
+echo "Performing docker build"
+docker login -u AWS -p $(aws ecr get-login-password --region $region) $ecrEndpoint;
+
+docker buildx build \
+  --platform linux/arm64 \
+  --progress plain \
+  --no-cache \
+  --build-arg CACHEBUST="$(date)" \
+  -f Dockerfile.prod \
+  -t "$imageName:$commitSha" \
+  -t "$imageName:latest" . \
+  || exit 1;
+  # > ./out.log 2>&1 \
+
+docker image tag "$imageName:$commitSha" "$ecrImageUrl:$commitSha";
+docker image tag "$imageName:latest" "$ecrImageUrl:latest";
+
+# echo "Publishing to ECR"
+docker push $ecrImageUrl --all-tags;

buildspecs/deploy.yml

@@ -0,0 +1,8 @@
+version: 0.2
+
+phases:
+  build:
+    commands:
+      - echo "Deploying $ECR_ENDPOINT:$IMAGE_VERSION_TAG to lambda function $LAMBDA_FUNCTION_NAME"
+      - aws lambda update-function-code --function-name $LAMBDA_FUNCTION_NAME --image-uri "$ECR_ENDPOINT:$IMAGE_VERSION_TAG"
+      - echo "Deployment complete"

cdktf/.gitignore

@@ -0,0 +1,11 @@
+*.d.ts
+*.js
+node_modules
+cdktf.out
+cdktf.log
+*terraform.*.tfstate*
+.gen
+.terraform
+tsconfig.tsbuildinfo
+!jest.config.js
+!setup.js

cdktf/__tests__/main-test.ts

@@ -0,0 +1,89 @@
+// Copyright (c) HashiCorp, Inc
+// SPDX-License-Identifier: MPL-2.0
+import "cdktf/lib/testing/adapters/jest"; // Load types for expect matchers
+// import { Testing } from "cdktf";
+
+describe("My CDKTF Application", () => {
+  // The tests below are example tests, you can find more information at
+  // https://cdk.tf/testing
+  it.todo("should be tested");
+
+  // // All Unit tests test the synthesised terraform code, it does not create real-world resources
+  // describe("Unit testing using assertions", () => {
+  //   it("should contain a resource", () => {
+  //     // import { Image,Container } from "./.gen/providers/docker"
+  //     expect(
+  //       Testing.synthScope((scope) => {
+  //         new MyApplicationsAbstraction(scope, "my-app", {});
+  //       })
+  //     ).toHaveResource(Container);
+
+  //     expect(
+  //       Testing.synthScope((scope) => {
+  //         new MyApplicationsAbstraction(scope, "my-app", {});
+  //       })
+  //     ).toHaveResourceWithProperties(Image, { name: "ubuntu:latest" });
+  //   });
+  // });
+
+  // describe("Unit testing using snapshots", () => {
+  //   it("Tests the snapshot", () => {
+  //     const app = Testing.app();
+  //     const stack = new TerraformStack(app, "test");
+
+  //     new TestProvider(stack, "provider", {
+  //       accessKey: "1",
+  //     });
+
+  //     new TestResource(stack, "test", {
+  //       name: "my-resource",
+  //     });
+
+  //     expect(Testing.synth(stack)).toMatchSnapshot();
+  //   });
+
+  //   it("Tests a combination of resources", () => {
+  //     expect(
+  //       Testing.synthScope((stack) => {
+  //         new TestDataSource(stack, "test-data-source", {
+  //           name: "foo",
+  //         });
+
+  //         new TestResource(stack, "test-resource", {
+  //           name: "bar",
+  //         });
+  //       })
+  //     ).toMatchInlineSnapshot();
+  //   });
+  // });
+
+  // describe("Checking validity", () => {
+  //   it("check if the produced terraform configuration is valid", () => {
+  //     const app = Testing.app();
+  //     const stack = new TerraformStack(app, "test");
+
+  //     new TestDataSource(stack, "test-data-source", {
+  //       name: "foo",
+  //     });
+
+  //     new TestResource(stack, "test-resource", {
+  //       name: "bar",
+  //     });
+  //     expect(Testing.fullSynth(app)).toBeValidTerraform();
+  //   });
+
+  //   it("check if this can be planned", () => {
+  //     const app = Testing.app();
+  //     const stack = new TerraformStack(app, "test");
+
+  //     new TestDataSource(stack, "test-data-source", {
+  //       name: "foo",
+  //     });
+
+  //     new TestResource(stack, "test-resource", {
+  //       name: "bar",
+  //     });
+  //     expect(Testing.fullSynth(app)).toPlanSuccessfully();
+  //   });
+  // });
+});