Refactor project structure into sub-stacks

Add ability to selectively deploy stacks
Use cfn-lint instead of sam validate

wip

wip

.github/workflows/test-node.yml

@@ -3,11 +3,11 @@ on:
   push:
     paths:
       - ".github/workflows/test-node.yml"
-      - "node/**"
+      - "api/**"
   workflow_dispatch:
 defaults:
   run:
-    working-directory: ./node
+    working-directory: ./api
 jobs:
   test:
     runs-on: ubuntu-latest
@@ -20,7 +20,7 @@ jobs:
         with:
           node-version: 20.x
           cache: "npm"
-          cache-dependency-path: 'node/package-lock.json'
+          cache-dependency-path: 'api/package-lock.json'
       - run: npm ci
       - name: Check code style
         run: npm run lint && npm run prettier

.github/workflows/validate-template.yml

@@ -16,13 +16,15 @@ jobs:
         - uses: actions/setup-python@v4
           with:
             python-version: '3.12' 
+        - name: Install cfn-lint
+          run: pip install cfn-lint
         - uses: aws-actions/setup-sam@v1
-        - name: sam fix https://github.com/aws/aws-sam-cli/issues/4527
-          run: $(dirname $(readlink $(which sam)))/pip install --force-reinstall "cryptography==38.0.4"
+        # - name: sam fix https://github.com/aws/aws-sam-cli/issues/4527
+        #   run: $(dirname $(readlink $(which sam)))/pip install --force-reinstall "cryptography==38.0.4"
         - uses: aws-actions/configure-aws-credentials@master
           with:
             role-to-assume: arn:aws:iam::${{ secrets.AwsAccount }}:role/github-actions-role
             aws-region: us-east-1
         - uses: actions/checkout@v3
         - name: Validate template
-          run: sam build && sam validate
+          run: make build && make validate

Makefile

@@ -26,29 +26,39 @@ help:
 	echo "make test-python     | run python tests"
 	echo "make cover-node      | run node tests with coverage"
 	echo "make cover-python    | run python tests with coverage"
-.aws-sam/build.toml: ./template.yaml node/package-lock.json node/src/package-lock.json chat/dependencies/requirements.txt chat/src/requirements.txt
-	sed -Ei.orig 's/^(\s+)#\*\s/\1/' template.yaml
-	sed -Ei.orig 's/^(\s+)#\*\s/\1/' chat/template.yaml
-	sam build --cached --parallel
-	mv template.yaml.orig template.yaml
-	mv chat/template.yaml.orig chat/template.yaml
+
+.aws-sam/build.toml: ./template.yaml api/package-lock.json api/src/package-lock.json chat/dependencies/requirements.txt chat/src/requirements.txt
+	sed -Ei.orig 's/"dependencies"/"devDependencies"/' api/src/package.json
+	cp api/src/package-lock.json api/src/package-lock.json.orig
+	cd api/src && npm i --package-lock-only && cd -
+	for d in . api av-download chat docs ; do \
+		sed -Ei.orig 's/^(\s+)#\*\s/\1/' $$d/template.yaml; \
+	done
+
+	-sam build --cached --parallel
+
+	for d in . api av-download chat docs ; do \
+		mv $$d/template.yaml.orig $$d/template.yaml; \
+	done
+	mv api/src/package.json.orig api/src/package.json
+	mv api/src/package-lock.json.orig api/src/package-lock.json
 deps-node:
-	cd node/src ;\
+	cd api/src ;\
 	npm list >/dev/null 2>&1 ;\
 	src_deps=$$? ;\
 	cd .. ;\
 	npm list >/dev/null 2>&1 ;\
 	dev_deps=$$? ;\
 	test $$src_deps -eq 0 -a $$dev_deps -eq 0 || npm ci
 
-	cd lambdas ;\
+	cd av-download/lambdas ;\
 	npm list >/dev/null 2>&1 || npm ci
 cover-node: deps-node
-	cd node && npm run test:coverage
+	cd api && npm run test:coverage
 style-node: deps-node
-	cd node && npm run prettier
+	cd api && npm run prettier
 test-node: deps-node
-	cd node && npm run test
+	cd api && npm run test
 deps-python:
 	cd chat/src && pip install -r requirements.txt && pip install -r requirements-dev.txt
 cover-python: deps-python
@@ -63,16 +73,19 @@ test-python: deps-python
 	cd chat && __SKIP_SECRETS__=true SKIP_LLM_REQUEST=True PYTHONPATH=src:test python -m unittest discover -v
 python-version:
 	cd chat && python --version
-build: .aws-sam/build.toml
+build:
+	bash -c "trap 'trap - SIGINT SIGTERM ERR; $(MAKE) reset; exit 1' SIGINT SIGTERM ERR; $(MAKE) .aws-sam/build.toml reset"
+validate:
+	cfn-lint template.yaml **/template.yaml --ignore-checks E3510
 serve-http: deps-node
 	@printf '\033[0;31mWARNING: Serving only the local HTTP API. The chat websocket API is not available in local mode.\033[0m\n'
 	rm -rf .aws-sam
-	sam local start-api --host 0.0.0.0 --log-file dc-api.log ${SERVE_PARAMS}
+	sam local start-api -t api/template.yaml --env-vars $$PWD/env.json --host 0.0.0.0 --log-file dc-api.log ${SERVE_PARAMS}
 serve-https: SERVE_PARAMS = --port 3002 --ssl-cert-file $$HOME/.dev_cert/dev.rdc.cert.pem --ssl-key-file $$HOME/.dev_cert/dev.rdc.key.pem
 serve-https: serve-http
 serve: serve-https
 start-with-step: deps-node
-	sam local start-lambda --host 0.0.0.0 --port 3005 --env-vars env.json --log-file lambda.log & \
+	sam local start-lambda -t av-download/template.yaml --host 0.0.0.0 --port 3005 --env-vars $$PWD/env.json --log-file lambda.log & \
 	echo $$! > .sam-pids ;\
 	sam local start-api --host 0.0.0.0 --port 3002 --log-file dc-api.log \
 		--ssl-cert-file $$HOME/.dev_cert/dev.rdc.cert.pem --ssl-key-file $$HOME/.dev_cert/dev.rdc.key.pem & \
@@ -89,4 +102,6 @@ env:
 secrets:
 	ln -s ../tfvars/dc-api/* .
 clean:
-	rm -rf .aws-sam node/node_modules node/src/node_modules python/**/__pycache__ python/.coverage python/.ruff_cache
+	rm -rf .aws-sam api/node_modules api/src/node_modules python/**/__pycache__ python/.coverage python/.ruff_cache
+reset:
+	for f in $$(find . -maxdepth 2 -name '*.orig'); do mv $$f $${f%%.orig}; done

node/package.json → api/package.json

@@ -10,7 +10,7 @@
   },
   "scripts": {
     "lint": "eslint src/**/*.js test/**/*.js",
-    "preinstall": "cd src && npm i && cd - && cd ../lambdas && npm i && cd -",
+    "preinstall": "cd src && npm i && cd - && cd ../av-download/lambdas && npm i && cd -",
     "prettier": "prettier -c src test",
     "prettier:fix": "prettier -cw src test",
     "test": "mocha",

node/src/handlers/get-file-set-download.js → api/src/handlers/get-file-set-download.js

@@ -1,3 +1,7 @@
+const {
+  GetSecretValueCommand,
+  SecretsManagerClient,
+} = require("@aws-sdk/client-secrets-manager");
 const { SFNClient, StartExecutionCommand } = require("@aws-sdk/client-sfn");
 const { wrap } = require("./middleware");
 const { getFileSet } = require("../api/opensearch");
@@ -12,10 +16,15 @@ const mime = require("mime-types");
 const opensearchResponse = require("../api/response/opensearch");
 const path = require("path");
 
+let Secrets;
 /**
  * Handler for download file set endpoint
  */
-exports.handler = wrap(async (event) => {
+exports.handler = wrap(async (event, context) => {
+  const secretsManagerClient =
+    context?.injections?.secretsManagerClient || new SecretsManagerClient({});
+  await loadSecrets(secretsManagerClient);
+
   const id = event.pathParameters.id;
   const email = event.queryStringParameters?.email;
   const referer = event.headers?.referer;
@@ -57,6 +66,23 @@ exports.handler = wrap(async (event) => {
   }
 });
 
+async function loadSecrets(client) {
+  if (Secrets) return Secrets;
+
+  const { SECRETS_PATH } = process.env;
+  const SecretId = `${SECRETS_PATH}/config/av-download`;
+  try {
+    const cmd = new GetSecretValueCommand({ SecretId });
+    const secretsResponse = await client.send(cmd);
+    if (secretsResponse.SecretString) {
+      Secrets = JSON.parse(secretsResponse.SecretString);
+    }
+  } catch (err) {
+    console.warn("Error loading secrets from", SecretId);
+  }
+  return Secrets;
+}
+
 function isAltFileDownload(doc) {
   const acceptedTypes = [
     "application/pdf",
@@ -109,7 +135,7 @@ function derivativeKey(doc) {
 
 async function getDownloadLink(doc) {
   const clientParams = {};
-  const bucket = process.env.PYRAMID_BUCKET;
+  const bucket = Secrets.pyramid_bucket;
   const key = derivativeKey(doc);
 
   const getObjectParams = {
@@ -173,16 +199,16 @@ const IIIFImageRequest = async (doc) => {
 };
 
 async function processAVDownload(doc, email, referer) {
-  const stepFunctionConfig = process.env.STEP_FUNCTION_ENDPOINT
-    ? { endpoint: process.env.STEP_FUNCTION_ENDPOINT }
+  const stepFunctionConfig = Secrets.step_function_endpoint
+    ? { endpoint: Secrets.step_function_endpoint }
     : {};
   const client = new SFNClient(stepFunctionConfig);
 
   const fileSet = doc._source;
   const url = new URL(fileSet.streaming_url);
 
   const sourceLocation = s3Location(fileSet.streaming_url);
-  const destinationBucket = process.env.MEDIA_CONVERT_DESTINATION_BUCKET;
+  const destinationBucket = Secrets.media_convert_destination_bucket;
   const fileSetId = path.parse(url.pathname).name;
   const fileSetLabel = fileSet.label;
   const workId = fileSet.work_id;
@@ -197,14 +223,14 @@ async function processAVDownload(doc, email, referer) {
   const filename = isAudio(doc) ? `${fileSetId}.mp3` : `${fileSetId}.mp4`;
 
   var params = {
-    stateMachineArn: process.env.AV_DOWNLOAD_STATE_MACHINE_ARN,
+    stateMachineArn: Secrets.av_download_state_machine_arn,
     input: JSON.stringify({
       configuration: {
-        startAudioTranscodeFunction: process.env.START_AUDIO_TRANSCODE_FUNCTION,
-        startTranscodeFunction: process.env.START_TRANSCODE_FUNCTION,
-        transcodeStatusFunction: process.env.TRANSCODE_STATUS_FUNCTION,
-        getDownloadLinkFunction: process.env.GET_DOWNLOAD_LINK_FUNCTION,
-        sendTemplatedEmailFunction: process.env.SEND_TEMPLATED_EMAIL_FUNCTION,
+        startAudioTranscodeFunction: Secrets.start_audio_transcode_function,
+        startTranscodeFunction: Secrets.start_transcode_function,
+        transcodeStatusFunction: Secrets.transcode_status_function,
+        getDownloadLinkFunction: Secrets.get_download_link_function,
+        sendTemplatedEmailFunction: Secrets.send_templated_email_function,
       },
       transcodeInput: {
         settings: settings,
@@ -221,8 +247,8 @@ async function processAVDownload(doc, email, referer) {
       },
       sendEmailInput: {
         to: email,
-        template: process.env.AV_DOWNLOAD_EMAIL_TEMPLATE,
-        from: process.env.REPOSITORY_EMAIL,
+        template: Secrets.av_download_email_template,
+        from: Secrets.repository_email,
         params: {
           downloadLink: "",
           fileSetId,
@@ -253,7 +279,7 @@ async function processAVDownload(doc, email, referer) {
 
 function s3Location(streaming_url) {
   const url = new URL(streaming_url);
-  return `s3://${process.env.STREAMING_BUCKET}${url.pathname}`;
+  return `s3://${Secrets.streaming_bucket}${url.pathname}`;
 }
 
 function invalidRequest(code, message) {