-
Notifications
You must be signed in to change notification settings - Fork 13
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
chore: Utility to generate responses for development environment (#2790)
- Loading branch information
1 parent
534bb6e
commit f5044f6
Showing
8 changed files
with
1,275 additions
and
176 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,5 +1,6 @@ | ||
| /// <reference types="next" /> | ||
| /// <reference types="next/image-types/global" /> | ||
| /// <reference types="next/navigation-types/compat/navigation" /> | ||
|
|
||
| // NOTE: This file should not be edited | ||
| // see https://nextjs.org/docs/basic-features/typescript for more information. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| # Copy to .env | ||
|
|
||
| LOCAL_LAMBDA_ENDPOINT=http://127.0.0.1:3001 | ||
| LOCAL_AWS_ENDPOINT=http://localhost:4566 | ||
| RELIABILITY_FILE_STORAGE=forms-local-reliability-file-storage | ||
|
|
||
| AWS_ACCESS_KEY_ID=test | ||
| AWS_SECRET_ACCESS_KEY=test | ||
| AWS_REGION=ca-central-1 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,161 @@ | ||
| /* eslint-disable no-console */ | ||
| import readline from "readline"; | ||
| import { | ||
| SQSClient, | ||
| GetQueueUrlCommand, | ||
| ReceiveMessageCommand, | ||
| DeleteMessageCommand, | ||
| } from "@aws-sdk/client-sqs"; | ||
| import { LambdaClient, InvokeCommand } from "@aws-sdk/client-lambda"; | ||
| import { config } from "dotenv"; | ||
|
|
||
| const decoder = new TextDecoder(); | ||
|
|
||
| function getValue(query: string) { | ||
| const rl = readline.createInterface({ | ||
| input: process.stdin, | ||
| output: process.stdout, | ||
| }); | ||
|
|
||
| return new Promise<string>((resolve) => | ||
| rl.question(query, (ans) => { | ||
| rl.close(); | ||
| resolve(ans); | ||
| }) | ||
| ); | ||
| } | ||
|
|
||
| const twirlTimer = () => { | ||
| var P = ["\\", "|", "/", "-"]; | ||
| var x = 0; | ||
| return setInterval(function () { | ||
| process.stdout.write("\r" + P[x++]); | ||
| x &= 3; | ||
| }, 250); | ||
| }; | ||
|
|
||
| function writeWaitingPercent(current: number, total: number) { | ||
| readline.cursorTo(process.stdout, 0); | ||
| process.stdout.write(`waiting ... ${Math.round((current / total) * 100)}%`); | ||
| } | ||
|
|
||
| const getQueueURL = async (client: SQSClient) => { | ||
| const data = await client.send( | ||
| new GetQueueUrlCommand({ | ||
| QueueName: "submission_processing.fifo", | ||
| }) | ||
| ); | ||
| return data.QueueUrl; | ||
| }; | ||
|
|
||
| const main = async () => { | ||
| try { | ||
| const formID = await getValue("Form ID to generate responses for:"); | ||
| const numberOfResponses = parseInt(await getValue("Number of responses to generate:"), 10); | ||
|
|
||
| const encoder = new TextEncoder(); | ||
|
|
||
| // Setup all required services | ||
|
|
||
| const sqsClient = new SQSClient({ | ||
| region: process.env.AWS_REGION ?? "ca-central-1", | ||
| endpoint: process.env.LOCAL_AWS_ENDPOINT, | ||
| }); | ||
|
|
||
| const sqsQueueUrl = await getQueueURL(sqsClient); | ||
|
|
||
| const lambdaClient = new LambdaClient({ | ||
| region: "ca-central-1", | ||
| retryMode: "standard", | ||
| endpoint: process.env.LOCAL_LAMBDA_ENDPOINT, | ||
| }); | ||
|
|
||
| // Generate and submit responses | ||
| console.log("Generating responses for form."); | ||
|
|
||
| for (let response = 0; response < numberOfResponses; response++) { | ||
| const command = new InvokeCommand({ | ||
| FunctionName: "Submission", | ||
| Payload: encoder.encode( | ||
| JSON.stringify({ | ||
| formID, | ||
| responses: {}, | ||
| language: "en", | ||
| securityAttribute: "Protected A", | ||
| }) | ||
| ), | ||
| }); | ||
| try { | ||
| const response = await lambdaClient.send(command); | ||
|
|
||
| const payload = decoder.decode(response.Payload); | ||
| if (response.FunctionError || !JSON.parse(payload).status) { | ||
| throw new Error("Submission API could not process form response"); | ||
| } | ||
| } catch (err) { | ||
| console.error(err as Error); | ||
| throw new Error("Could not process request with Lambda Submission function"); | ||
| } | ||
| writeWaitingPercent(response + 1, numberOfResponses); | ||
| } | ||
|
|
||
| // Retrieve and process responses from Reliabilty Queue | ||
| let messagesWaiting = true; | ||
|
|
||
| console.log("\nProcessing responses in Reliability Queue"); | ||
|
|
||
| const workingOnProcessing = twirlTimer(); | ||
| while (messagesWaiting) { | ||
| try { | ||
| const receiveCommand = new ReceiveMessageCommand({ | ||
| QueueUrl: sqsQueueUrl, | ||
| MaxNumberOfMessages: 1, | ||
| VisibilityTimeout: 30, | ||
| WaitTimeSeconds: 5, | ||
| }); | ||
| const { Messages } = await sqsClient.send(receiveCommand); | ||
| // If there are no messages to process stop the loop | ||
| if (!Messages) { | ||
| messagesWaiting = false; | ||
| break; | ||
| } | ||
| const { Body: formResponse, ReceiptHandle } = Messages[0]; | ||
|
|
||
| const reliabilityCommand = new InvokeCommand({ | ||
| FunctionName: "Reliability", | ||
| Payload: encoder.encode( | ||
| JSON.stringify({ | ||
| Records: [ | ||
| { | ||
| body: formResponse, | ||
| }, | ||
| ], | ||
| }) | ||
| ), | ||
| }); | ||
| const response = await lambdaClient.send(reliabilityCommand); | ||
|
|
||
| if (response.FunctionError) { | ||
| throw new Error("Submission API could not process form response"); | ||
| } | ||
|
|
||
| const deleteCommand = new DeleteMessageCommand({ | ||
| QueueUrl: sqsQueueUrl, | ||
| ReceiptHandle, | ||
| }); | ||
| sqsClient.send(deleteCommand); | ||
| } catch (err) { | ||
| console.error(err as Error); | ||
| throw new Error("Could not process request with Lambda Submission function"); | ||
| } | ||
| } | ||
| clearInterval(workingOnProcessing); | ||
| console.log(`\nData generation completed for ${numberOfResponses} responses.`); | ||
| } catch (e) { | ||
| console.log(e); | ||
| } | ||
| }; | ||
| // config() adds the .env variables to process.env | ||
| config(); | ||
|
|
||
| main(); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| { | ||
| "name": "generate_responses", | ||
| "version": "0.1.0", | ||
| "description": "Generate responses to the Vault", | ||
| "main": "generate_responses.ts", | ||
| "scripts": { | ||
| "generate": "ts-node generate_responses.ts" | ||
| }, | ||
| "keywords": [], | ||
| "author": "Bryan Robitaille", | ||
| "dependencies": { | ||
| "@aws-sdk/client-lambda": "^3.429.0", | ||
| "@aws-sdk/client-sqs": "^3.429.0", | ||
| "@aws-sdk/types": "^3.428.0", | ||
| "@types/node": "^20.8.6", | ||
| "dotenv": "^16.3.1", | ||
| "ts-node": "^10.9.1", | ||
| "typescript": "^5.2.2" | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| { | ||
| "compilerOptions": { | ||
| /* Visit https://aka.ms/tsconfig to read more about this file */ | ||
|
|
||
| /* Language and Environment */ | ||
| "target": "es2016" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */, | ||
|
|
||
| /* Modules */ | ||
| "module": "commonjs" /* Specify what module code is generated. */, | ||
|
|
||
| /* Interop Constraints */ | ||
| "esModuleInterop": true /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */, | ||
| "forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */, | ||
|
|
||
| /* Type Checking */ | ||
| "strict": true /* Enable all strict type-checking options. */, | ||
|
|
||
| /* Completeness */ | ||
| "skipLibCheck": true /* Skip type checking all .d.ts files. */ | ||
| } | ||
| } |
Oops, something went wrong.