Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: create only one instance of DynamoDBDocumentClient #3565

Merged
merged 1 commit into from
May 7, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 8 additions & 21 deletions app/api/id/[form]/submission/confirm/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,21 +2,16 @@ import { NextResponse } from "next/server";
import { logMessage } from "@lib/logger";
import { middleware, jsonValidator, sessionExists } from "@lib/middleware";
import uuidArraySchema from "@lib/middleware/schemas/uuid-array.schema.json";
import {
DynamoDBDocumentClient,
BatchGetCommand,
TransactWriteCommand,
} from "@aws-sdk/lib-dynamodb";
import { BatchGetCommand, TransactWriteCommand } from "@aws-sdk/lib-dynamodb";
import { MiddlewareProps, WithRequired } from "@lib/types";
import { connectToDynamo } from "@lib/integration/dynamodbConnector";
import { dynamodbClient } from "@lib/integration/dynamodbConnector";
import { AccessControlError, createAbility } from "@lib/privileges";
import { checkUserHasTemplateOwnership } from "@lib/templates";
import { logEvent } from "@lib/auditLogs";

async function getSubmissionsFromConfirmationCodes(
formId: string,
confirmationCodes: string[],
dynamoDbClient: DynamoDBDocumentClient
confirmationCodes: string[]
): Promise<{
submissionsToConfirm: { name: string; confirmationCode: string }[];
confirmationCodesAlreadyUsed: string[];
Expand Down Expand Up @@ -44,7 +39,7 @@ async function getSubmissionsFromConfirmationCodes(
});

// eslint-disable-next-line no-await-in-loop
const response = await dynamoDbClient.send(request);
const response = await dynamodbClient.send(request);

if (response.Responses?.Vault) {
response.Responses.Vault.forEach((record) => {
Expand Down Expand Up @@ -91,8 +86,7 @@ async function getSubmissionsFromConfirmationCodes(

async function confirm(
formId: string,
submissionsToConfirm: { name: string; confirmationCode: string }[],
dynamoDbClient: DynamoDBDocumentClient
submissionsToConfirm: { name: string; confirmationCode: string }[]
): Promise<void> {
const confirmationTimestamp = Date.now();
const removalDate = confirmationTimestamp + 2592000000; // 2592000000 milliseconds = 30 days
Expand Down Expand Up @@ -136,7 +130,7 @@ async function confirm(
}),
});

await dynamoDbClient.send(request);
await dynamodbClient.send(request);
}

export const PUT = middleware(
Expand Down Expand Up @@ -184,22 +178,15 @@ export const PUT = middleware(
}

try {
const dynamoDbClient = connectToDynamo();

// max 100 confirmation codes per request
const submissionsFromConfirmationCodes = await getSubmissionsFromConfirmationCodes(
formId,
confirmationCodes,
dynamoDbClient
confirmationCodes
);

if (submissionsFromConfirmationCodes.submissionsToConfirm.length > 0) {
// max 50 submissions per request
await confirm(
formId,
submissionsFromConfirmationCodes.submissionsToConfirm,
dynamoDbClient
);
await confirm(formId, submissionsFromConfirmationCodes.submissionsToConfirm);
// Done asychronously to not block response back to client
submissionsFromConfirmationCodes.submissionsToConfirm.forEach((confirmation) =>
logEvent(
Expand Down
25 changes: 8 additions & 17 deletions app/api/id/[form]/submission/report/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,9 @@ import { logMessage } from "@lib/logger";
import { middleware, jsonValidator, sessionExists } from "@lib/middleware";
import { createTicket } from "@lib/integration/freshdesk";
import downloadReportProblemSchema from "@lib/middleware/schemas/download-report-problem-schema.json";
import {
BatchGetCommand,
DynamoDBDocumentClient,
TransactWriteCommand,
} from "@aws-sdk/lib-dynamodb";
import { BatchGetCommand, TransactWriteCommand } from "@aws-sdk/lib-dynamodb";
import { MiddlewareProps, VaultStatus, WithRequired } from "@lib/types";
import { connectToDynamo } from "@lib/integration/dynamodbConnector";
import { dynamodbClient } from "@lib/integration/dynamodbConnector";
import { createAbility, AccessControlError } from "@lib/privileges";
import { checkUserHasTemplateOwnership } from "@lib/templates";
import { logEvent } from "@lib/auditLogs";
Expand All @@ -18,8 +14,7 @@ const MAXIMUM_SUBMISSION_NAMES_PER_REQUEST = 20;

async function getSubmissionsFromSubmissionNames(
formId: string,
submissionNames: string[],
dynamoDbClient: DynamoDBDocumentClient
submissionNames: string[]
): Promise<{
submissionsToReport: { name: string; confirmationCode: string }[];
submissionNamesAlreadyUsed: string[];
Expand Down Expand Up @@ -48,7 +43,7 @@ async function getSubmissionsFromSubmissionNames(
});

// eslint-disable-next-line no-await-in-loop
const response = await dynamoDbClient.send(request);
const response = await dynamodbClient.send(request);

if (response.Responses?.Vault) {
response.Responses.Vault.forEach((record) => {
Expand Down Expand Up @@ -95,8 +90,7 @@ async function getSubmissionsFromSubmissionNames(

async function report(
formId: string,
submissionsToReport: { name: string; confirmationCode: string }[],
dynamoDbClient: DynamoDBDocumentClient
submissionsToReport: { name: string; confirmationCode: string }[]
): Promise<void> {
const request = new TransactWriteCommand({
TransactItems: submissionsToReport.flatMap((submission) => {
Expand Down Expand Up @@ -133,7 +127,7 @@ async function report(
}),
});

await dynamoDbClient.send(request);
await dynamodbClient.send(request);
}

async function notifySupport(
Expand Down Expand Up @@ -229,16 +223,13 @@ export const PUT = middleware(
}

try {
const dynamoDbClient = connectToDynamo();

const submissionsFromSubmissionNames = await getSubmissionsFromSubmissionNames(
formId,
entries,
dynamoDbClient
entries
);

if (submissionsFromSubmissionNames.submissionsToReport.length > 0) {
await report(formId, submissionsFromSubmissionNames.submissionsToReport, dynamoDbClient);
await report(formId, submissionsFromSubmissionNames.submissionsToReport);
// Note: may throw an error and handled in below catch e.g. if api key missing
await notifySupport(
formId,
Expand Down
6 changes: 2 additions & 4 deletions app/api/notify-callback/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import { SQSClient, GetQueueUrlCommand, SendMessageCommand } from "@aws-sdk/clie
import { middleware } from "@lib/middleware";
import { MiddlewareRequest, MiddlewareReturn } from "@lib/types";
import { UpdateCommand } from "@aws-sdk/lib-dynamodb";
import { connectToDynamo } from "@lib/integration/dynamodbConnector";
import { dynamodbClient } from "@lib/integration/dynamodbConnector";
import { headers } from "next/headers";
const SQS_REPROCESS_SUBMISSION_QUEUE_NAME = "reprocess_submission_queue.fifo";

Expand All @@ -28,8 +28,6 @@ const getQueueURL = async () => {
* @returns void
*/
async function removeProcessedMark(submissionID: string) {
const documentClient = connectToDynamo();

const updateItem = {
TableName: "ReliabilityQueue",
Key: {
Expand All @@ -42,7 +40,7 @@ async function removeProcessedMark(submissionID: string) {
ReturnValues: "NONE" as const,
};

return documentClient.send(new UpdateCommand(updateItem));
return dynamodbClient.send(new UpdateCommand(updateItem));
}

/**
Expand Down
18 changes: 6 additions & 12 deletions lib/integration/dynamodbConnector.ts
Original file line number Diff line number Diff line change
@@ -1,15 +1,9 @@
import { DynamoDBClient } from "@aws-sdk/client-dynamodb";
import { DynamoDBDocumentClient } from "@aws-sdk/lib-dynamodb";

/**
* Helper function to instantiate DynamoDB and Document client.
* https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/dynamodb-example-document-client.html
*/
export function connectToDynamo(): DynamoDBDocumentClient {
return DynamoDBDocumentClient.from(
new DynamoDBClient({
region: process.env.AWS_REGION ?? "ca-central-1",
...(process.env.LOCAL_AWS_ENDPOINT && { endpoint: process.env.LOCAL_AWS_ENDPOINT }),
})
);
}
export const dynamodbClient = DynamoDBDocumentClient.from(
new DynamoDBClient({
region: process.env.AWS_REGION ?? "ca-central-1",
...(process.env.LOCAL_AWS_ENDPOINT && { endpoint: process.env.LOCAL_AWS_ENDPOINT }),
})
);
23 changes: 7 additions & 16 deletions lib/vault.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import {
unprocessedSubmissionsCacheCheck,
unprocessedSubmissionsCachePut,
} from "./cache/unprocessedSubmissionsCache";
import { connectToDynamo } from "./integration/dynamodbConnector";
import { dynamodbClient } from "./integration/dynamodbConnector";
import { logMessage } from "./logger";
import { AccessControlError, checkPrivileges } from "./privileges";
import { chunkArray } from "@lib/utils";
Expand Down Expand Up @@ -86,8 +86,6 @@ const submissionTypeExists = async (ability: UserAbility, formID: string, status
);
throw e;
});
const documentClient = connectToDynamo();

const getItemsDbParams: QueryCommandInput = {
TableName: "Vault",
IndexName: "Status",
Expand All @@ -109,7 +107,7 @@ const submissionTypeExists = async (ability: UserAbility, formID: string, status
};
const queryCommand = new QueryCommand(getItemsDbParams);
// eslint-disable-next-line no-await-in-loop
const response = await documentClient.send(queryCommand);
const response = await dynamodbClient.send(queryCommand);
return Boolean(response.Items?.length);
};

Expand Down Expand Up @@ -150,8 +148,6 @@ export async function listAllSubmissions(
// We're going to request one more than the limit so we can consistently determine if there are more responses
const responseRetrievalLimit = responseDownloadLimit + 1;

const documentClient = connectToDynamo();

let accumulatedResponses: VaultSubmissionList[] = [];
let submissionsRemaining = false;
let paginationLastEvaluatedKey = null;
Expand Down Expand Up @@ -179,7 +175,7 @@ export async function listAllSubmissions(
};
const queryCommand = new QueryCommand(getItemsDbParams);
// eslint-disable-next-line no-await-in-loop
const response = await documentClient.send(queryCommand);
const response = await dynamodbClient.send(queryCommand);

if (response.Items?.length) {
accumulatedResponses = accumulatedResponses.concat(
Expand Down Expand Up @@ -278,7 +274,6 @@ export async function retrieveSubmissions(
// DynamoDB BatchGetItem can only retrieve 100 items at a time
// Create function that will run the batch in parallel
const dbQuery = async (keys: Record<string, string>[]) => {
const documentClient = connectToDynamo();
let accumulatedResponses: VaultSubmission[] = [];
while (keys && keys.length > 0) {
const queryCommand = new BatchGetCommand({
Expand All @@ -296,7 +291,7 @@ export async function retrieveSubmissions(
});

// eslint-disable-next-line no-await-in-loop
const response = await documentClient.send(queryCommand);
const response = await dynamodbClient.send(queryCommand);

if (response.Responses?.Vault.length) {
accumulatedResponses = accumulatedResponses.concat(
Expand Down Expand Up @@ -370,8 +365,6 @@ export async function updateLastDownloadedBy(
formID: string,
email: string
) {
const documentClient = connectToDynamo();

// TransactWriteItem can only update 100 items at a time
const asyncUpdateRequests = chunkArray(responses, 100).map((chunkedResponses) => {
const request = new TransactWriteCommand({
Expand Down Expand Up @@ -402,7 +395,7 @@ export async function updateLastDownloadedBy(
}),
});

return () => documentClient.send(request);
return () => dynamodbClient.send(request);
});

// @todo - handle errors that can result from failing TransactWriteCommand operations
Expand Down Expand Up @@ -444,8 +437,6 @@ export async function unprocessedSubmissions(
*/
export async function deleteDraftFormResponses(ability: UserAbility, formID: string) {
try {
const documentClient = connectToDynamo();

// Ensure users are owners of the form
await checkAbilityToAccessSubmissions(ability, formID).catch((error) => {
if (error instanceof AccessControlError) {
Expand Down Expand Up @@ -491,7 +482,7 @@ export async function deleteDraftFormResponses(ability: UserAbility, formID: str
};
const queryCommand = new QueryCommand(getItemsDbParams);
// eslint-disable-next-line no-await-in-loop
const response = await documentClient.send(queryCommand);
const response = await dynamodbClient.send(queryCommand);

if (response.Items?.length) {
accumulatedResponses = accumulatedResponses.concat(
Expand All @@ -509,7 +500,7 @@ export async function deleteDraftFormResponses(ability: UserAbility, formID: str

const asyncDeleteRequests = chunkArray(accumulatedResponses, 25).map((request) => {
return () =>
documentClient.send(
dynamodbClient.send(
new BatchWriteCommand({
RequestItems: {
Vault: request.map((entryName) => ({
Expand Down
Loading