From 5382443bca63ea373d4ce46ce237527a32792a78 Mon Sep 17 00:00:00 2001 From: nitish-egov Date: Wed, 24 Jul 2024 16:59:44 +0530 Subject: [PATCH 1/5] updated readmeconfig for sheet --- .../project-factory/src/server/utils/excelUtils.ts | 4 ++-- .../src/server/utils/genericUtils.ts | 14 ++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/utilities/project-factory/src/server/utils/excelUtils.ts b/utilities/project-factory/src/server/utils/excelUtils.ts index e8ee221b781..8f7a0acc614 100644 --- a/utilities/project-factory/src/server/utils/excelUtils.ts +++ b/utilities/project-factory/src/server/utils/excelUtils.ts @@ -71,8 +71,8 @@ function updateFontNameToRoboto(worksheet: ExcelJS.Worksheet) { function formatWorksheet(worksheet: any, datas: any, headerSet: any) { // Add empty rows after the main header - worksheet.addRow([]); - worksheet.addRow([]); + // worksheet.addRow([]); + // worksheet.addRow([]); worksheet.addRow([]); // Add the data rows with text wrapping diff --git a/utilities/project-factory/src/server/utils/genericUtils.ts b/utilities/project-factory/src/server/utils/genericUtils.ts index 6c276c37092..6b8ca7c24e5 100644 --- a/utilities/project-factory/src/server/utils/genericUtils.ts +++ b/utilities/project-factory/src/server/utils/genericUtils.ts @@ -521,13 +521,15 @@ async function createReadMeSheet(request: any, workbook: any, mainHeader: any, l const headerSet = new Set(); - const datas = readMeConfig.texts.flatMap((text: any) => { - const descriptions = text.descriptions.map((description: any) => { - return getLocalizedName(description.text, localizationMap); + const datas = readMeConfig.texts + .filter((text: any) => text?.inSheet) // Filter out texts with inSheet set to false + .flatMap((text: any) => { + const descriptions = text.descriptions.map((description: any) => { + return getLocalizedName(description.text, localizationMap); + }); + headerSet.add(getLocalizedName(text.header, localizationMap)); + return [getLocalizedName(text.header, localizationMap), ...descriptions, ""]; }); - headerSet.add(getLocalizedName(text.header, localizationMap)); - return [getLocalizedName(text.header, localizationMap), ...descriptions, "", "", "", ""]; - }); // Create the worksheet and add the main header const worksheet = workbook.addWorksheet(getLocalizedName("HCM_README_SHEETNAME", localizationMap)); From daa9e0f001022a5f983cb53a8367b7eee459b291 Mon Sep 17 00:00:00 2001 From: nitish-egov Date: Thu, 25 Jul 2024 15:35:48 +0530 Subject: [PATCH 2/5] added cache for generating target template when only delivery conditions change --- .../src/server/service/dataManageService.ts | 57 +++++++++++++------ .../src/server/utils/campaignUtils.ts | 9 ++- .../src/server/utils/generateUtils.ts | 13 +++-- .../src/server/utils/genericUtils.ts | 16 +++--- .../src/server/utils/targetUtils.ts | 2 +- 5 files changed, 65 insertions(+), 32 deletions(-) diff --git a/utilities/project-factory/src/server/service/dataManageService.ts b/utilities/project-factory/src/server/service/dataManageService.ts index 12f60b61441..c7c45474ea7 100644 --- a/utilities/project-factory/src/server/service/dataManageService.ts +++ b/utilities/project-factory/src/server/service/dataManageService.ts @@ -3,12 +3,15 @@ import { processGenericRequest } from "../api/campaignApis"; import { createAndUploadFile, getBoundarySheetData } from "../api/genericApis"; import { getLocalizedName, processDataSearchRequest } from "../utils/campaignUtils"; import { addDataToSheet, enrichResourceDetails, getLocalizedMessagesHandler, searchGeneratedResources, processGenerate, throwError } from "../utils/genericUtils"; -import { logger } from "../utils/logger"; +import { getFormattedStringForDebug, logger } from "../utils/logger"; import { validateCreateRequest, validateDownloadRequest, validateSearchRequest } from "../validators/campaignValidators"; import { validateGenerateRequest } from "../validators/genericValidator"; import { getLocalisationModuleName } from "../utils/localisationUtils"; import { getBoundaryTabName } from "../utils/boundaryUtils"; import { getNewExcelWorkbook } from "../utils/excelUtils"; +import { redis, checkRedisConnection } from "../utils/redisUtils"; // Importing checkRedisConnection function +import config from '../config/index' + const generateDataService = async (request: express.Request) => { @@ -38,23 +41,43 @@ const downloadDataService = async (request: express.Request) => { } const getBoundaryDataService = async ( - request: express.Request) => { + request: express.Request, enableCaching = false) => { try { - const workbook = getNewExcelWorkbook(); - const { hierarchyType } = request?.query; - const localizationMapHierarchy = hierarchyType && await getLocalizedMessagesHandler(request, request?.query?.tenantId, getLocalisationModuleName(hierarchyType)); - const localizationMapModule = await getLocalizedMessagesHandler(request, request?.query?.tenantId); - const localizationMap = { ...localizationMapHierarchy, ...localizationMapModule }; - // Retrieve boundary sheet data - const boundarySheetData: any = await getBoundarySheetData(request, localizationMap); - - const localizedBoundaryTab = getLocalizedName(getBoundaryTabName(), localizationMap); - const boundarySheet = workbook.addWorksheet(localizedBoundaryTab); - addDataToSheet(boundarySheet, boundarySheetData); - const BoundaryFileDetails: any = await createAndUploadFile(workbook, request); - // Return boundary file details - logger.info("RETURNS THE BOUNDARY RESPONSE"); - return BoundaryFileDetails; + const { hierarchyType, campaignId } = request?.query; + const cacheTTL = 300; // TTL in seconds (5 minutes) + const cacheKey = `${campaignId}-${hierarchyType}`; + const isRedisConnected = await checkRedisConnection(); + const cachedData = await redis.get(cacheKey); // Get cached data + if (cachedData && enableCaching && isRedisConnected) { + logger.info("CACHE HIT :: " + cacheKey); + logger.debug(`CACHED DATA :: ${getFormattedStringForDebug(cachedData)}`); + + // Reset the TTL for the cache key + if (config.cacheValues.resetCache) { + await redis.expire(cacheKey, cacheTTL); + } + + return JSON.parse(cachedData); // Return parsed cached data if available + } + else { + logger.info("NO CACHE FOUND :: REQUEST :: " + cacheKey); + const workbook = getNewExcelWorkbook(); + const localizationMapHierarchy = hierarchyType && await getLocalizedMessagesHandler(request, request?.query?.tenantId, getLocalisationModuleName(hierarchyType)); + const localizationMapModule = await getLocalizedMessagesHandler(request, request?.query?.tenantId); + const localizationMap = { ...localizationMapHierarchy, ...localizationMapModule }; + // Retrieve boundary sheet data + const boundarySheetData: any = await getBoundarySheetData(request, localizationMap); + const localizedBoundaryTab = getLocalizedName(getBoundaryTabName(), localizationMap); + const boundarySheet = workbook.addWorksheet(localizedBoundaryTab); + addDataToSheet(boundarySheet, boundarySheetData); + const boundaryFileDetails: any = await createAndUploadFile(workbook, request); + // Return boundary file details + logger.info("RETURNS THE BOUNDARY RESPONSE"); + if (cacheKey && isRedisConnected) { + await redis.set(cacheKey, JSON.stringify(boundaryFileDetails), "EX", cacheTTL); // Cache the response data with TTL + } + return boundaryFileDetails; + } } catch (e: any) { console.log(e) logger.error(String(e)) diff --git a/utilities/project-factory/src/server/utils/campaignUtils.ts b/utilities/project-factory/src/server/utils/campaignUtils.ts index 370c346c01d..f89eaf98703 100644 --- a/utilities/project-factory/src/server/utils/campaignUtils.ts +++ b/utilities/project-factory/src/server/utils/campaignUtils.ts @@ -17,7 +17,7 @@ import { getBoundaryColumnName, getBoundaryTabName } from "./boundaryUtils"; import { searchProjectTypeCampaignService } from "../service/campaignManageService"; import { validateBoundaryOfResouces } from "../validators/campaignValidators"; import { getExcelWorkbookFromFileURL, getNewExcelWorkbook, lockTargetFields, updateFontNameToRoboto } from "./excelUtils"; -import { callGenerateIfBoundariesDiffer } from "./generateUtils"; +import { areBoundariesSame, callGenerateIfBoundariesDiffer } from "./generateUtils"; import { createProcessTracks, persistTrack } from "./processTrackUtils"; import { generateDynamicTargetHeaders, isDynamicTargetTemplateForProjectType, updateTargetColumnsIfDeliveryConditionsDifferForSMC } from "./targetUtils"; const _ = require('lodash'); @@ -589,8 +589,13 @@ function enrichInnerCampaignDetails(request: any, updatedInnerCampaignDetails: a async function enrichAndPersistCampaignForUpdate(request: any, firstPersist: boolean = false) { const action = request?.body?.CampaignDetails?.action; + const existingCampaignDetails = request?.body?.ExistingCampaignDetails; callGenerateIfBoundariesDiffer(request); - updateTargetColumnsIfDeliveryConditionsDifferForSMC(request); + if (existingCampaignDetails) { + if (areBoundariesSame(existingCampaignDetails?.boundaries, request?.body?.CampaignDetails?.boundaries)) { + updateTargetColumnsIfDeliveryConditionsDifferForSMC(request); + } + } const ExistingCampaignDetails = request?.body?.ExistingCampaignDetails; var updatedInnerCampaignDetails = {} enrichInnerCampaignDetails(request, updatedInnerCampaignDetails) diff --git a/utilities/project-factory/src/server/utils/generateUtils.ts b/utilities/project-factory/src/server/utils/generateUtils.ts index e6b7ee59e40..606f59f1b0c 100644 --- a/utilities/project-factory/src/server/utils/generateUtils.ts +++ b/utilities/project-factory/src/server/utils/generateUtils.ts @@ -62,7 +62,7 @@ async function callGenerateIfBoundariesDiffer(request: any) { } } -async function callGenerate(request: any, type: any) { +async function callGenerate(request: any, type: any, enableCaching = false) { logger.info(`calling generate api for type ${type}`); if (type === "facilityWithBoundary" || type == "userWithBoundary") { const { hierarchyType } = request.query; @@ -74,12 +74,17 @@ async function callGenerate(request: any, type: any) { const localizationMapModule = await getLocalizedMessagesHandler(request, request.query.tenantId); const localizationMap = { ...localizationMapHierarchy, ...localizationMapModule }; const filteredBoundary = await getBoundarySheetData(request, localizationMap); - await processGenerate(request, filteredBoundary); + await processGenerate(request, enableCaching, filteredBoundary); } else { - await processGenerate(request); + if (enableCaching) { + await processGenerate(request, true); + } + else { + await processGenerate(request); + } } } -export { callGenerateIfBoundariesDiffer, callGenerate } +export { callGenerateIfBoundariesDiffer, callGenerate, areBoundariesSame } diff --git a/utilities/project-factory/src/server/utils/genericUtils.ts b/utilities/project-factory/src/server/utils/genericUtils.ts index 6b8ca7c24e5..1e3f6166c85 100644 --- a/utilities/project-factory/src/server/utils/genericUtils.ts +++ b/utilities/project-factory/src/server/utils/genericUtils.ts @@ -361,7 +361,7 @@ async function getFinalUpdatedResponse(result: any, responseData: any, request: -async function fullProcessFlowForNewEntry(newEntryResponse: any, generatedResource: any, request: any, filteredBoundary?: any) { +async function fullProcessFlowForNewEntry(newEntryResponse: any, generatedResource: any, request: any, enableCaching = false, filteredBoundary?: any) { try { const { type, hierarchyType } = request?.query; generatedResource = { generatedResource: newEntryResponse } @@ -374,7 +374,7 @@ async function fullProcessFlowForNewEntry(newEntryResponse: any, generatedResour if (type === 'boundary') { // get boundary data from boundary relationship search api logger.info("Generating Boundary Data") - const boundaryDataSheetGeneratedBeforeDifferentTabSeparation = await getBoundaryDataService(request); + const boundaryDataSheetGeneratedBeforeDifferentTabSeparation = await getBoundaryDataService(request,enableCaching); logger.info(`Boundary data generated successfully: ${JSON.stringify(boundaryDataSheetGeneratedBeforeDifferentTabSeparation)}`); // get boundary sheet data after being generated logger.info("generating different tabs logic ") @@ -764,9 +764,9 @@ async function processGenerateRequest(request: any, localizationMap?: { [key: st } } -async function processGenerateForNew(request: any, generatedResource: any, newEntryResponse: any, filteredBoundary?: any) { +async function processGenerateForNew(request: any, generatedResource: any, newEntryResponse: any, enableCaching = false, filteredBoundary?: any) { request.body.generatedResource = newEntryResponse; - await fullProcessFlowForNewEntry(newEntryResponse, generatedResource, request, filteredBoundary); + await fullProcessFlowForNewEntry(newEntryResponse, generatedResource, request, enableCaching, filteredBoundary); return request.body.generatedResource; } @@ -786,7 +786,7 @@ function handleGenerateError(newEntryResponse: any, generatedResource: any, erro produceModifiedMessages(generatedResource, updateGeneratedResourceTopic); } -async function updateAndPersistGenerateRequest(newEntryResponse: any, oldEntryResponse: any, responseData: any, request: any, filteredBoundary?: any) { +async function updateAndPersistGenerateRequest(newEntryResponse: any, oldEntryResponse: any, responseData: any, request: any, enableCaching = false, filteredBoundary?: any) { const { forceUpdate } = request.query; const forceUpdateBool: boolean = forceUpdate === 'true'; let generatedResource: any; @@ -797,7 +797,7 @@ async function updateAndPersistGenerateRequest(newEntryResponse: any, oldEntryRe request.body.generatedResource = oldEntryResponse; } if (responseData.length === 0 || forceUpdateBool) { - processGenerateForNew(request, generatedResource, newEntryResponse, filteredBoundary) + processGenerateForNew(request, generatedResource, newEntryResponse, enableCaching, filteredBoundary) } else { request.body.generatedResource = responseData @@ -806,7 +806,7 @@ async function updateAndPersistGenerateRequest(newEntryResponse: any, oldEntryRe /* */ -async function processGenerate(request: any, filteredBoundary?: any) { +async function processGenerate(request: any, enableCaching = false, filteredBoundary?: any) { // fetch the data from db to check any request already exists const responseData = await searchGeneratedResources(request); // modify response from db @@ -816,7 +816,7 @@ async function processGenerate(request: any, filteredBoundary?: any) { // make old data status as expired const oldEntryResponse = await updateExistingResourceExpired(modifiedResponse, request); // generate data - await updateAndPersistGenerateRequest(newEntryResponse, oldEntryResponse, responseData, request, filteredBoundary); + await updateAndPersistGenerateRequest(newEntryResponse, oldEntryResponse, responseData, request, enableCaching, filteredBoundary); } /* TODO add comments @nitish-egov diff --git a/utilities/project-factory/src/server/utils/targetUtils.ts b/utilities/project-factory/src/server/utils/targetUtils.ts index 244bcd3016b..160c7a0cf6b 100644 --- a/utilities/project-factory/src/server/utils/targetUtils.ts +++ b/utilities/project-factory/src/server/utils/targetUtils.ts @@ -111,7 +111,7 @@ async function updateTargetColumnsIfDeliveryConditionsDifferForSMC(request: any) const newParamsBoundary = { ...query, ...params, type: "boundary" }; const newRequestBoundary = replicateRequest(request, newRequestBody, newParamsBoundary); - await callGenerate(newRequestBoundary, "boundary"); + await callGenerate(newRequestBoundary, "boundary", true); } } } From 728ff981f466073f0c2b44afd03fe49a2c1890c0 Mon Sep 17 00:00:00 2001 From: nitish-egov Date: Wed, 31 Jul 2024 10:38:32 +0530 Subject: [PATCH 3/5] added logic for having only 18 target columns if exceed i will create one column with header OTHER_TARGETS --- utilities/project-factory/src/server/utils/targetUtils.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/utilities/project-factory/src/server/utils/targetUtils.ts b/utilities/project-factory/src/server/utils/targetUtils.ts index 160c7a0cf6b..06f9b5f079c 100644 --- a/utilities/project-factory/src/server/utils/targetUtils.ts +++ b/utilities/project-factory/src/server/utils/targetUtils.ts @@ -65,13 +65,15 @@ function modifyDeliveryConditions(dataa: any[]): any { function generateTargetColumnsBasedOnDeliveryConditions(uniqueDeliveryConditions: any, localizationMap?: any) { const targetColumnsBasedOnDeliveryConditions: string[] = []; - - uniqueDeliveryConditions.forEach((str: any) => { + uniqueDeliveryConditions.forEach((str: any, index: number) => { const uniqueDeliveryConditionsObject = JSON.parse(str); // Parse JSON string into object const targetColumnString = createTargetString(uniqueDeliveryConditionsObject, localizationMap); targetColumnsBasedOnDeliveryConditions.push(targetColumnString); }); - + if (targetColumnsBasedOnDeliveryConditions.length > 18) { + targetColumnsBasedOnDeliveryConditions.splice(18); + targetColumnsBasedOnDeliveryConditions.push(getLocalizedName("OTHER_TARGETS", localizationMap)); + } return targetColumnsBasedOnDeliveryConditions; } From 62d84e15bc5741459982eb3b5914dddad18e9e60 Mon Sep 17 00:00:00 2001 From: nitish-egov Date: Wed, 31 Jul 2024 16:38:51 +0530 Subject: [PATCH 4/5] updated config to fetch from devops accordingly --- utilities/project-factory/src/server/config/index.ts | 8 ++++---- utilities/project-factory/src/server/utils/targetUtils.ts | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/utilities/project-factory/src/server/config/index.ts b/utilities/project-factory/src/server/config/index.ts index 5d454e48b7b..263414e9e2a 100644 --- a/utilities/project-factory/src/server/config/index.ts +++ b/utilities/project-factory/src/server/config/index.ts @@ -17,11 +17,11 @@ const getDBSchemaName = (dbSchema = "") => { // Configuration object containing various environment variables const config = { cacheTime : 300, - enableDynamicTemplateFor: ["MR-DN"], - isCallGenerateWhenDeliveryConditionsDiffer: true, + enableDynamicTemplateFor: process.env.ENABLE_DYNAMIC_TEMPLATE_FOR || "MR-DN", + isCallGenerateWhenDeliveryConditionsDiffer: process.env.IS_CALL_GENERATE_WHEN_DELIVERY_CONDITIONS_DIFFER || true, prefixForMicroplanCampaigns: "MP", - excludeHierarchyTypeFromBoundaryCodes: false, - excludeBoundaryNameAtLastFromBoundaryCodes: false, + excludeHierarchyTypeFromBoundaryCodes: process.env.EXCLUDE_HIERARCHY_TYPE_FROM_BOUNDARY_CODES || false, + excludeBoundaryNameAtLastFromBoundaryCodes: process.env.EXCLUDE_BOUNDARY_NAME_AT_LAST_FROM_BOUNDARY_CODES || false, masterNameForSchemaOfColumnHeaders: "adminSchema", masterNameForSplitBoundariesOn: "hierarchyConfig", boundary: { diff --git a/utilities/project-factory/src/server/utils/targetUtils.ts b/utilities/project-factory/src/server/utils/targetUtils.ts index 06f9b5f079c..74a91782dba 100644 --- a/utilities/project-factory/src/server/utils/targetUtils.ts +++ b/utilities/project-factory/src/server/utils/targetUtils.ts @@ -120,7 +120,8 @@ async function updateTargetColumnsIfDeliveryConditionsDifferForSMC(request: any) function isDynamicTargetTemplateForProjectType(projectType: string) { const projectTypesFromConfig = config?.enableDynamicTemplateFor; - return projectTypesFromConfig?.includes(projectType) ?? false; + const projectTypesArray = projectTypesFromConfig ? projectTypesFromConfig.split(',') : []; + return projectTypesArray.includes(projectType); } From 5486dee93250c11a645ab856a5c12ac40f5592e2 Mon Sep 17 00:00:00 2001 From: nitish-egov Date: Thu, 1 Aug 2024 14:36:23 +0530 Subject: [PATCH 5/5] updated config for project -factory --- utilities/project-factory/src/server/config/index.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utilities/project-factory/src/server/config/index.ts b/utilities/project-factory/src/server/config/index.ts index 598ea165e4b..87cf8d4c4fd 100644 --- a/utilities/project-factory/src/server/config/index.ts +++ b/utilities/project-factory/src/server/config/index.ts @@ -18,10 +18,10 @@ const getDBSchemaName = (dbSchema = "") => { const config = { cacheTime : 300, enableDynamicTemplateFor: process.env.ENABLE_DYNAMIC_TEMPLATE_FOR || "MR-DN", - isCallGenerateWhenDeliveryConditionsDiffer: process.env.IS_CALL_GENERATE_WHEN_DELIVERY_CONDITIONS_DIFFER || false, + isCallGenerateWhenDeliveryConditionsDiffer: (process.env.IS_CALL_GENERATE_WHEN_DELIVERY_CONDITIONS_DIFFER === "true") || false, prefixForMicroplanCampaigns: "MP", - excludeHierarchyTypeFromBoundaryCodes: process.env.EXCLUDE_HIERARCHY_TYPE_FROM_BOUNDARY_CODES || false, - excludeBoundaryNameAtLastFromBoundaryCodes: process.env.EXCLUDE_BOUNDARY_NAME_AT_LAST_FROM_BOUNDARY_CODES || false, + excludeHierarchyTypeFromBoundaryCodes: (process.env.EXCLUDE_HIERARCHY_TYPE_FROM_BOUNDARY_CODES === "true") || false, + excludeBoundaryNameAtLastFromBoundaryCodes: (process.env.EXCLUDE_BOUNDARY_NAME_AT_LAST_FROM_BOUNDARY_CODES === "true") || false, masterNameForSchemaOfColumnHeaders: "adminSchema", masterNameForSplitBoundariesOn: "hierarchyConfig", boundary: {