Redis integration (#940)

* Feat : added redis * Feat : added redis retry
egovernments · Jun 19, 2024 · c519505 · c519505
1 parent 1b4c1b3
commit c519505
Show file tree

Hide file tree

Showing 11 changed files with 430 additions and 428 deletions.
diff --git a/utilities/project-factory/package.json b/utilities/project-factory/package.json
@@ -31,6 +31,7 @@
     "hash-sum": "2.0.0",
     "helmet": "7.1.0",
     "http-proxy-middleware": "^3.0.0",
+    "ioredis": "^5.4.1",
     "jaeger-client": "^3.19.0",
     "jsonpath": "1.1.1",
     "kafka-node": "5.0.0",

diff --git a/utilities/project-factory/src/server/api/campaignApis.ts b/utilities/project-factory/src/server/api/campaignApis.ts
@@ -9,7 +9,6 @@ import { callMdmsTypeSchema, getCampaignNumber } from "./genericApis";
 import { boundaryBulkUpload, convertToTypeData, generateHierarchy, generateProcessedFileAndPersist, getLocalizedName, reorderBoundariesOfDataAndValidate } from "../utils/campaignUtils";
 const _ = require('lodash');
 import { produceModifiedMessages } from "../kafka/Listener";
-import { userRoles } from "../config/constants";
 import { createDataService } from "../service/dataManageService";
 import { searchProjectTypeCampaignService } from "../service/campaignManageService";
 import { getExcelWorkbookFromFileURL } from "../utils/excelUtils";
@@ -543,7 +542,8 @@ function convertUserRoles(employees: any[], request: any) {
       var newRoles: any[] = []
       const rolesArray = employee.user.roles.split(',').map((role: any) => role.trim());
       for (const role of rolesArray) {
-        newRoles.push({ name: role, code: userRoles[role], tenantId: request?.body?.ResourceDetails?.tenantId })
+        const code = role.toUpperCase().split(' ').join('_')
+        newRoles.push({ name: role, code: code, tenantId: request?.body?.ResourceDetails?.tenantId })
       }
       employee.user.roles = newRoles
     }
@@ -771,7 +771,7 @@ async function handleResouceDetailsError(request: any, error: any) {
     };
     const persistMessage: any = { ResourceDetails: request.body.ResourceDetails }
     if (request?.body?.ResourceDetails?.action == "create") {
-       persistMessage.ResourceDetails.additionalDetails = { error: stringifiedError }
+      persistMessage.ResourceDetails.additionalDetails = { error: stringifiedError }
     }
     produceModifiedMessages(persistMessage, config?.kafka?.KAFKA_UPDATE_RESOURCE_DETAILS_TOPIC);
   }

diff --git a/utilities/project-factory/src/server/api/genericApis.ts b/utilities/project-factory/src/server/api/genericApis.ts
@@ -1,7 +1,7 @@
 // Import necessary modules and libraries
 import config from "../config"; // Import configuration settings
 import FormData from "form-data"; // Import FormData for handling multipart/form-data requests
-import { httpRequest } from "../utils/request"; // Import httpRequest function for making HTTP requests
+import { defaultheader, httpRequest } from "../utils/request"; // Import httpRequest function for making HTTP requests
 import { getFormattedStringForDebug, logger } from "../utils/logger"; // Import logger for logging
 import { correctParentValues, findMapValue, generateActivityMessage, getBoundaryRelationshipData, getDataSheetReady, getLocalizedHeaders, sortCampaignDetails, throwError } from "../utils/genericUtils"; // Import utility functions
 import { extractCodesFromBoundaryRelationshipResponse, generateFilteredBoundaryData, getConfigurableColumnHeadersBasedOnCampaignType, getFiltersFromCampaignSearchResponse, getLocalizedName } from '../utils/campaignUtils'; // Import utility functions
@@ -923,15 +923,19 @@ async function confirmBoundaryParentCreation(request: any, code: any) {
     const searchBody = {
       RequestInfo: request.body.RequestInfo,
     }
-    const params = {
+    const params: any = {
       hierarchyType: request?.body?.ResourceDetails?.hierarchyType,
       tenantId: request?.body?.ResourceDetails?.tenantId,
       codes: code
     }
     var retry = 6;
     var boundaryFound = false;
+    const header = {
+      ...defaultheader,
+      cachekey: `boundaryRelationShipSearch${params?.hierarchyType}${params?.tenantId}${params.codes || ''}${params?.includeChildren || ''}`,
+    }
     while (!boundaryFound && retry >= 0) {
-      const response = await httpRequest(config.host.boundaryHost + config.paths.boundaryRelationship, searchBody, params);
+      const response = await httpRequest(config.host.boundaryHost + config.paths.boundaryRelationship, searchBody, params, undefined, undefined, header);
       if (response?.TenantBoundary?.[0].boundary?.[0]) {
         boundaryFound = true;
       }
@@ -969,8 +973,12 @@ async function createBoundaryRelationship(request: any, boundaryMap: Map<{ key:
       "includeChildren": true,
       "hierarchyType": request?.body?.ResourceDetails?.hierarchyType
     };
+    const header = {
+      ...defaultheader,
+      cachekey: `boundaryRelationShipSearch${params?.hierarchyType}${params?.tenantId}${params.codes || ''}${params?.includeChildren || ''}`,
+    }
 
-    const boundaryRelationshipResponse = await httpRequest(url, request.body, params);
+    const boundaryRelationshipResponse = await httpRequest(url, request.body, params, undefined, undefined, header);
     const boundaryData = boundaryRelationshipResponse?.TenantBoundary?.[0]?.boundary;
     const allCodes = extractCodesFromBoundaryRelationshipResponse(boundaryData);
 
@@ -1053,33 +1061,6 @@ async function callMdmsData(
   return response;
 }
 
-
-
-async function callMdmsV2Data(
-  request: any,
-  moduleName: string,
-  masterName: string,
-  tenantId: string, filters: any) {
-  try {
-    const { RequestInfo = {} } = request?.body || {};
-    const requestBody = {
-      RequestInfo,
-      MdmsCriteria: {
-        tenantId: tenantId,
-        filters,
-        schemaCode: moduleName + "." + config?.masterNameForSchemaOfColumnHeaders,
-        limit: 10,
-        offset: 0
-      },
-    };
-    const url = config.host.mdmsV2 + config.paths.mdms_v2_search;
-    const response = await httpRequest(url, requestBody, { tenantId: tenantId });
-    return response;
-  } catch (error: any) {
-    throwError("MDMS", 400, "MDMS_DATA_NOT_FOUND_ERROR", `Mdms Data not found for ${moduleName}"-"${masterName}`)
-  }
-}
-
 function enrichSchema(data: any, properties: any, required: any, columns: any, unique: any, columnsNotToBeFreezed: any, errorMessage: any) {
 
   // Sort columns based on orderNumber, using name as tie-breaker if orderNumbers are equal
@@ -1168,7 +1149,11 @@ async function callMdmsTypeSchema(
     }
   };
   const url = config.host.mdmsV2 + config.paths.mdms_v2_search;
-  const response = await httpRequest(url, requestBody);
+  const header = {
+    ...defaultheader,
+    cachekey: `mdmsv2Seacrh${requestBody?.MdmsCriteria?.tenantId}${campaignType}${type}.${campaignType}${requestBody?.MdmsCriteria?.schemaCode}`
+  }
+  const response = await httpRequest(url, requestBody, undefined, undefined, undefined, header);
   if (!response?.mdms?.[0]?.data) {
     throwError("COMMON", 500, "INTERNAL_SERVER_ERROR", "Error occured during schema search");
   }
@@ -1202,6 +1187,5 @@ export {
   getTargetSheetDataAfterCode,
   callMdmsData,
   getMDMSV1Data,
-  callMdmsV2Data,
   callMdmsTypeSchema
 }
diff --git a/utilities/project-factory/src/server/config/constants.ts b/utilities/project-factory/src/server/config/constants.ts
@@ -9,7 +9,7 @@ export const CONSTANTS: any = {
             INTERNAL_SERVER_ERROR: "Internal server error",
             INVALID_PAGINATION: "Invalid pagination",
             KAFKA_ERROR: "Some error occured in kafka",
-            SCHEMA_ERROR : " Schema related error",
+            SCHEMA_ERROR: " Schema related error",
             RESPONSE_NOT_FOUND_ERROR: "Response not found"
         },
         FILE: {
@@ -86,15 +86,6 @@ const getMessage = (key: any) => {
     return unknownError;
 }
 
-export const userRoles: any = {
-    "Registrar": "REGISTRAR",
-    "Distributor": "DISTRIBUTOR",
-    "Supervisor": "SUPERVISOR",
-    "Help Desk": "HELPDESK_USER",
-    "Monitor Local": "MONITOR_LOCAL",
-    "Logistical officer": "LOGISTICAL_OFFICER",
-}
-
 export const campaignStatuses: any = {
     drafted: "drafted",
     started: "creating",

diff --git a/utilities/project-factory/src/server/config/index.ts b/utilities/project-factory/src/server/config/index.ts
@@ -36,6 +36,11 @@ const config = {
     userDefaultPassword: process.env.USER_DEFAULT_PASSWORD || "eGov@123",
     userPasswordAutoGenerate: process.env.USER_PASSWORD_AUTO_GENERATE || "true",
   },
+  cacheValues: {
+    cacheEnabled: process.env.CACHE_ENABLED,
+    resetCache: process.env.RESET_CACHE,
+    redisPort: process.env.REDIS_PORT || "6379",
+  },
   kafka: {
     // Kafka topics
     KAFKA_SAVE_PROJECT_CAMPAIGN_DETAILS_TOPIC: process.env.KAFKA_SAVE_PROJECT_CAMPAIGN_DETAILS_TOPIC || "save-project-campaign-details",
@@ -85,6 +90,7 @@ const config = {
     serverHost: HOST,
     // Kafka broker host
     KAFKA_BROKER_HOST: process.env.KAFKA_BROKER_HOST || "kafka-v2.kafka-cluster:9092",
+    redisHost: process.env.REDIS_HOST || "localhost",
     mdms: process.env.EGOV_MDMS_HOST || "https://unified-dev.digit.org/",
     mdmsV2: process.env.EGOV_MDMS_V2_HOST || "https://unified-dev.digit.org/",
     filestore: process.env.EGOV_FILESTORE_SERVICE_HOST || "https://unified-dev.digit.org/",
@@ -146,7 +152,7 @@ const config = {
     matchFacilityData: false,
     retryCount: process.env.CREATE_RESOURCE_RETRY_COUNT || "3",
     notCreateUserIfAlreadyThere: process.env.NOT_CREATE_USER_IF_ALREADY_THERE || false,
-    maxHttpRetries: "4"
+    maxHttpRetries: process.env.MAX_HTTP_RETRIES || "4"
   }
 };
 // Exporting getErrorCodes function and config object

diff --git a/utilities/project-factory/src/server/utils/campaignUtils.ts b/utilities/project-factory/src/server/utils/campaignUtils.ts
@@ -1,5 +1,5 @@
 
-import { httpRequest } from "./request";
+import { defaultheader, httpRequest } from "./request";
 import config from "../config/index";
 import { v4 as uuidv4 } from 'uuid';
 import { produceModifiedMessages } from '../kafka/Listener'
@@ -1085,7 +1085,11 @@ async function addBoundariesForData(request: any, CampaignDetails: any) {
             hierarchyType: request?.body?.ResourceDetails?.hierarchyType,
             includeChildren: true
         }
-        const boundaryResponse = await httpRequest(config.host.boundaryHost + config.paths.boundaryRelationship, request.body, params);
+        const header = {
+            ...defaultheader,
+            cachekey: `boundaryRelationShipSearch${params?.hierarchyType}${params?.tenantId}${params.codes || ''}${params?.includeChildren || ''}`,
+        }
+        const boundaryResponse = await httpRequest(config.host.boundaryHost + config.paths.boundaryRelationship, request.body, params, undefined, undefined, header);
         if (boundaryResponse?.TenantBoundary?.[0]?.boundary?.[0]) {
             var boundaryChildren = boundaries.reduce((acc: any, boundary: any) => {
                 acc[boundary.code] = boundary?.includeAllChildren;
@@ -1160,7 +1164,11 @@ async function reorderBoundaries(request: any, localizationMap?: any) {
             hierarchyType: request?.body?.CampaignDetails?.hierarchyType,
             includeChildren: true
         }
-        const boundaryResponse = await httpRequest(config.host.boundaryHost + config.paths.boundaryRelationship, request.body, params);
+        const header = {
+            ...defaultheader,
+            cachekey: `boundaryRelationShipSearch${params?.hierarchyType}${params?.tenantId}${params.codes || ''}${params?.includeChildren || ''}`,
+        }
+        const boundaryResponse = await httpRequest(config.host.boundaryHost + config.paths.boundaryRelationship, request.body, params, undefined, undefined, header);
         if (boundaryResponse?.TenantBoundary?.[0]?.boundary?.[0]) {
             const codesTargetMapping = await getCodesTarget(request, localizationMap)
             mapTargets(boundaryResponse?.TenantBoundary?.[0]?.boundary, codesTargetMapping)
@@ -1733,7 +1741,6 @@ const getConfigurableColumnHeadersBasedOnCampaignType = async (request: any, loc
         const campaignType = responseFromCampaignSearch?.CampaignDetails[0]?.projectType;
 
         const mdmsResponse = await callMdmsTypeSchema(request, request?.query?.tenantId || request?.body?.ResourceDetails?.tenantId, request?.query?.type || request?.body?.ResourceDetails?.type, campaignType)
-        // const mdmsResponse = await callMdmsV2Data(request, config?.values?.moduleName, request?.query?.type || request?.body?.ResourceDetails?.type, request?.query?.tenantId || request?.body?.ResourceDetails?.tenantId, filters);
         if (!mdmsResponse || mdmsResponse?.columns.length === 0) {
             logger.error(`Campaign Type ${campaignType} has not any columns configured in schema`)
             throwError("COMMON", 400, "SCHEMA_ERROR", `Campaign Type ${campaignType} has not any columns configured in schema`);

diff --git a/utilities/project-factory/src/server/utils/genericUtils.ts b/utilities/project-factory/src/server/utils/genericUtils.ts
@@ -1,5 +1,5 @@
 import { NextFunction, Request, Response } from "express";
-import { httpRequest } from "./request";
+import { httpRequest, defaultheader } from "./request";
 import config, { getErrorCodes } from "../config/index";
 import { v4 as uuidv4 } from 'uuid';
 import { produceModifiedMessages } from "../kafka/Listener";
@@ -835,7 +835,11 @@ async function getDataFromSheet(request: any, fileStoreId: any, tenantId: any, c
 async function getBoundaryRelationshipData(request: any, params: any) {
   logger.info("Boundary relationship search initiated")
   const url = `${config.host.boundaryHost}${config.paths.boundaryRelationship}`;
-  const boundaryRelationshipResponse = await httpRequest(url, request.body, params);
+  const header = {
+    ...defaultheader,
+    cachekey: `boundaryRelationShipSearch${params?.hierarchyType}${params?.tenantId}${params.codes || ''}${params?.includeChildren || ''}`,
+  }
+  const boundaryRelationshipResponse = await httpRequest(url, request.body, params, undefined, undefined, header);
   logger.info("Boundary relationship search response received")
   return boundaryRelationshipResponse?.TenantBoundary?.[0]?.boundary;
 }

diff --git a/utilities/project-factory/src/server/utils/redisUtils.ts b/utilities/project-factory/src/server/utils/redisUtils.ts
@@ -0,0 +1,29 @@
+import Redis from "ioredis";
+import config from "../config";
+
+const redis = new Redis({
+    host: config.host.redisHost,
+    port: parseInt(config.cacheValues.redisPort),
+    retryStrategy(times) {
+        if (times > 1) {
+            return null; // Stop retrying after one attempt
+        }
+        return 500; // Delay before retrying (in milliseconds)
+    },
+    maxRetriesPerRequest: 1, // Allow only 1 retry per request
+    reconnectOnError(err) {
+        return false; // Do not reconnect on errors
+    },
+});
+
+async function checkRedisConnection(): Promise<boolean> {
+    try {
+        await redis.ping();
+        return true;
+    } catch (error) {
+        console.error("Redis connection error:", error);
+        return false;
+    }
+}
+
+export { redis, checkRedisConnection };