fix missing ga data

IntersectMBO · Apr 24, 2024 · 687a8dd · 687a8dd
1 parent d5f7f23
commit 687a8dd
Show file tree

Hide file tree

Showing 17 changed files with 87 additions and 26 deletions.
diff --git a/govtool/frontend/src/consts/externalDataModalConfig.ts b/govtool/frontend/src/consts/externalDataModalConfig.ts
@@ -36,6 +36,7 @@ export const storageInformationErrorModals: Record<
   >["state"]
 > = {
   [MetadataValidationStatus.URL_NOT_FOUND]: urlCannotBeFound,
+  [MetadataValidationStatus.INCORRECT_FORMAT]: externalDataDoesntMatchModal,
   [MetadataValidationStatus.INVALID_JSONLD]: externalDataDoesntMatchModal,
   [MetadataValidationStatus.INVALID_HASH]: externalDataDoesntMatchModal,
 };
diff --git a/govtool/frontend/src/hooks/queries/useGetProposalsInfiniteQuery.ts b/govtool/frontend/src/hooks/queries/useGetProposalsInfiniteQuery.ts
@@ -24,11 +24,16 @@ export const useGetProposalsInfiniteQuery = ({
     });
     const mappedElements = await Promise.all(
       data.elements.map(async (proposal: ActionType) => {
-        const isDataMissing = await checkIsMissingGAMetadata({
+        const { metadata, status } = await checkIsMissingGAMetadata({
           hash: proposal?.metadataHash ?? "",
           url: proposal?.url ?? "",
         });
-        return { ...proposal, isDataMissing };
+        // workaround for the missing data in db-sync
+        return {
+          ...proposal,
+          ...metadata,
+          isDataMissing: status || false,
+        };
       }),
     );
 

diff --git a/govtool/frontend/src/hooks/queries/useGetProposalsQuery.ts b/govtool/frontend/src/hooks/queries/useGetProposalsQuery.ts
@@ -33,11 +33,12 @@ export const useGetProposalsQuery = ({
       allProposals
         .flatMap((proposal) => proposal.elements)
         .map(async (proposal) => {
-          const isDataMissing = await checkIsMissingGAMetadata({
+          const { metadata, status } = await checkIsMissingGAMetadata({
             hash: proposal?.metadataHash ?? "",
             url: proposal?.url ?? "",
           });
-          return { ...proposal, isDataMissing };
+          // workaround for the missing data in db-sync
+          return { ...proposal, ...metadata, isDataMissing: status || false };
         }),
     );
 

diff --git a/govtool/frontend/src/i18n/locales/en.ts b/govtool/frontend/src/i18n/locales/en.ts
@@ -751,7 +751,7 @@ export const en = {
     dataMissingErrors: {
       dataMissing: "Data Missing",
       notVerifiable: "Not Verifiable",
-      incorrectFormat: "Incorrect Format",
+      incorrectFormat: "Data Formatted Incorrectly",
     },
     about: "About",
     abstain: "Abstain",

diff --git a/govtool/frontend/src/models/metadataValidation.ts b/govtool/frontend/src/models/metadataValidation.ts
@@ -9,6 +9,8 @@ export enum MetadataValidationStatus {
 export type ValidateMetadataResult = {
   status?: MetadataValidationStatus;
   valid: boolean;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  metadata?: any;
 };
 
 export enum MetadataStandard {

diff --git a/govtool/frontend/src/services/API.ts b/govtool/frontend/src/services/API.ts
@@ -6,11 +6,14 @@ import { PATHS } from "@consts";
 const TIMEOUT_IN_SECONDS = 30 * 1000; // 1000 ms is 1 s then its 10 s
 const BASE_URL = import.meta.env.VITE_BASE_URL;
 
+// Validation should be performed directly on the server
+// than no metadata service is needed and `/api` might be removed
 export const API = axios.create({
   baseURL: `${BASE_URL}/api`,
   timeout: TIMEOUT_IN_SECONDS,
 });
 
+// TODO: Remove this service and use the API service
 export const METADATA_VALIDATION_API = axios.create({
   baseURL: `${BASE_URL}/metadata-validation`,
   timeout: TIMEOUT_IN_SECONDS,

diff --git a/govtool/frontend/src/services/requests/getDRepVotes.ts b/govtool/frontend/src/services/requests/getDRepVotes.ts
@@ -21,14 +21,19 @@ export const getDRepVotes = async ({
 
   const mappedData = (await Promise.all(
     data.map(async (proposal) => {
-      const isDataMissing = await checkIsMissingGAMetadata({
+      const { metadata, status } = await checkIsMissingGAMetadata({
         hash: proposal?.proposal?.metadataHash,
         url: proposal?.proposal?.url,
       });
 
       return {
         vote: proposal.vote,
-        proposal: { ...proposal.proposal, isDataMissing },
+        proposal: {
+          ...proposal.proposal,
+          // workaround for the missing data in db-sync
+          ...metadata,
+          isDataMissing: status || false,
+        },
       };
     }),
   )) as VotedProposalToDisplay[];

diff --git a/govtool/frontend/src/services/requests/getProposal.ts b/govtool/frontend/src/services/requests/getProposal.ts
@@ -8,10 +8,10 @@ export const getProposal = async (proposalId: string, drepId?: string) => {
     `/proposal/get/${encodedHash}?drepId=${drepId}`,
   );
 
-  const isDataMissing = await checkIsMissingGAMetadata({
+  const { metadata, status } = await checkIsMissingGAMetadata({
     hash: data?.proposal.metadataHash,
     url: data?.proposal.url,
   });
-
-  return { ...data, isDataMissing };
+  // workaround for the missing data in db-sync
+  return { ...data, ...metadata, isDataMissing: status || false };
 };
diff --git a/govtool/frontend/src/utils/getMetadataDataMissingStatusTranslation.ts b/govtool/frontend/src/utils/getMetadataDataMissingStatusTranslation.ts
@@ -13,6 +13,7 @@ export const getMetadataDataMissingStatusTranslation = (
   const errorKey = {
     [MetadataValidationStatus.URL_NOT_FOUND]: "dataMissing",
     [MetadataValidationStatus.INVALID_JSONLD]: "incorrectFormat",
+    [MetadataValidationStatus.INCORRECT_FORMAT]: "incorrectFormat",
     [MetadataValidationStatus.INVALID_HASH]: "notVerifiable",
   }[status] as "dataMissing" | "incorrectFormat" | "notVerifiable";
   return i18n.t(`dataMissingErrors.${errorKey || "dataMissing"}`);

diff --git a/govtool/frontend/src/utils/tests/getMetadataDataMissingStatusTranslation.test.ts b/govtool/frontend/src/utils/tests/getMetadataDataMissingStatusTranslation.test.ts
@@ -13,7 +13,7 @@ describe("getMetadataDataMissingStatusTranslation", () => {
     const translation = getMetadataDataMissingStatusTranslation(
       MetadataValidationStatus.INVALID_JSONLD,
     );
-    expect(translation).toBe("Incorrect Format");
+    expect(translation).toBe("Data Formatted Incorrectly");
   });
 
   it("should return the correct translation for INVALID_HASH status", () => {

diff --git a/govtool/frontend/src/utils/tests/validateMetadataHash.test.ts b/govtool/frontend/src/utils/tests/validateMetadataHash.test.ts
@@ -13,14 +13,18 @@ const mockPostValidate = postValidate as jest.MockedFunction<
 >;
 
 describe("checkIsMissingGAMetadata", () => {
-  it("returns false when there are no issues with the metadata", async () => {
+  it("returns metadata when there are no issues with the validation", async () => {
     mockPostValidate.mockResolvedValueOnce({
       valid: true,
+      metadata: { some: "metadata" },
     });
 
     const result = await checkIsMissingGAMetadata({ url, hash });
 
-    expect(result).toBe(false);
+    expect(result).toStrictEqual({
+      valid: true,
+      metadata: { some: "metadata" },
+    });
     expect(mockPostValidate).toHaveBeenCalledWith({
       url,
       hash,
@@ -36,7 +40,7 @@ describe("checkIsMissingGAMetadata", () => {
 
     const result = await checkIsMissingGAMetadata({ url, hash });
 
-    expect(result).toBe(MetadataValidationStatus.INVALID_HASH);
+    expect(result.status).toBe(MetadataValidationStatus.INVALID_HASH);
     expect(mockPostValidate).toHaveBeenCalledWith({
       url,
       hash,
@@ -52,7 +56,7 @@ describe("checkIsMissingGAMetadata", () => {
 
     const result = await checkIsMissingGAMetadata({ url, hash });
 
-    expect(result).toBe(MetadataValidationStatus.INVALID_JSONLD);
+    expect(result.status).toBe(MetadataValidationStatus.INVALID_JSONLD);
     expect(mockPostValidate).toHaveBeenCalledWith({
       url,
       hash,
@@ -65,7 +69,7 @@ describe("checkIsMissingGAMetadata", () => {
 
     const result = await checkIsMissingGAMetadata({ url, hash });
 
-    expect(result).toBe(MetadataValidationStatus.URL_NOT_FOUND);
+    expect(result.status).toBe(MetadataValidationStatus.URL_NOT_FOUND);
     expect(mockPostValidate).toHaveBeenCalledWith({
       url,
       hash,

diff --git a/govtool/frontend/src/utils/validateMetadataHash.ts b/govtool/frontend/src/utils/validateMetadataHash.ts
@@ -2,24 +2,31 @@ import { postValidate } from "@services";
 
 import { MetadataStandard, MetadataValidationStatus } from "@/models";
 
+type CheckIsMissingGAMetadataResponse = {
+  status?: MetadataValidationStatus;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  metadata?: any;
+  valid: boolean;
+};
+
 export const checkIsMissingGAMetadata = async ({
   url,
   hash,
 }: {
   url: string;
   hash: string;
-}): Promise<boolean | MetadataValidationStatus> => {
+}): Promise<CheckIsMissingGAMetadataResponse> => {
   try {
-    const { status } = await postValidate({
+    const { status, metadata, valid } = await postValidate({
       url,
       hash,
       standard: MetadataStandard.CIP108,
     });
     if (status) {
-      return status;
+      return { status, valid };
     }
-    return false;
+    return { metadata, valid };
   } catch (error) {
-    return MetadataValidationStatus.URL_NOT_FOUND;
+    return { status: MetadataValidationStatus.URL_NOT_FOUND, valid: false };
   }
 };
diff --git a/govtool/metadata-validation/src/app.service.ts b/govtool/metadata-validation/src/app.service.ts
@@ -5,7 +5,7 @@ import * as blake from 'blakejs';
 
 import { ValidateMetadataDTO } from '@dto';
 import { MetadataValidationStatus } from '@enums';
-import { canonizeJSON, validateMetadataStandard } from '@utils';
+import { canonizeJSON, validateMetadataStandard, parseMetadata } from '@utils';
 import { ValidateMetadataResult } from '@types';
 
 @Injectable()
@@ -18,6 +18,7 @@ export class AppService {
     standard,
   }: ValidateMetadataDTO): Promise<ValidateMetadataResult> {
     let status: MetadataValidationStatus;
+    let metadata: any;
     try {
       const { data } = await firstValueFrom(
         this.httpService.get(url).pipe(
@@ -31,6 +32,8 @@ export class AppService {
         await validateMetadataStandard(data, standard);
       }
 
+      metadata = parseMetadata(data.body, standard);
+
       let canonizedMetadata;
       try {
         canonizedMetadata = await canonizeJSON(data);
@@ -48,6 +51,6 @@ export class AppService {
       }
     }
 
-    return { status, valid: !Boolean(status) };
+    return { status, valid: !Boolean(status), metadata };
   }
 }
diff --git a/govtool/metadata-validation/src/schemas/cipStandardSchema.ts b/govtool/metadata-validation/src/schemas/cipStandardSchema.ts
@@ -32,9 +32,13 @@ export const cipStandardSchema: StandardSpecification = {
       references: Joi.array().items(
         Joi.object({
           '@type': Joi.string(),
-          label: Joi.object({ '@value': Joi.string().required() }),
-          uri: Joi.object({ '@value': Joi.string().uri().required() }),
-          referenceHash: Joi.object({
+          'CIP108:reference-label': Joi.object({
+            '@value': Joi.string().required(),
+          }),
+          'CIP108:reference-uri': Joi.object({
+            '@value': Joi.string().uri().required(),
+          }),
+          'CIP108:reference-hash': Joi.object({
             hashDigest: Joi.string().required(),
             hashAlgorithm: Joi.string().required(),
           }),

diff --git a/govtool/metadata-validation/src/types/validateMetadata.ts b/govtool/metadata-validation/src/types/validateMetadata.ts
@@ -7,4 +7,5 @@ export enum MetadataStandard {
 export type ValidateMetadataResult = {
   status?: MetadataValidationStatus;
   valid: boolean;
+  metadata: any;
 };
diff --git a/govtool/metadata-validation/src/utils/index.ts b/govtool/metadata-validation/src/utils/index.ts
@@ -1,2 +1,3 @@
 export * from './canonizeJSON';
 export * from './validateMetadataStandard';
+export * from './parseMetadata';
diff --git a/govtool/metadata-validation/src/utils/parseMetadata.ts b/govtool/metadata-validation/src/utils/parseMetadata.ts
@@ -0,0 +1,23 @@
+import { MetadataStandard } from '@/types';
+
+const CIP_108_VALUE_KEYS = ['abstract', 'motivation', 'rationale', 'title'];
+export const parseMetadata = (metadata: any, standard: MetadataStandard) => {
+  const parsedMetadata = {};
+  switch (standard) {
+    case MetadataStandard.CIP108:
+      for (const [key, value] of Object.entries(metadata)) {
+        if (CIP_108_VALUE_KEYS.includes(key)) {
+          parsedMetadata[key] = value['@value'];
+        }
+
+        if (key === 'references') {
+          parsedMetadata[key] = (Array.isArray(value) ? value : [])?.map(
+            (reference) => reference['CIP108:reference-uri']['@value'],
+          );
+        }
+      }
+      return parsedMetadata;
+    default:
+      return;
+  }
+};