From 93284fd778a062605500be32fa2e3221f8b3420e Mon Sep 17 00:00:00 2001 From: tyler17 Date: Wed, 27 Sep 2023 15:51:52 -0700 Subject: [PATCH 1/3] only use matterWrapper --- lib/matter.ts | 24 +++++++++ modules/avcs/api/fetchAvcs.ts | 4 +- modules/delegates/api/fetchGithubDelegates.ts | 10 ++-- modules/executive/api/parseExecutive.ts | 4 +- modules/polling/api/fetchPollMetadata.ts | 4 +- .../__tests__/parsePollMetadata.spec.ts | 6 +-- .../__tests__/validatePollParameters.spec.ts | 52 +++++++++---------- modules/polling/helpers/parsePollMetadata.ts | 4 +- modules/polling/helpers/validator.ts | 4 +- pages/executive/create.tsx | 4 +- 10 files changed, 70 insertions(+), 46 deletions(-) create mode 100644 lib/matter.ts diff --git a/lib/matter.ts b/lib/matter.ts new file mode 100644 index 000000000..d49a71d2c --- /dev/null +++ b/lib/matter.ts @@ -0,0 +1,24 @@ +import matter from 'gray-matter'; + +interface MatterResult { + data: any; + content: string; +} + +export function matterWrapper(document: string): MatterResult { + const { data, content } = matter(document || '', { + engines: { + javascript: { + parse: function () { + console.log('Parsing JavaScript is not allowed'); + return {}; + }, + stringify: function () { + console.log('Stringifying JavaScript is not allowed'); + return ''; + } + } + } + }); + return { data, content }; +} \ No newline at end of file diff --git a/modules/avcs/api/fetchAvcs.ts b/modules/avcs/api/fetchAvcs.ts index 6679ff3fb..2a26c442b 100644 --- a/modules/avcs/api/fetchAvcs.ts +++ b/modules/avcs/api/fetchAvcs.ts @@ -16,7 +16,7 @@ import { allGithubAvcs } from 'modules/gql/queries/allGithubAvcs'; import logger from 'lib/logger'; import { GraphQlQueryResponseData } from '@octokit/graphql'; import { RepositoryInfo } from 'modules/delegates/api/getDelegatesRepositoryInfo'; -import matter from 'gray-matter'; +import { matterWrapper } from 'lib/matter'; import { markdownToHtml } from 'lib/markdown'; import { AvcOrderByEnum } from '../avcs.constants'; import { fetchDelegatesPaginated } from 'modules/delegates/api/fetchDelegates'; @@ -50,7 +50,7 @@ async function extractGithubInformationGraphQL( const { content, data: { external_profile_url, avc_name } - } = matter(profileMdDoc); + } = matterWrapper(profileMdDoc); const html = await markdownToHtml(content); const avcDescription = html.split('

$delegate_name

')[0]; diff --git a/modules/delegates/api/fetchGithubDelegates.ts b/modules/delegates/api/fetchGithubDelegates.ts index 2f2e62e3b..ce4953d8a 100644 --- a/modules/delegates/api/fetchGithubDelegates.ts +++ b/modules/delegates/api/fetchGithubDelegates.ts @@ -6,7 +6,7 @@ SPDX-License-Identifier: AGPL-3.0-or-later */ -import matter from 'gray-matter'; +import { matterWrapper } from 'lib/matter'; import { GraphQlQueryResponseData } from '@octokit/graphql'; import { SupportedNetworks } from 'modules/web3/constants/networks'; import { cacheGet, cacheSet } from 'modules/cache/cache'; @@ -45,13 +45,13 @@ async function extractGithubInformation( const { content, data: { name, external_profile_url, tags, avc_name } - } = matter(profileMdDoc); + } = matterWrapper(profileMdDoc); let metricsMdDoc; let metricsData; if (metricsMd) { metricsMdDoc = await (await fetch(metricsMd?.download_url)).text(); - const { data } = matter(metricsMdDoc); + const { data } = matterWrapper(metricsMdDoc); metricsData = data; } @@ -110,10 +110,10 @@ async function extractGithubInformationGraphQL( const { content, data: { name, external_profile_url, tags, avc_name } - } = matter(profileMdDoc); + } = matterWrapper(profileMdDoc); const metricsMdDoc = metricsMd?.object?.text; - const { data } = matter(metricsMdDoc); + const { data } = matterWrapper(metricsMdDoc); const metricsData = data; let cuMember = false; diff --git a/modules/executive/api/parseExecutive.ts b/modules/executive/api/parseExecutive.ts index 8d580539b..bafab1a8e 100644 --- a/modules/executive/api/parseExecutive.ts +++ b/modules/executive/api/parseExecutive.ts @@ -6,7 +6,7 @@ SPDX-License-Identifier: AGPL-3.0-or-later */ -import matter from 'gray-matter'; +import { matterWrapper } from 'lib/matter'; import { CMSProposal } from 'modules/executive/types'; import { ethers } from 'ethers'; import { slugify } from 'lib/utils'; @@ -22,7 +22,7 @@ export function parseExecutive( const { content, data: { title, summary, address, date } - } = matter(proposalDoc); + } = matterWrapper(proposalDoc); // Remove empty docs if (!(content && title && summary && address && date)) { logger.warn( diff --git a/modules/polling/api/fetchPollMetadata.ts b/modules/polling/api/fetchPollMetadata.ts index 3605c1d5d..3769f3f33 100644 --- a/modules/polling/api/fetchPollMetadata.ts +++ b/modules/polling/api/fetchPollMetadata.ts @@ -9,7 +9,7 @@ SPDX-License-Identifier: AGPL-3.0-or-later import { Poll, PartialPoll } from '../types/poll'; import { backoffRetry, timeoutPromise } from 'lib/utils'; -import matter from 'gray-matter'; +import { matterWrapper } from 'lib/matter'; import { parsePollMetadata } from '../helpers/parsePollMetadata'; import logger from 'lib/logger'; @@ -22,7 +22,7 @@ async function fetchPollGithubDocument(url: string): Promise { return resp?.text(); }); - if (!(document.length > 0 && Object.keys(matter(document).data?.options)?.length > 0)) { + if (!(document.length > 0 && Object.keys(matterWrapper(document).data?.options)?.length > 0)) { throw new Error('Invalid poll document'); } diff --git a/modules/polling/helpers/__tests__/parsePollMetadata.spec.ts b/modules/polling/helpers/__tests__/parsePollMetadata.spec.ts index ec2de51f6..51090b138 100644 --- a/modules/polling/helpers/__tests__/parsePollMetadata.spec.ts +++ b/modules/polling/helpers/__tests__/parsePollMetadata.spec.ts @@ -10,7 +10,7 @@ import { parsePollMetadata, spockPollToPartialPoll } from '../parsePollMetadata' import pollJson327 from './__helpers__/poll-327.js'; import pollJson431 from './__helpers__/poll-431.js'; import fs from 'fs'; -import matter from 'gray-matter'; +import { matterWrapper } from 'lib/matter'; import { PollSpock } from '../../types/pollSpock'; import { getPollTags, getPollTagsMapping } from 'modules/polling/api/getPollTags'; import { PollInputFormat, PollResultDisplay, PollVictoryConditions } from 'modules/polling/polling.constants'; @@ -76,7 +76,7 @@ describe('Parse poll metadata', () => { }) ); - expect(actual.content).toEqual(matter(pollMetadata431).content); + expect(actual.content).toEqual(matterWrapper(pollMetadata431).content); }); test('return the expected values for an old uncategorized poll', async () => { @@ -111,6 +111,6 @@ describe('Parse poll metadata', () => { }) ); - expect(actual.content).toEqual(matter(pollMetadata327).content); + expect(actual.content).toEqual(matterWrapper(pollMetadata327).content); }); }); diff --git a/modules/polling/helpers/__tests__/validatePollParameters.spec.ts b/modules/polling/helpers/__tests__/validatePollParameters.spec.ts index a114351b0..0eea7b15e 100644 --- a/modules/polling/helpers/__tests__/validatePollParameters.spec.ts +++ b/modules/polling/helpers/__tests__/validatePollParameters.spec.ts @@ -6,7 +6,7 @@ SPDX-License-Identifier: AGPL-3.0-or-later */ -import matter from 'gray-matter'; +import { matterWrapper } from 'lib/matter'; import { POLL_VOTE_TYPE } from 'modules/polling/polling.constants'; import { ERRORS_VALIDATE_POLL_PARAMETERS, @@ -22,7 +22,7 @@ parameters: --- `; - const parametersMarkdown = matter(parameters); + const parametersMarkdown = matterWrapper(parameters); // Returns correct if is correct const [parsed, errors] = validatePollParameters(parametersMarkdown.data.parameters); expect(parsed).toBe(null); @@ -39,7 +39,7 @@ parameters: # hello `; - const parametersMarkdown = matter(parameters); + const parametersMarkdown = matterWrapper(parameters); // Returns correct if is correct const [parsed, errors] = validatePollParameters(parametersMarkdown.data.parameters); expect(parsed).toBe(null); @@ -57,7 +57,7 @@ parameters: # hello `; - const parametersMarkdown = matter(parameters); + const parametersMarkdown = matterWrapper(parameters); // Returns correct if is correct const [parsed, errors] = validatePollParameters(parametersMarkdown.data.parameters); expect(parsed).toBe(null); @@ -76,7 +76,7 @@ parameters: # hello `; - const parametersMarkdown = matter(parameters); + const parametersMarkdown = matterWrapper(parameters); // Returns correct if is correct const [parsed, errors] = validatePollParameters(parametersMarkdown.data.parameters); expect(parsed).toBe(null); @@ -97,7 +97,7 @@ parameters: # hello `; - const parametersMarkdown = matter(parameters); + const parametersMarkdown = matterWrapper(parameters); // Returns correct if is correct const [parsed, errors] = validatePollParameters(parametersMarkdown.data.parameters); expect(parsed).toBe(null); @@ -120,7 +120,7 @@ parameters: # hello `; - const parametersMarkdown = matter(parameters); + const parametersMarkdown = matterWrapper(parameters); // Returns correct if is correct const [parsed, errors] = validatePollParameters(parametersMarkdown.data.parameters); expect(parsed).toBe(null); @@ -142,7 +142,7 @@ parameters: # hello `; - const parametersMarkdown = matter(parameters); + const parametersMarkdown = matterWrapper(parameters); // Returns correct if is correct const [parsed, errors] = validatePollParameters(parametersMarkdown.data.parameters); expect(parsed).toBe(null); @@ -162,7 +162,7 @@ parameters: # hello `; - const parametersMarkdown = matter(parameters); + const parametersMarkdown = matterWrapper(parameters); // Returns correct if is correct const [parsed, errors] = validatePollParameters(parametersMarkdown.data.parameters); expect(parsed).toBe(null); @@ -182,7 +182,7 @@ parameters: # hello `; - const parametersMarkdown = matter(parameters); + const parametersMarkdown = matterWrapper(parameters); // Returns correct if is correct const [parsed, errors] = validatePollParameters(parametersMarkdown.data.parameters); expect(parsed).toBe(null); @@ -202,7 +202,7 @@ parameters: # hello `; - const parametersMarkdown = matter(parameters); + const parametersMarkdown = matterWrapper(parameters); // Returns correct if is correct const [parsed, errors] = validatePollParameters(parametersMarkdown.data.parameters); expect(parsed).toBe(null); @@ -222,7 +222,7 @@ parameters: # hello `; - const parametersMarkdown = matter(parameters); + const parametersMarkdown = matterWrapper(parameters); // Returns correct if is correct const [parsed, errors] = validatePollParameters(parametersMarkdown.data.parameters); expect(parsed).toEqual({ @@ -248,7 +248,7 @@ parameters: # hello `; - const parametersMarkdown = matter(parameters); + const parametersMarkdown = matterWrapper(parameters); // Returns correct if is correct const [parsed, errors] = validatePollParameters(parametersMarkdown.data.parameters); expect(parsed).toBe(null); @@ -268,7 +268,7 @@ parameters: # hello `; - const parametersMarkdown = matter(parameters); + const parametersMarkdown = matterWrapper(parameters); // Returns correct if is correct const [parsed, errors] = validatePollParameters(parametersMarkdown.data.parameters); expect(parsed).toEqual({ @@ -293,7 +293,7 @@ parameters: # hello `; - const parametersMarkdown = matter(parameters); + const parametersMarkdown = matterWrapper(parameters); // Returns correct if is correct const [parsed, errors] = validatePollParameters(parametersMarkdown.data.parameters); expect(parsed).toBe(null); @@ -313,7 +313,7 @@ parameters: # hello `; - const parametersMarkdown = matter(parameters); + const parametersMarkdown = matterWrapper(parameters); const [parsed, errors] = validatePollParameters(parametersMarkdown.data.parameters); expect(parsed).toBe(null); @@ -333,7 +333,7 @@ parameters: # hello `; - const parametersMarkdown = matter(parameters); + const parametersMarkdown = matterWrapper(parameters); // Returns correct if is correct const [parsed, errors] = validatePollParameters(parametersMarkdown.data.parameters); expect(parsed).toBe(null); @@ -355,7 +355,7 @@ parameters: # hello `; - const parametersMarkdown = matter(parameters); + const parametersMarkdown = matterWrapper(parameters); // Returns correct if is correct const [parsed, errors] = validatePollParameters(parametersMarkdown.data.parameters); expect(parsed).toEqual({ @@ -384,7 +384,7 @@ parameters: # hello `; - const parametersMarkdown = matter(parameters); + const parametersMarkdown = matterWrapper(parameters); // Returns correct if is correct const [parsed, errors] = validatePollParameters(parametersMarkdown.data.parameters); expect(parsed).toEqual({ @@ -413,7 +413,7 @@ parameters: # hello `; - const parametersMarkdown = matter(parameters); + const parametersMarkdown = matterWrapper(parameters); // Returns correct if is correct const [parsed, errors] = validatePollParameters(parametersMarkdown.data.parameters); expect(parsed).toEqual(null); @@ -434,7 +434,7 @@ parameters: # hello `; - const parametersMarkdown = matter(parameters); + const parametersMarkdown = matterWrapper(parameters); // Returns correct if is correct const [parsed, errors] = validatePollParameters(parametersMarkdown.data.parameters); expect(parsed).toEqual(null); @@ -455,7 +455,7 @@ parameters: # hello `; - const parametersMarkdown = matter(parameters); + const parametersMarkdown = matterWrapper(parameters); // Returns correct if is correct const [parsed, errors] = validatePollParameters(parametersMarkdown.data.parameters); expect(parsed).toEqual(null); @@ -478,7 +478,7 @@ parameters: # hello `; - const parametersMarkdown = matter(parameters); + const parametersMarkdown = matterWrapper(parameters); // Returns correct if is correct const [parsed, errors] = validatePollParameters(parametersMarkdown.data.parameters); expect(parsed).toEqual(null); @@ -503,7 +503,7 @@ parameters: # hello `; - const parametersMarkdown = matter(parameters); + const parametersMarkdown = matterWrapper(parameters); // Returns correct if is correct const [parsed, errors] = validatePollParameters(parametersMarkdown.data.parameters); expect(parsed).toEqual(null); @@ -525,7 +525,7 @@ parameters: # hello `; - const parametersMarkdown = matter(parameters); + const parametersMarkdown = matterWrapper(parameters); // Returns correct if is correct const [parsed, errors] = validatePollParameters(parametersMarkdown.data.parameters); expect(parsed).toEqual(null); @@ -543,7 +543,7 @@ parameters: // - { type : default, options : [2] } // result_display: single-vote-breakdown // `; - // const parametersMarkdown = matter(parameters); + // const parametersMarkdown = matterWrapper(parameters); // // Returns correct if is correct // const [parsed, errors] = validatePollParameters(parametersMarkdown.data); // }); diff --git a/modules/polling/helpers/parsePollMetadata.ts b/modules/polling/helpers/parsePollMetadata.ts index 60385f575..3106b04c1 100644 --- a/modules/polling/helpers/parsePollMetadata.ts +++ b/modules/polling/helpers/parsePollMetadata.ts @@ -6,7 +6,7 @@ SPDX-License-Identifier: AGPL-3.0-or-later */ -import matter from 'gray-matter'; +import { matterWrapper } from 'lib/matter'; import validUrl from 'valid-url'; import { Poll, PartialPoll, PollVoteType } from 'modules/polling/types'; import { POLL_VOTE_TYPE } from '../polling.constants'; @@ -32,7 +32,7 @@ export async function parsePollMetadata( [key: number]: string[]; } ): Promise { - const { data: pollMeta, content } = matter(document || '', { + const { data: pollMeta, content } = matterWrapper(document || '', { engines: { javascript: { parse: function () { diff --git a/modules/polling/helpers/validator.ts b/modules/polling/helpers/validator.ts index b344cbe57..b111dbf22 100644 --- a/modules/polling/helpers/validator.ts +++ b/modules/polling/helpers/validator.ts @@ -6,7 +6,7 @@ SPDX-License-Identifier: AGPL-3.0-or-later */ -import matter from 'gray-matter'; +import { matterWrapper } from 'lib/matter'; import isEmpty from 'lodash/isEmpty'; import { Poll, PartialPoll } from 'modules/polling/types'; import { parsePollMetadata } from './parsePollMetadata'; @@ -35,7 +35,7 @@ export async function validatePollFromRawURL(url: string, poll?: PartialPoll): P export function validatePollMarkdown(text: string): ValidationResult { try { - const { data, content } = matter(text); + const { data, content } = matterWrapper(text); if (!content) return { valid: false, errors: ['Document is blank'] }; if (isEmpty(data)) return { valid: false, errors: ['Front matter is blank'] }; const errors: string[] = []; diff --git a/pages/executive/create.tsx b/pages/executive/create.tsx index 7bb7cceff..051df4006 100644 --- a/pages/executive/create.tsx +++ b/pages/executive/create.tsx @@ -11,7 +11,7 @@ import PrimaryLayout from 'modules/app/components/layout/layouts/Primary'; import Stack from 'modules/app/components/layout/layouts/Stack'; import { useState } from 'react'; import { ethers } from 'ethers'; -import matter from 'gray-matter'; +import { matterWrapper } from 'lib/matter'; import { markdownToHtml } from 'lib/markdown'; import { SupportedNetworks } from 'modules/web3/constants/networks'; import { HeadComponent } from 'modules/app/components/layout/Head'; @@ -54,7 +54,7 @@ const ExecutiveCreate = (): JSX.Element => { setFetchFinished(false); try { const rawMd = await (await fetch(url, { cache: 'no-cache' })).text(); - const { data, content } = matter(rawMd); + const { data, content } = matterWrapper(rawMd); metadata = data; execMarkdown = content; } catch (e) { From 9c0ca89dfce4bac924fa162cedaf2139c7305f75 Mon Sep 17 00:00:00 2001 From: tyler17 Date: Wed, 27 Sep 2023 15:54:37 -0700 Subject: [PATCH 2/3] fix --- modules/polling/helpers/parsePollMetadata.ts | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/modules/polling/helpers/parsePollMetadata.ts b/modules/polling/helpers/parsePollMetadata.ts index 3106b04c1..cab168a43 100644 --- a/modules/polling/helpers/parsePollMetadata.ts +++ b/modules/polling/helpers/parsePollMetadata.ts @@ -32,20 +32,7 @@ export async function parsePollMetadata( [key: number]: string[]; } ): Promise { - const { data: pollMeta, content } = matterWrapper(document || '', { - engines: { - javascript: { - parse: function () { - console.log('Parsing JavaScript is not allowed'); - return {}; - }, - stringify: function () { - console.log('Stringifying JavaScript is not allowed'); - return ''; - } - } - } - }); + const { data: pollMeta, content } = matterWrapper(document || ''); const summary = pollMeta?.summary || ''; const title = pollMeta?.title || ''; const options = pollMeta.options; From 27df9f16ad62439248d4a71d8d23966640910521 Mon Sep 17 00:00:00 2001 From: tyler17 Date: Wed, 27 Sep 2023 16:50:25 -0700 Subject: [PATCH 3/3] fix unit tests --- modules/polling/helpers/__tests__/validator.spec.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/polling/helpers/__tests__/validator.spec.ts b/modules/polling/helpers/__tests__/validator.spec.ts index bc7a2c410..f49699b0f 100644 --- a/modules/polling/helpers/__tests__/validator.spec.ts +++ b/modules/polling/helpers/__tests__/validator.spec.ts @@ -24,7 +24,6 @@ test('reject a blank document', () => { test('reject null', () => { const result = validatePollMarkdown(null); expect(result.valid).toBeFalsy(); - expect(result.errors).toContain('expected input to be a string or buffer'); }); test('reject a document with no options', () => {