From 4005ef8a8c844b29431cf95b87e320dd5b61b561 Mon Sep 17 00:00:00 2001 From: Tim Seckinger Date: Wed, 13 Nov 2019 17:38:13 +0000 Subject: [PATCH] request all (if more than 1000) blog posts from Contentful --- .babelrc | 8 +++++ .babelrc.js | 8 ----- .../__tests__/filter-posts-to-process.test.js | 2 +- .../get-all-contentful-blog-post.test.js | 33 +++++++++++++++++++ .../__tests__/parse-xml-to-json.test.js | 2 +- .../__tests__/publish-to-contentful.test.js | 2 +- .../transform-html-to-markdown.test.js | 2 +- .../__tests__/transform-meta-data.test.js | 2 +- .../__tests__/validate-mdx.test.js | 2 +- .../__tests__/add-frontmatter.test.js | 2 +- .../custom-mdx/__tests__/index.test.js | 20 ++++------- .../__tests__/transform-iframes.test.js | 2 +- .../__tests__/transform-image-data.test.js | 9 ++--- .../__tests__/transform-strings.test.js | 2 +- .../upload-image-to-contentful.test.js | 2 +- .../custom-mdx/transform-strings.js | 4 +-- .../get-all-contentful-blog-posts.js | 19 +++++++++++ src/functions/blogMethods/index.js | 7 ++-- 18 files changed, 83 insertions(+), 45 deletions(-) create mode 100644 .babelrc delete mode 100644 .babelrc.js create mode 100644 src/functions/blogMethods/__tests__/get-all-contentful-blog-post.test.js rename src/functions/blogMethods/{ => custom-mdx}/__tests__/upload-image-to-contentful.test.js (94%) create mode 100644 src/functions/blogMethods/get-all-contentful-blog-posts.js diff --git a/.babelrc b/.babelrc new file mode 100644 index 000000000..020f52e28 --- /dev/null +++ b/.babelrc @@ -0,0 +1,8 @@ +{ + "presets": ["babel-preset-gatsby"], + "env": { + "test": { + "plugins": ["require-context-hook"] + } + } +} diff --git a/.babelrc.js b/.babelrc.js deleted file mode 100644 index cc025d3ba..000000000 --- a/.babelrc.js +++ /dev/null @@ -1,8 +0,0 @@ -module.exports = { - presets: ['babel-preset-gatsby'], - env: { - test: { - plugins: ['require-context-hook'], - }, - }, -}; diff --git a/src/functions/blogMethods/__tests__/filter-posts-to-process.test.js b/src/functions/blogMethods/__tests__/filter-posts-to-process.test.js index e3d71f391..f92120e0a 100644 --- a/src/functions/blogMethods/__tests__/filter-posts-to-process.test.js +++ b/src/functions/blogMethods/__tests__/filter-posts-to-process.test.js @@ -1,5 +1,5 @@ /* eslint-disable no-console */ -const FilterPostsToProcess = require('../../../../src/functions/blogMethods/filter-posts-to-process'); +const FilterPostsToProcess = require('../filter-posts-to-process'); const originalConsoleInfo = console.info; beforeEach(() => { diff --git a/src/functions/blogMethods/__tests__/get-all-contentful-blog-post.test.js b/src/functions/blogMethods/__tests__/get-all-contentful-blog-post.test.js new file mode 100644 index 000000000..bb29b10df --- /dev/null +++ b/src/functions/blogMethods/__tests__/get-all-contentful-blog-post.test.js @@ -0,0 +1,33 @@ +const GetAllContentfulBlogPosts = require('../get-all-contentful-blog-posts'); + +const posts = [...Array(2 ** 16).keys()].map(i => ({ slug: `post-${i}` })); + +const mockGetEntries = jest.fn(({ skip, limit }) => ({ + skip, + limit, + total: posts.length, + items: posts.slice(skip, skip + limit), +})); +const environment = { getEntries: mockGetEntries }; + +beforeEach(() => { + mockGetEntries.mockClear(); +}); + +it('requests blog posts', async () => { + await GetAllContentfulBlogPosts(environment); + mockGetEntries.mock.calls.map(([{ content_type }]) => { + expect(content_type).toBe('blogPost'); + }); +}); + +it('requests until it has all entries and returns them', async () => { + expect(await GetAllContentfulBlogPosts(environment)).toEqual(posts); +}); + +it('does not exceed the contentful entry limit', async () => { + await GetAllContentfulBlogPosts(environment); + mockGetEntries.mock.calls.map(([{ limit }]) => { + expect(limit).toBeLessThanOrEqual(1000); + }); +}); diff --git a/src/functions/blogMethods/__tests__/parse-xml-to-json.test.js b/src/functions/blogMethods/__tests__/parse-xml-to-json.test.js index d6b5f073e..74e9c9d06 100644 --- a/src/functions/blogMethods/__tests__/parse-xml-to-json.test.js +++ b/src/functions/blogMethods/__tests__/parse-xml-to-json.test.js @@ -1,4 +1,4 @@ -const ParseXMLToJSON = require('../../../../src/functions/blogMethods/parse-xml-to-json'); +const ParseXMLToJSON = require('../parse-xml-to-json'); const { readFileSync } = require('fs'); const { resolve } = require('path'); diff --git a/src/functions/blogMethods/__tests__/publish-to-contentful.test.js b/src/functions/blogMethods/__tests__/publish-to-contentful.test.js index b496d4a7c..8f9c66dc3 100644 --- a/src/functions/blogMethods/__tests__/publish-to-contentful.test.js +++ b/src/functions/blogMethods/__tests__/publish-to-contentful.test.js @@ -1,5 +1,5 @@ /* eslint-disable no-console */ -const PublishToContentful = require('../../../../src/functions/blogMethods/publish-to-contentful'); +const PublishToContentful = require('../publish-to-contentful'); const until = require('async-wait-until'); const originalConsoleInfo = console.info; diff --git a/src/functions/blogMethods/__tests__/transform-html-to-markdown.test.js b/src/functions/blogMethods/__tests__/transform-html-to-markdown.test.js index 4df3ac450..0e82a5922 100644 --- a/src/functions/blogMethods/__tests__/transform-html-to-markdown.test.js +++ b/src/functions/blogMethods/__tests__/transform-html-to-markdown.test.js @@ -1,4 +1,4 @@ -const TransformHTMLToMarkdown = require('../../../../src/functions/blogMethods/transform-html-to-markdown'); +const TransformHTMLToMarkdown = require('../transform-html-to-markdown'); const tweetBlockquote = `
diff --git a/src/functions/blogMethods/__tests__/transform-meta-data.test.js b/src/functions/blogMethods/__tests__/transform-meta-data.test.js index 8dedf36fc..652435b62 100644 --- a/src/functions/blogMethods/__tests__/transform-meta-data.test.js +++ b/src/functions/blogMethods/__tests__/transform-meta-data.test.js @@ -1,4 +1,4 @@ -const TransformMetaData = require('../../../../src/functions/blogMethods/transform-meta-data'); +const TransformMetaData = require('../transform-meta-data'); const headerImage = { sys: { diff --git a/src/functions/blogMethods/__tests__/validate-mdx.test.js b/src/functions/blogMethods/__tests__/validate-mdx.test.js index 15b95bb28..05d4378d9 100644 --- a/src/functions/blogMethods/__tests__/validate-mdx.test.js +++ b/src/functions/blogMethods/__tests__/validate-mdx.test.js @@ -1,4 +1,4 @@ -const ValidateMdx = require('../../../../src/functions/blogMethods/validate-mdx'); +const ValidateMdx = require('../validate-mdx'); it.each` description | mdx diff --git a/src/functions/blogMethods/custom-mdx/__tests__/add-frontmatter.test.js b/src/functions/blogMethods/custom-mdx/__tests__/add-frontmatter.test.js index 77546b39e..f07cf0574 100644 --- a/src/functions/blogMethods/custom-mdx/__tests__/add-frontmatter.test.js +++ b/src/functions/blogMethods/custom-mdx/__tests__/add-frontmatter.test.js @@ -1,4 +1,4 @@ -const AddFrontMatter = require('../../../../../src/functions/blogMethods/custom-mdx/add-frontmatter'); +const AddFrontMatter = require('../add-frontmatter'); const post = { title: 'Blog Title', diff --git a/src/functions/blogMethods/custom-mdx/__tests__/index.test.js b/src/functions/blogMethods/custom-mdx/__tests__/index.test.js index e274ec805..02a09ae09 100644 --- a/src/functions/blogMethods/custom-mdx/__tests__/index.test.js +++ b/src/functions/blogMethods/custom-mdx/__tests__/index.test.js @@ -1,18 +1,10 @@ -const TransformCustomMdx = require('../../../../../src/functions/blogMethods/custom-mdx'); -const TransformImageData = require('../../../../../src/functions/blogMethods/custom-mdx/transform-image-data'); +const TransformCustomMdx = require('..'); +const TransformImageData = require('../transform-image-data'); -jest.mock( - '../../../../../src/functions/blogMethods/custom-mdx/add-frontmatter', -); -jest.mock( - '../../../../../src/functions/blogMethods/custom-mdx/transform-iframes', -); -jest.mock( - '../../../../../src/functions/blogMethods/custom-mdx/transform-image-data', -); -jest.mock( - '../../../../../src/functions/blogMethods/custom-mdx/transform-strings', -); +jest.mock('../add-frontmatter'); +jest.mock('../transform-iframes'); +jest.mock('../transform-image-data'); +jest.mock('../transform-strings'); const post = { slug: 'Post', diff --git a/src/functions/blogMethods/custom-mdx/__tests__/transform-iframes.test.js b/src/functions/blogMethods/custom-mdx/__tests__/transform-iframes.test.js index 5995eb82f..dfa8104d6 100644 --- a/src/functions/blogMethods/custom-mdx/__tests__/transform-iframes.test.js +++ b/src/functions/blogMethods/custom-mdx/__tests__/transform-iframes.test.js @@ -1,4 +1,4 @@ -const TransformIframes = require('../../../../../src/functions/blogMethods/custom-mdx/transform-iframes'); +const TransformIframes = require('../transform-iframes'); const Got = require('got'); jest.mock('got'); diff --git a/src/functions/blogMethods/custom-mdx/__tests__/transform-image-data.test.js b/src/functions/blogMethods/custom-mdx/__tests__/transform-image-data.test.js index 682b3adf0..36a79c8f2 100644 --- a/src/functions/blogMethods/custom-mdx/__tests__/transform-image-data.test.js +++ b/src/functions/blogMethods/custom-mdx/__tests__/transform-image-data.test.js @@ -1,4 +1,4 @@ -jest.mock('../../../../../src/functions/utils/is-prod', () => true); +jest.mock('../../../utils/is-prod', () => true); const mockUploadToContentful = jest.fn(({ name }) => { return { @@ -7,16 +7,13 @@ const mockUploadToContentful = jest.fn(({ name }) => { }; }); -jest.mock( - '../../../../../src/functions/blogMethods/custom-mdx/upload-image-to-contentful', - () => mockUploadToContentful, -); +jest.mock('../upload-image-to-contentful', () => mockUploadToContentful); beforeEach(() => { mockUploadToContentful.mockClear(); }); -const TransformImageData = require('../../../../../src/functions/blogMethods/custom-mdx/transform-image-data'); +const TransformImageData = require('../transform-image-data'); const img1 = { caption: 'Image 1 caption', diff --git a/src/functions/blogMethods/custom-mdx/__tests__/transform-strings.test.js b/src/functions/blogMethods/custom-mdx/__tests__/transform-strings.test.js index 440ce6460..24edee347 100644 --- a/src/functions/blogMethods/custom-mdx/__tests__/transform-strings.test.js +++ b/src/functions/blogMethods/custom-mdx/__tests__/transform-strings.test.js @@ -1,4 +1,4 @@ -const TransformStrings = require('../../../../../src/functions/blogMethods/custom-mdx/transform-strings'); +const TransformStrings = require('../transform-strings'); it.each([ 'https://medium.com/yld-blog/', diff --git a/src/functions/blogMethods/__tests__/upload-image-to-contentful.test.js b/src/functions/blogMethods/custom-mdx/__tests__/upload-image-to-contentful.test.js similarity index 94% rename from src/functions/blogMethods/__tests__/upload-image-to-contentful.test.js rename to src/functions/blogMethods/custom-mdx/__tests__/upload-image-to-contentful.test.js index 07245d5b4..33254d663 100644 --- a/src/functions/blogMethods/__tests__/upload-image-to-contentful.test.js +++ b/src/functions/blogMethods/custom-mdx/__tests__/upload-image-to-contentful.test.js @@ -1,4 +1,4 @@ -const UploadImageToContentful = require('../../../../src/functions/blogMethods/custom-mdx/upload-image-to-contentful'); +const UploadImageToContentful = require('../upload-image-to-contentful'); const mockPublish = jest.fn(function publish() { return { diff --git a/src/functions/blogMethods/custom-mdx/transform-strings.js b/src/functions/blogMethods/custom-mdx/transform-strings.js index 8c087f3a5..4687af554 100644 --- a/src/functions/blogMethods/custom-mdx/transform-strings.js +++ b/src/functions/blogMethods/custom-mdx/transform-strings.js @@ -11,9 +11,9 @@ const postPathSegment = new RegExp(`(${slug.source}-${hexUid.source})?`); const path = new RegExp(`${blogPath.source}(\\/${postPathSegment.source})?`); const mdLink = new RegExp(`\\(${origin.source}${path.source}\\)`, 'gi'); -const mdLinkFull = /\(https?:\/\/medium\.com\/yld(-engineering)?-blog(\/((?[^)]+)-[0-9a-f]{8,})?)?\)/; +const mdLinkFull = /\(https?:\/\/medium\.com\/yld(-engineering)?-blog(\/((?[^)]+)-[0-9a-f]{8,})?)?\)/gi; // composite mdLink and mdLinkFull are provided because one may find either more legible - this ensures they are kept in sync -assert.strictEqual(mdLink.source, mdLinkFull.source); +assert.deepStrictEqual(mdLink, mdLinkFull); const transformStrings = content => content.replace(mdLink, function() { diff --git a/src/functions/blogMethods/get-all-contentful-blog-posts.js b/src/functions/blogMethods/get-all-contentful-blog-posts.js new file mode 100644 index 000000000..08af315d1 --- /dev/null +++ b/src/functions/blogMethods/get-all-contentful-blog-posts.js @@ -0,0 +1,19 @@ +const getAllBlogPosts = async environment => { + let total = Number.POSITIVE_INFINITY; + let items = []; + + while (items.length < total) { + const { items: cmsPosts, total: cmsTotal } = await environment.getEntries({ + content_type: 'blogPost', + limit: 1000, + skip: items.length, + }); + + items.push(...cmsPosts); + total = cmsTotal; + } + + return items; +}; + +module.exports = getAllBlogPosts; diff --git a/src/functions/blogMethods/index.js b/src/functions/blogMethods/index.js index 1ca35eeaf..df0e95036 100644 --- a/src/functions/blogMethods/index.js +++ b/src/functions/blogMethods/index.js @@ -6,6 +6,7 @@ const TransformHtmlToMd = require('./transform-html-to-markdown'); const TransformCustomMDX = require('./custom-mdx'); const PublishToContentful = require('./publish-to-contentful'); const FilterPostsToProcess = require('./filter-posts-to-process'); +const GetAllContentfulBlogPosts = require('./get-all-contentful-blog-posts'); const ValidateMdx = require('./validate-mdx'); const TransformMetaData = require('./transform-meta-data'); @@ -25,11 +26,7 @@ const syncMediumToContentful = async data => { const { allFields, requiredFields } = GetContentTypeFields(contentType); - const { items: cmsBlogPosts } = await environment.getEntries({ - limit: 1000, - content_type: 'blogPost', - }); - + const cmsBlogPosts = await GetAllContentfulBlogPosts(environment); const parsedPostsInJson = await ParseXMLToJSON(data); const postsToProcess = FilterPostsToProcess(parsedPostsInJson, {