Skip to content

Commit

Permalink
[1.0] Contentful fixes (#1263)
Browse files Browse the repository at this point in the history 
* Some fixes for source-contentful

* Update yarn.lock
  • Loading branch information
@KyleAMathews
KyleAMathews authored Jun 27, 2017
1 parent 4431751 commit c64a932
Show file tree
Hide file tree
Showing 9 changed files with 197 additions and 143 deletions.
29 changes: 15 additions & 14 deletions examples/using-contentful/src/pages/image-api.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,8 @@ export default props => {
You can also set the{` `}
<a href="https://www.contentful.com/developers/docs/references/images-api/#/reference/resizing-&-cropping/change-the-resizing-behavior">
resizing behavior
</a>{` `}
</a>
{` `}
and{` `}
<a href="https://www.contentful.com/developers/docs/references/images-api/#/reference/resizing-&-cropping/specify-focus-area-for-resizing">
resizing focus area
Expand Down Expand Up @@ -180,19 +181,19 @@ export default props => {
a <code>maxWidth</code>, the max width the container of the images
reaches.
</p>
{assets.map(({ node: { title, responsiveSizes } }) => (
<img
alt={title}
src={responsiveSizes.src}
srcSet={responsiveSizes.srcSet}
sizes={responsiveSizes.sizes}
style={{
marginRight: rhythm(1 / 2),
marginBottom: rhythm(1 / 2),
border: `1px solid tomato`,
}}
/>
))}
{assets.map(({ node: { title, responsiveSizes } }) =>
<img
alt={title}
src={responsiveSizes.src}
srcSet={responsiveSizes.srcSet}
sizes={responsiveSizes.sizes}
style={{
marginRight: rhythm(1 / 2),
marginBottom: rhythm(1 / 2),
border: `1px solid tomato`,
}}
/>
)}
<h4>GraphQL query</h4>
<pre style={{ background: `#efeded`, padding: rhythm(3 / 4) }}>
<code
Expand Down
18 changes: 9 additions & 9 deletions packages/gatsby-source-contentful/src/__tests__/extend-node-type.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,17 +22,17 @@ describe(`contentful extend node type`, () => {
const image = {
defaultLocale: `en-US`,
file: {
url: `//images.contentful.com/ubriaw6jfhm1/10TkaLheGeQG6qQGqWYqUI/5421d3108cbb699561acabd594fa2cb0/ryugj83mqwa1asojwtwb.jpg`,
fileName: `ryugj83mqwa1asojwtwb.jpg`,
contentType: `image/jpeg`,
details: {
size: 28435,
image: {
width: `4500`,
height: `6000`,
},
url: `//images.contentful.com/ubriaw6jfhm1/10TkaLheGeQG6qQGqWYqUI/5421d3108cbb699561acabd594fa2cb0/ryugj83mqwa1asojwtwb.jpg`,
fileName: `ryugj83mqwa1asojwtwb.jpg`,
contentType: `image/jpeg`,
details: {
size: 28435,
image: {
width: `4500`,
height: `6000`,
},
},
},
}
describe(`resolveResponsiveResolution`, () => {
it(`generates responsive resolution data for images`, async () => {
Expand Down
188 changes: 108 additions & 80 deletions packages/gatsby-source-contentful/src/gatsby-node.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,16 @@ const _ = require(`lodash`)
exports.setFieldsOnGraphQLNodeType = require(`./extend-node-type`).extendNodeType

exports.sourceNodes = async (
{ boundActionCreators, getNode, hasNodeChanged, store },
{ boundActionCreators, getNodes, hasNodeChanged, store },
{ spaceId, accessToken }
) => {
const { createNode, setPluginStatus } = boundActionCreators

const {
createNode,
deleteNodes,
touchNode,
setPluginStatus,
} = boundActionCreators

// Fetch articles.
console.time(`Fetch Contentful data`)
console.log(`Starting to fetch data from Contentful`)
Expand All @@ -25,50 +30,48 @@ exports.sourceNodes = async (
space: spaceId,
accessToken,
})
// the structure of the sync payload is a bit different
// all field will be in this format { fieldName: {'locale': value} } so we need to get the space and its default local
// this can be extended later to support multiple locals
// The sync API puts the locale in all fields in this format { fieldName:
// {'locale': value} } so we need to get the space and its default local.
//
// We'll extend this soon to support multiple locales.
let space
let defaultLocale = `en-US`
try {
console.log(`Fetching default locale`)
space = await client.getSpace()
defaultLocale = _.find(space.locales, { 'default': true }).code
defaultLocale = _.find(space.locales, { default: true }).code
console.log(`default local is : ${defaultLocale}`)
} catch (e) {
console.log(`can't get space`)
// TODO maybe return here
console.log(
`Accessing your Contentful space failed. Perhaps you're offline or the spaceId/accessToken is incorrect.`
)
// TODO perhaps continue if there's cached data? That would let
// someone develop a contentful site even if not connected to the internet.
// For prod builds though always fail if we can't get the latest data.
process.exit(1)
}

// Get sync token if it exists
// Get sync token if it exists.
let syncToken
let lastSyncedData = { entries: [], assets: [], deletedEntries: [], deletedAssets: [], nextSyncToken: null }
if (
store.getState().status.plugins &&
store.getState().status.plugins[`gatsby-source-contentful`]
) {
lastSyncedData = store.getState().status.plugins[`gatsby-source-contentful`].status
.lastSyncedData
syncToken = lastSyncedData.nextSyncToken
syncToken = store.getState().status.plugins[`gatsby-source-contentful`]
.status.syncToken
}

// The SDK will map the entities to the following object
// {
// entries,
// assets,
// deletedEntries,
// deletedAssets
// }
let currentSyncData
try {
let query = syncToken ? { nextSyncToken: syncToken } : { initial: true }
currentSyncData = await client.sync(query)
currentSyncData = await client.sync(query)
} catch (e) {
currentSyncData = { entries: [], assets: [], deletedEntries: [], deletedAssets:[] }
console.log(`error fetching contentful data`, e)
process.exit(1)
}

// We fetch content types normaly since we don't receive it in the sync data
// We need to fetch content types with the non-sync API as the sync API
// doesn't support this.
let contentTypes
try {
contentTypes = await pagedGet(client, `getContentTypes`)
Expand All @@ -78,47 +81,39 @@ exports.sourceNodes = async (
console.log(`contentTypes fetched`, contentTypes.items.length)

const contentTypeItems = contentTypes.items
// remove outdated entries
lastSyncedData.entries.filter(entry => {
return _.find(
currentSyncData.entries, (newEntry) => newEntry.sys.id === entry.sys.id
||
_.find(
currentSyncData.deletedEntries, (deletedEntry) => deletedEntry.sys.id === entry.sys.id
)
)
})

// merge entries
lastSyncedData.entries = lastSyncedData.entries.concat(currentSyncData.entries)
let entryList = contentTypeItems.map(contentType => {
return lastSyncedData.entries.filter(entry => entry.sys.contentType.sys.id === contentType.sys.id )
})
// Remove deleted entries & assets.
// TODO figure out if entries referencing now deleted entries/assets
// are "updated" so will get updated here.
deleteNodes(currentSyncData.deletedEntries.map(e => e.sys.id))
deleteNodes(currentSyncData.deletedAssets.map(e => e.sys.id))

lastSyncedData.assets.filter(asset => {
return _.find(
currentSyncData.assets, (newAsset) => newAsset.sys.id === asset.sys.id
)
||
_.find(
currentSyncData.deletedAssets, (deletedAsset) => deletedAsset.sys.id === asset.sys.id
)
})
const existingNodes = getNodes().filter(
n => n.internal.owner === `gatsby-source-contentful`
)
existingNodes.forEach(n => touchNode(n.id))

let entryList = contentTypeItems.map(contentType =>
currentSyncData.entries.filter(
entry => entry.sys.contentType.sys.id === contentType.sys.id
)
)

const assets = currentSyncData.assets

lastSyncedData.assets = lastSyncedData.assets.concat(currentSyncData.assets)
let assets = lastSyncedData.assets

console.log(`Total assets `, assets.length)
console.log(`Updated entries `, currentSyncData.entries.length)
console.log(`Deleted entries `, currentSyncData.deletedEntries.length)
console.log(`Updated assets `, currentSyncData.assets.length)
console.log(`Deleted assets `, currentSyncData.deletedAssets.length)
console.timeEnd(`fetch Contentful data`)

// update syncToken
lastSyncedData.nextSyncToken = currentSyncData.nextSyncToken
// cache the data

// Update syncToken
const nextSyncToken = currentSyncData.nextSyncToken

// Store our sync state for the next sync.
setPluginStatus({
status: {
lastSyncedData: lastSyncedData,
syncToken: nextSyncToken,
},
})

Expand All @@ -135,17 +130,40 @@ exports.sourceNodes = async (
}
})

// entryList = entryList.map(entries => entries.map(entryItem => {
// entryItem.defaultLocale = defaultLocale
// return new Proxy(entryItem, localeProxyHandler)
// }))
// Build foreign reference map before starting to insert any nodes
// Build foreign reference map before starting to insert any nodes
const foreignReferenceMap = processAPIData.buildForeignReferenceMap({
contentTypeItems,
entryList,
notResolvable,
defaultLocale,
})

// Update existing entry nodes that weren't updated but that need reverse
// links added.
const newOrUpdatedEntries = []
entryList.forEach(entries =>
entries.forEach(entry => newOrUpdatedEntries.push(entry.sys.id))
)
Object.keys(foreignReferenceMap)
existingNodes
.filter(n => _.includes(newOrUpdatedEntries, n.id))
.forEach(n => {
if (foreignReferenceMap[n.id]) {
foreignReferenceMap[n.id].forEach(foreignReference => {
// Add reverse links
if (n[foreignReference.name]) {
n[foreignReference.name].push(foreignReference.id)
// It might already be there so we'll uniquify after pushing.
n[foreignReference.name] = _.uniq(n[foreignReference.name])
} else {
// If is one foreign reference, there can always be many.
// Best to be safe and put it in an array to start with.
n[foreignReference.name] = [foreignReference.id]
}
})
}
})

contentTypeItems.forEach((contentTypeItem, i) => {
processAPIData.createContentTypeNodes({
contentTypeItem,
Expand All @@ -170,22 +188,32 @@ exports.sourceNodes = async (
* The first call will have no aggregated response. Subsequent calls will
* concatenate the new responses to the original one.
*/
function pagedGet (client, method, query = {}, skip = 0, pageLimit = 1000, aggregatedResponse = null) {
return client[method]({
...query,
skip: skip,
limit: pageLimit,
order: `sys.createdAt`,
})
.then((response) => {
if (!aggregatedResponse) {
aggregatedResponse = response
} else {
aggregatedResponse.items = aggregatedResponse.items.concat(response.items)
}
if (skip + pageLimit <= response.total) {
return pagedGet(client, method, skip + pageLimit, aggregatedResponse)
}
return aggregatedResponse
})
function pagedGet(
client,
method,
query = {},
skip = 0,
pageLimit = 1000,
aggregatedResponse = null
) {
return client
[method]({
...query,
skip: skip,
limit: pageLimit,
order: `sys.createdAt`,
})
.then(response => {
if (!aggregatedResponse) {
aggregatedResponse = response
} else {
aggregatedResponse.items = aggregatedResponse.items.concat(
response.items
)
}
if (skip + pageLimit <= response.total) {
return pagedGet(client, method, skip + pageLimit, aggregatedResponse)
}
return aggregatedResponse
})
}
25 changes: 18 additions & 7 deletions packages/gatsby-source-contentful/src/process-api-data.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@ exports.buildForeignReferenceMap = ({
entryList[i].forEach(entryItem => {
const entryItemFields = entryItem.fields
Object.keys(entryItemFields).forEach(entryItemFieldKey => {
let entryItemFieldValue = entryItemFields[entryItemFieldKey][defaultLocale]
console.log(`${entryItemFieldKey} => ${String(entryItemFieldValue)}`)
let entryItemFieldValue =
entryItemFields[entryItemFieldKey][defaultLocale]
if (Array.isArray(entryItemFieldValue)) {
if (
entryItemFieldValue[0].sys &&
Expand Down Expand Up @@ -120,7 +120,8 @@ exports.createContentTypeNodes = ({

// Add linkages to other nodes based on foreign references
Object.keys(entryItemFields).forEach(entryItemFieldKey => {
const entryItemFieldValue = entryItemFields[entryItemFieldKey]
const entryItemFieldValue =
entryItemFields[entryItemFieldKey][defaultLocale]
if (Array.isArray(entryItemFieldValue)) {
if (
entryItemFieldValue[0].sys &&
Expand Down Expand Up @@ -168,10 +169,22 @@ exports.createContentTypeNodes = ({
children: [],
internal: {
type: `${makeTypeName(contentTypeItemId)}`,
mediaType: `application/json`,
mediaType: `application/x-contentful`,
},
}

// Use default locale field.
Object.keys(entryItemFields).forEach(entryItemFieldKey => {
// Ignore fields with "___node" as they're already handled
// and won't be a text field.
if (entryItemFieldKey.split(`___`).length > 1) {
return
}

entryItemFields[entryItemFieldKey] =
entryItemFields[entryItemFieldKey][defaultLocale]
})

// Replace text fields with text nodes so we can process their markdown
// into HTML.
Object.keys(entryItemFields).forEach(entryItemFieldKey => {
Expand All @@ -188,11 +201,10 @@ exports.createContentTypeNodes = ({
: f.id) === entryItemFieldKey
).type
if (fieldType === `Text`) {
console.log(entryItemFields[entryItemFieldKey])
entryItemFields[`${entryItemFieldKey}___NODE`] = createTextNode(
entryNode,
entryItemFieldKey,
entryItemFields[entryItemFieldKey][defaultLocale],
entryItemFields[entryItemFieldKey],
createNode
)

Expand Down Expand Up @@ -242,7 +254,6 @@ exports.createAssetNodes = ({ assetItem, createNode, defaultLocale }) => {
title: assetItem.fields.title[defaultLocale],
description: assetItem.fields.description[defaultLocale],
}
console.log(assetItem)
const assetNode = {
id: assetItem.sys.id,
parent: `__SOURCE__`,
Expand Down
Loading

0 comments on commit c64a932

Please sign in to comment.