feat: Improve metadata job (#1035)

improve metadta job

src/jobs/compute-traits.ts

@@ -4,7 +4,7 @@ import { redisConfig } from './common'
 export type JobData = { collectionId: string }
 
 export const traitsQueue = new Queue<JobData>('traitsQueue', {
-    defaultJobOptions: { delay: 2000, attempts: 2, removeOnComplete: true },
+    defaultJobOptions: { delay: 3000, attempts: 2, removeOnComplete: true },
     redis: redisConfig,
     settings: {
         maxStalledCount: 3,

src/jobs/process-metadata.ts

@@ -4,7 +4,15 @@ import { redisConfig } from './common'
 export type JobData = { resourceId: string; type: 'token' | 'collection'; force: boolean; allTokens: boolean }
 
 export const metadataQueue = new Queue<JobData>('metadataQueue', {
-    defaultJobOptions: { delay: 1000, attempts: 1, removeOnComplete: true },
+    defaultJobOptions: {
+        delay: 100,
+        attempts: 3,
+        backoff: {
+            type: 'exponential',
+            delay: 3000,
+        },
+        removeOnComplete: true,
+    },
     redis: redisConfig,
     settings: {
         maxStalledCount: 2,

src/mappings/util/metadata.ts

@@ -24,8 +24,15 @@ export async function fetchMetadata(url: string, job: Queue.Job) {
         maxRedirects: url.startsWith('https://platform.production.enjinusercontent.com/') ? 3 : 1,
         httpsAgent: new https.Agent({ keepAlive: true, rejectUnauthorized: false }),
     })
+
+    let finalUrl = url.replace('ipfs://', 'https://ipfs.io/ipfs/')
+    if (job.attemptsMade > 1) {
+        finalUrl = url.replace('ipfs://', 'https://cloudflare-ipfs.com/ipfs/')
+        job.log(`Fetching metadata from ${finalUrl} attempt ${job.attemptsMade}`)
+    }
+
     try {
-        const { status, data } = await api.get(url.replace('ipfs://', 'https://ipfs.io/ipfs/'))
+        const { status, data } = await api.get(finalUrl)
         if (status < 400) {
             return data
         }