From 73508ff532d6c59df79d1412bfbb5d681ae889a4 Mon Sep 17 00:00:00 2001 From: Shiba <44804845+DeepDoge@users.noreply.github.com> Date: Sat, 8 Jan 2022 23:32:49 +0000 Subject: [PATCH 1/5] =?UTF-8?q?=F0=9F=8D=A3=20added=20back=20memory=20cach?= =?UTF-8?q?e=20temporarily?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - added back manual memory cache temporarily - and this time its working as expected - also disabled use of browser cache for now - thought we are still saying browser to cache it for the future version --- src/common/yt.ts | 6 +++--- src/scripts/tabOnUpdated.ts | 11 ++++++++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/common/yt.ts b/src/common/yt.ts index 130f5a6..929eea5 100644 --- a/src/common/yt.ts +++ b/src/common/yt.ts @@ -106,7 +106,7 @@ export const ytService = { if (channelId) return { id: channelId, type: 'channel' }; return null; }, - + /** * @param descriptorsWithIndex YT resource IDs to check * @returns a promise with the list of channels that were found on lbry @@ -163,7 +163,7 @@ export const ytService = { if (!descriptor.id) break url.searchParams.set(urlResolverFunction.paramName, descriptor.id) - const apiResponse = await fetch(url.toString(), { cache: 'force-cache' }); + const apiResponse = await fetch(url.toString(), { cache: 'reload' }); if (!apiResponse.ok) break const value = followResponsePath(await apiResponse.json(), urlResolverFunction.responsePath) if (value) results[descriptor.index] = value @@ -184,7 +184,7 @@ export const ytService = { .filter((descriptorId) => descriptorId) .join(urlResolverFunction.paramArraySeperator) ) - const apiResponse = await fetch(url.toString(), { cache: 'force-cache' }); + const apiResponse = await fetch(url.toString(), { cache: 'reload' }); if (!apiResponse.ok) break const values = followResponsePath(await apiResponse.json(), urlResolverFunction.responsePath) values.forEach((value, index) => { diff --git a/src/scripts/tabOnUpdated.ts b/src/scripts/tabOnUpdated.ts index c3b7fde..62e5f53 100644 --- a/src/scripts/tabOnUpdated.ts +++ b/src/scripts/tabOnUpdated.ts @@ -16,20 +16,25 @@ async function resolveYT(descriptor: YtIdResolverDescriptor) { return segments.join('/'); } +const ctxFromURLIdCache: Record> = {} async function ctxFromURL(href: string): Promise { if (!href) return; - + const url = new URL(href); if (!getSourcePlatfromSettingsFromHostname(url.hostname)) return if (!(url.pathname.startsWith('/watch') || url.pathname.startsWith('/channel'))) return - const { redirect, targetPlatform } = await getExtensionSettingsAsync('redirect', 'targetPlatform'); const descriptor = ytService.getId(href); if (!descriptor) return; // couldn't get the ID, so we're done - const res = await resolveYT(descriptor); // NOTE: API call cached by the browser + // NOTE: API call cached by the browser for the future version + // But cache busting is active for now + // Manual memory cache will be removed later + const promise = ctxFromURLIdCache[descriptor.id] ?? (ctxFromURLIdCache[descriptor.id] = resolveYT(descriptor)) + const res = await promise; if (!res) return; // couldn't find it on lbry, so we're done + const { redirect, targetPlatform } = await getExtensionSettingsAsync('redirect', 'targetPlatform'); return { descriptor, lbryPathname: res, redirect, targetPlatform }; } From 685f9615caf70608179040549d98b990fa5f9bda Mon Sep 17 00:00:00 2001 From: Shiba <44804845+DeepDoge@users.noreply.github.com> Date: Sun, 9 Jan 2022 11:54:53 +0000 Subject: [PATCH 2/5] =?UTF-8?q?=F0=9F=8D=B1=20URL=20resolver=20api=20cachi?= =?UTF-8?q?ng=20handled=20by=20the=20client?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Told browser to not to cache the api request - Removed manual memory cache - Implimented caching using indexedDB - Cache time is 1 day --- src/common/yt.ts | 101 ++++++++++++++++++++++++++++++++---- src/scripts/tabOnUpdated.ts | 8 +-- 2 files changed, 94 insertions(+), 15 deletions(-) diff --git a/src/common/yt.ts b/src/common/yt.ts index 929eea5..370231b 100644 --- a/src/common/yt.ts +++ b/src/common/yt.ts @@ -38,6 +38,69 @@ export interface YtIdResolverDescriptor { type: 'channel' | 'video' } +const URLResolverCache = (() => +{ + const openRequest = indexedDB.open("yt-url-resolver-cache") + + if (typeof self.indexedDB !== 'undefined') + { + openRequest.addEventListener('upgradeneeded', () => + { + const db = openRequest.result + const store = db.createObjectStore("store") + store.createIndex("expireAt", "expireAt") + }) + + // Delete Expired + openRequest.addEventListener('success', () => + { + const db = openRequest.result + const transaction = db.transaction("store", "readwrite") + const range = IDBKeyRange.upperBound(new Date()) + + const expireAtCursorRequest = transaction.objectStore("store").index("expireAt").openCursor(range) + expireAtCursorRequest.addEventListener('success', () => + { + const timestampCursor = expireAtCursorRequest.result + if (!timestampCursor) return + console.log("deleting: " + timestampCursor) + timestampCursor.delete() + timestampCursor.continue() + }) + }) + } + else console.warn(`IndexedDB not supported`) + + async function put(url: string, id: string) : Promise + { + return await new Promise((resolve, reject) => + { + const db = openRequest.result + console.log('new put!') + if (!db) return resolve() + console.log('new put') + const store = db.transaction("store", "readwrite").objectStore("store") + const putRequest = store.put({ value: url, expireAt: new Date(Date.now() + 24 * 60 * 60 * 1000) }, id) + putRequest.addEventListener('success', () => resolve()) + putRequest.addEventListener('error', () => reject(putRequest.error)) + }) + } + async function get(id: string): Promise + { + return (await new Promise((resolve, reject) => + { + const db = openRequest.result + if (!db) return resolve(null) + const store = db.transaction("store", "readonly").objectStore("store") + const getRequest = store.get(id) + getRequest.addEventListener('success', () => resolve(getRequest.result)) + getRequest.addEventListener('error', () => reject(getRequest.error)) + }) as any)?.value + } + + return { put, get } +})() + export const ytService = { /** @@ -113,9 +176,24 @@ export const ytService = { */ async resolveById(descriptors: YtIdResolverDescriptor[], progressCallback?: (progress: number) => void): Promise { const descriptorsWithIndex: (YtIdResolverDescriptor & { index: number })[] = descriptors.map((descriptor, index) => ({...descriptor, index})) - - const descriptorsChunks = chunk(descriptorsWithIndex, QUERY_CHUNK_SIZE); + descriptors = null as any const results: string[] = [] + + await Promise.all(descriptorsWithIndex.map(async (descriptor, index) => { + if (!descriptor) return + const cache = await URLResolverCache.get(descriptor.id) + + // Cache can be null, if there is no lbry url yet + if (cache !== undefined) { + // Directly setting it to results + results[index] = cache + + // We remove it so we dont ask it to API + descriptorsWithIndex.splice(index, 1) + } + })) + + const descriptorsChunks = chunk(descriptorsWithIndex, QUERY_CHUNK_SIZE); let progressCount = 0; await Promise.all(descriptorsChunks.map(async (descriptorChunk) => { @@ -154,6 +232,7 @@ export const ytService = { async function requestGroup(urlResolverFunction: YtUrlResolveFunction, descriptorsGroup: typeof descriptorsWithIndex) { url.pathname = urlResolverFunction.pathname + if (urlResolverFunction.paramArraySeperator === SingleValueAtATime) { await Promise.all(descriptorsGroup.map(async (descriptor) => { @@ -162,11 +241,13 @@ export const ytService = { default: if (!descriptor.id) break url.searchParams.set(urlResolverFunction.paramName, descriptor.id) - - const apiResponse = await fetch(url.toString(), { cache: 'reload' }); + + const apiResponse = await fetch(url.toString(), { cache: 'no-store' }); if (!apiResponse.ok) break + const value = followResponsePath(await apiResponse.json(), urlResolverFunction.responsePath) if (value) results[descriptor.index] = value + await URLResolverCache.put(value, descriptor.id) } progressCount++ if (progressCallback) progressCallback(progressCount / descriptorsWithIndex.length) @@ -184,13 +265,15 @@ export const ytService = { .filter((descriptorId) => descriptorId) .join(urlResolverFunction.paramArraySeperator) ) - const apiResponse = await fetch(url.toString(), { cache: 'reload' }); + const apiResponse = await fetch(url.toString(), { cache: 'no-store' }); if (!apiResponse.ok) break const values = followResponsePath(await apiResponse.json(), urlResolverFunction.responsePath) - values.forEach((value, index) => { - const descriptorIndex = descriptorsGroup[index].index - if (value) (results[descriptorIndex] = value) - }) + console.log('hellooo') + await Promise.all(values.map(async (value, index) => { + const descriptor = descriptorsGroup[index] + if (value) results[descriptor.index] = value + await URLResolverCache.put(value, descriptor.id) + })) } progressCount += descriptorsGroup.length if (progressCallback) progressCallback(progressCount / descriptorsWithIndex.length) diff --git a/src/scripts/tabOnUpdated.ts b/src/scripts/tabOnUpdated.ts index 62e5f53..bebcf01 100644 --- a/src/scripts/tabOnUpdated.ts +++ b/src/scripts/tabOnUpdated.ts @@ -16,7 +16,6 @@ async function resolveYT(descriptor: YtIdResolverDescriptor) { return segments.join('/'); } -const ctxFromURLIdCache: Record> = {} async function ctxFromURL(href: string): Promise { if (!href) return; @@ -27,11 +26,8 @@ async function ctxFromURL(href: string): Promise { const descriptor = ytService.getId(href); if (!descriptor) return; // couldn't get the ID, so we're done - // NOTE: API call cached by the browser for the future version - // But cache busting is active for now - // Manual memory cache will be removed later - const promise = ctxFromURLIdCache[descriptor.id] ?? (ctxFromURLIdCache[descriptor.id] = resolveYT(descriptor)) - const res = await promise; + // NOTE: API call cached by resolveYT method automatically + const res = await resolveYT(descriptor); if (!res) return; // couldn't find it on lbry, so we're done const { redirect, targetPlatform } = await getExtensionSettingsAsync('redirect', 'targetPlatform'); From ff83dfc62d816d0e1f11a98ea26470209ba29032 Mon Sep 17 00:00:00 2001 From: Shiba <44804845+DeepDoge@users.noreply.github.com> Date: Sun, 9 Jan 2022 12:20:24 +0000 Subject: [PATCH 3/5] =?UTF-8?q?=F0=9F=8D=B1=20Removed=20logs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/common/yt.ts | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/common/yt.ts b/src/common/yt.ts index 370231b..1f00e64 100644 --- a/src/common/yt.ts +++ b/src/common/yt.ts @@ -61,11 +61,10 @@ const URLResolverCache = (() => const expireAtCursorRequest = transaction.objectStore("store").index("expireAt").openCursor(range) expireAtCursorRequest.addEventListener('success', () => { - const timestampCursor = expireAtCursorRequest.result - if (!timestampCursor) return - console.log("deleting: " + timestampCursor) - timestampCursor.delete() - timestampCursor.continue() + const expireCursor = expireAtCursorRequest.result + if (!expireCursor) return + expireCursor.delete() + expireCursor.continue() }) }) } @@ -76,9 +75,7 @@ const URLResolverCache = (() => return await new Promise((resolve, reject) => { const db = openRequest.result - console.log('new put!') if (!db) return resolve() - console.log('new put') const store = db.transaction("store", "readwrite").objectStore("store") const putRequest = store.put({ value: url, expireAt: new Date(Date.now() + 24 * 60 * 60 * 1000) }, id) putRequest.addEventListener('success', () => resolve()) @@ -265,10 +262,11 @@ export const ytService = { .filter((descriptorId) => descriptorId) .join(urlResolverFunction.paramArraySeperator) ) + const apiResponse = await fetch(url.toString(), { cache: 'no-store' }); if (!apiResponse.ok) break const values = followResponsePath(await apiResponse.json(), urlResolverFunction.responsePath) - console.log('hellooo') + await Promise.all(values.map(async (value, index) => { const descriptor = descriptorsGroup[index] if (value) results[descriptor.index] = value From 475c38ba0c12f9282495d33dd7c198401db7fa60 Mon Sep 17 00:00:00 2001 From: Shiba <44804845+DeepDoge@users.noreply.github.com> Date: Sun, 9 Jan 2022 12:46:18 +0000 Subject: [PATCH 4/5] =?UTF-8?q?=F0=9F=8D=99=20URL=20resolver=20caching=20i?= =?UTF-8?q?mprovements?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Updated types for url because it can also be null - Cached 404 as null --- src/common/yt.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/common/yt.ts b/src/common/yt.ts index 1f00e64..6d255e2 100644 --- a/src/common/yt.ts +++ b/src/common/yt.ts @@ -70,7 +70,7 @@ const URLResolverCache = (() => } else console.warn(`IndexedDB not supported`) - async function put(url: string, id: string) : Promise + async function put(url: string | null, id: string) : Promise { return await new Promise((resolve, reject) => { @@ -82,7 +82,7 @@ const URLResolverCache = (() => putRequest.addEventListener('error', () => reject(putRequest.error)) }) } - async function get(id: string): Promise + async function get(id: string): Promise { return (await new Promise((resolve, reject) => { @@ -171,10 +171,10 @@ export const ytService = { * @param descriptorsWithIndex YT resource IDs to check * @returns a promise with the list of channels that were found on lbry */ - async resolveById(descriptors: YtIdResolverDescriptor[], progressCallback?: (progress: number) => void): Promise { + async resolveById(descriptors: YtIdResolverDescriptor[], progressCallback?: (progress: number) => void): Promise<(string | null)[]> { const descriptorsWithIndex: (YtIdResolverDescriptor & { index: number })[] = descriptors.map((descriptor, index) => ({...descriptor, index})) descriptors = null as any - const results: string[] = [] + const results: (string | null)[] = [] await Promise.all(descriptorsWithIndex.map(async (descriptor, index) => { if (!descriptor) return @@ -240,7 +240,11 @@ export const ytService = { url.searchParams.set(urlResolverFunction.paramName, descriptor.id) const apiResponse = await fetch(url.toString(), { cache: 'no-store' }); - if (!apiResponse.ok) break + if (!apiResponse.ok) { + // Some API might not respond with 200 if it can't find the url + if (apiResponse.status === 404) await URLResolverCache.put(null, descriptor.id) + break + } const value = followResponsePath(await apiResponse.json(), urlResolverFunction.responsePath) if (value) results[descriptor.index] = value From b750c86b880f2154a52c296e4b5430b79fc775f5 Mon Sep 17 00:00:00 2001 From: Shiba <44804845+DeepDoge@users.noreply.github.com> Date: Sun, 9 Jan 2022 13:12:05 +0000 Subject: [PATCH 5/5] =?UTF-8?q?=F0=9F=8D=98=20ctxFromURL=20runs=20once=20f?= =?UTF-8?q?or=20the=20same=20url=20at=20a=20time?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/scripts/tabOnUpdated.ts | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/scripts/tabOnUpdated.ts b/src/scripts/tabOnUpdated.ts index bebcf01..cf67bfc 100644 --- a/src/scripts/tabOnUpdated.ts +++ b/src/scripts/tabOnUpdated.ts @@ -16,6 +16,7 @@ async function resolveYT(descriptor: YtIdResolverDescriptor) { return segments.join('/'); } +const ctxFromURLOnGoingPromise: Record> = {} async function ctxFromURL(href: string): Promise { if (!href) return; @@ -26,12 +27,18 @@ async function ctxFromURL(href: string): Promise { const descriptor = ytService.getId(href); if (!descriptor) return; // couldn't get the ID, so we're done - // NOTE: API call cached by resolveYT method automatically - const res = await resolveYT(descriptor); - if (!res) return; // couldn't find it on lbry, so we're done + // Don't create a new Promise for same ID until on going one is over. + const promise = ctxFromURLOnGoingPromise[descriptor.id] ?? (ctxFromURLOnGoingPromise[descriptor.id] = (async () => { + // NOTE: API call cached by resolveYT method automatically + const res = await resolveYT(descriptor) + if (!res) return // couldn't find it on lbry, so we're done - const { redirect, targetPlatform } = await getExtensionSettingsAsync('redirect', 'targetPlatform'); - return { descriptor, lbryPathname: res, redirect, targetPlatform }; + const { redirect, targetPlatform } = await getExtensionSettingsAsync('redirect', 'targetPlatform') + return { descriptor, lbryPathname: res, redirect, targetPlatform } + })()) + await promise + delete ctxFromURLOnGoingPromise[descriptor.id] + return await promise } // handles lbry.tv -> lbry app redirect