Properly pass loginCookie in all requests

src/Downloader.ts

@@ -34,7 +34,13 @@
 imageminOptions.set('webp', new Map())
 
 imageminOptions.get('default').set('image/png', {
-  plugins: [(imageminPngquant as any)({ speed: 3, strip: true, dithering: 0 }), imageminAdvPng({ optimizationLevel: 4, iterations: 5 })],
+  plugins: [
+    (imageminPngquant as any)({ speed: 3, strip: true, dithering: 0 }),
+    imageminAdvPng({
+      optimizationLevel: 4,
+      iterations: 5,
+    }),
+  ],
 })
 imageminOptions.get('default').set('image/jpeg', {
   plugins: [imageminJpegoptim({ max: 60, stripAll: true })],
@@ -81,6 +87,7 @@
 }
 
 type URLDirector = WikimediaDesktopURLDirector | WikimediaMobileURLDirector | VisualEditorURLDirector | RestApiURLDirector
+
 /**
  * Downloader is a class providing content retrieval functionalities for both Mediawiki and S3 remote instances.
  */
@@ -258,7 +265,13 @@
     return finalProcessedResp
   }
 
-  public async getArticleDetailsNS(ns: number, gapcontinue = ''): Promise<{ gapContinue: string; articleDetails: QueryMwRet }> {
+  public async getArticleDetailsNS(
+    ns: number,
+    gapcontinue = '',
+  ): Promise<{
+    gapContinue: string
+    articleDetails: QueryMwRet
+  }> {
     let queryContinuation: QueryContinueOpts
     let finalProcessedResp: QueryMwRet
     let gCont: string = null
@@ -366,7 +379,13 @@
     })
   }
 
-  public async downloadContent(_url: string, retry = true): Promise<{ content: Buffer | string; responseHeaders: any }> {
+  public async downloadContent(
+    _url: string,
+    retry = true,
+  ): Promise<{
+    content: Buffer | string
+    responseHeaders: any
+  }> {
     if (!_url) {
       throw new Error(`Parameter [${_url}] is not a valid url`)
     }
@@ -616,7 +635,16 @@
     handler(err)
   }
 
-  private async getSubCategories(articleId: string, continueStr = ''): Promise<Array<{ pageid: number; ns: number; title: string }>> {
+  private async getSubCategories(
+    articleId: string,
+    continueStr = '',
+  ): Promise<
+    Array<{
+      pageid: number
+      ns: number
+      title: string
+    }>
+  > {
     const apiUrlDirector = new ApiURLDirector(MediaWiki.actionApiUrl.href)
 
     const { query, continue: cont } = await this.getJSON<any>(apiUrlDirector.buildSubCategoriesURL(articleId, continueStr))
@@ -694,7 +722,11 @@
     jsConfigVars = jsConfigVars.replace('nosuchaction', 'view') // to replace the wgAction config that is set to 'nosuchaction' from api but should be 'view'
 
     // Download mobile page dependencies only once
-    if ((await MediaWiki.hasWikimediaMobileApi()) && this.wikimediaMobileJsDependenciesList.length === 0 && this.wikimediaMobileStyleDependenciesList.length === 0) {
+    if (
+      (await MediaWiki.hasWikimediaMobileApi(this.loginCookie)) &&
+      this.wikimediaMobileJsDependenciesList.length === 0 &&
+      this.wikimediaMobileStyleDependenciesList.length === 0
+    ) {
       try {
         // TODO: An arbitrary title can be placed since all Wikimedia wikis have the same mobile offline resources
         const mobileModulesData = await this.getJSON<any>(`${MediaWiki.mobileModulePath}Test`)

src/MediaWiki.ts

@@ -172,41 +172,41 @@
     this.initializeMediaWikiDefaults()
   }
 
-  public async hasWikimediaDesktopApi(): Promise<boolean> {
+  public async hasWikimediaDesktopApi(loginCookie: string): Promise<boolean> {
     if (this.#hasWikimediaDesktopApi === null) {
       this.wikimediaDesktopUrlDirector = new WikimediaDesktopURLDirector(this.wikimediaDesktopApiUrl.href)
-      this.#hasWikimediaDesktopApi = await checkApiAvailability(this.wikimediaDesktopUrlDirector.buildArticleURL(this.apiCheckArticleId))
+      this.#hasWikimediaDesktopApi = await checkApiAvailability(this.wikimediaDesktopUrlDirector.buildArticleURL(this.apiCheckArticleId), loginCookie)
       return this.#hasWikimediaDesktopApi
     }
     return this.#hasWikimediaDesktopApi
   }
 
-  public async hasWikimediaMobileApi(): Promise<boolean> {
+  public async hasWikimediaMobileApi(loginCookie: string): Promise<boolean> {
     if (this.#hasWikimediaMobileApi === null) {
       this.wikimediaMobileUrlDirector = new WikimediaMobileURLDirector(this.wikimediaMobileApiUrl.href)
-      this.#hasWikimediaMobileApi = await checkApiAvailability(this.wikimediaMobileUrlDirector.buildArticleURL(this.apiCheckArticleId))
+      this.#hasWikimediaMobileApi = await checkApiAvailability(this.wikimediaMobileUrlDirector.buildArticleURL(this.apiCheckArticleId), loginCookie)
       return this.#hasWikimediaMobileApi
     }
     return this.#hasWikimediaMobileApi
   }
 
-  public async hasVisualEditorApi(): Promise<boolean> {
+  public async hasVisualEditorApi(loginCookie: string): Promise<boolean> {
     if (this.#hasVisualEditorApi === null) {
       this.visualEditorUrlDirector = new VisualEditorURLDirector(this.visualEditorApiUrl.href)
       this.#hasVisualEditorApi = await checkApiAvailability(
         this.visualEditorUrlDirector.buildArticleURL(this.apiCheckArticleId),
-        '' /* empty login cookie */,
+        loginCookie,
         this.visualEditorUrlDirector.validMimeTypes,
       )
       return this.#hasVisualEditorApi
     }
     return this.#hasVisualEditorApi
   }
 
-  public async hasRestApi(): Promise<boolean> {
+  public async hasRestApi(loginCookie: string): Promise<boolean> {
     if (this.#hasRestApi === null) {
       this.restApiUrlDirector = new RestApiURLDirector(this.restApiUrl.href)
-      this.#hasRestApi = await checkApiAvailability(this.restApiUrlDirector.buildArticleURL(this.apiCheckArticleId))
+      this.#hasRestApi = await checkApiAvailability(this.restApiUrlDirector.buildArticleURL(this.apiCheckArticleId), loginCookie)
       return this.#hasRestApi
     }
     return this.#hasRestApi
@@ -289,6 +289,7 @@
           }
 
           downloader.loginCookie = resp.headers['set-cookie'].join(';')
+          downloader.jsonRequestOptions.headers.cookie = downloader.loginCookie
         })
         .catch((err) => {
           throw err

src/mwoffliner.lib.ts

@@ -212,10 +212,10 @@ async function execute(argv: any) {
 
   MediaWiki.apiCheckArticleId = mwMetaData.mainPage
   await MediaWiki.hasCoordinates(downloader)
-  await MediaWiki.hasWikimediaDesktopApi()
-  const hasWikimediaMobileApi = await MediaWiki.hasWikimediaMobileApi()
-  await MediaWiki.hasRestApi()
-  await MediaWiki.hasVisualEditorApi()
+  await MediaWiki.hasWikimediaDesktopApi(downloader.loginCookie)
+  const hasWikimediaMobileApi = await MediaWiki.hasWikimediaMobileApi(downloader.loginCookie)
+  await MediaWiki.hasRestApi(downloader.loginCookie)
+  await MediaWiki.hasVisualEditorApi(downloader.loginCookie)
 
   RedisStore.setOptions(argv.redis || config.defaults.redisPath)
   await RedisStore.connect()

src/renderers/renderer.builder.ts

@@ -8,14 +8,14 @@ import { RendererBuilderOptions } from './abstract.renderer.js'
 import * as logger from './../Logger.js'
 
 export class RendererBuilder {
-  public async createRenderer(options: RendererBuilderOptions): Promise<Renderer> {
+  public async createRenderer(options: RendererBuilderOptions, loginCookie: string): Promise<Renderer> {
     const { renderType, renderName } = options
 
     const [hasVisualEditorApi, hasWikimediaDesktopApi, hasWikimediaMobileApi, hasRestApi] = await Promise.all([
-      MediaWiki.hasVisualEditorApi(),
-      MediaWiki.hasWikimediaDesktopApi(),
-      MediaWiki.hasWikimediaMobileApi(),
-      MediaWiki.hasRestApi(),
+      MediaWiki.hasVisualEditorApi(loginCookie),
+      MediaWiki.hasWikimediaDesktopApi(loginCookie),
+      MediaWiki.hasWikimediaMobileApi(loginCookie),
+      MediaWiki.hasRestApi(loginCookie),
     ])
 
     switch (renderType) {

src/util/saveArticles.ts

@@ -54,7 +54,12 @@
       } finally {
         if (isFailed) {
           if (doRetry && resp.status !== 404) {
-            await retryStore.set(resp.path, { url: resp.url, namespace: resp.namespace, mult: resp.mult, width: resp.width })
+            await retryStore.set(resp.path, {
+              url: resp.url,
+              namespace: resp.namespace,
+              mult: resp.mult,
+              width: resp.width,
+            })
           } else {
             logger.warn(`Error downloading file [${urlHelper.deserializeUrl(resp.url)}], skipping`)
             dump.status.files.fail += 1
@@ -242,17 +247,23 @@
   let articlesRenderer
   if (forceRender) {
     // All articles and main page will use the same renderer if 'forceRender' is specified
-    const renderer = await rendererBuilder.createRenderer({
-      renderType: 'specific',
-      renderName: forceRender,
-    })
+    const renderer = await rendererBuilder.createRenderer(
+      {
+        renderType: 'specific',
+        renderName: forceRender,
+      },
+      downloader.loginCookie,
+    )
     mainPageRenderer = renderer
     articlesRenderer = renderer
   } else {
-    mainPageRenderer = await rendererBuilder.createRenderer({ renderType: 'desktop' })
-    articlesRenderer = await rendererBuilder.createRenderer({
-      renderType: hasWikimediaMobileApi ? 'mobile' : 'auto',
-    })
+    mainPageRenderer = await rendererBuilder.createRenderer({ renderType: 'desktop' }, downloader.loginCookie)
+    articlesRenderer = await rendererBuilder.createRenderer(
+      {
+        renderType: hasWikimediaMobileApi ? 'mobile' : 'auto',
+      },
+      downloader.loginCookie,
+    )
   }
   downloader.setUrlsDirectors(mainPageRenderer, articlesRenderer)
 
@@ -390,7 +401,11 @@
   logger.log(`Done with downloading a total of [${articlesTotal}] articles`)
 
   if (jsConfigVars) {
-    const jsConfigVarArticle = new ZimArticle({ url: jsPath('jsConfigVars', config.output.dirs.mediawiki), data: jsConfigVars, ns: '-' })
+    const jsConfigVarArticle = new ZimArticle({
+      url: jsPath('jsConfigVars', config.output.dirs.mediawiki),
+      data: jsConfigVars,
+      ns: '-',
+    })
     zimCreator.addArticle(jsConfigVarArticle)
   }
 

test/unit/downloader.test.ts

@@ -34,10 +34,10 @@ describe('Downloader class', () => {
 
     await MediaWiki.getMwMetaData(downloader)
     await MediaWiki.hasCoordinates(downloader)
-    await MediaWiki.hasWikimediaDesktopApi()
-    await MediaWiki.hasWikimediaMobileApi()
-    await MediaWiki.hasRestApi()
-    await MediaWiki.hasVisualEditorApi()
+    await MediaWiki.hasWikimediaDesktopApi(downloader.loginCookie)
+    await MediaWiki.hasWikimediaMobileApi(downloader.loginCookie)
+    await MediaWiki.hasRestApi(downloader.loginCookie)
+    await MediaWiki.hasVisualEditorApi(downloader.loginCookie)
   })
 
   test('Test Action API version 2 response in comparison with version 1', async () => {

test/unit/mwApi.test.ts

@@ -18,9 +18,9 @@ afterAll(stopRedis)
 const initMW = async (downloader: Downloader) => {
   await MediaWiki.getMwMetaData(downloader)
   await MediaWiki.hasCoordinates(downloader)
-  await MediaWiki.hasWikimediaDesktopApi()
-  await MediaWiki.hasRestApi()
-  await MediaWiki.hasVisualEditorApi()
+  await MediaWiki.hasWikimediaDesktopApi(downloader.loginCookie)
+  await MediaWiki.hasRestApi(downloader.loginCookie)
+  await MediaWiki.hasVisualEditorApi(downloader.loginCookie)
 
   await MediaWiki.getNamespaces([], downloader)
 }

test/unit/mwApiCapabilities.test.ts

@@ -14,75 +14,75 @@ describe('Checking Mediawiki capabilities', () => {
 
   test('test capabilities of en.wikipedia.org', async () => {
     MediaWiki.base = 'https://en.wikipedia.org'
-    expect(await MediaWiki.hasWikimediaDesktopApi()).toBe(true)
-    expect(await MediaWiki.hasWikimediaMobileApi()).toBe(true)
-    expect(await MediaWiki.hasRestApi()).toBe(true)
-    expect(await MediaWiki.hasVisualEditorApi()).toBe(true)
+    expect(await MediaWiki.hasWikimediaDesktopApi('')).toBe(true)
+    expect(await MediaWiki.hasWikimediaMobileApi('')).toBe(true)
+    expect(await MediaWiki.hasRestApi('')).toBe(true)
+    expect(await MediaWiki.hasVisualEditorApi('')).toBe(true)
   })
 
   test('test capabilities of wiki.openstreetmap.org', async () => {
     MediaWiki.base = 'https://wiki.openstreetmap.org'
 
-    expect(await MediaWiki.hasWikimediaDesktopApi()).toBe(false)
-    expect(await MediaWiki.hasWikimediaMobileApi()).toBe(false)
-    expect(await MediaWiki.hasRestApi()).toBe(true)
-    expect(await MediaWiki.hasVisualEditorApi()).toBe(true)
+    expect(await MediaWiki.hasWikimediaDesktopApi('')).toBe(false)
+    expect(await MediaWiki.hasWikimediaMobileApi('')).toBe(false)
+    expect(await MediaWiki.hasRestApi('')).toBe(true)
+    expect(await MediaWiki.hasVisualEditorApi('')).toBe(true)
   })
 
   test('test capabilities of fo.wikisource.org', async () => {
     MediaWiki.base = 'https://fo.wikisource.org'
 
-    expect(await MediaWiki.hasWikimediaDesktopApi()).toBe(true)
-    expect(await MediaWiki.hasWikimediaMobileApi()).toBe(false)
-    expect(await MediaWiki.hasRestApi()).toBe(true)
-    expect(await MediaWiki.hasVisualEditorApi()).toBe(true)
+    expect(await MediaWiki.hasWikimediaDesktopApi('')).toBe(true)
+    expect(await MediaWiki.hasWikimediaMobileApi('')).toBe(false)
+    expect(await MediaWiki.hasRestApi('')).toBe(true)
+    expect(await MediaWiki.hasVisualEditorApi('')).toBe(true)
   })
 
   test('test capabilities of minecraft.wiki with correct VisualEditor receipt', async () => {
     MediaWiki.base = 'https://minecraft.wiki'
     MediaWiki.wikiPath = '/'
     MediaWiki.actionApiPath = 'api.php'
 
-    expect(await MediaWiki.hasWikimediaDesktopApi()).toBe(false)
-    expect(await MediaWiki.hasWikimediaMobileApi()).toBe(false)
-    expect(await MediaWiki.hasRestApi()).toBe(false)
-    expect(await MediaWiki.hasVisualEditorApi()).toBe(true)
+    expect(await MediaWiki.hasWikimediaDesktopApi('')).toBe(false)
+    expect(await MediaWiki.hasWikimediaMobileApi('')).toBe(false)
+    expect(await MediaWiki.hasRestApi('')).toBe(false)
+    expect(await MediaWiki.hasVisualEditorApi('')).toBe(true)
   })
 
   test('test capabilities of pokemon.fandom.com with correct VisualEditor receipt', async () => {
     MediaWiki.base = 'https://pokemon.fandom.com/'
     MediaWiki.wikiPath = '/'
     MediaWiki.actionApiPath = 'api.php'
 
-    expect(await MediaWiki.hasWikimediaDesktopApi()).toBe(false)
-    expect(await MediaWiki.hasWikimediaMobileApi()).toBe(false)
-    expect(await MediaWiki.hasRestApi()).toBe(false)
-    expect(await MediaWiki.hasVisualEditorApi()).toBe(true)
+    expect(await MediaWiki.hasWikimediaDesktopApi('')).toBe(false)
+    expect(await MediaWiki.hasWikimediaMobileApi('')).toBe(false)
+    expect(await MediaWiki.hasRestApi('')).toBe(false)
+    expect(await MediaWiki.hasVisualEditorApi('')).toBe(true)
   })
 
   test('test capabilities of pokemon.fandom.com with default receipt', async () => {
     MediaWiki.base = 'https://pokemon.fandom.com/'
 
-    expect(await MediaWiki.hasWikimediaDesktopApi()).toBe(false)
-    expect(await MediaWiki.hasWikimediaMobileApi()).toBe(false)
-    expect(await MediaWiki.hasRestApi()).toBe(false)
-    expect(await MediaWiki.hasVisualEditorApi()).toBe(false)
+    expect(await MediaWiki.hasWikimediaDesktopApi('')).toBe(false)
+    expect(await MediaWiki.hasWikimediaMobileApi('')).toBe(false)
+    expect(await MediaWiki.hasRestApi('')).toBe(false)
+    expect(await MediaWiki.hasVisualEditorApi('')).toBe(false)
   })
 
   test('test capabilities of pokemon.fandom.com with RestApi receipt', async () => {
     MediaWiki.base = 'https://pokemon.fandom.com/'
     MediaWiki.wikiPath = '/'
     MediaWiki.restApiPath = 'rest.php'
 
-    expect(await MediaWiki.hasWikimediaDesktopApi()).toBe(false)
-    expect(await MediaWiki.hasWikimediaMobileApi()).toBe(false)
-    expect(await MediaWiki.hasVisualEditorApi()).toBe(false)
+    expect(await MediaWiki.hasWikimediaDesktopApi('')).toBe(false)
+    expect(await MediaWiki.hasWikimediaMobileApi('')).toBe(false)
+    expect(await MediaWiki.hasVisualEditorApi('')).toBe(false)
 
     /* TODO:
       Title MediaWiki:Sidebar does not exist for Mediawiki Rest Api in pokemon.fandom.com for some reason. This will lead to incorrect capability check
       See: https://pokemon.fandom.com/rest.php/v1/page/MediaWiki%3ASidebar/html
     */
     MediaWiki.apiCheckArticleId = 'Volcarona'
-    expect(await MediaWiki.hasRestApi()).toBe(true)
+    expect(await MediaWiki.hasRestApi('')).toBe(true)
   })
 })