refactor: Improve mods metadata refresh mechanism

xmcl-electron-app/main/definedPlugins.ts

@@ -7,6 +7,7 @@ import { pluginGameDataPath } from '@xmcl/runtime/app/pluginGameDataPath'
 import { pluginMediaProtocol } from '@xmcl/runtime/base/pluginMediaProtocol'
 import { pluginClientToken } from '@xmcl/runtime/clientToken/pluginClientToken'
 import { pluginCurseforgeClient } from '@xmcl/runtime/curseforge/pluginCurseforgeClient'
+import { pluinModrinthClient } from '@xmcl/runtime/modrinth/pluginModrinthClient'
 import { pluginEncodingWorker } from '@xmcl/runtime/encoding/pluginEncodingWorker'
 import { pluginResourceWorker } from '@xmcl/runtime/resource/pluginResourceWorker'
 import { pluginNativeReplacer } from '@xmcl/runtime/nativeReplacer/pluginNativeReplacer'
@@ -32,7 +33,6 @@ import { pluginYggdrasilHandler } from '@xmcl/runtime/yggdrasilServer/pluginYggd
 import { pluginLaunchPrecheck } from '@xmcl/runtime/launch/pluginLaunchPrecheck'
 import { pluginUncaughtError } from '@xmcl/runtime/uncaughtError/pluginUncaughtError'
 import { elyByPlugin } from '@xmcl/runtime/elyby/elyByPlugin'
-import { pluginInstanceModDiscover } from '@xmcl/runtime/mod/pluginInstanceModDiscover'
 
 import { LauncherAppPlugin } from '~/app'
 import { definedServices } from './definedServices'
@@ -51,7 +51,6 @@ export const definedPlugins: LauncherAppPlugin[] = [
   pluginNvdiaGPULinux,
   pluginUncaughtError,
   pluginNativeReplacer,
-  pluginInstanceModDiscover,
   elyByPlugin,
 
   pluginMediaProtocol,
@@ -63,6 +62,7 @@ export const definedPlugins: LauncherAppPlugin[] = [
   pluginModrinthModpackHandler,
   pluginClientToken,
   pluginCurseforgeClient,
+  pluinModrinthClient,
   pluginServicesHandler(definedServices),
   pluginGameDataPath,
   pluginTelemetry,

xmcl-keystone-ui/src/composables/cache.ts

@@ -1,3 +1,4 @@
+import { MaybeRef } from '@vueuse/core'
 import { Ref, ref, watch } from 'vue'
 
 export interface LocalStorageOptions {
@@ -45,7 +46,7 @@ export function useLocalStorageCacheFloat(key: string, defaultValue: number): Re
   return useLocalStorageCache(key, () => defaultValue, (n) => n.toString(), (s) => Number.parseFloat(s))
 }
 
-export function useLocalStorageCacheInt(key: string, defaultValue: number): Ref<number> {
+export function useLocalStorageCacheInt(key: MaybeRef<string>, defaultValue: number): Ref<number> {
   return useLocalStorageCache(key, () => defaultValue, (n) => n.toString(), (s) => Number.parseInt(s, 10))
 }
 

xmcl-keystone-ui/src/composables/instanceMods.ts

@@ -1,14 +1,52 @@
 import { ModFile, getModFileFromResource } from '@/util/mod'
-import { InstanceModUpdatePayloadAction, InstanceModsServiceKey, InstanceModsState, JavaRecord, PartialResourceHash, Resource, RuntimeVersions, applyUpdateToResource } from '@xmcl/runtime-api'
+import { useEventListener } from '@vueuse/core'
+import { InstanceModUpdatePayloadAction, InstanceModsServiceKey, InstanceModsState, JavaRecord, MutableState, PartialResourceHash, Resource, RuntimeVersions, applyUpdateToResource } from '@xmcl/runtime-api'
+import debounce from 'lodash.debounce'
 import { InjectionKey, Ref, set } from 'vue'
+import { useLocalStorageCache, useLocalStorageCacheInt } from './cache'
 import { useService } from './service'
 import { useState } from './syncableState'
 
 export const kInstanceModsContext: InjectionKey<ReturnType<typeof useInstanceMods>> = Symbol('instance-mods')
 
+function useInstanceModsMetadataRefresh(instancePath: Ref<string>, state: Ref<MutableState<InstanceModsState> | undefined>) {
+  const lastUpdateMetadata = useLocalStorageCache<Record<string, number>>('instanceModsLastRefreshMetadata', () => ({}), JSON.stringify, JSON.parse)
+  const { refreshMetadata } = useService(InstanceModsServiceKey)
+  const expireTime = 1000 * 30 * 60 // 0.5 hour
+
+  async function checkAndUpdate() {
+    const last = lastUpdateMetadata.value[instancePath.value] || 0
+    if ((Date.now() - last) > expireTime) {
+      await update()
+    }
+  }
+
+  async function update() {
+    lastUpdateMetadata.value[instancePath.value] = Date.now()
+    await refreshMetadata(instancePath.value)
+  }
+
+  const debounced = debounce(checkAndUpdate, 1000)
+
+  watch(state, (s) => {
+    if (!s) return
+    s.subscribe('instanceModUpdates', () => {
+      debounced()
+    })
+    checkAndUpdate()
+  }, { immediate: true })
+
+  useEventListener('focus', checkAndUpdate)
+
+  return {
+    checkAndUpdate,
+    update,
+  }
+}
+
 export function useInstanceMods(instancePath: Ref<string>, instanceRuntime: Ref<RuntimeVersions>, java: Ref<JavaRecord | undefined>) {
   const { watch: watchMods } = useService(InstanceModsServiceKey)
-  const { isValidating, error, state } = useState(async () => {
+  const { isValidating, error, state, revalidate } = useState(async () => {
     const inst = instancePath.value
     if (!inst) { return undefined }
     console.time('[watchMods] ' + inst)
@@ -74,15 +112,15 @@ export function useInstanceMods(instancePath: Ref<string>, instanceRuntime: Ref<
       reset()
       return
     }
-    console.log('update instance mods by state')
+    console.log('[instanceMods] update by state')
     updateItems(state.value?.mods, instanceRuntime.value)
   })
   watch(instanceRuntime, () => {
     if (!state.value?.mods) {
       reset()
       return
     }
-    console.log('update instance mods by runtime')
+    console.log('[instanceMods] update by runtime')
     updateItems(state.value?.mods, instanceRuntime.value)
   }, { deep: true })
 
@@ -110,16 +148,15 @@ export function useInstanceMods(instancePath: Ref<string>, instanceRuntime: Ref<
     provideRuntime.value = runtime
   }
 
-  function revalidate() {
-    state.value?.revalidate()
-  }
+  const { update: updateMetadata } = useInstanceModsMetadataRefresh(instancePath, state)
 
   return {
     mods,
     modsIconsMap,
     provideRuntime,
     enabledModCounts,
     isValidating,
+    updateMetadata,
     error,
     revalidate,
   }

xmcl-keystone-ui/src/composables/instanceUpdate.ts

@@ -14,6 +14,7 @@ export type InstanceInstallOptions = {
   type: 'updates'
   updates: InstanceFileUpdate[]
   id: string
+  selectOnlyAdd?: boolean
 }
 
 export const InstanceInstallDialog: DialogKey<InstanceInstallOptions> = 'instance-install'

xmcl-keystone-ui/src/composables/modDependenciesCheck.ts

@@ -16,9 +16,10 @@ import { getSWRV } from '@/util/swrvGet'
 import { getModrinthVersionModel } from './modrinthVersions'
 import { kSWRVConfig } from './swrvConfig'
 import { TaskItem } from '@/entities/task'
+import { filter as fuzzy } from 'fuzzy'
 
 export function useModDependenciesCheck(path: Ref<string>, runtime: Ref<RuntimeVersions>) {
-  const { mods: instanceMods } = injection(kInstanceModsContext)
+  const { mods: instanceMods, updateMetadata } = injection(kInstanceModsContext)
   const updates = ref([] as InstanceFileUpdate[])
   const checked = ref(false)
   const config = inject(kSWRVConfig)
@@ -112,6 +113,9 @@ export function useModDependenciesCheck(path: Ref<string>, runtime: Ref<RuntimeV
   }
 
   const { refresh, refreshing, error } = useRefreshable(async () => {
+    await updateMetadata()
+    await new Promise((resolve) => setTimeout(resolve, 500))
+
     const result: InstanceFileUpdate[] = []
     const mods = instanceMods.value
     const _path = path.value
@@ -122,6 +126,29 @@ export function useModDependenciesCheck(path: Ref<string>, runtime: Ref<RuntimeV
       checkCurseforgeDependencies(mods, runtimes, result),
     ])
 
+    const similarAppend: InstanceFileUpdate[] = []
+    for (const f of result) {
+      similarAppend.push(f)
+      if (f.file.path.startsWith('/mods')) {
+        const fileName = f.file.path.substring(6)
+        const filtered = fuzzy(fileName, mods, {
+          extract: (m) => m.path.substring(6),
+        })
+        const bestMatched = filtered[0]
+        if (bestMatched) {
+          similarAppend.push({
+            operation: 'remove',
+            file: {
+              path: bestMatched.original.path,
+              hashes: {
+                sha1: bestMatched.original.hash,
+              },
+            },
+          })
+        }
+      }
+    }
+
     updates.value = result
     checked.value = true
     operationId = crypto.getRandomValues(new Uint8Array(8)).join('')
@@ -140,6 +167,7 @@ export function useModDependenciesCheck(path: Ref<string>, runtime: Ref<RuntimeV
       type: 'updates',
       updates: updates.value,
       id: operationId,
+      selectOnlyAdd: true,
     })
   }
 

xmcl-keystone-ui/src/composables/syncableState.ts

@@ -44,9 +44,9 @@ export function useState<T extends object>(fetcher: (abortSignal: AbortSignal) =
     }
   }
   watchEffect(mutate)
-  const revalidateCall = () => {
+  const revalidateCall = async () => {
     if (isValidating.value) return
-    state.value?.revalidate()
+    await state.value?.revalidate()
   }
   useEventListener(document, 'visibilitychange', revalidateCall, false)
   useEventListener(window, 'focus', revalidateCall, false)

xmcl-keystone-ui/src/views/HomeInstanceInstallDialog.vue

@@ -312,7 +312,13 @@ const { refresh, refreshing, error } = useRefreshable<InstanceInstallOptions>(as
   }
 
   if (upgrade.value) {
-    selected.value = upgrade.value.files.map(f => f.file.path)
+    if (param.type === 'updates' && param.selectOnlyAdd) {
+      selected.value = upgrade.value.files
+        .filter(f => f.operation === 'add')
+        .map(f => f.file.path)
+    } else {
+      selected.value = upgrade.value.files.map(f => f.file.path)
+    }
   }
 })
 

xmcl-runtime-api/src/services/InstanceModsService.ts

@@ -77,6 +77,10 @@ export interface InstanceModsService {
    * Read all mods under the current instance
    */
   watch(instancePath: string): Promise<MutableState<InstanceModsState>>
+  /**
+   * Refresh the metadata of the instance mods
+   */
+  refreshMetadata(instancePath: string): Promise<void>
   /**
    * Show instance /mods dictionary
    * @param instancePath The instance path

xmcl-runtime/mod/InstanceModsService.ts

@@ -4,13 +4,15 @@ import debounce from 'lodash.debounce'
 import watch from 'node-watch'
 import { dirname, join } from 'path'
 import { Inject, LauncherAppKey } from '~/app'
-import { ResourceService } from '~/resource'
+import { kResourceWorker, ResourceService } from '~/resource'
 import { shouldIgnoreFile } from '~/resource/core/pathUtils'
 import { AbstractService, ExposeServiceKey, ServiceStateManager } from '~/service'
 import { AnyError, isSystemError } from '~/util/error'
 import { LauncherApp } from '../app/LauncherApp'
 import { AggregateExecutor } from '../util/aggregator'
 import { linkWithTimeoutOrCopy, readdirIfPresent } from '../util/fs'
+import { ModrinthV2Client } from '@xmcl/modrinth'
+import { CurseforgeV1Client } from '@xmcl/curseforge'
 
 /**
  * Provide the abilities to import mods and resource packs files to instance
@@ -30,6 +32,90 @@ export class InstanceModsService extends AbstractService implements IInstanceMod
     })
   }
 
+  async refreshMetadata(instancePath: string): Promise<void> {
+    const stateManager = await this.app.registry.get(ServiceStateManager)
+    const state: MutableState<InstanceModsState> | undefined = await stateManager.get(getInstanceModStateKey(instancePath))
+    if (state) {
+      const modrinthClient = await this.app.registry.getOrCreate(ModrinthV2Client)
+      const curseforgeClient = await this.app.registry.getOrCreate(CurseforgeV1Client)
+      const worker = await this.app.registry.getOrCreate(kResourceWorker)
+
+      const onRefreshModrinth = async (all: Resource[]) => {
+        try {
+          const versions = await modrinthClient.getProjectVersionsByHash(all.map(v => v.hash))
+          const options = Object.entries(versions).map(([hash, version]) => {
+            const f = all.find(f => f.hash === hash)
+            if (f) return { hash: f.hash, metadata: { modrinth: { projectId: version.project_id, versionId: version.id } } }
+            return undefined
+          }).filter((v): v is any => !!v)
+          if (options.length > 0) {
+            await this.resourceService.updateResources(options)
+            state.instanceModUpdates([[options, InstanceModUpdatePayloadAction.Update]])
+          }
+        } catch (e) {
+          this.error(e as any)
+        }
+      }
+      const onRefreshCurseforge = async (all: Resource[]) => {
+        try {
+          const chunkSize = 8
+          const allChunks = [] as Resource[][]
+          for (let i = 0; i < all.length; i += chunkSize) {
+            allChunks.push(all.slice(i, i + chunkSize))
+          }
+
+          const allPrints: Record<number, Resource> = {}
+          for (const chunk of allChunks) {
+            const prints = (await Promise.all(chunk.map(async (v) => ({ fingerprint: await worker.fingerprint(v.path), file: v }))))
+            for (const { fingerprint, file } of prints) {
+              if (fingerprint in allPrints) {
+                this.error(new Error(`Duplicated fingerprint ${fingerprint} for ${file.path} and ${allPrints[fingerprint].path}`))
+                continue
+              }
+              allPrints[fingerprint] = file
+            }
+          }
+          const result = await curseforgeClient.getFingerprintsMatchesByGameId(432, Object.keys(allPrints).map(v => parseInt(v, 10)))
+          const options = [] as { hash: string; metadata: { curseforge: { projectId: number; fileId: number } } }[]
+          for (const f of result.exactMatches) {
+            const r = allPrints[f.file.fileFingerprint] || Object.values(allPrints).find(v => v.hash === f.file.hashes.find(a => a.algo === 1)?.value)
+            if (r) {
+              r.metadata.curseforge = { projectId: f.file.modId, fileId: f.file.id }
+              options.push({
+                hash: r.hash,
+                metadata: {
+                  curseforge: { projectId: f.file.modId, fileId: f.file.id },
+                },
+              })
+            }
+          }
+
+          if (options.length > 0) {
+            await this.resourceService.updateResources(options)
+            state.instanceModUpdates([[options, InstanceModUpdatePayloadAction.Update]])
+          }
+        } catch (e) {
+          this.error(e as any)
+        }
+      }
+
+      const refreshCurseforge: Resource[] = []
+      const refreshModrinth: Resource[] = []
+      for (const mod of state.mods.filter(v => !v.metadata.curseforge || !v.metadata.modrinth)) {
+        if (!mod.metadata.curseforge) {
+          refreshCurseforge.push(mod)
+        }
+        if (!mod.metadata.modrinth) {
+          refreshModrinth.push(mod)
+        }
+      }
+      await Promise.allSettled([
+        refreshCurseforge.length > 0 ? onRefreshCurseforge(refreshCurseforge) : undefined,
+        refreshModrinth.length > 0 ? onRefreshModrinth(refreshModrinth) : undefined,
+      ])
+    }
+  }
+
   async showDirectory(path: string): Promise<void> {
     await this.app.shell.openDirectory(join(path, 'mods'))
   }