diff --git a/Sources/Brave/Frontend/Browser/BrowserViewController.swift b/Sources/Brave/Frontend/Browser/BrowserViewController.swift index 1b518507a39..529eef617c4 100644 --- a/Sources/Brave/Frontend/Browser/BrowserViewController.swift +++ b/Sources/Brave/Frontend/Browser/BrowserViewController.swift @@ -420,7 +420,7 @@ public class BrowserViewController: UIViewController { ScriptFactory.shared.clearCaches() Task { - await AdBlockStats.shared.clearCaches() + await AdBlockStats.shared.didReceiveMemoryWarning() } for tab in tabManager.tabsForCurrentMode where tab.id != tabManager.selectedTab?.id { diff --git a/Sources/Brave/Frontend/Settings/Features/ShieldsPrivacy/FilterLists/FilterListsView.swift b/Sources/Brave/Frontend/Settings/Features/ShieldsPrivacy/FilterLists/FilterListsView.swift index 42fa9616162..5ca097d31cd 100644 --- a/Sources/Brave/Frontend/Settings/Features/ShieldsPrivacy/FilterLists/FilterListsView.swift +++ b/Sources/Brave/Frontend/Settings/Features/ShieldsPrivacy/FilterLists/FilterListsView.swift @@ -16,53 +16,17 @@ struct FilterListsView: View { @ObservedObject private var customFilterListStorage = CustomFilterListStorage.shared @Environment(\.editMode) private var editMode @State private var showingAddSheet = false + @State private var expectedEnabledSources: Set = Set(AdBlockStats.shared.enabledSources) private let dateFormatter = RelativeDateTimeFormatter() + private var reachedMaxLimit: Bool { + expectedEnabledSources.count >= AdBlockStats.maxNumberOfAllowedFilterLists + } + var body: some View { List { Section { - ForEach($customFilterListStorage.filterListsURLs) { $filterListURL in - VStack(alignment: .leading, spacing: 4) { - Toggle(isOn: $filterListURL.setting.isEnabled) { - VStack(alignment: .leading, spacing: 4) { - Text(filterListURL.title) - .foregroundColor(Color(.bravePrimary)) - .truncationMode(.middle) - .lineLimit(1) - - switch filterListURL.downloadStatus { - case .downloaded(let downloadDate): - Text(String.localizedStringWithFormat( - Strings.filterListsLastUpdated, - dateFormatter.localizedString(for: downloadDate, relativeTo: Date()))) - .font(.caption) - .foregroundColor(Color(.braveLabel)) - case .failure: - Text(Strings.filterListsDownloadFailed) - .font(.caption) - .foregroundColor(.red) - case .pending: - Text(Strings.filterListsDownloadPending) - .font(.caption) - .foregroundColor(Color(.braveLabel)) - } - } - } - .disabled(editMode?.wrappedValue.isEditing == true) - .toggleStyle(SwitchToggleStyle(tint: .accentColor)) - .onChange(of: filterListURL.setting.isEnabled) { value in - Task { - CustomFilterListSetting.save(inMemory: !customFilterListStorage.persistChanges) - } - } - - Text(filterListURL.setting.externalURL.absoluteDisplayString) - .font(.caption) - .foregroundColor(Color(.secondaryBraveLabel)) - .allowsTightening(true) - }.listRowBackground(Color(.secondaryBraveGroupedBackground)) - } - .onDelete(perform: onDeleteHandling) + customFilterListView Button { showingAddSheet = true @@ -71,27 +35,16 @@ struct FilterListsView: View { .foregroundColor(Color(.braveBlurpleTint)) } .disabled(editMode?.wrappedValue.isEditing == true) - .listRowBackground(Color(.secondaryBraveGroupedBackground)) .popover(isPresented: $showingAddSheet, content: { FilterListAddURLView() }) } header: { Text(Strings.customFilterLists) } + .toggleStyle(SwitchToggleStyle(tint: .accentColor)) Section { - ForEach($filterListStorage.filterLists) { $filterList in - Toggle(isOn: $filterList.isEnabled) { - VStack(alignment: .leading) { - Text(filterList.entry.title) - .foregroundColor(Color(.bravePrimary)) - Text(filterList.entry.desc) - .font(.caption) - .foregroundColor(Color(.secondaryBraveLabel)) - } - }.toggleStyle(SwitchToggleStyle(tint: .accentColor)) - .listRowBackground(Color(.secondaryBraveGroupedBackground)) - } + filterListView } header: { VStack(alignment: .leading, spacing: 4) { Text(Strings.defaultFilterLists) @@ -101,6 +54,8 @@ struct FilterListsView: View { } } } + .toggleStyle(SwitchToggleStyle(tint: .accentColor)) + .listRowBackground(Color(.secondaryBraveGroupedBackground)) .animation(.default, value: customFilterListStorage.filterListsURLs) .listBackgroundColor(Color(UIColor.braveGroupedBackground)) .listStyle(.insetGrouped) @@ -111,6 +66,84 @@ struct FilterListsView: View { editMode?.wrappedValue.isEditing == false ) } + .onDisappear { + Task.detached { + await AdBlockStats.shared.removeDisabledEngines() + await AdBlockStats.shared.ensureEnabledEngines() + } + } + } + + @ViewBuilder private var filterListView: some View { + ForEach($filterListStorage.filterLists) { $filterList in + Toggle(isOn: $filterList.isEnabled) { + VStack(alignment: .leading) { + Text(filterList.entry.title) + .foregroundColor(Color(.bravePrimary)) + Text(filterList.entry.desc) + .font(.caption) + .foregroundColor(Color(.secondaryBraveLabel)) + } + } + .disabled(!filterList.isEnabled && reachedMaxLimit) + .onChange(of: filterList.isEnabled) { isEnabled in + if isEnabled { + expectedEnabledSources.insert(filterList.engineSource) + } else { + expectedEnabledSources.remove(filterList.engineSource) + } + } + } + } + + @ViewBuilder private var customFilterListView: some View { + ForEach($customFilterListStorage.filterListsURLs) { $filterListURL in + VStack(alignment: .leading, spacing: 4) { + Toggle(isOn: $filterListURL.setting.isEnabled) { + VStack(alignment: .leading, spacing: 4) { + Text(filterListURL.title) + .foregroundColor(Color(.bravePrimary)) + .truncationMode(.middle) + .lineLimit(1) + + switch filterListURL.downloadStatus { + case .downloaded(let downloadDate): + Text(String.localizedStringWithFormat( + Strings.filterListsLastUpdated, + dateFormatter.localizedString(for: downloadDate, relativeTo: Date()))) + .font(.caption) + .foregroundColor(Color(.braveLabel)) + case .failure: + Text(Strings.filterListsDownloadFailed) + .font(.caption) + .foregroundColor(.red) + case .pending: + Text(Strings.filterListsDownloadPending) + .font(.caption) + .foregroundColor(Color(.braveLabel)) + } + } + } + .disabled(reachedMaxLimit && !filterListURL.setting.isEnabled) + .onChange(of: filterListURL.setting.isEnabled) { isEnabled in + if isEnabled { + expectedEnabledSources.insert(filterListURL.setting.engineSource) + } else { + expectedEnabledSources.remove(filterListURL.setting.engineSource) + } + + Task { + CustomFilterListSetting.save(inMemory: !customFilterListStorage.persistChanges) + } + } + + Text(filterListURL.setting.externalURL.absoluteDisplayString) + .font(.caption) + .foregroundColor(Color(.secondaryBraveLabel)) + .allowsTightening(true) + } + } + .onDelete(perform: onDeleteHandling) } private func onDeleteHandling(offsets: IndexSet) { diff --git a/Sources/Brave/WebFilters/FilterListCustomURLDownloader.swift b/Sources/Brave/WebFilters/FilterListCustomURLDownloader.swift index bb3f3905a28..f96cd767614 100644 --- a/Sources/Brave/WebFilters/FilterListCustomURLDownloader.swift +++ b/Sources/Brave/WebFilters/FilterListCustomURLDownloader.swift @@ -58,22 +58,29 @@ actor FilterListCustomURLDownloader: ObservableObject { self.startedService = true await CustomFilterListStorage.shared.loadCachedFilterLists() - await CustomFilterListStorage.shared.filterListsURLs.asyncConcurrentForEach { customURL in - let resource = await customURL.setting.resource - - do { - if let cachedResult = try resource.cachedResult() { - await self.handle(downloadResult: cachedResult, for: customURL) + do { + try await CustomFilterListStorage.shared.filterListsURLs.asyncConcurrentForEach { customURL in + let resource = await customURL.setting.resource + + do { + if let cachedResult = try resource.cachedResult() { + await self.handle(downloadResult: cachedResult, for: customURL) + } + } catch { + let uuid = await customURL.setting.uuid + ContentBlockerManager.log.error( + "Failed to cached data for custom filter list `\(uuid)`: \(error)" + ) } - } catch { - let uuid = await customURL.setting.uuid - ContentBlockerManager.log.error( - "Failed to cached data for custom filter list `\(uuid)`: \(error)" - ) + + // Always fetch this resource so it's ready if the user enables it. + await self.startFetching(filterListCustomURL: customURL) + + // Sleep for 1ms. This drastically reduces memory usage without much impact to usability + try await Task.sleep(nanoseconds: 1000000) } - - // Always fetch this resource so it's ready if the user enables it. - await self.startFetching(filterListCustomURL: customURL) + } catch { + // Ignore cancellation errors } } @@ -105,32 +112,33 @@ actor FilterListCustomURLDownloader: ObservableObject { } // Add/remove the resource depending on if it is enabled/disabled - if await filterListCustomURL.setting.isEnabled { - guard let resourcesInfo = await FilterListResourceDownloader.shared.resourcesInfo else { - assertionFailure("This should not have been called if the resources are not ready") - return - } - - let version = fileVersionDateFormatter.string(from: downloadResult.date) - let filterListInfo = CachedAdBlockEngine.FilterListInfo( - source: .filterListURL(uuid: uuid), - localFileURL: downloadResult.fileURL, - version: version, fileType: .text + let source = CachedAdBlockEngine.Source.filterListURL(uuid: uuid) + guard let resourcesInfo = await FilterListResourceDownloader.shared.resourcesInfo else { + assertionFailure("This should not have been called if the resources are not ready") + return + } + + let version = fileVersionDateFormatter.string(from: downloadResult.date) + let filterListInfo = CachedAdBlockEngine.FilterListInfo( + source: .filterListURL(uuid: uuid), + localFileURL: downloadResult.fileURL, + version: version, fileType: .text + ) + + guard await AdBlockStats.shared.isEagerlyLoaded(source: source) else { + // Don't compile unless eager + await AdBlockStats.shared.updateIfNeeded(resourcesInfo: resourcesInfo) + await AdBlockStats.shared.updateIfNeeded(filterListInfo: filterListInfo, isAlwaysAggressive: true) + return + } + + do { + try await AdBlockStats.shared.compileDelayed( + filterListInfo: filterListInfo, resourcesInfo: resourcesInfo, + isAlwaysAggressive: true, delayed: true ) - - do { - let engine = try await CachedAdBlockEngine.compile( - filterListInfo: filterListInfo, resourcesInfo: resourcesInfo, - isAlwaysAggressive: true - ) - - await AdBlockStats.shared.add(engine: engine) - ContentBlockerManager.log.debug("Compiled engine for custom filter list `\(uuid)` v\(version)") - } catch { - ContentBlockerManager.log.error("Failed to compile engine for custom filter list `\(uuid)` v\(version): \(String(describing: error))") - } - } else { - await AdBlockStats.shared.removeEngine(for: .filterListURL(uuid: uuid)) + } catch { + // Don't handle cancellation errors } } diff --git a/Sources/Brave/WebFilters/FilterListResourceDownloader.swift b/Sources/Brave/WebFilters/FilterListResourceDownloader.swift index 28eb44959dd..e8ec2fcd5d5 100644 --- a/Sources/Brave/WebFilters/FilterListResourceDownloader.swift +++ b/Sources/Brave/WebFilters/FilterListResourceDownloader.swift @@ -72,20 +72,28 @@ public actor FilterListResourceDownloader { await FilterListStorage.shared.loadFilterListSettings() let filterListSettings = await FilterListStorage.shared.allFilterListSettings - await filterListSettings.asyncConcurrentForEach { setting in - guard await setting.isEnabled == true else { return } - guard let componentId = await setting.componentId else { return } - - // Try to load the filter list folder. We always have to compile this at start - guard let folderURL = await setting.folderURL, FileManager.default.fileExists(atPath: folderURL.path) else { - return + do { + try await filterListSettings.asyncConcurrentForEach { setting in + guard await setting.isEnabled == true else { return } + guard let componentId = await setting.componentId else { return } + guard FilterList.disabledComponentIDs.contains(componentId) else { return } + + // Try to load the filter list folder. We always have to compile this at start + guard let folderURL = await setting.folderURL, FileManager.default.fileExists(atPath: folderURL.path) else { + return + } + + await self.compileFilterListEngineIfNeeded( + fromComponentId: componentId, folderURL: folderURL, + isAlwaysAggressive: setting.isAlwaysAggressive, + resourcesInfo: resourcesInfo + ) + + // Sleep for 1ms. This drastically reduces memory usage without much impact to usability + try await Task.sleep(nanoseconds: 1000000) } - - await self.compileFilterListEngineIfNeeded( - fromComponentId: componentId, folderURL: folderURL, - isAlwaysAggressive: setting.isAlwaysAggressive, - resourcesInfo: resourcesInfo - ) + } catch { + // Ignore the cancellation. } } @@ -154,17 +162,10 @@ public actor FilterListResourceDownloader { return } - do { - let engine = try await CachedAdBlockEngine.compile( - filterListInfo: filterListInfo, resourcesInfo: resourcesInfo, - isAlwaysAggressive: false - ) - - ContentBlockerManager.log.debug("Compiled default engine v\(version)") - await AdBlockStats.shared.add(engine: engine) - } catch { - ContentBlockerManager.log.debug("Failed to compile default engine v\(version): \(String(describing: error))") - } + await AdBlockStats.shared.compile( + filterListInfo: filterListInfo, resourcesInfo: resourcesInfo, + isAlwaysAggressive: false + ) } /// Load general filter lists (shields) from the given `AdblockService` `shieldsInstallPath` `URL`. @@ -174,26 +175,34 @@ public actor FilterListResourceDownloader { resourcesInfo: CachedAdBlockEngine.ResourcesInfo ) async { let version = folderURL.lastPathComponent + let source = CachedAdBlockEngine.Source.filterList(componentId: componentId) let filterListInfo = CachedAdBlockEngine.FilterListInfo( - source: .filterList(componentId: componentId), + source: source, localFileURL: folderURL.appendingPathComponent("list.txt", conformingTo: .text), version: version, fileType: .text ) + guard await AdBlockStats.shared.isEagerlyLoaded(source: source) else { + // Don't compile unless eager + await AdBlockStats.shared.updateIfNeeded(resourcesInfo: resourcesInfo) + await AdBlockStats.shared.updateIfNeeded(filterListInfo: filterListInfo, isAlwaysAggressive: isAlwaysAggressive) + return + } + guard await AdBlockStats.shared.needsCompilation(for: filterListInfo, resourcesInfo: resourcesInfo) else { // Don't compile unless needed return } + let isImportant = await AdBlockStats.shared.criticalSources.contains(source) + do { - let engine = try await CachedAdBlockEngine.compile( + try await AdBlockStats.shared.compileDelayed( filterListInfo: filterListInfo, resourcesInfo: resourcesInfo, - isAlwaysAggressive: isAlwaysAggressive + isAlwaysAggressive: isAlwaysAggressive, delayed: !isImportant ) - ContentBlockerManager.log.debug("Compiled engine for `\(componentId)` v\(version)") - await AdBlockStats.shared.add(engine: engine) } catch { - ContentBlockerManager.log.error("Failed to compile engine for `\(componentId)` v\(version): \(String(describing: error))") + // Don't handle cancellation errors } } diff --git a/Sources/Brave/WebFilters/ShieldStats/Adblock/AdBlockEngine+Extensions.swift b/Sources/Brave/WebFilters/ShieldStats/Adblock/AdBlockEngine+Extensions.swift index 00ed3b8f465..2ad3cf11cba 100644 --- a/Sources/Brave/WebFilters/ShieldStats/Adblock/AdBlockEngine+Extensions.swift +++ b/Sources/Brave/WebFilters/ShieldStats/Adblock/AdBlockEngine+Extensions.swift @@ -14,26 +14,14 @@ extension AdblockEngine { } convenience init(textFileURL fileURL: URL, resourcesFileURL: URL) throws { - guard let data = FileManager.default.contents(atPath: fileURL.path) else { - throw CompileError.fileNotFound - } - - guard let rules = String(data: data, encoding: .utf8) else { - throw CompileError.fileNotFound - } - - try self.init(rules: rules) + try self.init(rules: String(contentsOf: fileURL)) try useResources(fromFileURL: resourcesFileURL) } convenience init(datFileURL fileURL: URL, resourcesFileURL: URL) throws { - guard let data = FileManager.default.contents(atPath: fileURL.path) else { - throw CompileError.fileNotFound - } - self.init() - if !deserialize(data: data) { + if try !deserialize(data: Data(contentsOf: fileURL)) { throw CompileError.couldNotDeserializeDATFile } @@ -62,8 +50,7 @@ extension AdblockEngine { private func useResources(fromFileURL fileURL: URL) throws { // Add scriplets if available - if let data = FileManager.default.contents(atPath: fileURL.path), - let json = try Self.validateJSON(data) { + if let json = try Self.validateJSON(Data(contentsOf: fileURL)) { useResources(json) } } diff --git a/Sources/Brave/WebFilters/ShieldStats/Adblock/AdBlockStats.swift b/Sources/Brave/WebFilters/ShieldStats/Adblock/AdBlockStats.swift index 29966a1360c..648af310393 100644 --- a/Sources/Brave/WebFilters/ShieldStats/Adblock/AdBlockStats.swift +++ b/Sources/Brave/WebFilters/ShieldStats/Adblock/AdBlockStats.swift @@ -11,51 +11,212 @@ import os.log /// This object holds on to our adblock engines and returns information needed for stats tracking as well as some conveniences /// for injected scripts needed during web navigation and cosmetic filters models needed by the `SelectorsPollerScript.js` script. public actor AdBlockStats { + static let maxNumberOfAllowedFilterLists = 30 typealias CosmeticFilterModelTuple = (isAlwaysAggressive: Bool, model: CosmeticFilterModel) public static let shared = AdBlockStats() + + /// An object containing the basic information to allow us to compile an engine + struct LazyFilterListInfo { + let filterListInfo: CachedAdBlockEngine.FilterListInfo + let isAlwaysAggressive: Bool + } - // Adblock engine for general adblock lists. - private var cachedEngines: [CachedAdBlockEngine] + /// A list of filter list info that are available for compilation. This information is used for lazy loading. + private(set) var availableFilterLists: [CachedAdBlockEngine.Source: LazyFilterListInfo] + /// The info for the resource file. This is a shared file used by all filter lists that contain scriplets. This information is used for lazy loading. + private(set) var resourcesInfo: CachedAdBlockEngine.ResourcesInfo? + /// Adblock engine for general adblock lists. + private(set) var cachedEngines: [CachedAdBlockEngine.Source: CachedAdBlockEngine] + /// The current task that is compiling. + private var currentCompileTask: Task<(), Never>? + + /// Return all the critical sources + /// + /// Critical sources are those that are enabled and are "on" by default. Giving us the most important sources. + /// Used for memory managment so we know which filter lists to disable upon a memory warning + @MainActor var criticalSources: [CachedAdBlockEngine.Source] { + var enabledSources: [CachedAdBlockEngine.Source] = [.adBlock] + enabledSources.append(contentsOf: FilterListStorage.shared.ciriticalSources) + return enabledSources + } + + @MainActor var enabledSources: [CachedAdBlockEngine.Source] { + var enabledSources: [CachedAdBlockEngine.Source] = [.adBlock] + enabledSources.append(contentsOf: FilterListStorage.shared.enabledSources) + enabledSources.append(contentsOf: CustomFilterListStorage.shared.enabledSources) + return enabledSources + } + + /// Tells us if we reached the max limit of already compiled filter lists + var reachedMaxLimit: Bool { + return cachedEngines.count >= Self.maxNumberOfAllowedFilterLists + } init() { - cachedEngines = [] + cachedEngines = [:] + availableFilterLists = [:] } - /// Clear the caches on all of the engines - func clearCaches() { - cachedEngines.forEach({ $0.clearCaches() }) + /// Handle memory warnings by freeing up some memory + func didReceiveMemoryWarning() async { + cachedEngines.values.forEach({ $0.clearCaches() }) + await removeDisabledEngines() } - func add(engine: CachedAdBlockEngine) { - if let index = cachedEngines.firstIndex(where: { $0.filterListInfo.source == engine.filterListInfo.source }) { - cachedEngines[index] = engine + public func compileDelayed( + filterListInfo: CachedAdBlockEngine.FilterListInfo, resourcesInfo: CachedAdBlockEngine.ResourcesInfo, + isAlwaysAggressive: Bool, delayed: Bool + ) async throws { + if delayed { + Task.detached(priority: .background) { + ContentBlockerManager.log.debug("Delaying \(filterListInfo.source.debugDescription)") + await self.compile( + filterListInfo: filterListInfo, resourcesInfo: resourcesInfo, + isAlwaysAggressive: isAlwaysAggressive + ) + } } else { - cachedEngines.append(engine) + await self.compile( + filterListInfo: filterListInfo, resourcesInfo: resourcesInfo, + isAlwaysAggressive: isAlwaysAggressive + ) } } + /// Create and add an engine from the given resources. + /// If an engine already exists for the given source, it will be replaced. + /// + /// - Note: This method will ensure syncronous compilation + public func compile( + filterListInfo: CachedAdBlockEngine.FilterListInfo, resourcesInfo: CachedAdBlockEngine.ResourcesInfo, + isAlwaysAggressive: Bool + ) async { + await currentCompileTask?.value + + guard needsCompilation(for: filterListInfo, resourcesInfo: resourcesInfo) else { + // Ensure we only compile if we need to. This prevents two lazy loads from recompiling + return + } + + currentCompileTask = Task { + if reachedMaxLimit && cachedEngines[filterListInfo.source] == nil { + ContentBlockerManager.log.error("Failed to compile engine for \(filterListInfo.source.debugDescription): Reached maximum!") + return + } + + do { + let engine = try CachedAdBlockEngine.compile( + filterListInfo: filterListInfo, resourcesInfo: resourcesInfo, isAlwaysAggressive: isAlwaysAggressive + ) + + add(engine: engine) + } catch { + ContentBlockerManager.log.error("Failed to compile engine for \(filterListInfo.source.debugDescription)") + } + } + + await currentCompileTask?.value + } + + /// Add a new engine to the list. + /// If an engine already exists for the same source, it will be replaced instead. + private func add(engine: CachedAdBlockEngine) { + cachedEngines[engine.filterListInfo.source] = engine + updateIfNeeded(resourcesInfo: engine.resourcesInfo) + updateIfNeeded(filterListInfo: engine.filterListInfo, isAlwaysAggressive: engine.isAlwaysAggressive) + ContentBlockerManager.log.debug("Added engine for \(engine.filterListInfo.debugDescription)") + } + + /// Add or update `filterListInfo` if it is a newer version. This information is used for lazy loading. + func updateIfNeeded(filterListInfo: CachedAdBlockEngine.FilterListInfo, isAlwaysAggressive: Bool) { + if let existingLazyInfo = availableFilterLists[filterListInfo.source] { + guard filterListInfo.version > existingLazyInfo.filterListInfo.version else { return } + } + + availableFilterLists[filterListInfo.source] = LazyFilterListInfo( + filterListInfo: filterListInfo, isAlwaysAggressive: isAlwaysAggressive + ) + } + + /// Add or update `resourcesInfo` if it is a newer version. This information is used for lazy loading. + func updateIfNeeded(resourcesInfo: CachedAdBlockEngine.ResourcesInfo) { + guard self.resourcesInfo == nil || resourcesInfo.version > self.resourcesInfo!.version else { return } + self.resourcesInfo = resourcesInfo + } + + /// Remove an engine for the given source. func removeEngine(for source: CachedAdBlockEngine.Source) { - cachedEngines.removeAll { cachedEngine in - cachedEngine.filterListInfo.source == source + if let filterListInfo = cachedEngines[source]?.filterListInfo { + ContentBlockerManager.log.debug("Removed engine for \(filterListInfo.debugDescription)") } + + cachedEngines.removeValue(forKey: source) } - func needsCompilation(for filterListInfo: CachedAdBlockEngine.FilterListInfo, resourcesInfo: CachedAdBlockEngine.ResourcesInfo) -> Bool { - return !cachedEngines.contains { cachedEngine in - if cachedEngine.filterListInfo.source == filterListInfo.source { - return cachedEngine.filterListInfo.version < filterListInfo.version - || cachedEngine.resourcesInfo.version < resourcesInfo.version - } else { - return true + /// Remove all the engines + func removeAllEngines() { + cachedEngines.removeAll() + } + + /// Remove all engines that have disabled sources + func removeDisabledEngines() async { + let sources = await Set(enabledSources) + + for source in cachedEngines.keys { + guard !sources.contains(source) else { continue } + removeEngine(for: source) + } + } + + /// Remove all engines that have disabled sources + func ensureEnabledEngines() async { + do { + var count = 0 + + for source in await enabledSources { + guard cachedEngines[source] == nil else { continue } + guard let availableFilterList = availableFilterLists[source] else { continue } + guard let resourcesInfo = self.resourcesInfo else { continue } + + try await compileDelayed( + filterListInfo: availableFilterList.filterListInfo, + resourcesInfo: resourcesInfo, + isAlwaysAggressive: availableFilterList.isAlwaysAggressive, + delayed: count > 5 + ) + + // Sleep for 1ms. This drastically reduces memory usage without much impact to usability + count += 1 + try await Task.sleep(nanoseconds: 1000000) } + } catch { + // Ignore cancellation errors } } - /// Checks the general and regional engines to see if the request should be blocked + /// Tells us if this source should be eagerly loaded. /// - /// - Warning: This method needs to be synced on `AdBlockStatus.adblockSerialQueue` + /// Eagerness is determined by several factors: + /// * If the source represents a fitler list or a custom filter list, it is eager if it is enabled + /// * If the source represents the `adblock` default filter list, it is always eager regardless of shield settings + func isEagerlyLoaded(source: CachedAdBlockEngine.Source) async -> Bool { + return await enabledSources.contains(source) + } + + /// Tells us if an engine needs compilation if it's missing or if its resources are outdated + func needsCompilation(for filterListInfo: CachedAdBlockEngine.FilterListInfo, resourcesInfo: CachedAdBlockEngine.ResourcesInfo) -> Bool { + if let cachedEngine = cachedEngines[filterListInfo.source] { + return cachedEngine.filterListInfo.version < filterListInfo.version + && cachedEngine.resourcesInfo.version < resourcesInfo.version + } else { + return true + } + } + + /// Checks the general and regional engines to see if the request should be blocked func shouldBlock(requestURL: URL, sourceURL: URL, resourceType: AdblockEngine.ResourceType, isAggressiveMode: Bool) async -> Bool { - return await cachedEngines.asyncConcurrentMap({ cachedEngine in + let sources = await self.enabledSources + return await cachedEngines(for: sources).asyncConcurrentMap({ cachedEngine in return await cachedEngine.shouldBlock( requestURL: requestURL, sourceURL: sourceURL, @@ -82,15 +243,17 @@ public actor AdBlockStats { }) } - /// Returns all the models for this frame URL - func cachedEngines(for domain: Domain) async -> [CachedAdBlockEngine] { - let enabledSources = await enabledSources(for: domain) - - let engines = await cachedEngines.asyncFilter({ cachedEngine in - return enabledSources.contains(cachedEngine.filterListInfo.source) - }) - - return engines + /// Returns all appropriate engines for the given domain + @MainActor func cachedEngines(for domain: Domain) async -> [CachedAdBlockEngine] { + let sources = enabledSources(for: domain) + return await cachedEngines(for: sources) + } + + /// Return all the cached engines for the given sources. If any filter list is not yet loaded, it will be lazily loaded + private func cachedEngines(for sources: [CachedAdBlockEngine.Source]) -> [CachedAdBlockEngine] { + return sources.compactMap { source -> CachedAdBlockEngine? in + return cachedEngines[source] + } } /// Returns all the models for this frame URL @@ -108,41 +271,72 @@ public actor AdBlockStats { } } - @MainActor private func enabledSources(for domain: Domain) -> Set { - var enabledSources = FilterListStorage.shared.enabledSources.union( - CustomFilterListStorage.shared.enabledSources - ) - enabledSources.insert(.adBlock) + /// Give us all the enabled sources for the given domain + @MainActor private func enabledSources(for domain: Domain) -> [CachedAdBlockEngine.Source] { + let enabledSources = self.enabledSources return enabledSources.filter({ $0.isEnabled(for: domain )}) } } +extension FilterListSetting { + @MainActor var engineSource: CachedAdBlockEngine.Source? { + guard let componentId = componentId else { return nil } + return .filterList(componentId: componentId) + } +} + +extension FilterList { + @MainActor var engineSource: CachedAdBlockEngine.Source { + return .filterList(componentId: entry.componentId) + } +} + private extension FilterListStorage { - @MainActor var enabledSources: Set { - let sources = allFilterListSettings.compactMap { setting -> CachedAdBlockEngine.Source? in - guard setting.isEnabled else { return nil } - - if let componentId = setting.componentId { - return .filterList(componentId: componentId) - } else if let filterList = filterLists.first(where: { $0.uuid == setting.uuid }) { - return .filterList(componentId: filterList.entry.componentId) - } else { - return nil + /// Gives us source representations of all the critical filter lists + /// + /// Critical filter lists are those that are enabled and are "on" by default. Giving us the most important filter lists. + /// Used for memory managment so we know which filter lists to disable upon a memory warning + @MainActor var ciriticalSources: [CachedAdBlockEngine.Source] { + return enabledSources.filter { source in + switch source { + case .filterList(let componentId): + return FilterList.defaultOnComponentIds.contains(componentId) + default: + return false + } + } + } + + /// Gives us source representations of all the enabled filter lists + @MainActor var enabledSources: [CachedAdBlockEngine.Source] { + if !filterLists.isEmpty { + return filterLists.compactMap { filterList -> CachedAdBlockEngine.Source? in + guard filterList.isEnabled else { return nil } + return filterList.engineSource + } + } else { + // We may not have the filter lists loaded yet. In which case we load the settings + return allFilterListSettings.compactMap { setting -> CachedAdBlockEngine.Source? in + guard setting.isEnabled else { return nil } + return setting.engineSource } } - - return Set(sources) + } +} + +extension CustomFilterListSetting { + @MainActor var engineSource: CachedAdBlockEngine.Source { + return .filterListURL(uuid: uuid) } } private extension CustomFilterListStorage { - @MainActor var enabledSources: Set { - let sources = filterListsURLs.compactMap { filterList -> CachedAdBlockEngine.Source? in + /// Gives us source representations of all the enabled custom filter lists + @MainActor var enabledSources: [CachedAdBlockEngine.Source] { + return filterListsURLs.compactMap { filterList -> CachedAdBlockEngine.Source? in guard filterList.setting.isEnabled else { return nil } - return .filterListURL(uuid: filterList.setting.uuid) + return filterList.setting.engineSource } - - return Set(sources) } } diff --git a/Sources/Brave/WebFilters/ShieldStats/Adblock/CachedAdBlockEngine.swift b/Sources/Brave/WebFilters/ShieldStats/Adblock/CachedAdBlockEngine.swift index 14676218a69..67306e8be30 100644 --- a/Sources/Brave/WebFilters/ShieldStats/Adblock/CachedAdBlockEngine.swift +++ b/Sources/Brave/WebFilters/ShieldStats/Adblock/CachedAdBlockEngine.swift @@ -11,21 +11,40 @@ import Preferences /// An object that wraps around an `AdblockEngine` and caches some results /// and ensures information is always returned on the correct thread on the engine. public class CachedAdBlockEngine { - public enum Source: Hashable { + public enum Source: Hashable, CustomDebugStringConvertible { case adBlock case filterList(componentId: String) case filterListURL(uuid: String) + + public var debugDescription: String { + switch self { + case .adBlock: return "adBlock" + case .filterList(let componentId): return "filterList(\(componentId))" + case .filterListURL(let uuid): return "filterListURL(\(uuid))" + } + } } - public enum FileType: Hashable { + public enum FileType: Hashable, CustomDebugStringConvertible { case dat, text + + public var debugDescription: String { + switch self { + case .dat: return "dat" + case .text: return "txt" + } + } } - public struct FilterListInfo: Hashable, Equatable { + public struct FilterListInfo: Hashable, Equatable, CustomDebugStringConvertible { let source: Source let localFileURL: URL let version: String let fileType: FileType + + public var debugDescription: String { + return "`\(source.debugDescription)` v\(version) (\(fileType.debugDescription))" + } } public struct ResourcesInfo: Hashable, Equatable { @@ -180,32 +199,22 @@ public class CachedAdBlockEngine { /// Create an engine from the given resources public static func compile( filterListInfo: FilterListInfo, resourcesInfo: ResourcesInfo, isAlwaysAggressive: Bool - ) async throws -> CachedAdBlockEngine { - return try await withCheckedThrowingContinuation { continuation in + ) throws -> CachedAdBlockEngine { + switch filterListInfo.fileType { + case .dat: + let engine = try AdblockEngine(datFileURL: filterListInfo.localFileURL, resourcesFileURL: resourcesInfo.localFileURL) let serialQueue = DispatchQueue(label: "com.brave.WrappedAdBlockEngine.\(UUID().uuidString)") - - serialQueue.async { - do { - switch filterListInfo.fileType { - case .dat: - let engine = try AdblockEngine(datFileURL: filterListInfo.localFileURL, resourcesFileURL: resourcesInfo.localFileURL) - let cachedEngine = CachedAdBlockEngine( - engine: engine, filterListInfo: filterListInfo, resourcesInfo: resourcesInfo, - serialQueue: serialQueue, isAlwaysAggressive: isAlwaysAggressive - ) - continuation.resume(returning: cachedEngine) - case .text: - let engine = try AdblockEngine(textFileURL: filterListInfo.localFileURL, resourcesFileURL: resourcesInfo.localFileURL) - let cachedEngine = CachedAdBlockEngine( - engine: engine, filterListInfo: filterListInfo, resourcesInfo: resourcesInfo, - serialQueue: serialQueue, isAlwaysAggressive: isAlwaysAggressive - ) - continuation.resume(returning: cachedEngine) - } - } catch { - continuation.resume(throwing: error) - } - } + return CachedAdBlockEngine( + engine: engine, filterListInfo: filterListInfo, resourcesInfo: resourcesInfo, + serialQueue: serialQueue, isAlwaysAggressive: isAlwaysAggressive + ) + case .text: + let engine = try AdblockEngine(textFileURL: filterListInfo.localFileURL, resourcesFileURL: resourcesInfo.localFileURL) + let serialQueue = DispatchQueue(label: "com.brave.WrappedAdBlockEngine.\(UUID().uuidString)") + return CachedAdBlockEngine( + engine: engine, filterListInfo: filterListInfo, resourcesInfo: resourcesInfo, + serialQueue: serialQueue, isAlwaysAggressive: isAlwaysAggressive + ) } } } diff --git a/Tests/ClientTests/Web Filters/CachedAdBlockEngineTests.swift b/Tests/ClientTests/Web Filters/CachedAdBlockEngineTests.swift index 27b171e1a12..ddde3cd15c5 100644 --- a/Tests/ClientTests/Web Filters/CachedAdBlockEngineTests.swift +++ b/Tests/ClientTests/Web Filters/CachedAdBlockEngineTests.swift @@ -65,7 +65,7 @@ final class CachedAdBlockEngineTests: XCTestCase { await filterListInfos.asyncConcurrentForEach { filterListInfo in do { - let engine = try await CachedAdBlockEngine.compile( + let engine = try CachedAdBlockEngine.compile( filterListInfo: filterListInfo, resourcesInfo: resourcesInfo, isAlwaysAggressive: false ) @@ -121,29 +121,21 @@ final class CachedAdBlockEngineTests: XCTestCase { options.iterationCount = 10 measure(metrics: [XCTClockMetric(), XCTCPUMetric(), XCTMemoryMetric()], options: options) { - let exp = expectation(description: "Finished") - - Task { - let uuid = UUID().uuidString - - let filterListInfo = CachedAdBlockEngine.FilterListInfo( - source: .filterList(componentId: uuid), - localFileURL: sampleFilterListURL, - version: "bundled", fileType: .text + let uuid = UUID().uuidString + + let filterListInfo = CachedAdBlockEngine.FilterListInfo( + source: .filterList(componentId: uuid), + localFileURL: sampleFilterListURL, + version: "bundled", fileType: .text + ) + + do { + _ = try CachedAdBlockEngine.compile( + filterListInfo: filterListInfo, resourcesInfo: resourcesInfo, isAlwaysAggressive: false ) - - do { - _ = try await CachedAdBlockEngine.compile( - filterListInfo: filterListInfo, resourcesInfo: resourcesInfo, isAlwaysAggressive: false - ) - } catch { - XCTFail(error.localizedDescription) - } - - exp.fulfill() + } catch { + XCTFail(error.localizedDescription) } - - wait(for: [exp], timeout: 20) } } } diff --git a/fastlane/Fastfile b/fastlane/Fastfile index 202b07b387a..85c6ff9f4b3 100644 --- a/fastlane/Fastfile +++ b/fastlane/Fastfile @@ -51,7 +51,7 @@ platform :ios do "ClientTests/ContentBlockerTests", "ClientTests/HttpCookieExtensionTest/testSaveAndLoadCookie", "ClientTests/UserAgentTests", - "ClientTests/AdBlockEngineManagerTests/testPerformance", + "ClientTests/CachedAdBlockEngineTests/testPerformance", "DataTests", "BraveWalletTests/ManageSiteConnectionsStoreTests/testRemoveAllPermissions", "BraveWalletTests/ManageSiteConnectionsStoreTests/testRemovePermissions",