From 2bbe919b78b2a102403f34dce2b0c3d9daf8c3e1 Mon Sep 17 00:00:00 2001 From: qingzhuozhen <84748495+qingzhuozhen@users.noreply.github.com> Date: Thu, 14 Mar 2024 17:23:17 -0700 Subject: [PATCH] feat: migrate storage to for better thread safety (#129) --- Amplitude-Swift.xcodeproj/project.pbxproj | 35 ++- Sources/Amplitude/Amplitude.swift | 12 +- Sources/Amplitude/Configuration.swift | 10 +- .../Storages/PersistentStorage.swift | 212 +++++++++++--- .../Amplitude/Utilities/Diagonostics.swift | 59 ++++ .../Utilities/DispatchQueueHolder.swift | 12 + .../Amplitude/Utilities/EventPipeline.swift | 2 +- Sources/Amplitude/Utilities/HttpClient.swift | 12 +- .../Utilities/OutputFileStream.swift | 26 +- .../Amplitude/Utilities/UrlExtension.swift | 13 + Tests/AmplitudeTests/AmplitudeTests.swift | 37 ++- .../StoragePrefixMigrationTests.swift | 31 ++- .../Storages/PersistentStorageTests.swift | 261 +++++++++++++++++- .../Utilities/DiagnosticsTests.swift | 57 ++++ .../Utilities/EventPipelineTests.swift | 2 +- .../Utilities/HttpClientTests.swift | 23 +- .../Utilities/IdentifyInterceptorTests.swift | 2 +- ...ersistentStorageResponseHandlerTests.swift | 10 +- 18 files changed, 709 insertions(+), 107 deletions(-) create mode 100644 Sources/Amplitude/Utilities/Diagonostics.swift create mode 100644 Sources/Amplitude/Utilities/DispatchQueueHolder.swift create mode 100644 Tests/AmplitudeTests/Utilities/DiagnosticsTests.swift diff --git a/Amplitude-Swift.xcodeproj/project.pbxproj b/Amplitude-Swift.xcodeproj/project.pbxproj index 455f23ac..ded506a3 100644 --- a/Amplitude-Swift.xcodeproj/project.pbxproj +++ b/Amplitude-Swift.xcodeproj/project.pbxproj @@ -3,7 +3,7 @@ archiveVersion = 1; classes = { }; - objectVersion = 54; + objectVersion = 52; objects = { /* Begin PBXAggregateTarget section */ @@ -21,6 +21,9 @@ /* End PBXAggregateTarget section */ /* Begin PBXBuildFile section */ + 3E281B8C2B967F19009D913B /* Diagonostics.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3E281B8B2B967F19009D913B /* Diagonostics.swift */; }; + 3E281B8E2B96833D009D913B /* DiagnosticsTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3E281B8D2B96833D009D913B /* DiagnosticsTests.swift */; }; + 3E281B912B9BCC14009D913B /* DispatchQueueHolder.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3E281B902B9BCC14009D913B /* DispatchQueueHolder.swift */; }; 8EDEC02B99EE2092B567A61D /* ObjCIngestionMetadata.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8EDEC500EBDA8B813056E2DB /* ObjCIngestionMetadata.swift */; }; 8EDEC1073A308B12B5CCD975 /* AnalyticsConnectorPlugin.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8EDECD39BAA97DD4320C0AA5 /* AnalyticsConnectorPlugin.swift */; }; 8EDEC10C56FA7F7DEEB48B6F /* ObjCBaseEvent.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8EDECCFD935A0C5A6FE85E87 /* ObjCBaseEvent.swift */; }; @@ -141,6 +144,9 @@ /* End PBXContainerItemProxy section */ /* Begin PBXFileReference section */ + 3E281B8B2B967F19009D913B /* Diagonostics.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Diagonostics.swift; sourceTree = ""; }; + 3E281B8D2B96833D009D913B /* DiagnosticsTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DiagnosticsTests.swift; sourceTree = ""; }; + 3E281B902B9BCC14009D913B /* DispatchQueueHolder.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DispatchQueueHolder.swift; sourceTree = ""; }; 8EDEC0630C3B587334275D9B /* AmplitudeSessionTests.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AmplitudeSessionTests.swift; sourceTree = ""; }; 8EDEC1160D95DC3F0E48DDF7 /* ObjCPlugin.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ObjCPlugin.swift; sourceTree = ""; }; 8EDEC1576C95A2EB2FEF00A8 /* ObjCAmplitude.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ObjCAmplitude.swift; sourceTree = ""; }; @@ -415,6 +421,8 @@ BA9BEA4A299FB43B00BC0F7C /* IdentifyInterceptor.swift */, 8EDEC54AB4DF9E1074C3D6A4 /* Weak.swift */, D010435E2B6C59EE00F8173C /* SandboxHelper.swift */, + 3E281B8B2B967F19009D913B /* Diagonostics.swift */, + 3E281B902B9BCC14009D913B /* DispatchQueueHolder.swift */, ); path = Utilities; sourceTree = ""; @@ -522,6 +530,7 @@ BA9BEA4C299FB4BB00BC0F7C /* IdentifyInterceptorTests.swift */, 8EDEC4F83BFAA664749FAEF0 /* QueueTimeTests.swift */, D01043602B6C5A8500F8173C /* SandboxHelperTests.swift */, + 3E281B8D2B96833D009D913B /* DiagnosticsTests.swift */, ); path = Utilities; sourceTree = ""; @@ -544,6 +553,7 @@ buildPhases = ( OBJ_87 /* Sources */, OBJ_125 /* Frameworks */, + 3E281B8F2B98EC92009D913B /* ShellScript */, ); buildRules = ( ); @@ -633,6 +643,26 @@ }; /* End PBXResourcesBuildPhase section */ +/* Begin PBXShellScriptBuildPhase section */ + 3E281B8F2B98EC92009D913B /* ShellScript */ = { + isa = PBXShellScriptBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + inputFileListPaths = ( + ); + inputPaths = ( + ); + outputFileListPaths = ( + ); + outputPaths = ( + ); + runOnlyForDeploymentPostprocessing = 0; + shellPath = /bin/sh; + shellScript = "if which swiftlint >/dev/null; then\n swiftlint\nelse\n echo \"warning: SwiftLint not installed, download from https://github.com/realm/SwiftLint\"\nfi\n"; + }; +/* End PBXShellScriptBuildPhase section */ + /* Begin PBXSourcesBuildPhase section */ OBJ_130 /* Sources */ = { isa = PBXSourcesBuildPhase; @@ -662,6 +692,7 @@ OBJ_153 /* TimelineTests.swift in Sources */, OBJ_154 /* TypesTests.swift in Sources */, OBJ_155 /* EventPipelineTests.swift in Sources */, + 3E281B8E2B96833D009D913B /* DiagnosticsTests.swift in Sources */, B6F338A32B685793006179E2 /* NetworkConnectivityCheckerPluginTests.swift in Sources */, OBJ_156 /* HttpClientTests.swift in Sources */, D01043612B6C5A8500F8173C /* SandboxHelperTests.swift in Sources */, @@ -705,6 +736,7 @@ OBJ_110 /* State.swift in Sources */, B6CCC6CD2B6B14510004B203 /* ObjCNetworkConnectivityCheckerPlugin.swift in Sources */, OBJ_111 /* InMemoryStorage.swift in Sources */, + 3E281B912B9BCC14009D913B /* DispatchQueueHolder.swift in Sources */, OBJ_112 /* PersistentStorage.swift in Sources */, BA9BEA4B299FB43B00BC0F7C /* IdentifyInterceptor.swift in Sources */, BA0639F62A4DD491000F1CEE /* LegacyDatabaseStorage.swift in Sources */, @@ -720,6 +752,7 @@ OBJ_122 /* QueueTimer.swift in Sources */, OBJ_124 /* UrlExtension.swift in Sources */, 8EDECFCCF4219767F26210D6 /* Sessions.swift in Sources */, + 3E281B8C2B967F19009D913B /* Diagonostics.swift in Sources */, 8EDECD602E181B3E2E85D4DF /* StoragePrefixMigration.swift in Sources */, 8EDEC8F8DD2CDCD6568512F8 /* RemnantDataMigration.swift in Sources */, 8EDEC977C03AA2676724F436 /* BasePlugins.swift in Sources */, diff --git a/Sources/Amplitude/Amplitude.swift b/Sources/Amplitude/Amplitude.swift index e75df5ce..fd0c0309 100644 --- a/Sources/Amplitude/Amplitude.swift +++ b/Sources/Amplitude/Amplitude.swift @@ -349,12 +349,12 @@ public class Amplitude { } configuration.loggerProvider.debug(message: "Running migrateApiKeyStorages") if let persistentStorage = configuration.storageProvider as? PersistentStorage { - let apiKeyStorage = PersistentStorage(storagePrefix: "\(PersistentStorage.DEFAULT_STORAGE_PREFIX)-\(configuration.apiKey)") + let apiKeyStorage = PersistentStorage(storagePrefix: "\(PersistentStorage.DEFAULT_STORAGE_PREFIX)-\(configuration.apiKey)", logger: self.logger, diagonostics: configuration.diagonostics) StoragePrefixMigration(source: apiKeyStorage, destination: persistentStorage, logger: logger).execute() } if let persistentIdentifyStorage = configuration.identifyStorageProvider as? PersistentStorage { - let apiKeyIdentifyStorage = PersistentStorage(storagePrefix: "\(PersistentStorage.DEFAULT_STORAGE_PREFIX)-identify-\(configuration.apiKey)") + let apiKeyIdentifyStorage = PersistentStorage(storagePrefix: "\(PersistentStorage.DEFAULT_STORAGE_PREFIX)-identify-\(configuration.apiKey)", logger: self.logger, diagonostics: configuration.diagonostics) StoragePrefixMigration(source: apiKeyIdentifyStorage, destination: persistentIdentifyStorage, logger: logger).execute() } } @@ -367,12 +367,12 @@ public class Amplitude { configuration.loggerProvider.debug(message: "Running migrateDefaultInstanceStorages") let legacyDefaultInstanceName = "default_instance" if let persistentStorage = configuration.storageProvider as? PersistentStorage { - let legacyStorage = PersistentStorage(storagePrefix: "storage-\(legacyDefaultInstanceName)") + let legacyStorage = PersistentStorage(storagePrefix: "storage-\(legacyDefaultInstanceName)", logger: self.logger, diagonostics: configuration.diagonostics) StoragePrefixMigration(source: legacyStorage, destination: persistentStorage, logger: logger).execute() } if let persistentIdentifyStorage = configuration.identifyStorageProvider as? PersistentStorage { - let legacyIdentifyStorage = PersistentStorage(storagePrefix: "identify-\(legacyDefaultInstanceName)") + let legacyIdentifyStorage = PersistentStorage(storagePrefix: "identify-\(legacyDefaultInstanceName)", logger: self.logger, diagonostics: configuration.diagonostics) StoragePrefixMigration(source: legacyIdentifyStorage, destination: persistentIdentifyStorage, logger: logger).execute() } } @@ -393,7 +393,7 @@ public class Amplitude { let instanceName = configuration.getNormalizeInstanceName() if let persistentStorage = configuration.storageProvider as? PersistentStorage { let instanceOnlyEventPrefix = "\(PersistentStorage.DEFAULT_STORAGE_PREFIX)-storage-\(instanceName)" - let instanceNameOnlyStorage = PersistentStorage(storagePrefix: instanceOnlyEventPrefix) + let instanceNameOnlyStorage = PersistentStorage(storagePrefix: instanceOnlyEventPrefix, logger: self.logger, diagonostics: configuration.diagonostics) StoragePrefixMigration( source: instanceNameOnlyStorage, destination: persistentStorage, @@ -403,7 +403,7 @@ public class Amplitude { if let persistentIdentifyStorage = configuration.identifyStorageProvider as? PersistentStorage { let instanceOnlyIdentifyPrefix = "\(PersistentStorage.DEFAULT_STORAGE_PREFIX)-identify-\(instanceName)" - let instanceNameOnlyIdentifyStorage = PersistentStorage(storagePrefix: instanceOnlyIdentifyPrefix) + let instanceNameOnlyIdentifyStorage = PersistentStorage(storagePrefix: instanceOnlyIdentifyPrefix, logger: self.logger, diagonostics: configuration.diagonostics) StoragePrefixMigration( source: instanceNameOnlyIdentifyStorage, destination: persistentIdentifyStorage, diff --git a/Sources/Amplitude/Configuration.swift b/Sources/Amplitude/Configuration.swift index d7d89277..2e2f8410 100644 --- a/Sources/Amplitude/Configuration.swift +++ b/Sources/Amplitude/Configuration.swift @@ -34,6 +34,7 @@ public class Configuration { public internal(set) var migrateLegacyData: Bool public var defaultTracking: DefaultTrackingOptions public var offline: Bool? + internal let diagonostics: Diagnostics public init( apiKey: String, @@ -71,12 +72,13 @@ public class Configuration { self.flushIntervalMillis = flushIntervalMillis self.instanceName = normalizedInstanceName self.optOut = optOut - self.storageProvider = storageProvider - ?? PersistentStorage(storagePrefix: PersistentStorage.getEventStoragePrefix(apiKey, normalizedInstanceName)) - self.identifyStorageProvider = identifyStorageProvider - ?? PersistentStorage(storagePrefix: PersistentStorage.getIdentifyStoragePrefix(apiKey, normalizedInstanceName)) + self.diagonostics = Diagnostics() self.logLevel = logLevel self.loggerProvider = loggerProvider + self.storageProvider = storageProvider + ?? PersistentStorage(storagePrefix: PersistentStorage.getEventStoragePrefix(apiKey, normalizedInstanceName), logger: self.loggerProvider, diagonostics: self.diagonostics) + self.identifyStorageProvider = identifyStorageProvider + ?? PersistentStorage(storagePrefix: PersistentStorage.getIdentifyStoragePrefix(apiKey, normalizedInstanceName), logger: self.loggerProvider, diagonostics: self.diagonostics) self.minIdLength = minIdLength self.partnerId = partnerId self.callback = callback diff --git a/Sources/Amplitude/Storages/PersistentStorage.swift b/Sources/Amplitude/Storages/PersistentStorage.swift index 22eeceee..0086a598 100644 --- a/Sources/Amplitude/Storages/PersistentStorage.swift +++ b/Sources/Amplitude/Storages/PersistentStorage.swift @@ -22,21 +22,27 @@ class PersistentStorage: Storage { let userDefaults: UserDefaults? let fileManager: FileManager private var outputStream: OutputFileStream? - internal weak var amplitude: Amplitude? // Store event.callback in memory as it cannot be ser/deser in files. private var eventCallbackMap: [String: EventCallback] private var appPath: String! - let syncQueue = DispatchQueue(label: "syncPersistentStorage.amplitude.com") + let syncQueue = DispatchQueueHolder.storageQueue + let storageVersionKey: String + let logger: (any Logger)? + let diagonostics: Diagnostics - init(storagePrefix: String) { + init(storagePrefix: String, logger: (any Logger)?, diagonostics: Diagnostics) { self.storagePrefix = storagePrefix == PersistentStorage.DEFAULT_STORAGE_PREFIX || storagePrefix.starts(with: "\(PersistentStorage.DEFAULT_STORAGE_PREFIX)-") ? storagePrefix : "\(PersistentStorage.DEFAULT_STORAGE_PREFIX)-\(storagePrefix)" self.userDefaults = UserDefaults(suiteName: "\(PersistentStorage.AMP_STORAGE_PREFIX).\(self.storagePrefix)") self.fileManager = FileManager.default self.eventCallbackMap = [String: EventCallback]() + self.storageVersionKey = "\(PersistentStorage.STORAGE_VERSION).\(self.storagePrefix)" + self.logger = logger + self.diagonostics = diagonostics // Make sure Amplitude data is sandboxed per app self.appPath = isStorageSandboxed() ? "" : "\(Bundle.main.bundleIdentifier!)/" + handleV1Files() } func write(key: StorageKey, value: Any?) throws { @@ -77,8 +83,15 @@ class PersistentStorage: Storage { var content: String? do { content = try String(contentsOf: eventBlock, encoding: .utf8) + if eventBlock.hasPrefix(PersistentStorage.STORAGE_V2_PREFIX) { + // v2 file + return readV2File(content: content!) + } else { + // handle v1 file + return readV1File(content: content!) + } } catch { - amplitude?.logger?.error(message: error.localizedDescription) + logger?.error(message: error.localizedDescription) } return content } @@ -88,7 +101,7 @@ class PersistentStorage: Storage { do { try fileManager.removeItem(atPath: eventBlock.path) } catch { - amplitude?.logger?.error(message: error.localizedDescription) + logger?.error(message: error.localizedDescription) } } } @@ -104,7 +117,7 @@ class PersistentStorage: Storage { do { try fileManager.removeItem(atPath: eventBlock.path) } catch { - amplitude?.logger?.error(message: error.localizedDescription) + logger?.error(message: error.localizedDescription) } } } @@ -188,6 +201,9 @@ extension PersistentStorage { static let AMP_STORAGE_PREFIX = "com.amplitude.storage" static let MAX_FILE_SIZE = 975000 // 975KB static let TEMP_FILE_EXTENSION = "tmp" + static let DELMITER = "\u{0000}" + static let STORAGE_VERSION = "amplitude.events.storage.version" + static let STORAGE_V2_PREFIX = "v2-" enum Exception: Error { case unsupportedType @@ -200,6 +216,16 @@ extension PersistentStorage { } private func getCurrentEventFile() -> URL { + let allOpenFiles = try? fileManager.contentsOfDirectory( + at: getEventsStorageDirectory(), + includingPropertiesForKeys: [], + options: .skipsHiddenFiles + ).filter { (file) -> Bool in + return file.pathExtension == PersistentStorage.TEMP_FILE_EXTENSION + } + if allOpenFiles != nil && allOpenFiles!.count > 0 { + return allOpenFiles![0] + } var currentFileIndex = 0 let index: Int = getCurrentEventFileIndex() ?? 0 userDefaults?.set(index, forKey: eventsFileKey) @@ -213,7 +239,7 @@ extension PersistentStorage { private func getEventsFile(index: Int) -> URL { let dir = getEventsStorageDirectory() - let fileURL = dir.appendingPathComponent("\(index)").appendingPathExtension( + let fileURL = dir.appendingPathComponent("\(PersistentStorage.STORAGE_V2_PREFIX)\(index)").appendingPathExtension( PersistentStorage.TEMP_FILE_EXTENSION ) return fileURL @@ -228,8 +254,8 @@ extension PersistentStorage { } // finish out any file in progress - let index = getCurrentEventFileIndex() ?? 0 - finish(file: getEventsFile(index: index)) + let currentFile = getCurrentEventFile() + finish(file: currentFile) let allFiles = try? fileManager.contentsOfDirectory( at: getEventsStorageDirectory(), @@ -243,7 +269,7 @@ extension PersistentStorage { } } let sorted = files?.sorted { (left, right) -> Bool in - return left.lastPathComponent > right.lastPathComponent + return left.lastPathComponent < right.lastPathComponent } if let s = sorted { result = s @@ -276,10 +302,8 @@ extension PersistentStorage { private func storeEvent(toFile file: URL, event: BaseEvent) { var storeFile = file - var newFile = false if fileManager.fileExists(atPath: storeFile.path) == false { start(file: storeFile) - newFile = true } else if outputStream == nil { // this can happen if an instance was terminated before finishing a file. open(file: storeFile) @@ -294,21 +318,17 @@ extension PersistentStorage { // Set the new file path storeFile = getCurrentEventFile() start(file: storeFile) - newFile = true } - let jsonString = event.toString() + let jsonString = event.toString().replacingOccurrences(of: PersistentStorage.DELMITER, with: "") do { if outputStream == nil { - amplitude?.logger?.error(message: "OutputStream is nil with file: \(storeFile)") + logger?.error(message: "OutputStream is nil with file: \(storeFile)") } - if newFile == false { - // prepare for the next entry - try outputStream?.write(",") - } - try outputStream?.write(jsonString) + try outputStream?.write("\(jsonString)\(PersistentStorage.DELMITER)") } catch { - amplitude?.logger?.error(message: error.localizedDescription) + diagonostics.addErrorLog(error.localizedDescription) + logger?.error(message: error.localizedDescription) } } @@ -316,30 +336,30 @@ extension PersistentStorage { let storeFile = file guard fileManager.fileExists(atPath: storeFile.path) != true else { + diagonostics.addErrorLog("Splited file duplicate for path: \(storeFile.path)") return } start(file: storeFile) - let jsonString = events.map { $0.toString() }.joined(separator: ", ") + + let jsonString = events.map { $0.toString().replacingOccurrences(of: PersistentStorage.DELMITER, with: "") }.joined(separator: PersistentStorage.DELMITER) do { if outputStream == nil { - amplitude?.logger?.error(message: "OutputStream is nil with file: \(storeFile)") + logger?.error(message: "OutputStream is nil with file: \(storeFile)") } - try outputStream?.write(jsonString) + try outputStream?.write("\(jsonString)\(PersistentStorage.DELMITER)") } catch { - amplitude?.logger?.error(message: error.localizedDescription) + logger?.error(message: error.localizedDescription) } finish(file: storeFile) } private func start(file: URL) { - let contents = "[" do { outputStream = try OutputFileStream(fileURL: file) try outputStream?.create() - try outputStream?.write(contents) } catch { - amplitude?.logger?.error(message: error.localizedDescription) + logger?.error(message: error.localizedDescription) } } @@ -352,7 +372,8 @@ extension PersistentStorage { try outputStream.open() } } catch { - amplitude?.logger?.error(message: error.localizedDescription) + diagonostics.addErrorLog(error.localizedDescription) + logger?.error(message: error.localizedDescription) } } } @@ -362,23 +383,142 @@ extension PersistentStorage { return } - let fileEnding = "]" do { - try outputStream.write(fileEnding) try outputStream.close() } catch { - amplitude?.logger?.error(message: error.localizedDescription) + diagonostics.addErrorLog(error.localizedDescription) + logger?.error(message: error.localizedDescription) } self.outputStream = nil + rename(file) + let currentFileIndex: Int = (getCurrentEventFileIndex() ?? 0) + 1 + userDefaults?.set(currentFileIndex, forKey: eventsFileKey) + } + + private func rename(_ file: URL) { let fileWithoutTemp = file.deletingPathExtension() + var updatedFile = fileWithoutTemp + if !fileManager.fileExists(atPath: file.path) { + logger?.debug(message: "Try to rename non exist file.") + return + } + if fileManager.fileExists(atPath: fileWithoutTemp.path) { + logger?.debug(message: "File already exists \(fileWithoutTemp.path), handle gracefully.") + let suffix = "-\(Date().timeIntervalSince1970)-\(Int.random(in: 0..<1000))" + updatedFile = fileWithoutTemp.appendFileNameSuffix(suffix: suffix) + } do { - try fileManager.moveItem(at: file, to: fileWithoutTemp) + try fileManager.moveItem(at: file, to: updatedFile) } catch { - amplitude?.logger?.error(message: "Unable to rename file: \(file.path)") + diagonostics.addErrorLog(error.localizedDescription) + logger?.error(message: "Unable to rename file: \(file.path)") } + } - let currentFileIndex: Int = (getCurrentEventFileIndex() ?? 0) + 1 - userDefaults?.set(currentFileIndex, forKey: eventsFileKey) + private func handleV1Files() { + syncQueue.sync { + let currentStorageVersion = (userDefaults?.object(forKey: self.storageVersionKey) as? Int) ?? 0 + if currentStorageVersion > 1 { + return + } + let allFiles = self.getEventFiles(includeUnfinished: true) + for file in allFiles { + do { + if file.hasPrefix(PersistentStorage.STORAGE_V2_PREFIX) { + logger?.debug(message: "file already migrated") + return + } + let content = try String(contentsOf: file, encoding: .utf8) + if content.hasSuffix(PersistentStorage.DELMITER) { + break // already handled and in v2 format + } + + let normalizedContent = "[\(content.trimmingCharacters(in: ["[", ",", "]"]))]" + let events = BaseEvent.fromArrayString(jsonString: normalizedContent) + if events != nil { + migrateFile(file: file, events: events!) + } + if file.pathExtension != "" { + finish(file: file) + } + migrateFileName(file) + } catch { + diagonostics.addErrorLog("Error migrating file: \(file.path) for \(error.localizedDescription)") + logger?.error(message: error.localizedDescription) + } + } + userDefaults?.setValue(2, forKey: self.storageVersionKey) + } + } + + private func migrateFile(file: URL, events: [BaseEvent]) { + guard fileManager.fileExists(atPath: file.path) == true else { + diagonostics.addErrorLog("File to migrate not exists any more : \(file.path)") + return + } + + do { + let jsonString = events.map { $0.toString().replacingOccurrences(of: PersistentStorage.DELMITER, with: "") }.joined(separator: PersistentStorage.DELMITER) + let finalString = "\(jsonString)\(PersistentStorage.DELMITER)" + try finalString.write(to: file, atomically: true, encoding: .utf8) + } catch { + diagonostics.addErrorLog(error.localizedDescription) + logger?.error(message: error.localizedDescription) + } + } + + private func migrateFileName(_ file: URL) { + let fileNameV2 = file.appendFileNamePrefix(prefix: PersistentStorage.STORAGE_V2_PREFIX).deletingPathExtension() + if fileManager.fileExists(atPath: fileNameV2.path) { + logger?.debug(message: "Migrate found an existing file.") + return + } + do { + try fileManager.moveItem(at: file, to: fileNameV2) + } catch { + diagonostics.addErrorLog(error.localizedDescription) + logger?.error(message: "Unable to migrate file: \(file.path)") + } + } + + private func readV2File(content: String) -> String { + var events: [BaseEvent] = [BaseEvent]() + content.components(separatedBy: PersistentStorage.DELMITER).forEach{ + let currentString = String($0) + if currentString.isEmpty { + return + } + if let event = BaseEvent.fromString(jsonString: String($0)) { + events.append(event) + } else { + diagonostics.addMalformedEvent(String($0)) + } + } + return eventsToJSONString(events: events) + } + + private func eventsToJSONString(events: [BaseEvent]) -> String { + var result = "" + do { + let encoder = JSONEncoder() + let json = try encoder.encode(events) + if let printed = String(data: json, encoding: .utf8) { + result = printed + } + } catch { + diagonostics.addErrorLog(error.localizedDescription) + } + return result + } + + private func readV1File(content: String) -> String { + var result = "" + let normalizedContent = "[\(content.trimmingCharacters(in: ["[", ",", "]"]))]" + let events = BaseEvent.fromArrayString(jsonString: normalizedContent) + if events != nil { + result = normalizedContent + } + return result } } diff --git a/Sources/Amplitude/Utilities/Diagonostics.swift b/Sources/Amplitude/Utilities/Diagonostics.swift new file mode 100644 index 00000000..8135cfa6 --- /dev/null +++ b/Sources/Amplitude/Utilities/Diagonostics.swift @@ -0,0 +1,59 @@ +// +// Diagonostics.swift +// Amplitude-Swift +// +// Created by Qingzhuo Zhen on 3/4/24. +// + +import Foundation + +public class Diagnostics { + private var malformedEvents: [String]? + private var errorLogs: [String]? + + init(){} + + func addMalformedEvent(_ event: String) { + if malformedEvents == nil { + malformedEvents = [String]() + } + malformedEvents?.append(event) + } + + func addErrorLog(_ log: String) { + if errorLogs == nil { + errorLogs = [String]() + } + errorLogs?.append(log) + } + + func hasDiagnostics() -> Bool { + return (malformedEvents != nil && malformedEvents!.count > 0) || (errorLogs != nil && errorLogs!.count > 0) + } + + /** + * Extracts the diagnostics as a JSON string. + * Warning: This will clear stored diagnostics. + * @return JSON string of diagnostics or empty if no diagnostics are present. + */ + func extractDiagonosticsToString() -> String { + if !hasDiagnostics() { + return "" + } + var diagnostics = [String: [String]]() + if malformedEvents != nil && malformedEvents!.count > 0 { + diagnostics["malformed_events"] = malformedEvents + } + if errorLogs != nil && errorLogs!.count > 0 { + diagnostics["error_logs"] = errorLogs + } + do { + let data = try JSONSerialization.data(withJSONObject: diagnostics, options: []) + malformedEvents?.removeAll() + errorLogs?.removeAll() + return String(data: data, encoding: .utf8) ?? "" + } catch { + return "" + } + } +} diff --git a/Sources/Amplitude/Utilities/DispatchQueueHolder.swift b/Sources/Amplitude/Utilities/DispatchQueueHolder.swift new file mode 100644 index 00000000..a31d517f --- /dev/null +++ b/Sources/Amplitude/Utilities/DispatchQueueHolder.swift @@ -0,0 +1,12 @@ +// +// DispatchQueueHolder.swift +// Amplitude-Swift +// +// Created by Qingzhuo Zhen on 3/8/24. +// + +import Foundation + +class DispatchQueueHolder { + static let storageQueue = DispatchQueue(label: "syncPersistentStorage.amplitude.com") +} diff --git a/Sources/Amplitude/Utilities/EventPipeline.swift b/Sources/Amplitude/Utilities/EventPipeline.swift index d409e7a9..afd1d656 100644 --- a/Sources/Amplitude/Utilities/EventPipeline.swift +++ b/Sources/Amplitude/Utilities/EventPipeline.swift @@ -23,7 +23,7 @@ public class EventPipeline { init(amplitude: Amplitude) { self.amplitude = amplitude - self.httpClient = HttpClient(configuration: amplitude.configuration) + self.httpClient = HttpClient(configuration: amplitude.configuration, diagnostics: amplitude.configuration.diagonostics) self.flushTimer = QueueTimer(interval: getFlushInterval()) { [weak self] in self?.flush() } diff --git a/Sources/Amplitude/Utilities/HttpClient.swift b/Sources/Amplitude/Utilities/HttpClient.swift index aa722986..06ee732b 100644 --- a/Sources/Amplitude/Utilities/HttpClient.swift +++ b/Sources/Amplitude/Utilities/HttpClient.swift @@ -10,6 +10,7 @@ import Foundation class HttpClient { let configuration: Configuration internal let session: URLSession + let diagnostics: Diagnostics private lazy var dateFormatter: ISO8601DateFormatter = { let formatter = ISO8601DateFormatter() @@ -17,8 +18,9 @@ class HttpClient { return formatter }() - init(configuration: Configuration) { + init(configuration: Configuration, diagnostics: Diagnostics) { self.configuration = configuration + self.diagnostics = diagnostics let sessionConfiguration = URLSessionConfiguration.default sessionConfiguration.httpMaximumConnectionsPerHost = 2 @@ -87,6 +89,14 @@ class HttpClient { ,"options":{"min_id_length":\(minIdLength)} """ } + if diagnostics.hasDiagnostics() { + let diagnosticsInfo = diagnostics.extractDiagonosticsToString() + if !diagnosticsInfo.isEmpty { + requestPayload += """ + ,"request_metadata":{"sdk":\(diagnosticsInfo)} + """ + } + } requestPayload += "}" return requestPayload.data(using: .utf8) } diff --git a/Sources/Amplitude/Utilities/OutputFileStream.swift b/Sources/Amplitude/Utilities/OutputFileStream.swift index 347abd4c..0bbddd44 100644 --- a/Sources/Amplitude/Utilities/OutputFileStream.swift +++ b/Sources/Amplitude/Utilities/OutputFileStream.swift @@ -50,18 +50,25 @@ internal class OutputFileStream { if fileHandle != nil { return } do { fileHandle = try FileHandle(forWritingTo: fileURL) - if #available(macOS 10.15.4, iOS 13.4, macCatalyst 13.4, tvOS 13.4, watchOS 6.2, *) { - _ = try? fileHandle?.seekToEnd() - } else if #available(tvOS 13.0, *) { - try? fileHandle?.seek(toOffset: .max) - } else { - fileHandle?.seekToEndOfFile() - } + seekToEnd() } catch { throw OutputStreamError.unableToOpen(fileURL.path) } } + func seekToEnd() { + if fileHandle == nil { + return + } + if #available(macOS 10.15.4, iOS 13.4, macCatalyst 13.4, tvOS 13.4, watchOS 6.2, *) { + _ = try? fileHandle?.seekToEnd() + } else if #available(tvOS 13.0, *) { + try? fileHandle?.seek(toOffset: .max) + } else { + fileHandle?.seekToEndOfFile() + } + } + func write(_ data: Data) throws { guard data.isEmpty == false else { return } if #available(macOS 10.15.4, iOS 13.4, macCatalyst 13.4, tvOS 13.4, watchOS 6.2, *) { @@ -75,9 +82,12 @@ internal class OutputFileStream { } } - func write(_ string: String) throws { + func write(_ string: String, _ append: Bool = true) throws { guard string.isEmpty == false else { return } if let data = string.data(using: .utf8) { + if append { + seekToEnd() + } try write(data) } } diff --git a/Sources/Amplitude/Utilities/UrlExtension.swift b/Sources/Amplitude/Utilities/UrlExtension.swift index a53d1ea2..b2abf12d 100644 --- a/Sources/Amplitude/Utilities/UrlExtension.swift +++ b/Sources/Amplitude/Utilities/UrlExtension.swift @@ -15,4 +15,17 @@ extension URL { } return deletingLastPathComponent().appendingPathComponent(filename) } + + func appendFileNamePrefix(prefix: String) -> URL { + var filename = "\(prefix)" + deletingPathExtension().lastPathComponent + if !pathExtension.isEmpty { + filename += ".\(pathExtension)" + } + return deletingLastPathComponent().appendingPathComponent(filename) + } + + func hasPrefix(_ prefix: String) -> Bool { + let filename = deletingPathExtension().lastPathComponent + return filename.hasPrefix(prefix) + } } diff --git a/Tests/AmplitudeTests/AmplitudeTests.swift b/Tests/AmplitudeTests/AmplitudeTests.swift index 57155ab8..51e7b9e3 100644 --- a/Tests/AmplitudeTests/AmplitudeTests.swift +++ b/Tests/AmplitudeTests/AmplitudeTests.swift @@ -16,6 +16,8 @@ final class AmplitudeTests: XCTestCase { private var storageTest: TestPersistentStorage! private var interceptStorageTest: TestPersistentStorage! + private let logger = ConsoleLogger() + private let diagonostics = Diagnostics() override func setUp() { super.setUp() @@ -23,8 +25,8 @@ final class AmplitudeTests: XCTestCase { configuration = Configuration(apiKey: apiKey) - storage = FakePersistentStorage(storagePrefix: "storage") - interceptStorage = FakePersistentStorage(storagePrefix: "intercept") + storage = FakePersistentStorage(storagePrefix: "storage", logger: self.logger, diagonostics: self.diagonostics) + interceptStorage = FakePersistentStorage(storagePrefix: "intercept", logger: self.logger, diagonostics: self.diagonostics) configurationWithFakeStorage = Configuration( apiKey: apiKey, storageProvider: storage, @@ -85,7 +87,7 @@ final class AmplitudeTests: XCTestCase { func testFilterAndEnrichmentPlugin() { let apiKey = "testFilterAndEnrichmentPlugin" let enrichedEventType = "Enriched Event" - storageTest = TestPersistentStorage(storagePrefix: "storage-\(apiKey)") + storageTest = TestPersistentStorage(storagePrefix: "storage-\(apiKey)", logger: self.logger, diagonostics: self.diagonostics) let amplitude = Amplitude(configuration: Configuration( apiKey: apiKey, storageProvider: storageTest @@ -193,15 +195,14 @@ final class AmplitudeTests: XCTestCase { func testInterceptedIdentifyWithPersistentStorage() { let apiKey = "testApiKeyPersist" - storageTest = TestPersistentStorage(storagePrefix: "storage") - interceptStorageTest = TestPersistentStorage(storagePrefix: "identify") - let amplitude = Amplitude( - configuration: Configuration( - apiKey: apiKey, - storageProvider: storageTest, - identifyStorageProvider: interceptStorageTest, - defaultTracking: DefaultTrackingOptions.NONE - )) + storageTest = TestPersistentStorage(storagePrefix: "storage", logger: self.logger, diagonostics: self.diagonostics) + interceptStorageTest = TestPersistentStorage(storagePrefix: "identify", logger: self.logger, diagonostics: self.diagonostics) + let amplitude = Amplitude(configuration: Configuration( + apiKey: apiKey, + storageProvider: storageTest, + identifyStorageProvider: interceptStorageTest, + defaultTracking: DefaultTrackingOptions.NONE + )) amplitude.setUserId(userId: "test-user") @@ -449,8 +450,8 @@ final class AmplitudeTests: XCTestCase { ) // Create storages using instance name only - let legacyEventStorage = PersistentStorage(storagePrefix: "storage-\(config.getNormalizeInstanceName())") - let legacyIdentityStorage = PersistentStorage(storagePrefix: "identify-\(config.getNormalizeInstanceName())") + let legacyEventStorage = PersistentStorage(storagePrefix: "storage-\(config.getNormalizeInstanceName())", logger: self.logger, diagonostics: self.diagonostics) + let legacyIdentityStorage = PersistentStorage(storagePrefix: "identify-\(config.getNormalizeInstanceName())", logger: self.logger, diagonostics: self.diagonostics) // Init Amplitude using legacy storage let legacyStorageAmplitude = FakeAmplitudeWithNoInstNameOnlyMigration( @@ -531,11 +532,9 @@ final class AmplitudeTests: XCTestCase { defaultTracking: DefaultTrackingOptions.NONE ) - // Create storages using instance name only - let legacyEventStorage = FakePersistentStorageAppSandboxEnabled( - storagePrefix: "storage-\(config.getNormalizeInstanceName())") - let legacyIdentityStorage = FakePersistentStorageAppSandboxEnabled( - storagePrefix: "identify-\(config.getNormalizeInstanceName())") + // Create storages using instance name only + let legacyEventStorage = FakePersistentStorageAppSandboxEnabled(storagePrefix: "storage-\(config.getNormalizeInstanceName())", logger: self.logger, diagonostics: self.diagonostics) + let legacyIdentityStorage = FakePersistentStorageAppSandboxEnabled(storagePrefix: "identify-\(config.getNormalizeInstanceName())", logger: self.logger, diagonostics: self.diagonostics) // Init Amplitude using legacy storage let legacyStorageAmplitude = FakeAmplitudeWithNoInstNameOnlyMigration( diff --git a/Tests/AmplitudeTests/Migration/StoragePrefixMigrationTests.swift b/Tests/AmplitudeTests/Migration/StoragePrefixMigrationTests.swift index 9b6a2304..0d16db9d 100644 --- a/Tests/AmplitudeTests/Migration/StoragePrefixMigrationTests.swift +++ b/Tests/AmplitudeTests/Migration/StoragePrefixMigrationTests.swift @@ -3,9 +3,12 @@ import XCTest @testable import AmplitudeSwift final class StoragePrefixMigrationTests: XCTestCase { + let logger = ConsoleLogger() + let diagonostics = Diagnostics() + func testUserDefaults() throws { - let source = PersistentStorage(storagePrefix: NSUUID().uuidString) - let destination = PersistentStorage(storagePrefix: NSUUID().uuidString) + let source = PersistentStorage(storagePrefix: NSUUID().uuidString, logger: self.logger, diagonostics: self.diagonostics) + let destination = PersistentStorage(storagePrefix: NSUUID().uuidString, logger: self.logger, diagonostics: self.diagonostics) try source.write(key: StorageKey.DEVICE_ID, value: "source-device") try source.write(key: StorageKey.USER_ID, value: "source-user") @@ -58,8 +61,8 @@ final class StoragePrefixMigrationTests: XCTestCase { } func testEventFiles() throws { - let source = PersistentStorage(storagePrefix: NSUUID().uuidString) - let destination = PersistentStorage(storagePrefix: NSUUID().uuidString) + let source = PersistentStorage(storagePrefix: NSUUID().uuidString, logger: self.logger, diagonostics: self.diagonostics) + let destination = PersistentStorage(storagePrefix: NSUUID().uuidString, logger: self.logger, diagonostics: self.diagonostics) try source.write(key: StorageKey.EVENTS, value: BaseEvent(eventType: "event-1")) try source.write(key: StorageKey.EVENTS, value: BaseEvent(eventType: "event-2")) @@ -92,8 +95,8 @@ final class StoragePrefixMigrationTests: XCTestCase { } func testMissingSource() throws { - let source = PersistentStorage(storagePrefix: NSUUID().uuidString) - let destination = PersistentStorage(storagePrefix: NSUUID().uuidString) + let source = PersistentStorage(storagePrefix: NSUUID().uuidString, logger: self.logger, diagonostics: self.diagonostics) + let destination = PersistentStorage(storagePrefix: NSUUID().uuidString, logger: self.logger, diagonostics: self.diagonostics) var destinationDeviceId: String? = destination.read(key: StorageKey.DEVICE_ID) var destinationLastEventId: Int? = destination.read(key: StorageKey.LAST_EVENT_ID) @@ -116,8 +119,8 @@ final class StoragePrefixMigrationTests: XCTestCase { } func testMoveEventFilesWithDuplicatedName() throws { - let source = PersistentStorage(storagePrefix: NSUUID().uuidString) - let destination = PersistentStorage(storagePrefix: NSUUID().uuidString) + let source = PersistentStorage(storagePrefix: NSUUID().uuidString, logger: self.logger, diagonostics: self.diagonostics) + let destination = PersistentStorage(storagePrefix: NSUUID().uuidString, logger: self.logger, diagonostics: self.diagonostics) try source.write(key: StorageKey.EVENTS, value: BaseEvent(eventType: "event-1")) source.rollover() @@ -143,12 +146,12 @@ final class StoragePrefixMigrationTests: XCTestCase { XCTAssertEqual(sourceEventFiles.count, 0) destinationEventFiles = destination.getEventFiles(includeUnfinished: true) - XCTAssertEqual(destinationEventFiles[0].lastPathComponent, "1") - XCTAssertEqual(destinationEventFiles[1].lastPathComponent.prefix(2), "0-") - XCTAssertEqual(destinationEventFiles[2].lastPathComponent, "0") - XCTAssertEqual(try getFileSize(destinationEventFiles[0]), sourceFileSizes[0]) - XCTAssertEqual(try getFileSize(destinationEventFiles[1]), sourceFileSizes[1]) - XCTAssertEqual(try getFileSize(destinationEventFiles[2]), destinationFileSizes[0]) + XCTAssertEqual(destinationEventFiles[0].lastPathComponent, "v2-0") + XCTAssertEqual(destinationEventFiles[1].lastPathComponent.prefix(5), "v2-0-") + XCTAssertEqual(destinationEventFiles[2].lastPathComponent, "v2-1") + XCTAssertEqual(try getFileSize(destinationEventFiles[1]), sourceFileSizes[0]) + XCTAssertEqual(try getFileSize(destinationEventFiles[2]), sourceFileSizes[1]) + XCTAssertEqual(try getFileSize(destinationEventFiles[0]), destinationFileSizes[0]) } private func getFileSize(_ url: URL) throws -> Int { diff --git a/Tests/AmplitudeTests/Storages/PersistentStorageTests.swift b/Tests/AmplitudeTests/Storages/PersistentStorageTests.swift index 7facc1af..107a8f00 100644 --- a/Tests/AmplitudeTests/Storages/PersistentStorageTests.swift +++ b/Tests/AmplitudeTests/Storages/PersistentStorageTests.swift @@ -10,8 +10,11 @@ import XCTest @testable import AmplitudeSwift final class PersistentStorageTests: XCTestCase { + let logger = ConsoleLogger() + let diagonostics = Diagnostics() + func testIsBasicType() { - let persistentStorage = PersistentStorage(storagePrefix: "storage") + let persistentStorage = PersistentStorage(storagePrefix: "storage", logger: self.logger, diagonostics: self.diagonostics) var isValueBasicType = persistentStorage.isBasicType(value: 111) XCTAssertEqual(isValueBasicType, true) @@ -35,7 +38,7 @@ final class PersistentStorageTests: XCTestCase { } func testWrite() { - let persistentStorage = PersistentStorage(storagePrefix: "xxx-instance") + let persistentStorage = PersistentStorage(storagePrefix: "xxx-instance", logger: self.logger, diagonostics: self.diagonostics) try? persistentStorage.write( key: StorageKey.EVENTS, value: BaseEvent(eventType: "test1") @@ -51,12 +54,12 @@ final class PersistentStorageTests: XCTestCase { } func testWriteWithTwoInstances() { - let persistentStorage1 = PersistentStorage(storagePrefix: "xxx-instance") + let persistentStorage1 = PersistentStorage(storagePrefix: "xxx-instance", logger: self.logger, diagonostics: self.diagonostics) try? persistentStorage1.write( key: StorageKey.EVENTS, value: BaseEvent(eventType: "test1") ) - let persistentStorage2 = PersistentStorage(storagePrefix: "xxx-instance") + let persistentStorage2 = PersistentStorage(storagePrefix: "xxx-instance", logger: self.logger, diagonostics: self.diagonostics) try? persistentStorage2.write( key: StorageKey.EVENTS, value: BaseEvent(eventType: "test2") @@ -77,9 +80,223 @@ final class PersistentStorageTests: XCTestCase { persistentStorage2.reset() } + func testRollover() { + let persistentStorage = PersistentStorage(storagePrefix: "xxx-instance", logger: self.logger, diagonostics: self.diagonostics) + let storeDirectory = persistentStorage.getEventsStorageDirectory(createDirectory: false) + try? persistentStorage.write( + key: StorageKey.EVENTS, + value: BaseEvent(eventType: "test1") + ) + let filesInStoreageDirectory = try? FileManager.default.contentsOfDirectory(at: storeDirectory, includingPropertiesForKeys: nil) + XCTAssertEqual(filesInStoreageDirectory?.count, 1) + XCTAssertEqual(filesInStoreageDirectory?[0].pathExtension, PersistentStorage.TEMP_FILE_EXTENSION) + let rawContentInFile = try? String(contentsOf: filesInStoreageDirectory![0], encoding: .utf8) + XCTAssertEqual(rawContentInFile, "\(BaseEvent(eventType: "test1").toString())\(PersistentStorage.DELMITER)") + persistentStorage.rollover() + let filesInStoreageDirectoryAfterRollover = try? FileManager.default.contentsOfDirectory(at: storeDirectory, includingPropertiesForKeys: nil) + XCTAssertEqual(filesInStoreageDirectoryAfterRollover?.count, 1) + XCTAssertEqual(filesInStoreageDirectoryAfterRollover?[0].pathExtension, "") + persistentStorage.reset() + } + + func testRemove() { + let persistentStorage = PersistentStorage(storagePrefix: "xxx-instance", logger: self.logger, diagonostics: self.diagonostics) + let storeDirectory = persistentStorage.getEventsStorageDirectory(createDirectory: false) + try? persistentStorage.write( + key: StorageKey.EVENTS, + value: BaseEvent(eventType: "test1") + ) + persistentStorage.rollover() + let filesInStoreageDirectory = try? FileManager.default.contentsOfDirectory(at: storeDirectory, includingPropertiesForKeys: nil) + XCTAssertEqual(filesInStoreageDirectory?.count, 1) + persistentStorage.remove(eventBlock: filesInStoreageDirectory![0]) + let filesInStoreageDirectoryAfterRemove = try? FileManager.default.contentsOfDirectory(at: storeDirectory, includingPropertiesForKeys: nil) + XCTAssertEqual(filesInStoreageDirectoryAfterRemove?.count, 0) + persistentStorage.reset() + } + + func testSplit() { + let persistentStorage = PersistentStorage(storagePrefix: "xxx-instance", logger: self.logger, diagonostics: self.diagonostics) + let storeDirectory = persistentStorage.getEventsStorageDirectory(createDirectory: false) + let event1 = BaseEvent(eventType: "test1") + let event2 = BaseEvent(eventType: "test2") + let events = [event1, event2] + try? persistentStorage.write( + key: StorageKey.EVENTS, + value: event1 + ) + try? persistentStorage.write( + key: StorageKey.EVENTS, + value: event2 + ) + persistentStorage.rollover() + let filesInStoreageDirectory = try? FileManager.default.contentsOfDirectory(at: storeDirectory, includingPropertiesForKeys: nil) + XCTAssertEqual(filesInStoreageDirectory?.count, 1) + let rawContentInFile = try? String(contentsOf: filesInStoreageDirectory![0], encoding: .utf8) + XCTAssertEqual(rawContentInFile, "\(event1.toString())\(PersistentStorage.DELMITER)\(event2.toString())\(PersistentStorage.DELMITER)") + persistentStorage.splitBlock(eventBlock: filesInStoreageDirectory![0], events: events) + let filesInStoreageDirectoryAfterSplit: [URL]? = persistentStorage.read(key: StorageKey.EVENTS) + XCTAssertEqual(filesInStoreageDirectoryAfterSplit?.count, 2) + let rawContentInFile1 = try? String(contentsOf: filesInStoreageDirectoryAfterSplit![0], encoding: .utf8) + let rawContentInFile2 = try? String(contentsOf: filesInStoreageDirectoryAfterSplit![1], encoding: .utf8) + XCTAssertEqual(rawContentInFile1, "\(event1.toString())\(PersistentStorage.DELMITER)") + XCTAssertEqual(rawContentInFile2, "\(event2.toString())\(PersistentStorage.DELMITER)") + persistentStorage.reset() + } + + func testDelimiterHandledGracefully() { + let persistentStorage = PersistentStorage(storagePrefix: "xxx-instance", logger: self.logger, diagonostics: self.diagonostics) + try? persistentStorage.write( + key: StorageKey.EVENTS, + value: BaseEvent(eventType: "test1\(PersistentStorage.DELMITER)") + ) + try? persistentStorage.write( + key: StorageKey.EVENTS, + value: BaseEvent(eventType: "test2\(PersistentStorage.DELMITER)") + ) + let eventFiles: [URL]? = persistentStorage.read(key: StorageKey.EVENTS) + XCTAssertEqual(eventFiles?.count, 1) + + let eventString = persistentStorage.getEventsString(eventBlock: (eventFiles?[0])!) + let decodedEvents = BaseEvent.fromArrayString(jsonString: eventString!) + XCTAssertEqual(decodedEvents!.count, 2) + XCTAssertEqual(decodedEvents![0].eventType, "test1\(PersistentStorage.DELMITER)") + XCTAssertEqual(decodedEvents![1].eventType, "test2\(PersistentStorage.DELMITER)") + persistentStorage.reset() + } + + func testMalformedEventInDiagnostics() { + let persistentStorage = PersistentStorage(storagePrefix: "xxx-instance", logger: self.logger, diagonostics: self.diagonostics) + let storeDirectory = persistentStorage.getEventsStorageDirectory(createDirectory: false) + let currentFile = storeDirectory.appendingPathComponent("\(PersistentStorage.STORAGE_V2_PREFIX)\(0)") + let event1 = BaseEvent(eventType: "test1") + let partial = "{\"event_type\":\"test1\",\"user_id\":\"159995596214061\",\"device_id\":\"9b935bb3cd75\"," + let malformedContent = "\(event1.toString())\(PersistentStorage.DELMITER)\(partial)\(PersistentStorage.DELMITER)" + writeContent(file: currentFile, content: malformedContent) + let rawFiles = try? FileManager.default.contentsOfDirectory(at: storeDirectory, includingPropertiesForKeys: nil) + let eventFiles: [URL]? = persistentStorage.read(key: StorageKey.EVENTS) + XCTAssertEqual(eventFiles?.count, 1) + + let eventString = persistentStorage.getEventsString(eventBlock: (eventFiles?[0])!) + let decodedEvents = BaseEvent.fromArrayString(jsonString: eventString!) + var malformedArr: [String] = [String]() + malformedArr.append(partial) + let data = try? JSONSerialization.data(withJSONObject: malformedArr, options: []) + let expectedPartial = String(data: data!, encoding: .utf8) ?? "" + XCTAssertEqual(decodedEvents!.count, 1) + XCTAssertTrue(self.diagonostics.hasDiagnostics() == true) + XCTAssertEqual(self.diagonostics.extractDiagonosticsToString(), "{\"malformed_events\":\(expectedPartial)}") + persistentStorage.reset() + } + + func testConcurrentWriteFromMultipleThreads() { + let persistentStorage = PersistentStorage(storagePrefix: "xxx-concurrent-instance", logger: self.logger, diagonostics: self.diagonostics) + persistentStorage.reset() + let dispatchGroup = DispatchGroup() + for i in 0..<100 { + Thread.detachNewThread { + dispatchGroup.enter() + for d in 0..<10 { + try? persistentStorage.write( + key: StorageKey.EVENTS, + value: BaseEvent(eventType: "test\(i)-\(d)") + ) + } + persistentStorage.rollover() + dispatchGroup.leave() + } + } + dispatchGroup.wait() + + var eventsCount = 0 + let eventFiles: [URL]? = persistentStorage.read(key: StorageKey.EVENTS) + XCTAssertNotNil(eventFiles) + for file in eventFiles! { + let eventString = persistentStorage.getEventsString(eventBlock: file) + let decodedEvents = BaseEvent.fromArrayString(jsonString: eventString!) + eventsCount += decodedEvents!.count + } + XCTAssertEqual(eventsCount, 1000) + persistentStorage.reset() + } + + func testConcurrentWriteOnMultipleInsances() { + let dispatchGroup = DispatchGroup() + for i in 0..<100 { + Thread.detachNewThread { + dispatchGroup.enter() + let persistentStorage = PersistentStorage(storagePrefix: "xxx-multiple-instance", logger: self.logger, diagonostics: self.diagonostics) + for d in 0..<10 { + try? persistentStorage.write( + key: StorageKey.EVENTS, + value: BaseEvent(eventType: "test\(i)-\(d)") + ) + } + persistentStorage.rollover() + dispatchGroup.leave() + } + } + dispatchGroup.wait() + let persistentStorage = PersistentStorage(storagePrefix: "xxx-multiple-instance", logger: self.logger, diagonostics: self.diagonostics) + let eventFiles: [URL]? = persistentStorage.read(key: StorageKey.EVENTS) + var eventsCount = 0 + XCTAssertNotNil(eventFiles) + for file in eventFiles! { + let eventString = persistentStorage.getEventsString(eventBlock: file) + let decodedEvents = BaseEvent.fromArrayString(jsonString: eventString!) + eventsCount += decodedEvents!.count + } + XCTAssertEqual(eventsCount, 1000) + persistentStorage.reset() + } + + func testHandleEarlierVersionFiles() { + let persistentStorageToGetDirectory = PersistentStorage(storagePrefix: "xxx-instance", logger: self.logger, diagonostics: self.diagonostics) + let storeDirectory = persistentStorageToGetDirectory.getEventsStorageDirectory(createDirectory: false) + persistentStorageToGetDirectory.reset() + createEarilierVersionFiles(storageDirectory: storeDirectory) + let persistentStorage = PersistentStorage(storagePrefix: "xxx-instance", logger: self.logger, diagonostics: self.diagonostics) + let eventFiles: [URL]? = persistentStorage.read(key: StorageKey.EVENTS) + XCTAssertEqual(eventFiles?.count, 6) + var eventsCount = 0 + eventFiles?.forEach({ + let eventString = persistentStorage.getEventsString(eventBlock: $0) + let decodedEvents = BaseEvent.fromArrayString(jsonString: eventString!) + eventsCount += decodedEvents?.count ?? 0 + }) + XCTAssertEqual(eventsCount, 10) + persistentStorage.reset() + } + + func testHandleEarlierVersionAndWriteEvents() { + let persistentStorageToGetDirectory = PersistentStorage(storagePrefix: "xxx-instance", logger: self.logger, diagonostics: self.diagonostics) + let storeDirectory = persistentStorageToGetDirectory.getEventsStorageDirectory(createDirectory: false) + persistentStorageToGetDirectory.reset() + createEarilierVersionFiles(storageDirectory: storeDirectory) + let persistentStorage = PersistentStorage(storagePrefix: "xxx-instance", logger: self.logger, diagonostics: self.diagonostics) + try? persistentStorage.write( + key: StorageKey.EVENTS, + value: BaseEvent(eventType: "test13") + ) + try? persistentStorage.write( + key: StorageKey.EVENTS, + value: BaseEvent(eventType: "test14") + ) + let eventFiles: [URL]? = persistentStorage.read(key: StorageKey.EVENTS) + XCTAssertEqual(eventFiles?.count, 7) + var eventsCount = 0 + eventFiles?.forEach({ + let eventString = persistentStorage.getEventsString(eventBlock: $0) + let decodedEvents = BaseEvent.fromArrayString(jsonString: eventString!) + eventsCount += decodedEvents?.count ?? 0 + }) + XCTAssertEqual(eventsCount, 12) + persistentStorage.reset() + } + #if os(macOS) func testMacOsStorageDirectorySandboxedWhenAppSandboxDisabled() { - let persistentStorage = PersistentStorage(storagePrefix: "mac-instance") + let persistentStorage = PersistentStorage(storagePrefix: "mac-instance", logger: self.logger, diagonostics: self.diagonostics) let bundleId = Bundle.main.bundleIdentifier! let storageUrl = persistentStorage.getEventsStorageDirectory(createDirectory: false) @@ -90,7 +307,7 @@ final class PersistentStorageTests: XCTestCase { } func testMacOsStorageDirectorySandboxedWhenAppSandboxEnabled() { - let persistentStorage = FakePersistentStorageAppSandboxEnabled(storagePrefix: "mac-app-sandbox-instance") + let persistentStorage = FakePersistentStorageAppSandboxEnabled(storagePrefix: "mac-app-sandbox-instance", logger: self.logger, diagonostics: self.diagonostics) let bundleId = Bundle.main.bundleIdentifier! let storageUrl = persistentStorage.getEventsStorageDirectory(createDirectory: false) @@ -100,4 +317,36 @@ final class PersistentStorageTests: XCTestCase { persistentStorage.reset() } #endif + + private func writeContent(file: URL, content: String) { + let outputStream = try? OutputFileStream(fileURL: file) + try? outputStream?.create() + try? outputStream?.write(content) + } + + private func createEarilierVersionFiles(storageDirectory: URL) { + let file0 = storageDirectory.appendingPathComponent("0") + let content0 = "[\(BaseEvent(eventType: "test1").toString()),\(BaseEvent(eventType: "test2").toString())]" + writeContent(file: file0, content: content0) + + let file1 = storageDirectory.appendingPathComponent("1") + let content1 = ",\(BaseEvent(eventType: "test3").toString()),\(BaseEvent(eventType: "test4").toString())]" + writeContent(file: file1, content: content1) + + let file2 = storageDirectory.appendingPathComponent("2") + let content2 = "[[\(BaseEvent(eventType: "test5").toString()),\(BaseEvent(eventType: "test6").toString())]]" + writeContent(file: file2, content: content2) + + let file3 = storageDirectory.appendingPathComponent("3") + let content3 = "\(BaseEvent(eventType: "test7").toString()),\(BaseEvent(eventType: "test8").toString())]" + writeContent(file: file3, content: content3) + + let file4 = storageDirectory.appendingPathComponent("4") + let content4 = "[\(BaseEvent(eventType: "test9").toString())],\(BaseEvent(eventType: "test10").toString())]" + writeContent(file: file4, content: content4) + + let file5 = storageDirectory.appendingPathComponent("5") + let content5 = "[\(BaseEvent(eventType: "test11").toString()),\(BaseEvent(eventType: "test12").toString())" + writeContent(file: file5, content: content5) + } } diff --git a/Tests/AmplitudeTests/Utilities/DiagnosticsTests.swift b/Tests/AmplitudeTests/Utilities/DiagnosticsTests.swift new file mode 100644 index 00000000..1cffe893 --- /dev/null +++ b/Tests/AmplitudeTests/Utilities/DiagnosticsTests.swift @@ -0,0 +1,57 @@ +// +// DiagnosticsTests.swift +// Amplitude-SwiftTests +// +// Created by Qingzhuo Zhen on 3/4/24. +// + +import XCTest + +@testable import AmplitudeSwift + +final class DiagnosticsTests: XCTestCase { + + func testAddMalformedEvent() { + let diagnostics = Diagnostics() + diagnostics.addMalformedEvent("event") + XCTAssertTrue(diagnostics.hasDiagnostics()) + XCTAssertEqual(diagnostics.extractDiagonosticsToString(), "{\"malformed_events\":[\"event\"]}") + } + + func testAddErrorLog() { + let diagnostics = Diagnostics() + diagnostics.addErrorLog("log") + XCTAssertTrue(diagnostics.hasDiagnostics()) + XCTAssertEqual(diagnostics.extractDiagonosticsToString(), "{\"error_logs\":[\"log\"]}") + } + + func testHasDiagonostics() { + let diagnostics = Diagnostics() + XCTAssertFalse(diagnostics.hasDiagnostics()) + diagnostics.addMalformedEvent("event") + XCTAssertTrue(diagnostics.hasDiagnostics()) + diagnostics.addErrorLog("log") + XCTAssertTrue(diagnostics.hasDiagnostics()) + } + + func testExtractDiagnostic() { + let diagnostics = Diagnostics() + XCTAssertEqual(diagnostics.extractDiagonosticsToString(), "") + diagnostics.addMalformedEvent("event") + diagnostics.addErrorLog("log") + let result = convertToDictionary(text: diagnostics.extractDiagonosticsToString()) + XCTAssertEqual((result?["malformed_events"] as? [String]) ?? [], ["event"]) + XCTAssertEqual((result?["error_logs"] as? [String]) ?? [], ["log"]) + } + + private func convertToDictionary(text: String) -> [String: Any]? { + if let data = text.data(using: .utf8) { + do { + return try JSONSerialization.jsonObject(with: data, options: []) as? [String: Any] + } catch { + print(error.localizedDescription) + } + } + return nil + } +} diff --git a/Tests/AmplitudeTests/Utilities/EventPipelineTests.swift b/Tests/AmplitudeTests/Utilities/EventPipelineTests.swift index bbc7ce4f..037352c9 100644 --- a/Tests/AmplitudeTests/Utilities/EventPipelineTests.swift +++ b/Tests/AmplitudeTests/Utilities/EventPipelineTests.swift @@ -25,7 +25,7 @@ final class EventPipelineTests: XCTestCase { storageProvider: storage ) let amplitude = Amplitude(configuration: configuration) - httpClient = FakeHttpClient(configuration: configuration) + httpClient = FakeHttpClient(configuration: configuration, diagnostics: configuration.diagonostics) pipeline = EventPipeline(amplitude: amplitude) pipeline.httpClient = httpClient } diff --git a/Tests/AmplitudeTests/Utilities/HttpClientTests.swift b/Tests/AmplitudeTests/Utilities/HttpClientTests.swift index 0fc84599..3af5c5c0 100644 --- a/Tests/AmplitudeTests/Utilities/HttpClientTests.swift +++ b/Tests/AmplitudeTests/Utilities/HttpClientTests.swift @@ -11,6 +11,7 @@ import XCTest final class HttpClientTests: XCTestCase { private var configuration: Configuration! + private let diagonostics: Diagnostics = Diagnostics() override func setUp() { super.setUp() @@ -18,21 +19,21 @@ final class HttpClientTests: XCTestCase { } func testGetUrlWithDefault() { - let httpClient = HttpClient(configuration: configuration) + let httpClient = HttpClient(configuration: configuration, diagnostics: diagonostics) XCTAssertEqual(httpClient.getUrl(), Constants.DEFAULT_API_HOST) } func testGetUrlWithCustomUrl() { let customUrl = "https//localhost.test" configuration.serverUrl = customUrl - let httpClient = HttpClient(configuration: configuration) + let httpClient = HttpClient(configuration: configuration, diagnostics: diagonostics) XCTAssertEqual(httpClient.getUrl(), customUrl) } func testGetRequestWithInvalidUrl() { let invalidUrl = "local host" configuration.serverUrl = invalidUrl - let httpClient = HttpClient(configuration: configuration) + let httpClient = HttpClient(configuration: configuration, diagnostics: diagonostics) XCTAssertThrowsError(try httpClient.getRequest()) { error in guard case HttpClient.Exception.invalidUrl(let url) = error else { @@ -43,7 +44,7 @@ final class HttpClientTests: XCTestCase { } func testGetRequestData() { - let httpClient = FakeHttpClient(configuration: configuration) + let httpClient = FakeHttpClient(configuration: configuration, diagnostics: diagonostics) let event = BaseEvent(userId: "unit-test user", eventType: "unit-test event") let expectedRequestPayload = """ @@ -55,9 +56,21 @@ final class HttpClientTests: XCTestCase { XCTAssertEqual(result, expectedRequestPayload) } + func testGetResponseDataWithDiagnostic() { + let httpClient = FakeHttpClient(configuration: configuration, diagnostics: diagonostics) + let event = BaseEvent(userId: "unit-test user", eventType: "unit-test event") + diagonostics.addMalformedEvent("malformed event") + let expectedRequestPayload: Data? = """ + {"api_key":"testApiKey","client_upload_time":"2023-10-24T18:16:24.000Z","events":[\(event.toString())],"request_metadata":{"sdk":{"malformed_events":["malformed event"]}}} + """.data(using: .utf8) + let result = httpClient.getRequestData(events: "[\(event.toString())]") + + XCTAssertEqual(result, expectedRequestPayload) + } + func testUploadWithInvalidApiKey() { // TODO: currently this test is sending request to real Amplitude host, update to mock for better stability - let httpClient = HttpClient(configuration: configuration) + let httpClient = HttpClient(configuration: configuration, diagnostics: diagonostics) let asyncExpectation = expectation(description: "Async function") let event1 = BaseEvent(userId: "unit-test user", deviceId: "unit-test device", eventType: "unit-test event") _ = httpClient.upload(events: "[\(event1.toString())]") { result in diff --git a/Tests/AmplitudeTests/Utilities/IdentifyInterceptorTests.swift b/Tests/AmplitudeTests/Utilities/IdentifyInterceptorTests.swift index 00b50b80..3fe8f2c2 100644 --- a/Tests/AmplitudeTests/Utilities/IdentifyInterceptorTests.swift +++ b/Tests/AmplitudeTests/Utilities/IdentifyInterceptorTests.swift @@ -28,7 +28,7 @@ final class IdentifyInterceptorTests: XCTestCase { let amplitude = Amplitude(configuration: configuration) mockPathCreation = MockPathCreation() amplitude.add(plugin: NetworkConnectivityCheckerPlugin(pathCreation: mockPathCreation)) - httpClient = FakeHttpClient(configuration: configuration) + httpClient = FakeHttpClient(configuration: configuration, diagnostics: configuration.diagonostics) pipeline = EventPipeline(amplitude: amplitude) pipeline.httpClient = httpClient interceptor = TestIdentifyInterceptor( diff --git a/Tests/AmplitudeTests/Utilities/PersistentStorageResponseHandlerTests.swift b/Tests/AmplitudeTests/Utilities/PersistentStorageResponseHandlerTests.swift index 48482e50..e2922cf0 100644 --- a/Tests/AmplitudeTests/Utilities/PersistentStorageResponseHandlerTests.swift +++ b/Tests/AmplitudeTests/Utilities/PersistentStorageResponseHandlerTests.swift @@ -16,10 +16,12 @@ final class PersistentStorageResponseHandlerTests: XCTestCase { private var eventPipeline: EventPipeline! private var eventBlock: URL! private var eventsString: String! + private let logger = ConsoleLogger() + private let diagonostics = Diagnostics() override func setUp() { super.setUp() - storage = PersistentStorage(storagePrefix: "storage") + storage = PersistentStorage(storagePrefix: "storage", logger: self.logger, diagonostics: self.diagonostics) configuration = Configuration(apiKey: "testApiKey", storageProvider: storage) amplitude = Amplitude(configuration: configuration) eventPipeline = EventPipeline(amplitude: amplitude) @@ -50,7 +52,7 @@ final class PersistentStorageResponseHandlerTests: XCTestCase { {"event_type":"test","insert_id":"c8d58999-7539-4184-8a7d-54302697baf0","user_id":"test-user"} ] """ - let fakePersistentStorage = FakePersistentStorage(storagePrefix: "storage") + let fakePersistentStorage = FakePersistentStorage(storagePrefix: "storage", logger: self.logger, diagonostics: self.diagonostics) let handler = PersistentStorageResponseHandler( configuration: configuration, storage: fakePersistentStorage, @@ -83,7 +85,7 @@ final class PersistentStorageResponseHandlerTests: XCTestCase { ] """ - let fakePersistentStorage = FakePersistentStorage(storagePrefix: "storage") + let fakePersistentStorage = FakePersistentStorage(storagePrefix: "storage", logger: self.logger, diagonostics: self.diagonostics) let handler = PersistentStorageResponseHandler( configuration: configuration, storage: fakePersistentStorage, @@ -108,7 +110,7 @@ final class PersistentStorageResponseHandlerTests: XCTestCase { ] """ - let fakePersistentStorage = FakePersistentStorage(storagePrefix: "storage") + let fakePersistentStorage = FakePersistentStorage(storagePrefix: "storage", logger: self.logger, diagonostics: self.diagonostics) let handler = PersistentStorageResponseHandler( configuration: configuration, storage: fakePersistentStorage,