fix: Use unique Cache files for symcaches with markers

crates/symbolicator-service/src/services/symcaches/markers.rs

@@ -1,5 +1,7 @@
+use std::fmt::Write;
 use std::io::SeekFrom;
 
+use crate::services::cacher::CacheKey;
 use crate::services::{bitcode::BcSymbolMapHandle, il2cpp::Il2cppHandle};
 
 /// This is the legacy marker that was used previously to flag a SymCache that was created
@@ -38,6 +40,15 @@ impl SymCacheMarkers {
         Self { markers }
     }
 
+    /// Appends the marker state to the [`CacheKey`] in order to have unique cache files depending
+    /// on the applied sources.
+    pub fn append_to_key(&self, mut key: CacheKey) -> CacheKey {
+        if self.markers > 0 {
+            write!(&mut key.cache_key, "_{}", self.markers).unwrap();
+        }
+        key
+    }
+
     /// Extracts the markers embedded in the given `data`.
     ///
     /// This is lenient and will return an empty set of markers if there is a parse error.

crates/symbolicator-service/src/services/symcaches/mod.rs

@@ -147,7 +147,8 @@ impl CacheItemRequest for FetchSymCacheInternal {
     const VERSIONS: CacheVersions = SYMCACHE_VERSIONS;
 
     fn get_cache_key(&self) -> CacheKey {
-        self.object_meta.cache_key()
+        let markers = SymCacheMarkers::from_sources(&self.secondary_sources);
+        markers.append_to_key(self.object_meta.cache_key())
     }
 
     fn compute<'a>(&'a self, temp_file: &'a mut NamedTempFile) -> BoxFuture<'a, CacheEntry> {
@@ -172,6 +173,9 @@ impl CacheItemRequest for FetchSymCacheInternal {
     }
 
     fn should_load(&self, data: &[u8]) -> bool {
+        // FIXME(swatinem): we can remove the whole `should_load` machinery once we bump this
+        // cache version, as afterwards we will never read a non-marked cache file that does have
+        // markers inside.
         SymCache::parse(data)
             .map(|_symcache| {
                 // NOTE: we do *not* check for the `is_latest` version here.