Merge pull request #2705 from OffchainLabs/fix-run-mode

[config change] use MessageCommitMode when executing future head block messages
OffchainLabs · Oct 9, 2024 · 32c3f4b · 32c3f4b
2 parents a48e7d9 + c99c24c
commit 32c3f4b
Show file tree

Hide file tree

Showing 14 changed files with 593 additions and 161 deletions.
diff --git a/arbitrator/stylus/src/cache.rs b/arbitrator/stylus/src/cache.rs
@@ -29,8 +29,16 @@ pub struct LruCounters {
     pub does_not_fit: u32,
 }
 
+pub struct LongTermCounters {
+    pub hits: u32,
+    pub misses: u32,
+}
+
 pub struct InitCache {
     long_term: HashMap<CacheKey, CacheItem>,
+    long_term_size_bytes: usize,
+    long_term_counters: LongTermCounters,
+
     lru: CLruCache<CacheKey, CacheItem, RandomState, CustomWeightScale>,
     lru_counters: LruCounters,
 }
@@ -91,6 +99,20 @@ pub struct LruCacheMetrics {
     pub does_not_fit: u32,
 }
 
+#[repr(C)]
+pub struct LongTermCacheMetrics {
+    pub size_bytes: u64,
+    pub count: u32,
+    pub hits: u32,
+    pub misses: u32,
+}
+
+#[repr(C)]
+pub struct CacheMetrics {
+    pub lru: LruCacheMetrics,
+    pub long_term: LongTermCacheMetrics,
+}
+
 pub fn deserialize_module(
     module: &[u8],
     version: u16,
@@ -117,6 +139,9 @@ impl InitCache {
     fn new(size_bytes: usize) -> Self {
         Self {
             long_term: HashMap::new(),
+            long_term_size_bytes: 0,
+            long_term_counters: LongTermCounters { hits: 0, misses: 0 },
+
             lru: CLruCache::with_config(
                 CLruCacheConfig::new(NonZeroUsize::new(size_bytes).unwrap())
                     .with_scale(CustomWeightScale),
@@ -136,22 +161,41 @@ impl InitCache {
     }
 
     /// Retrieves a cached value, updating items as necessary.
-    pub fn get(module_hash: Bytes32, version: u16, debug: bool) -> Option<(Module, Store)> {
-        let mut cache = cache!();
+    /// If long_term_tag is 1 and the item is only in LRU will insert to long term cache.
+    pub fn get(
+        module_hash: Bytes32,
+        version: u16,
+        long_term_tag: u32,
+        debug: bool,
+    ) -> Option<(Module, Store)> {
         let key = CacheKey::new(module_hash, version, debug);
+        let mut cache = cache!();
 
         // See if the item is in the long term cache
         if let Some(item) = cache.long_term.get(&key) {
-            return Some(item.data());
+            let data = item.data();
+            cache.long_term_counters.hits += 1;
+            return Some(data);
+        }
+        if long_term_tag == Self::ARBOS_TAG {
+            // only count misses only when we can expect to find the item in long term cache
+            cache.long_term_counters.misses += 1;
         }
 
         // See if the item is in the LRU cache, promoting if so
-        if let Some(item) = cache.lru.get(&key) {
-            let data = item.data();
+        if let Some(item) = cache.lru.peek(&key).cloned() {
             cache.lru_counters.hits += 1;
-            return Some(data);
+            if long_term_tag == Self::ARBOS_TAG {
+                cache.long_term_size_bytes += item.entry_size_estimate_bytes;
+                cache.long_term.insert(key, item.clone());
+            } else {
+                // only calls get to move the key to the head of the LRU list
+                cache.lru.get(&key);
+            }
+            return Some((item.module, Store::new(item.engine)));
         }
         cache.lru_counters.misses += 1;
+
         None
     }
 
@@ -174,6 +218,7 @@ impl InitCache {
         if let Some(item) = cache.lru.peek(&key).cloned() {
             if long_term_tag == Self::ARBOS_TAG {
                 cache.long_term.insert(key, item.clone());
+                cache.long_term_size_bytes += item.entry_size_estimate_bytes;
             } else {
                 // only calls get to move the key to the head of the LRU list
                 cache.lru.get(&key);
@@ -195,6 +240,7 @@ impl InitCache {
             };
         } else {
             cache.long_term.insert(key, item);
+            cache.long_term_size_bytes += entry_size_estimate_bytes;
         }
         Ok(data)
     }
@@ -207,6 +253,7 @@ impl InitCache {
         let key = CacheKey::new(module_hash, version, debug);
         let mut cache = cache!();
         if let Some(item) = cache.long_term.remove(&key) {
+            cache.long_term_size_bytes -= item.entry_size_estimate_bytes;
             if cache.lru.put_with_weight(key, item).is_err() {
                 eprintln!("{}", Self::DOES_NOT_FIT_MSG);
             }
@@ -225,23 +272,32 @@ impl InitCache {
                 eprintln!("{}", Self::DOES_NOT_FIT_MSG);
             }
         }
+        cache.long_term_size_bytes = 0;
     }
 
-    pub fn get_lru_metrics() -> LruCacheMetrics {
+    pub fn get_metrics() -> CacheMetrics {
         let mut cache = cache!();
 
-        let count = cache.lru.len();
-        let metrics = LruCacheMetrics {
-            // add 1 to each entry to account that we subtracted 1 in the weight calculation
-            size_bytes: (cache.lru.weight() + count).try_into().unwrap(),
+        let lru_count = cache.lru.len();
+        let lru_metrics = LruCacheMetrics {
+            // adds 1 to each entry to account that we subtracted 1 in the weight calculation
+            size_bytes: (cache.lru.weight() + lru_count).try_into().unwrap(),
 
-            count: count.try_into().unwrap(),
+            count: lru_count.try_into().unwrap(),
 
             hits: cache.lru_counters.hits,
             misses: cache.lru_counters.misses,
             does_not_fit: cache.lru_counters.does_not_fit,
         };
 
+        let long_term_metrics = LongTermCacheMetrics {
+            size_bytes: cache.long_term_size_bytes.try_into().unwrap(),
+            count: cache.long_term.len().try_into().unwrap(),
+
+            hits: cache.long_term_counters.hits,
+            misses: cache.long_term_counters.misses,
+        };
+
         // Empty counters.
         // go side, which is the only consumer of this function besides tests,
         // will read those counters and increment its own prometheus counters with them.
@@ -250,8 +306,12 @@ impl InitCache {
             misses: 0,
             does_not_fit: 0,
         };
+        cache.long_term_counters = LongTermCounters { hits: 0, misses: 0 };
 
-        metrics
+        CacheMetrics {
+            lru: lru_metrics,
+            long_term: long_term_metrics,
+        }
     }
 
     // only used for testing

diff --git a/arbitrator/stylus/src/lib.rs b/arbitrator/stylus/src/lib.rs
@@ -11,7 +11,7 @@ use arbutil::{
     format::DebugBytes,
     Bytes32,
 };
-use cache::{deserialize_module, InitCache, LruCacheMetrics};
+use cache::{deserialize_module, CacheMetrics, InitCache};
 use evm_api::NativeRequestHandler;
 use eyre::ErrReport;
 use native::NativeInstance;
@@ -364,10 +364,10 @@ pub unsafe extern "C" fn stylus_drop_vec(vec: RustBytes) {
     }
 }
 
-/// Gets lru cache metrics.
+/// Gets cache metrics.
 #[no_mangle]
-pub extern "C" fn stylus_get_lru_cache_metrics() -> LruCacheMetrics {
-    InitCache::get_lru_metrics()
+pub extern "C" fn stylus_get_cache_metrics() -> CacheMetrics {
+    InitCache::get_metrics()
 }
 
 /// Clears lru cache.
@@ -377,18 +377,23 @@ pub extern "C" fn stylus_clear_lru_cache() {
     InitCache::clear_lru_cache()
 }
 
-/// Gets lru entry size in bytes.
+/// Clears long term cache (for arbos_tag = 1)
 /// Only used for testing purposes.
 #[no_mangle]
-pub extern "C" fn stylus_get_lru_entry_size_estimate_bytes(
+pub extern "C" fn stylus_clear_long_term_cache() {
+    InitCache::clear_long_term(1);
+}
+
+/// Gets entry size in bytes.
+/// Only used for testing purposes.
+#[no_mangle]
+pub extern "C" fn stylus_get_entry_size_estimate_bytes(
     module: GoSliceData,
     version: u16,
     debug: bool,
 ) -> u64 {
     match deserialize_module(module.slice(), version, debug) {
         Err(error) => panic!("tried to get invalid asm!: {error}"),
-        Ok((_, _, lru_entry_size_estimate_bytes)) => {
-            lru_entry_size_estimate_bytes.try_into().unwrap()
-        }
+        Ok((_, _, entry_size_estimate_bytes)) => entry_size_estimate_bytes.try_into().unwrap(),
     }
 }
diff --git a/arbitrator/stylus/src/native.rs b/arbitrator/stylus/src/native.rs
@@ -121,13 +121,12 @@ impl<D: DataReader, E: EvmApi<D>> NativeInstance<D, E> {
         let compile = CompileConfig::version(version, debug);
         let env = WasmEnv::new(compile, None, evm, evm_data);
         let module_hash = env.evm_data.module_hash;
-
-        if let Some((module, store)) = InitCache::get(module_hash, version, debug) {
-            return Self::from_module(module, store, env);
-        }
         if !env.evm_data.cached {
             long_term_tag = 0;
         }
+        if let Some((module, store)) = InitCache::get(module_hash, version, long_term_tag, debug) {
+            return Self::from_module(module, store, env);
+        }
         let (module, store) =
             InitCache::insert(module_hash, module, version, long_term_tag, debug)?;
         Self::from_module(module, store, env)

diff --git a/arbos/block_processor.go b/arbos/block_processor.go
@@ -144,6 +144,7 @@ func ProduceBlock(
 	chainContext core.ChainContext,
 	chainConfig *params.ChainConfig,
 	isMsgForPrefetch bool,
+	runMode core.MessageRunMode,
 ) (*types.Block, types.Receipts, error) {
 	txes, err := ParseL2Transactions(message, chainConfig.ChainID)
 	if err != nil {
@@ -153,7 +154,7 @@ func ProduceBlock(
 
 	hooks := NoopSequencingHooks()
 	return ProduceBlockAdvanced(
-		message.Header, txes, delayedMessagesRead, lastBlockHeader, statedb, chainContext, chainConfig, hooks, isMsgForPrefetch,
+		message.Header, txes, delayedMessagesRead, lastBlockHeader, statedb, chainContext, chainConfig, hooks, isMsgForPrefetch, runMode,
 	)
 }
 
@@ -168,6 +169,7 @@ func ProduceBlockAdvanced(
 	chainConfig *params.ChainConfig,
 	sequencingHooks *SequencingHooks,
 	isMsgForPrefetch bool,
+	runMode core.MessageRunMode,
 ) (*types.Block, types.Receipts, error) {
 
 	state, err := arbosState.OpenSystemArbosState(statedb, nil, true)
@@ -318,6 +320,7 @@ func ProduceBlockAdvanced(
 				tx,
 				&header.GasUsed,
 				vm.Config{},
+				runMode,
 				func(result *core.ExecutionResult) error {
 					return hooks.PostTxFilter(header, state, tx, sender, dataGas, result)
 				},

diff --git a/arbos/programs/native.go b/arbos/programs/native.go
@@ -47,11 +47,16 @@ type rustBytes = C.RustBytes
 type rustSlice = C.RustSlice
 
 var (
-	stylusLRUCacheSizeBytesGauge        = metrics.NewRegisteredGauge("arb/arbos/stylus/cache/lru/size_bytes", nil)
-	stylusLRUCacheSizeCountGauge        = metrics.NewRegisteredGauge("arb/arbos/stylus/cache/lru/count", nil)
-	stylusLRUCacheSizeHitsCounter       = metrics.NewRegisteredCounter("arb/arbos/stylus/cache/lru/hits", nil)
-	stylusLRUCacheSizeMissesCounter     = metrics.NewRegisteredCounter("arb/arbos/stylus/cache/lru/misses", nil)
-	stylusLRUCacheSizeDoesNotFitCounter = metrics.NewRegisteredCounter("arb/arbos/stylus/cache/lru/does_not_fit", nil)
+	stylusLRUCacheSizeBytesGauge    = metrics.NewRegisteredGauge("arb/arbos/stylus/cache/lru/size_bytes", nil)
+	stylusLRUCacheCountGauge        = metrics.NewRegisteredGauge("arb/arbos/stylus/cache/lru/count", nil)
+	stylusLRUCacheHitsCounter       = metrics.NewRegisteredCounter("arb/arbos/stylus/cache/lru/hits", nil)
+	stylusLRUCacheMissesCounter     = metrics.NewRegisteredCounter("arb/arbos/stylus/cache/lru/misses", nil)
+	stylusLRUCacheDoesNotFitCounter = metrics.NewRegisteredCounter("arb/arbos/stylus/cache/lru/does_not_fit", nil)
+
+	stylusLongTermCacheSizeBytesGauge = metrics.NewRegisteredGauge("arb/arbos/stylus/cache/long_term/size_bytes", nil)
+	stylusLongTermCacheCountGauge     = metrics.NewRegisteredGauge("arb/arbos/stylus/cache/long_term/count", nil)
+	stylusLongTermCacheHitsCounter    = metrics.NewRegisteredCounter("arb/arbos/stylus/cache/long_term/hits", nil)
+	stylusLongTermCacheMissesCounter  = metrics.NewRegisteredCounter("arb/arbos/stylus/cache/long_term/misses", nil)
 )
 
 func activateProgram(
@@ -333,24 +338,52 @@ func SetWasmLruCacheCapacity(capacityBytes uint64) {
 	C.stylus_set_cache_lru_capacity(u64(capacityBytes))
 }
 
-// exported for testing
+func UpdateWasmCacheMetrics() {
+	metrics := C.stylus_get_cache_metrics()
+
+	stylusLRUCacheSizeBytesGauge.Update(int64(metrics.lru.size_bytes))
+	stylusLRUCacheCountGauge.Update(int64(metrics.lru.count))
+	stylusLRUCacheHitsCounter.Inc(int64(metrics.lru.hits))
+	stylusLRUCacheMissesCounter.Inc(int64(metrics.lru.misses))
+	stylusLRUCacheDoesNotFitCounter.Inc(int64(metrics.lru.does_not_fit))
+
+	stylusLongTermCacheSizeBytesGauge.Update(int64(metrics.long_term.size_bytes))
+	stylusLongTermCacheCountGauge.Update(int64(metrics.long_term.count))
+	stylusLongTermCacheHitsCounter.Inc(int64(metrics.long_term.hits))
+	stylusLongTermCacheMissesCounter.Inc(int64(metrics.long_term.misses))
+}
+
+// Used for testing
 type WasmLruCacheMetrics struct {
 	SizeBytes uint64
 	Count     uint32
 }
 
-func GetWasmLruCacheMetrics() *WasmLruCacheMetrics {
-	metrics := C.stylus_get_lru_cache_metrics()
+// Used for testing
+type WasmLongTermCacheMetrics struct {
+	SizeBytes uint64
+	Count     uint32
+}
 
-	stylusLRUCacheSizeBytesGauge.Update(int64(metrics.size_bytes))
-	stylusLRUCacheSizeCountGauge.Update(int64(metrics.count))
-	stylusLRUCacheSizeHitsCounter.Inc(int64(metrics.hits))
-	stylusLRUCacheSizeMissesCounter.Inc(int64(metrics.misses))
-	stylusLRUCacheSizeDoesNotFitCounter.Inc(int64(metrics.does_not_fit))
+// Used for testing
+type WasmCacheMetrics struct {
+	Lru      WasmLruCacheMetrics
+	LongTerm WasmLongTermCacheMetrics
+}
 
-	return &WasmLruCacheMetrics{
-		SizeBytes: uint64(metrics.size_bytes),
-		Count:     uint32(metrics.count),
+// Used for testing
+func GetWasmCacheMetrics() *WasmCacheMetrics {
+	metrics := C.stylus_get_cache_metrics()
+
+	return &WasmCacheMetrics{
+		Lru: WasmLruCacheMetrics{
+			SizeBytes: uint64(metrics.lru.size_bytes),
+			Count:     uint32(metrics.lru.count),
+		},
+		LongTerm: WasmLongTermCacheMetrics{
+			SizeBytes: uint64(metrics.long_term.size_bytes),
+			Count:     uint32(metrics.long_term.count),
+		},
 	}
 }
 
@@ -360,8 +393,13 @@ func ClearWasmLruCache() {
 }
 
 // Used for testing
-func GetLruEntrySizeEstimateBytes(module []byte, version uint16, debug bool) uint64 {
-	return uint64(C.stylus_get_lru_entry_size_estimate_bytes(goSlice(module), u16(version), cbool(debug)))
+func ClearWasmLongTermCache() {
+	C.stylus_clear_long_term_cache()
+}
+
+// Used for testing
+func GetEntrySizeEstimateBytes(module []byte, version uint16, debug bool) uint64 {
+	return uint64(C.stylus_get_entry_size_estimate_bytes(goSlice(module), u16(version), cbool(debug)))
 }
 
 const DefaultTargetDescriptionArm = "arm64-linux-unknown+neon"

diff --git a/cmd/replay/main.go b/cmd/replay/main.go
@@ -14,6 +14,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/consensus"
+	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
@@ -291,7 +292,7 @@ func main() {
 		message := readMessage(chainConfig.ArbitrumChainParams.DataAvailabilityCommittee)
 
 		chainContext := WavmChainContext{}
-		newBlock, _, err = arbos.ProduceBlock(message.Message, message.DelayedMessagesRead, lastBlockHeader, statedb, chainContext, chainConfig, false)
+		newBlock, _, err = arbos.ProduceBlock(message.Message, message.DelayedMessagesRead, lastBlockHeader, statedb, chainContext, chainConfig, false, core.MessageReplayMode)
 		if err != nil {
 			panic(err)
 		}