massa_module_cache/
hd_cache.rs

1use crate::types::{
2    ModuleInfo, ModuleMetadata, ModuleMetadataDeserializer, ModuleMetadataSerializer,
3};
4use massa_hash::{Hash, HASH_SIZE_BYTES};
5use massa_sc_runtime::{CondomLimits, GasCosts, RuntimeModule};
6use massa_serialization::{DeserializeError, Deserializer, Serializer};
7use rand::RngCore;
8use rocksdb::{Direction, IteratorMode, Options, WriteBatch, DB};
9use std::path::{Path, PathBuf};
10use tracing::{debug, warn};
11
12const OPEN_ERROR: &str = "critical: rocksdb open operation failed";
13const DESTROY_ERROR: &str = "critical: could not remove the previous module cache db";
14const CRUD_ERROR: &str = "critical: rocksdb crud operation failed";
15const DATA_SER_ERROR: &str = "critical: metadata serialization failed";
16const DATA_DESER_ERROR: &str = "critical: metadata deserialization failed";
17const MOD_SER_ERROR: &str = "critical: module serialization failed";
18const MOD_DESER_ERROR: &str = "critical: module deserialization failed";
19const MODULE_IDENT: u8 = 0u8;
20const DATA_IDENT: u8 = 1u8;
21
22/// Module key formatting macro
23#[macro_export]
24macro_rules! module_key {
25    ($bc_hash:expr) => {
26        [&$bc_hash.to_bytes()[..], &[MODULE_IDENT]].concat()
27    };
28}
29
30/// Delta key formatting macro
31#[macro_export]
32macro_rules! metadata_key {
33    ($bc_hash:expr) => {
34        [&$bc_hash.to_bytes()[..], &[DATA_IDENT]].concat()
35    };
36}
37
38/// Remove the db at `path` if there is one, so that the cache always starts empty.
39///
40/// `entry_count` starts at 0, so silently reopening a db that could not be destroyed would
41/// reuse stale entries (possibly left in a bad state) that are not accounted for.
42fn destroy_db(path: &Path) {
43    if !path.exists() {
44        return;
45    }
46    if let Err(e) = DB::destroy(&Options::default(), path) {
47        warn!(
48            "Failed to destroy the db: {:?}, removing its directory instead",
49            e
50        );
51        std::fs::remove_dir_all(path).expect(DESTROY_ERROR);
52    }
53}
54
55pub(crate) struct HDCache {
56    /// RocksDB database
57    db: Option<DB>,
58    /// How many entries are in the db. Count is initialized at creation time by iterating
59    /// over all the entries in the db then it is maintained in memory
60    entry_count: usize,
61    /// Maximum number of entries we want to keep in the db.
62    /// When this maximum is reached `snip_amount` entries are removed
63    max_entry_count: usize,
64    /// How many entries are removed when `entry_count` reaches `max_entry_count`
65    snip_amount: usize,
66    /// Module metadata serializer
67    meta_ser: ModuleMetadataSerializer,
68    /// Module metadata deserializer
69    meta_deser: ModuleMetadataDeserializer,
70    /// Test-only counter of `get` calls, used to assert that no redundant
71    /// RocksDB read/deserialize happens when a module is already in the LRU cache.
72    #[cfg(test)]
73    pub(crate) read_count: std::sync::atomic::AtomicUsize,
74}
75
76impl HDCache {
77    /// Create a new HDCache
78    ///
79    /// # Arguments
80    /// * path: where to store the db
81    /// * max_entry_count: maximum number of entries we want to keep in the db
82    /// * amount_to_remove: how many entries are removed when `entry_count` reaches `max_entry_count`
83    pub fn new(path: PathBuf, max_entry_count: usize, snip_amount: usize) -> Self {
84        // Reset the DB if it already exists
85        destroy_db(&path);
86        let db = DB::open_default(path).expect(OPEN_ERROR);
87        let entry_count = 0;
88
89        Self {
90            db: Some(db),
91            entry_count,
92            max_entry_count,
93            snip_amount,
94            meta_ser: ModuleMetadataSerializer::new(),
95            meta_deser: ModuleMetadataDeserializer::new(),
96            #[cfg(test)]
97            read_count: std::sync::atomic::AtomicUsize::new(0),
98        }
99    }
100
101    pub fn reset(&mut self) {
102        let path = self.db.as_ref().unwrap().path().to_path_buf();
103
104        // Close the existing database by dropping it
105        let _ = self.db.take();
106
107        // Destroy the database files
108        destroy_db(&path);
109        // Reopen the database
110        let db = DB::open_default(&path).expect(OPEN_ERROR);
111        self.db = Some(db);
112        self.entry_count = 0;
113    }
114
115    /// Whether a module is stored for `hash`.
116    /// A module key is always written together with its metadata key, see `insert`.
117    fn contains(&self, hash: Hash) -> bool {
118        self.db
119            .as_ref()
120            .expect(CRUD_ERROR)
121            .get_pinned(module_key!(hash))
122            .expect(CRUD_ERROR)
123            .is_some()
124    }
125
126    /// Insert a new module in the cache, or overwrite an already stored one
127    pub fn insert(&mut self, hash: Hash, module_info: ModuleInfo) {
128        // only count new entries, so that `entry_count` matches the stored modules
129        let is_new = !self.contains(hash);
130        if is_new && self.entry_count >= self.max_entry_count {
131            self.snip();
132        }
133
134        let mut ser_metadata = Vec::new();
135        let ser_module = match module_info {
136            ModuleInfo::Invalid(err_msg) => {
137                self.meta_ser
138                    .serialize(&ModuleMetadata::Invalid(err_msg), &mut ser_metadata)
139                    .expect(DATA_SER_ERROR);
140                Vec::new()
141            }
142            ModuleInfo::Module(module) => {
143                self.meta_ser
144                    .serialize(&ModuleMetadata::NotExecuted, &mut ser_metadata)
145                    .expect(DATA_SER_ERROR);
146                module.serialize().expect(MOD_SER_ERROR)
147            }
148            ModuleInfo::ModuleAndDelta((module, delta)) => {
149                self.meta_ser
150                    .serialize(&ModuleMetadata::Delta(delta), &mut ser_metadata)
151                    .expect(DATA_SER_ERROR);
152                module.serialize().expect(MOD_SER_ERROR)
153            }
154        };
155
156        let mut batch = WriteBatch::default();
157        batch.put(module_key!(hash), ser_module);
158        batch.put(metadata_key!(hash), ser_metadata);
159        self.db
160            .as_ref()
161            .expect(CRUD_ERROR)
162            .write(&batch)
163            .expect(CRUD_ERROR);
164
165        if is_new {
166            self.entry_count = self.entry_count.saturating_add(1);
167        }
168
169        debug!("(HD insert) entry_count is: {}", self.entry_count);
170    }
171
172    /// Sets the initialization cost of a given module separately
173    ///
174    /// # Arguments
175    /// * `hash`: hash associated to the module for which we want to set the cost
176    /// * `init_cost`: the new cost associated to the module
177    pub fn set_init_cost(&self, hash: Hash, init_cost: u64) {
178        // The module may be absent (snipped, or only in the LRU cache): writing its metadata
179        // alone would leave an orphan key, so skip it.
180        if !self.contains(hash) {
181            return;
182        }
183        let mut ser_metadata = Vec::new();
184        self.meta_ser
185            .serialize(&ModuleMetadata::Delta(init_cost), &mut ser_metadata)
186            .expect(DATA_SER_ERROR);
187        self.db
188            .as_ref()
189            .expect(CRUD_ERROR)
190            .put(metadata_key!(hash), ser_metadata)
191            .expect(CRUD_ERROR);
192    }
193
194    /// Sets a given module as invalid
195    pub fn set_invalid(&self, hash: Hash, err_msg: String) {
196        // see `set_init_cost`
197        if !self.contains(hash) {
198            return;
199        }
200        let mut ser_metadata = Vec::new();
201        self.meta_ser
202            .serialize(&ModuleMetadata::Invalid(err_msg), &mut ser_metadata)
203            .expect(DATA_SER_ERROR);
204        self.db
205            .as_ref()
206            .expect(CRUD_ERROR)
207            .put(metadata_key!(hash), ser_metadata)
208            .expect(CRUD_ERROR);
209    }
210
211    /// Retrieve a module
212    pub fn get(
213        &self,
214        hash: Hash,
215        gas_costs: GasCosts,
216        condom_limits: CondomLimits,
217    ) -> Option<ModuleInfo> {
218        #[cfg(test)]
219        self.read_count
220            .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
221
222        let mut iterator = self
223            .db
224            .as_ref()
225            .expect(CRUD_ERROR)
226            .iterator(IteratorMode::From(&module_key!(hash), Direction::Forward));
227
228        if let (Some(Ok((key_1, ser_module))), Some(Ok((key_2, ser_metadata)))) =
229            (iterator.next(), iterator.next())
230        {
231            if *key_1 == module_key!(hash) && *key_2 == metadata_key!(hash) {
232                let (_, metadata) = self
233                    .meta_deser
234                    .deserialize::<DeserializeError>(&ser_metadata)
235                    .expect(DATA_DESER_ERROR);
236                if let ModuleMetadata::Invalid(err_msg) = metadata {
237                    return Some(ModuleInfo::Invalid(err_msg));
238                }
239                let module = RuntimeModule::deserialize(
240                    &ser_module,
241                    gas_costs.max_instance_cost,
242                    gas_costs,
243                    condom_limits,
244                )
245                .expect(MOD_DESER_ERROR);
246                let result = match metadata {
247                    ModuleMetadata::Invalid(err_msg) => ModuleInfo::Invalid(err_msg),
248                    ModuleMetadata::NotExecuted => ModuleInfo::Module(module),
249                    ModuleMetadata::Delta(delta) => ModuleInfo::ModuleAndDelta((module, delta)),
250                };
251                Some(result)
252            } else {
253                None
254            }
255        } else {
256            None
257        }
258    }
259
260    /// Try to remove as much as `self.snip_amount` entries from the db.
261    ///
262    /// Walks the keys from a random point of the key space, wrapping around once, and removes
263    /// every visited key. It does not assume that module and metadata keys come in pairs, so a
264    /// stray key is removed instead of crashing the node, and only removed modules are counted.
265    fn snip(&mut self) {
266        let db = self.db.as_ref().expect(CRUD_ERROR);
267        let mut batch = WriteBatch::default();
268        let mut snipped_count: usize = 0;
269
270        // random starting point
271        let mut rbytes = [0u8; 16];
272        rand::thread_rng().fill_bytes(&mut rbytes);
273        let start = *Hash::compute_from(&rbytes).to_bytes();
274
275        let mut iter = db.raw_iterator();
276        iter.seek(start);
277        let mut wrapped = false;
278        while snipped_count < self.snip_amount {
279            let Some(key) = iter.key() else {
280                // end of the db (or read error): wrap around once
281                iter.status().expect(CRUD_ERROR);
282                if wrapped {
283                    break;
284                }
285                wrapped = true;
286                iter.seek_to_first();
287                continue;
288            };
289            if wrapped && key >= &start[..] {
290                // every key has been visited
291                break;
292            }
293
294            batch.delete(key);
295            if key.len() == HASH_SIZE_BYTES + 1 && key[HASH_SIZE_BYTES] == MODULE_IDENT {
296                // its metadata key comes right after: delete it now in case we stop here
297                let mut metadata_key = key.to_vec();
298                metadata_key[HASH_SIZE_BYTES] = DATA_IDENT;
299                batch.delete(metadata_key);
300                snipped_count += 1;
301            }
302            iter.next();
303        }
304
305        // delete the keys and reduce entry_count
306        db.write(&batch).expect(CRUD_ERROR);
307        self.entry_count = self.entry_count.saturating_sub(snipped_count);
308    }
309}
310
311#[cfg(test)]
312mod tests {
313    use super::*;
314    use massa_hash::Hash;
315    use massa_sc_runtime::{Compiler, GasCosts, RuntimeModule};
316    use rand::thread_rng;
317    use serial_test::serial;
318    use tempfile::TempDir;
319
320    fn make_default_module_info() -> ModuleInfo {
321        let bytecode: Vec<u8> = vec![
322            0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x01, 0x06, 0x01, 0x60, 0x01, 0x7f,
323            0x01, 0x7f, 0x03, 0x02, 0x01, 0x00, 0x07, 0x0b, 0x01, 0x07, 0x61, 0x64, 0x64, 0x5f,
324            0x6f, 0x6e, 0x65, 0x00, 0x00, 0x0a, 0x09, 0x01, 0x07, 0x00, 0x20, 0x00, 0x41, 0x01,
325            0x6a, 0x0b, 0x00, 0x1a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x01, 0x0a, 0x01, 0x00, 0x07,
326            0x61, 0x64, 0x64, 0x5f, 0x6f, 0x6e, 0x65, 0x02, 0x07, 0x01, 0x00, 0x01, 0x00, 0x02,
327            0x70, 0x30,
328        ];
329        ModuleInfo::Module(
330            RuntimeModule::new(
331                &bytecode,
332                GasCosts::default(),
333                Compiler::CL,
334                CondomLimits::default(),
335            )
336            .unwrap(),
337        )
338    }
339
340    fn setup() -> HDCache {
341        let tmp_path = TempDir::new().unwrap().path().to_path_buf();
342        HDCache::new(tmp_path, 1000, 10)
343    }
344
345    #[test]
346    #[serial]
347    fn test_basic_crud() {
348        let mut cache = setup();
349        let hash = Hash::compute_from(b"test_hash");
350        let module = make_default_module_info();
351
352        let init_cost = 100;
353        let gas_costs = GasCosts::default();
354        let condom_limits = CondomLimits::default();
355
356        cache.insert(hash, module);
357        let cached_module_v1 = cache
358            .get(hash, gas_costs.clone(), condom_limits.clone())
359            .unwrap();
360        assert!(matches!(cached_module_v1, ModuleInfo::Module(_)));
361
362        cache.set_init_cost(hash, init_cost);
363        let cached_module_v2 = cache
364            .get(hash, gas_costs.clone(), condom_limits.clone())
365            .unwrap();
366        assert!(matches!(cached_module_v2, ModuleInfo::ModuleAndDelta(_)));
367
368        let err_msg = "test_error".to_string();
369        cache.set_invalid(hash, err_msg.clone());
370        let cached_module_v3 = cache.get(hash, gas_costs, condom_limits.clone()).unwrap();
371        let ModuleInfo::Invalid(res_err) = cached_module_v3 else {
372            panic!("expected ModuleInfo::Invalid");
373        };
374        assert_eq!(res_err, err_msg);
375    }
376
377    #[test]
378    #[serial]
379    fn test_insert_more_than_max_entry() {
380        let mut cache = setup();
381        let module = make_default_module_info();
382
383        // fill the db: add cache.max_entry_count entries
384        for count in 0..cache.max_entry_count {
385            let key = Hash::compute_from(count.to_string().as_bytes());
386            cache.insert(key, module.clone());
387        }
388        assert_eq!(cache.entry_count, cache.max_entry_count);
389
390        // insert one more entry
391        let key = Hash::compute_from(cache.max_entry_count.to_string().as_bytes());
392        cache.insert(key, module);
393        assert_eq!(
394            cache.entry_count,
395            cache.max_entry_count - cache.snip_amount + 1
396        );
397        dbg!(cache.entry_count);
398    }
399
400    /// Number of module keys stored in the db, i.e. the actual entry count
401    fn stored_module_count(cache: &HDCache) -> usize {
402        cache
403            .db
404            .as_ref()
405            .unwrap()
406            .iterator(IteratorMode::Start)
407            .filter(|kv| {
408                let (key, _) = kv.as_ref().unwrap();
409                key.len() == HASH_SIZE_BYTES + 1 && key[HASH_SIZE_BYTES] == MODULE_IDENT
410            })
411            .count()
412    }
413
414    #[test]
415    #[serial]
416    fn test_metadata_update_of_missing_module_is_ignored() {
417        let cache = setup();
418        let hash = Hash::compute_from(b"never_inserted");
419
420        cache.set_init_cost(hash, 100);
421        cache.set_invalid(hash, "test_error".to_string());
422
423        // no orphan metadata key is written
424        let mut iter = cache.db.as_ref().unwrap().iterator(IteratorMode::Start);
425        assert!(iter.next().is_none());
426    }
427
428    #[test]
429    #[serial]
430    fn test_overwrite_does_not_grow_entry_count() {
431        let mut cache = setup();
432        let hash = Hash::compute_from(b"test_hash");
433        let module = make_default_module_info();
434
435        cache.insert(hash, module.clone());
436        cache.insert(hash, module);
437        assert_eq!(cache.entry_count, 1);
438        assert_eq!(stored_module_count(&cache), 1);
439    }
440
441    #[test]
442    #[serial]
443    fn test_snip_survives_stray_keys() {
444        let tmp_path = TempDir::new().unwrap().path().to_path_buf();
445        let mut cache = HDCache::new(tmp_path, 20, 5);
446        let module = make_default_module_info();
447
448        // stray keys: an orphan metadata key that is the smallest possible key (the previous
449        // implementation stepped back from it past the start of the db and panicked), and a key
450        // that does not follow the cache layout
451        let db = cache.db.as_ref().unwrap();
452        db.put(metadata_key!(Hash::from_bytes(&[0u8; 32])), [0u8])
453            .unwrap();
454        db.put([0xAAu8; 7], [0u8]).unwrap();
455
456        for count in 0..500 {
457            let key = Hash::compute_from(count.to_string().as_bytes());
458            cache.insert(key, module.clone());
459            assert!(cache.entry_count <= cache.max_entry_count);
460            assert_eq!(cache.entry_count, stored_module_count(&cache));
461        }
462    }
463
464    #[test]
465    #[serial]
466    fn test_new_and_reset_start_from_an_empty_db() {
467        let tmp_dir = TempDir::new().unwrap();
468        let path = tmp_dir.path().join("hd_cache");
469        let hash = Hash::compute_from(b"test_hash");
470        let gas_costs = GasCosts::default();
471        let condom_limits = CondomLimits::default();
472        let is_empty = |cache: &HDCache| {
473            let mut iter = cache.db.as_ref().unwrap().iterator(IteratorMode::Start);
474            iter.next().is_none()
475        };
476
477        // a previous run left an entry and a stray key on disk
478        {
479            let mut cache = HDCache::new(path.clone(), 1000, 10);
480            cache.insert(hash, make_default_module_info());
481            cache.db.as_ref().unwrap().put([0xAAu8; 7], [0u8]).unwrap();
482        }
483
484        // reopening starts from scratch
485        let mut cache = HDCache::new(path, 1000, 10);
486        assert!(is_empty(&cache));
487        assert!(cache
488            .get(hash, gas_costs.clone(), condom_limits.clone())
489            .is_none());
490
491        // so does a reset
492        cache.insert(hash, make_default_module_info());
493        cache.reset();
494        assert!(is_empty(&cache));
495        assert_eq!(cache.entry_count, 0);
496        assert!(cache.get(hash, gas_costs, condom_limits).is_none());
497    }
498
499    #[test]
500    #[serial]
501    fn test_snip_amount_larger_than_entry_count() {
502        let tmp_path = TempDir::new().unwrap().path().to_path_buf();
503        let mut cache = HDCache::new(tmp_path, 3, 10);
504        let module = make_default_module_info();
505
506        for count in 0..10 {
507            let key = Hash::compute_from(count.to_string().as_bytes());
508            cache.insert(key, module.clone());
509            assert_eq!(cache.entry_count, stored_module_count(&cache));
510        }
511    }
512
513    #[test]
514    #[serial]
515    fn test_missing_module() {
516        let mut cache = setup();
517        let module = make_default_module_info();
518
519        let gas_costs = GasCosts::default();
520        let condom_limits = CondomLimits::default();
521
522        for count in 0..cache.max_entry_count {
523            let key = Hash::compute_from(count.to_string().as_bytes());
524            cache.insert(key, module.clone());
525        }
526
527        for _ in 0..cache.max_entry_count {
528            let mut rbytes = [0u8; 16];
529            thread_rng().fill_bytes(&mut rbytes);
530            let get_key = Hash::compute_from(&rbytes);
531            let cached_module = cache.get(get_key, gas_costs.clone(), condom_limits.clone());
532            assert!(cached_module.is_none());
533        }
534    }
535}