1use crate::types::{
2 ModuleInfo, ModuleMetadata, ModuleMetadataDeserializer, ModuleMetadataSerializer,
3};
4use massa_hash::{Hash, HASH_SIZE_BYTES};
5use massa_sc_runtime::{CondomLimits, GasCosts, RuntimeModule};
6use massa_serialization::{DeserializeError, Deserializer, Serializer};
7use rand::RngCore;
8use rocksdb::{Direction, IteratorMode, Options, WriteBatch, DB};
9use std::path::{Path, PathBuf};
10use tracing::{debug, warn};
11
12const OPEN_ERROR: &str = "critical: rocksdb open operation failed";
13const DESTROY_ERROR: &str = "critical: could not remove the previous module cache db";
14const CRUD_ERROR: &str = "critical: rocksdb crud operation failed";
15const DATA_SER_ERROR: &str = "critical: metadata serialization failed";
16const DATA_DESER_ERROR: &str = "critical: metadata deserialization failed";
17const MOD_SER_ERROR: &str = "critical: module serialization failed";
18const MOD_DESER_ERROR: &str = "critical: module deserialization failed";
19const MODULE_IDENT: u8 = 0u8;
20const DATA_IDENT: u8 = 1u8;
21
22#[macro_export]
24macro_rules! module_key {
25 ($bc_hash:expr) => {
26 [&$bc_hash.to_bytes()[..], &[MODULE_IDENT]].concat()
27 };
28}
29
30#[macro_export]
32macro_rules! metadata_key {
33 ($bc_hash:expr) => {
34 [&$bc_hash.to_bytes()[..], &[DATA_IDENT]].concat()
35 };
36}
37
38fn destroy_db(path: &Path) {
43 if !path.exists() {
44 return;
45 }
46 if let Err(e) = DB::destroy(&Options::default(), path) {
47 warn!(
48 "Failed to destroy the db: {:?}, removing its directory instead",
49 e
50 );
51 std::fs::remove_dir_all(path).expect(DESTROY_ERROR);
52 }
53}
54
55pub(crate) struct HDCache {
56 db: Option<DB>,
58 entry_count: usize,
61 max_entry_count: usize,
64 snip_amount: usize,
66 meta_ser: ModuleMetadataSerializer,
68 meta_deser: ModuleMetadataDeserializer,
70 #[cfg(test)]
73 pub(crate) read_count: std::sync::atomic::AtomicUsize,
74}
75
76impl HDCache {
77 pub fn new(path: PathBuf, max_entry_count: usize, snip_amount: usize) -> Self {
84 destroy_db(&path);
86 let db = DB::open_default(path).expect(OPEN_ERROR);
87 let entry_count = 0;
88
89 Self {
90 db: Some(db),
91 entry_count,
92 max_entry_count,
93 snip_amount,
94 meta_ser: ModuleMetadataSerializer::new(),
95 meta_deser: ModuleMetadataDeserializer::new(),
96 #[cfg(test)]
97 read_count: std::sync::atomic::AtomicUsize::new(0),
98 }
99 }
100
101 pub fn reset(&mut self) {
102 let path = self.db.as_ref().unwrap().path().to_path_buf();
103
104 let _ = self.db.take();
106
107 destroy_db(&path);
109 let db = DB::open_default(&path).expect(OPEN_ERROR);
111 self.db = Some(db);
112 self.entry_count = 0;
113 }
114
115 fn contains(&self, hash: Hash) -> bool {
118 self.db
119 .as_ref()
120 .expect(CRUD_ERROR)
121 .get_pinned(module_key!(hash))
122 .expect(CRUD_ERROR)
123 .is_some()
124 }
125
126 pub fn insert(&mut self, hash: Hash, module_info: ModuleInfo) {
128 let is_new = !self.contains(hash);
130 if is_new && self.entry_count >= self.max_entry_count {
131 self.snip();
132 }
133
134 let mut ser_metadata = Vec::new();
135 let ser_module = match module_info {
136 ModuleInfo::Invalid(err_msg) => {
137 self.meta_ser
138 .serialize(&ModuleMetadata::Invalid(err_msg), &mut ser_metadata)
139 .expect(DATA_SER_ERROR);
140 Vec::new()
141 }
142 ModuleInfo::Module(module) => {
143 self.meta_ser
144 .serialize(&ModuleMetadata::NotExecuted, &mut ser_metadata)
145 .expect(DATA_SER_ERROR);
146 module.serialize().expect(MOD_SER_ERROR)
147 }
148 ModuleInfo::ModuleAndDelta((module, delta)) => {
149 self.meta_ser
150 .serialize(&ModuleMetadata::Delta(delta), &mut ser_metadata)
151 .expect(DATA_SER_ERROR);
152 module.serialize().expect(MOD_SER_ERROR)
153 }
154 };
155
156 let mut batch = WriteBatch::default();
157 batch.put(module_key!(hash), ser_module);
158 batch.put(metadata_key!(hash), ser_metadata);
159 self.db
160 .as_ref()
161 .expect(CRUD_ERROR)
162 .write(&batch)
163 .expect(CRUD_ERROR);
164
165 if is_new {
166 self.entry_count = self.entry_count.saturating_add(1);
167 }
168
169 debug!("(HD insert) entry_count is: {}", self.entry_count);
170 }
171
172 pub fn set_init_cost(&self, hash: Hash, init_cost: u64) {
178 if !self.contains(hash) {
181 return;
182 }
183 let mut ser_metadata = Vec::new();
184 self.meta_ser
185 .serialize(&ModuleMetadata::Delta(init_cost), &mut ser_metadata)
186 .expect(DATA_SER_ERROR);
187 self.db
188 .as_ref()
189 .expect(CRUD_ERROR)
190 .put(metadata_key!(hash), ser_metadata)
191 .expect(CRUD_ERROR);
192 }
193
194 pub fn set_invalid(&self, hash: Hash, err_msg: String) {
196 if !self.contains(hash) {
198 return;
199 }
200 let mut ser_metadata = Vec::new();
201 self.meta_ser
202 .serialize(&ModuleMetadata::Invalid(err_msg), &mut ser_metadata)
203 .expect(DATA_SER_ERROR);
204 self.db
205 .as_ref()
206 .expect(CRUD_ERROR)
207 .put(metadata_key!(hash), ser_metadata)
208 .expect(CRUD_ERROR);
209 }
210
211 pub fn get(
213 &self,
214 hash: Hash,
215 gas_costs: GasCosts,
216 condom_limits: CondomLimits,
217 ) -> Option<ModuleInfo> {
218 #[cfg(test)]
219 self.read_count
220 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
221
222 let mut iterator = self
223 .db
224 .as_ref()
225 .expect(CRUD_ERROR)
226 .iterator(IteratorMode::From(&module_key!(hash), Direction::Forward));
227
228 if let (Some(Ok((key_1, ser_module))), Some(Ok((key_2, ser_metadata)))) =
229 (iterator.next(), iterator.next())
230 {
231 if *key_1 == module_key!(hash) && *key_2 == metadata_key!(hash) {
232 let (_, metadata) = self
233 .meta_deser
234 .deserialize::<DeserializeError>(&ser_metadata)
235 .expect(DATA_DESER_ERROR);
236 if let ModuleMetadata::Invalid(err_msg) = metadata {
237 return Some(ModuleInfo::Invalid(err_msg));
238 }
239 let module = RuntimeModule::deserialize(
240 &ser_module,
241 gas_costs.max_instance_cost,
242 gas_costs,
243 condom_limits,
244 )
245 .expect(MOD_DESER_ERROR);
246 let result = match metadata {
247 ModuleMetadata::Invalid(err_msg) => ModuleInfo::Invalid(err_msg),
248 ModuleMetadata::NotExecuted => ModuleInfo::Module(module),
249 ModuleMetadata::Delta(delta) => ModuleInfo::ModuleAndDelta((module, delta)),
250 };
251 Some(result)
252 } else {
253 None
254 }
255 } else {
256 None
257 }
258 }
259
260 fn snip(&mut self) {
266 let db = self.db.as_ref().expect(CRUD_ERROR);
267 let mut batch = WriteBatch::default();
268 let mut snipped_count: usize = 0;
269
270 let mut rbytes = [0u8; 16];
272 rand::thread_rng().fill_bytes(&mut rbytes);
273 let start = *Hash::compute_from(&rbytes).to_bytes();
274
275 let mut iter = db.raw_iterator();
276 iter.seek(start);
277 let mut wrapped = false;
278 while snipped_count < self.snip_amount {
279 let Some(key) = iter.key() else {
280 iter.status().expect(CRUD_ERROR);
282 if wrapped {
283 break;
284 }
285 wrapped = true;
286 iter.seek_to_first();
287 continue;
288 };
289 if wrapped && key >= &start[..] {
290 break;
292 }
293
294 batch.delete(key);
295 if key.len() == HASH_SIZE_BYTES + 1 && key[HASH_SIZE_BYTES] == MODULE_IDENT {
296 let mut metadata_key = key.to_vec();
298 metadata_key[HASH_SIZE_BYTES] = DATA_IDENT;
299 batch.delete(metadata_key);
300 snipped_count += 1;
301 }
302 iter.next();
303 }
304
305 db.write(&batch).expect(CRUD_ERROR);
307 self.entry_count = self.entry_count.saturating_sub(snipped_count);
308 }
309}
310
311#[cfg(test)]
312mod tests {
313 use super::*;
314 use massa_hash::Hash;
315 use massa_sc_runtime::{Compiler, GasCosts, RuntimeModule};
316 use rand::thread_rng;
317 use serial_test::serial;
318 use tempfile::TempDir;
319
320 fn make_default_module_info() -> ModuleInfo {
321 let bytecode: Vec<u8> = vec![
322 0x00, 0x61, 0x73, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x01, 0x06, 0x01, 0x60, 0x01, 0x7f,
323 0x01, 0x7f, 0x03, 0x02, 0x01, 0x00, 0x07, 0x0b, 0x01, 0x07, 0x61, 0x64, 0x64, 0x5f,
324 0x6f, 0x6e, 0x65, 0x00, 0x00, 0x0a, 0x09, 0x01, 0x07, 0x00, 0x20, 0x00, 0x41, 0x01,
325 0x6a, 0x0b, 0x00, 0x1a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x01, 0x0a, 0x01, 0x00, 0x07,
326 0x61, 0x64, 0x64, 0x5f, 0x6f, 0x6e, 0x65, 0x02, 0x07, 0x01, 0x00, 0x01, 0x00, 0x02,
327 0x70, 0x30,
328 ];
329 ModuleInfo::Module(
330 RuntimeModule::new(
331 &bytecode,
332 GasCosts::default(),
333 Compiler::CL,
334 CondomLimits::default(),
335 )
336 .unwrap(),
337 )
338 }
339
340 fn setup() -> HDCache {
341 let tmp_path = TempDir::new().unwrap().path().to_path_buf();
342 HDCache::new(tmp_path, 1000, 10)
343 }
344
345 #[test]
346 #[serial]
347 fn test_basic_crud() {
348 let mut cache = setup();
349 let hash = Hash::compute_from(b"test_hash");
350 let module = make_default_module_info();
351
352 let init_cost = 100;
353 let gas_costs = GasCosts::default();
354 let condom_limits = CondomLimits::default();
355
356 cache.insert(hash, module);
357 let cached_module_v1 = cache
358 .get(hash, gas_costs.clone(), condom_limits.clone())
359 .unwrap();
360 assert!(matches!(cached_module_v1, ModuleInfo::Module(_)));
361
362 cache.set_init_cost(hash, init_cost);
363 let cached_module_v2 = cache
364 .get(hash, gas_costs.clone(), condom_limits.clone())
365 .unwrap();
366 assert!(matches!(cached_module_v2, ModuleInfo::ModuleAndDelta(_)));
367
368 let err_msg = "test_error".to_string();
369 cache.set_invalid(hash, err_msg.clone());
370 let cached_module_v3 = cache.get(hash, gas_costs, condom_limits.clone()).unwrap();
371 let ModuleInfo::Invalid(res_err) = cached_module_v3 else {
372 panic!("expected ModuleInfo::Invalid");
373 };
374 assert_eq!(res_err, err_msg);
375 }
376
377 #[test]
378 #[serial]
379 fn test_insert_more_than_max_entry() {
380 let mut cache = setup();
381 let module = make_default_module_info();
382
383 for count in 0..cache.max_entry_count {
385 let key = Hash::compute_from(count.to_string().as_bytes());
386 cache.insert(key, module.clone());
387 }
388 assert_eq!(cache.entry_count, cache.max_entry_count);
389
390 let key = Hash::compute_from(cache.max_entry_count.to_string().as_bytes());
392 cache.insert(key, module);
393 assert_eq!(
394 cache.entry_count,
395 cache.max_entry_count - cache.snip_amount + 1
396 );
397 dbg!(cache.entry_count);
398 }
399
400 fn stored_module_count(cache: &HDCache) -> usize {
402 cache
403 .db
404 .as_ref()
405 .unwrap()
406 .iterator(IteratorMode::Start)
407 .filter(|kv| {
408 let (key, _) = kv.as_ref().unwrap();
409 key.len() == HASH_SIZE_BYTES + 1 && key[HASH_SIZE_BYTES] == MODULE_IDENT
410 })
411 .count()
412 }
413
414 #[test]
415 #[serial]
416 fn test_metadata_update_of_missing_module_is_ignored() {
417 let cache = setup();
418 let hash = Hash::compute_from(b"never_inserted");
419
420 cache.set_init_cost(hash, 100);
421 cache.set_invalid(hash, "test_error".to_string());
422
423 let mut iter = cache.db.as_ref().unwrap().iterator(IteratorMode::Start);
425 assert!(iter.next().is_none());
426 }
427
428 #[test]
429 #[serial]
430 fn test_overwrite_does_not_grow_entry_count() {
431 let mut cache = setup();
432 let hash = Hash::compute_from(b"test_hash");
433 let module = make_default_module_info();
434
435 cache.insert(hash, module.clone());
436 cache.insert(hash, module);
437 assert_eq!(cache.entry_count, 1);
438 assert_eq!(stored_module_count(&cache), 1);
439 }
440
441 #[test]
442 #[serial]
443 fn test_snip_survives_stray_keys() {
444 let tmp_path = TempDir::new().unwrap().path().to_path_buf();
445 let mut cache = HDCache::new(tmp_path, 20, 5);
446 let module = make_default_module_info();
447
448 let db = cache.db.as_ref().unwrap();
452 db.put(metadata_key!(Hash::from_bytes(&[0u8; 32])), [0u8])
453 .unwrap();
454 db.put([0xAAu8; 7], [0u8]).unwrap();
455
456 for count in 0..500 {
457 let key = Hash::compute_from(count.to_string().as_bytes());
458 cache.insert(key, module.clone());
459 assert!(cache.entry_count <= cache.max_entry_count);
460 assert_eq!(cache.entry_count, stored_module_count(&cache));
461 }
462 }
463
464 #[test]
465 #[serial]
466 fn test_new_and_reset_start_from_an_empty_db() {
467 let tmp_dir = TempDir::new().unwrap();
468 let path = tmp_dir.path().join("hd_cache");
469 let hash = Hash::compute_from(b"test_hash");
470 let gas_costs = GasCosts::default();
471 let condom_limits = CondomLimits::default();
472 let is_empty = |cache: &HDCache| {
473 let mut iter = cache.db.as_ref().unwrap().iterator(IteratorMode::Start);
474 iter.next().is_none()
475 };
476
477 {
479 let mut cache = HDCache::new(path.clone(), 1000, 10);
480 cache.insert(hash, make_default_module_info());
481 cache.db.as_ref().unwrap().put([0xAAu8; 7], [0u8]).unwrap();
482 }
483
484 let mut cache = HDCache::new(path, 1000, 10);
486 assert!(is_empty(&cache));
487 assert!(cache
488 .get(hash, gas_costs.clone(), condom_limits.clone())
489 .is_none());
490
491 cache.insert(hash, make_default_module_info());
493 cache.reset();
494 assert!(is_empty(&cache));
495 assert_eq!(cache.entry_count, 0);
496 assert!(cache.get(hash, gas_costs, condom_limits).is_none());
497 }
498
499 #[test]
500 #[serial]
501 fn test_snip_amount_larger_than_entry_count() {
502 let tmp_path = TempDir::new().unwrap().path().to_path_buf();
503 let mut cache = HDCache::new(tmp_path, 3, 10);
504 let module = make_default_module_info();
505
506 for count in 0..10 {
507 let key = Hash::compute_from(count.to_string().as_bytes());
508 cache.insert(key, module.clone());
509 assert_eq!(cache.entry_count, stored_module_count(&cache));
510 }
511 }
512
513 #[test]
514 #[serial]
515 fn test_missing_module() {
516 let mut cache = setup();
517 let module = make_default_module_info();
518
519 let gas_costs = GasCosts::default();
520 let condom_limits = CondomLimits::default();
521
522 for count in 0..cache.max_entry_count {
523 let key = Hash::compute_from(count.to_string().as_bytes());
524 cache.insert(key, module.clone());
525 }
526
527 for _ in 0..cache.max_entry_count {
528 let mut rbytes = [0u8; 16];
529 thread_rng().fill_bytes(&mut rbytes);
530 let get_key = Hash::compute_from(&rbytes);
531 let cached_module = cache.get(get_key, gas_costs.clone(), condom_limits.clone());
532 assert!(cached_module.is_none());
533 }
534 }
535}