1use std::cmp::max;
3use std::collections::{BTreeMap, BTreeSet};
4use std::path::Path;
5use num_enum::IntoPrimitive;
7use rocksdb::{IteratorMode, Options, WriteBatch, DB};
8use tracing::{debug, warn};
9use crate::rocksdb_operator::counter_merge;
11use crate::ser_deser::{
12 SCOutputEventDeserializer, SCOutputEventDeserializerArgs, SCOutputEventSerializer,
13};
14use massa_models::address::Address;
15use massa_models::error::ModelsError;
16use massa_models::execution::EventFilter;
17use massa_models::operation::{OperationId, OperationIdSerializer};
18use massa_models::output_event::SCOutputEvent;
19use massa_models::slot::Slot;
20use massa_serialization::{DeserializeError, Deserializer, Serializer};
21
22const OPEN_ERROR: &str = "critical: rocksdb open operation failed";
23const DESTROY_ERROR: &str = "critical: rocksdb delete operation failed";
25const CRUD_ERROR: &str = "critical: rocksdb crud operation failed";
26const EVENT_DESER_ERROR: &str = "critical: event deserialization failed";
27const OPERATION_ID_DESER_ERROR: &str = "critical: deserialization failed for op id in rocksdb";
28const COUNTER_ERROR: &str = "critical: cannot get counter";
29const COUNTER_KEY_CREATION_ERROR: &str = "critical: cannot create counter key";
30
31#[allow(dead_code)]
32#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, IntoPrimitive)]
34#[repr(u8)]
35enum KeyIndent {
36 Counter = 0,
37 Event,
38 EmitterAddress,
39 OriginalCallerAddress,
40 OriginalOperationId,
41 IsError,
42 IsFinal,
43}
44
45enum KeyBuilderType<'a> {
47 Slot(&'a Slot),
48 Event(&'a Slot, u64),
49 Address(&'a Address),
50 OperationId(&'a OperationId),
51 Bool(bool),
52 None,
53}
54
55enum KeyKind {
56 Regular,
57 Prefix,
58 Counter,
59}
60
61struct DbKeyBuilder {
62 op_id_ser: OperationIdSerializer,
64}
65
66impl DbKeyBuilder {
67 fn new() -> Self {
68 Self {
69 op_id_ser: OperationIdSerializer::new(),
70 }
71 }
72
73 fn key(&self, indent: &KeyIndent, key_type: KeyBuilderType, key_kind: &KeyKind) -> Vec<u8> {
79 let mut key_base = if matches!(key_kind, KeyKind::Counter) {
80 vec![u8::from(KeyIndent::Counter), u8::from(*indent)]
81 } else {
82 vec![u8::from(*indent)]
83 };
84
85 match key_type {
86 KeyBuilderType::Slot(slot) => {
87 key_base.extend(slot.to_bytes_key());
88 }
89 KeyBuilderType::Event(slot, index) => {
90 key_base.extend(slot.to_bytes_key());
91 key_base.extend(index.to_be_bytes());
92 }
93 KeyBuilderType::Address(addr) => {
94 let addr_bytes = addr.to_prefixed_bytes();
95 let addr_bytes_len = addr_bytes.len();
96 key_base.extend(addr_bytes);
97 key_base.push(addr_bytes_len as u8);
98 }
99 KeyBuilderType::OperationId(op_id) => {
100 let mut buffer = Vec::new();
101 self.op_id_ser
102 .serialize(op_id, &mut buffer)
103 .expect(OPERATION_ID_DESER_ERROR);
104 key_base.extend(&buffer);
105 key_base.extend(u32::to_be_bytes(buffer.len() as u32));
106 }
107 KeyBuilderType::Bool(value) => {
108 key_base.push(u8::from(value));
109 }
110 KeyBuilderType::None => {}
111 }
112
113 key_base
114 }
115
116 fn key_from_event(
118 &self,
119 event: &SCOutputEvent,
120 indent: &KeyIndent,
121 key_kind: &KeyKind,
122 ) -> Option<Vec<u8>> {
123 let key = match indent {
133 KeyIndent::Event => {
134 let item = KeyBuilderType::Event(&event.context.slot, event.context.index_in_slot);
135 Some(self.key(indent, item, key_kind))
136 }
137 KeyIndent::EmitterAddress => {
138 if let Some(addr) = event.context.call_stack.back() {
139 let item = KeyBuilderType::Address(addr);
140 let mut key = self.key(indent, item, key_kind);
141 let item =
142 KeyBuilderType::Event(&event.context.slot, event.context.index_in_slot);
143 if matches!(key_kind, KeyKind::Regular) {
144 key.extend(self.key(&KeyIndent::Event, item, &KeyKind::Regular));
145 }
146 Some(key)
147 } else {
148 None
149 }
150 }
151 KeyIndent::OriginalCallerAddress => {
152 if let Some(addr) = event.context.call_stack.front() {
153 let item = KeyBuilderType::Address(addr);
154 let mut key = self.key(indent, item, key_kind);
155 let item =
156 KeyBuilderType::Event(&event.context.slot, event.context.index_in_slot);
157 if matches!(key_kind, KeyKind::Regular) {
158 key.extend(self.key(&KeyIndent::Event, item, &KeyKind::Regular));
159 }
160 Some(key)
161 } else {
162 None
163 }
164 }
165 KeyIndent::OriginalOperationId => {
166 if let Some(op_id) = event.context.origin_operation_id.as_ref() {
167 let item = KeyBuilderType::OperationId(op_id);
168 let mut key = self.key(indent, item, key_kind);
169 let item =
170 KeyBuilderType::Event(&event.context.slot, event.context.index_in_slot);
171 if matches!(key_kind, KeyKind::Regular) {
172 key.extend(self.key(&KeyIndent::Event, item, &KeyKind::Regular));
173 }
174 Some(key)
175 } else {
176 None
177 }
178 }
179 KeyIndent::IsError => {
180 let item = KeyBuilderType::Bool(event.context.is_error);
181 let mut key = self.key(indent, item, key_kind);
182 let item = KeyBuilderType::Event(&event.context.slot, event.context.index_in_slot);
183 if matches!(key_kind, KeyKind::Regular) {
184 key.extend(self.key(&KeyIndent::Event, item, &KeyKind::Regular));
185 }
186 Some(key)
187 }
188 _ => unreachable!(),
189 };
190
191 key
192 }
193
194 fn prefix_key_from_indent(&self, indent: &KeyIndent) -> Vec<u8> {
196 self.key(indent, KeyBuilderType::None, &KeyKind::Regular)
197 }
198
199 fn prefix_key_from_filter_item(&self, filter_item: &FilterItem, indent: &KeyIndent) -> Vec<u8> {
201 match (indent, filter_item) {
202 (KeyIndent::Event, FilterItem::SlotStartEnd(_start, _end)) => {
203 unimplemented!()
204 }
205 (KeyIndent::Event, FilterItem::SlotStart(start)) => {
206 self.key(indent, KeyBuilderType::Slot(start), &KeyKind::Prefix)
207 }
208 (KeyIndent::Event, FilterItem::SlotEnd(end)) => {
209 self.key(indent, KeyBuilderType::Slot(end), &KeyKind::Prefix)
210 }
211 (KeyIndent::EmitterAddress, FilterItem::EmitterAddress(addr)) => {
212 self.key(indent, KeyBuilderType::Address(addr), &KeyKind::Prefix)
213 }
214 (KeyIndent::OriginalCallerAddress, FilterItem::OriginalCallerAddress(addr)) => {
215 self.key(indent, KeyBuilderType::Address(addr), &KeyKind::Prefix)
216 }
217 (KeyIndent::OriginalOperationId, FilterItem::OriginalOperationId(op_id)) => {
218 self.key(indent, KeyBuilderType::OperationId(op_id), &KeyKind::Prefix)
219 }
220 (KeyIndent::IsError, FilterItem::IsError(v)) => {
221 self.key(indent, KeyBuilderType::Bool(*v), &KeyKind::Prefix)
222 }
223 _ => {
224 unreachable!()
225 }
226 }
227 }
228
229 fn counter_key_from_filter_item(
231 &self,
232 filter_item: &FilterItem,
233 indent: &KeyIndent,
234 ) -> Vec<u8> {
235 match (indent, filter_item) {
237 (KeyIndent::Event, FilterItem::SlotStartEnd(_start, _end)) => {
238 unimplemented!()
239 }
240 (KeyIndent::Event, FilterItem::SlotStart(start)) => {
241 self.key(indent, KeyBuilderType::Slot(start), &KeyKind::Counter)
242 }
243 (KeyIndent::Event, FilterItem::SlotEnd(end)) => {
244 self.key(indent, KeyBuilderType::Slot(end), &KeyKind::Counter)
245 }
246 (KeyIndent::EmitterAddress, FilterItem::EmitterAddress(addr)) => {
247 self.key(indent, KeyBuilderType::Address(addr), &KeyKind::Counter)
248 }
249 (KeyIndent::OriginalCallerAddress, FilterItem::OriginalCallerAddress(addr)) => {
250 self.key(indent, KeyBuilderType::Address(addr), &KeyKind::Counter)
251 }
252 (KeyIndent::OriginalOperationId, FilterItem::OriginalOperationId(op_id)) => self.key(
253 indent,
254 KeyBuilderType::OperationId(op_id),
255 &KeyKind::Counter,
256 ),
257 (KeyIndent::IsError, FilterItem::IsError(v)) => {
258 self.key(indent, KeyBuilderType::Bool(*v), &KeyKind::Counter)
259 }
260 _ => {
261 unreachable!()
262 }
263 }
264 }
265}
266
267pub(crate) struct EventCache {
269 db: DB,
271 entry_count: usize,
274 max_entry_count: usize,
277 snip_amount: usize,
279 event_ser: SCOutputEventSerializer,
281 event_deser: SCOutputEventDeserializer,
283 key_builder: DbKeyBuilder,
285 first_slot: Slot,
287 last_slot: Slot,
289 thread_count: u8,
291 max_events_per_operation: u64,
293 max_operations_per_block: u64,
295 max_events_per_query: usize,
297 max_scanned_keys_per_query: u64,
300}
301
302impl EventCache {
303 #[allow(clippy::too_many_arguments)]
304 pub fn new(
306 path: &Path,
307 max_entry_count: usize,
308 snip_amount: usize,
309 thread_count: u8,
310 max_recursive_call_depth: u16,
311 max_event_data_length: u64,
312 max_events_per_operation: u64,
313 max_operations_per_block: u64,
314 max_events_per_query: usize,
315 max_scanned_keys_per_query: u64,
316 ) -> Self {
317 if path.exists() {
319 DB::destroy(&Options::default(), path).expect(DESTROY_ERROR);
320 }
321 let options = {
322 let mut opts = Options::default();
323 opts.create_if_missing(true);
324 opts.set_merge_operator_associative("counter merge operator", counter_merge);
325 opts.set_max_open_files(128);
326 opts
327 };
328 let db = DB::open(&options, path).expect(OPEN_ERROR);
329
330 let key_builder_2 = DbKeyBuilder::new();
331
332 Self {
333 db,
334 entry_count: 0,
335 max_entry_count,
336 snip_amount,
337 event_ser: SCOutputEventSerializer::new(),
338 event_deser: SCOutputEventDeserializer::new(SCOutputEventDeserializerArgs {
339 thread_count,
340 max_call_stack_length: max_recursive_call_depth,
341 max_event_data_length,
342 }),
343 key_builder: key_builder_2,
344 first_slot: Slot::new(0, 0),
345 last_slot: Slot::new(0, 0),
346 thread_count,
347 max_events_per_operation,
348 max_operations_per_block,
349 max_events_per_query,
350 max_scanned_keys_per_query,
351 }
352 }
353
354 fn insert_into_batch(&mut self, event: SCOutputEvent, batch: &mut WriteBatch) {
356 let mut event_buffer = Vec::new();
357 self.event_ser.serialize(&event, &mut event_buffer).unwrap();
358
359 batch.put(
360 self.key_builder
361 .key_from_event(&event, &KeyIndent::Event, &KeyKind::Regular)
362 .unwrap(),
363 event_buffer,
364 );
365
366 if let Some(key) =
367 self.key_builder
368 .key_from_event(&event, &KeyIndent::EmitterAddress, &KeyKind::Regular)
369 {
370 let key_counter = self.key_builder.key_from_event(
371 &event,
372 &KeyIndent::EmitterAddress,
373 &KeyKind::Counter,
374 );
375 batch.put(key, vec![]);
376 let key_counter = key_counter.expect(COUNTER_KEY_CREATION_ERROR);
377 batch.merge(key_counter, 1i64.to_be_bytes());
378 }
379
380 if let Some(key) = self.key_builder.key_from_event(
381 &event,
382 &KeyIndent::OriginalCallerAddress,
383 &KeyKind::Regular,
384 ) {
385 let key_counter = self.key_builder.key_from_event(
386 &event,
387 &KeyIndent::OriginalCallerAddress,
388 &KeyKind::Counter,
389 );
390 batch.put(key, vec![]);
391 let key_counter = key_counter.expect(COUNTER_KEY_CREATION_ERROR);
392 batch.merge(key_counter, 1i64.to_be_bytes());
393 }
394
395 if let Some(key) = self.key_builder.key_from_event(
396 &event,
397 &KeyIndent::OriginalOperationId,
398 &KeyKind::Regular,
399 ) {
400 let key_counter = self.key_builder.key_from_event(
401 &event,
402 &KeyIndent::OriginalOperationId,
403 &KeyKind::Counter,
404 );
405 batch.put(key, vec![]);
406 let key_counter = key_counter.expect(COUNTER_KEY_CREATION_ERROR);
407 batch.merge(key_counter, 1i64.to_be_bytes());
408 }
409
410 {
411 if let Some(key) =
412 self.key_builder
413 .key_from_event(&event, &KeyIndent::IsError, &KeyKind::Regular)
414 {
415 let key_counter =
416 self.key_builder
417 .key_from_event(&event, &KeyIndent::IsError, &KeyKind::Counter);
418 let key_counter = key_counter.expect(COUNTER_KEY_CREATION_ERROR);
419 batch.put(key, vec![]);
420 batch.merge(key_counter, 1i64.to_be_bytes());
421 }
422 }
423
424 self.last_slot = max(self.last_slot, event.context.slot);
427 }
428
429 #[allow(dead_code)]
430 pub fn insert(&mut self, event: SCOutputEvent) {
432 if self.entry_count >= self.max_entry_count {
433 self.snip(None);
434 }
435
436 let mut batch = WriteBatch::default();
437 self.insert_into_batch(event, &mut batch);
438 self.db.write(&batch).expect(CRUD_ERROR);
439
440 self.entry_count = self.entry_count.saturating_add(1);
443
444 debug!("(Event insert) entry_count is: {}", self.entry_count);
445 }
446
447 pub fn insert_multi_it(
449 &mut self,
450 events: impl ExactSizeIterator<Item = SCOutputEvent> + Clone,
451 ) {
452 let events_len = events.len();
453
454 if self.entry_count + events_len >= self.max_entry_count {
455 let snip_amount = max(self.snip_amount, events_len);
456 self.snip(Some(snip_amount));
457 }
458
459 let mut batch = WriteBatch::default();
460 for event in events {
461 self.insert_into_batch(event, &mut batch);
462 }
463 self.db.write(&batch).expect(CRUD_ERROR);
464 self.entry_count = self.entry_count.saturating_add(events_len);
467
468 debug!("(Events insert) entry_count is: {}", self.entry_count);
469 }
470
471 pub(crate) fn get_filtered_sc_output_events(
478 &self,
479 filter: &EventFilter,
480 ) -> Result<(Vec<u64>, Vec<SCOutputEvent>), String> {
481 if filter.is_final == Some(false) {
484 return Ok((vec![], vec![]));
485 }
486
487 let mut filter_items = from_event_filter(filter);
494
495 if filter_items.is_empty() {
496 filter_items.push((KeyIndent::Event, FilterItem::SlotStart(Slot::new(0, 0))));
499 }
500
501 let it = filter_items.iter().map(|(key_indent, filter_item)| {
502 let count = self
503 .filter_item_estimate_count(key_indent, filter_item)
504 .unwrap_or_else(|e| {
505 warn!(
506 "Could not estimate count for key indent: {:?} - filter_item: {:?}: {}",
507 key_indent, filter_item, e
508 );
509 self.max_entry_count as u64
510 });
511 ((count, key_indent), filter_item)
512 });
513
514 let map = BTreeMap::from_iter(it);
515 debug!("Filter items map: {:?}", map);
516
517 let mut query_counts = Vec::with_capacity(map.len());
520 let mut filter_res_prev = None;
521 let mut scan_budget = self.max_scanned_keys_per_query;
522 let last_pass = map.len().saturating_sub(1);
523 for (pass, ((_counter, indent), filter_item)) in map.iter().enumerate() {
524 let mut filter_res = BTreeSet::new();
525 let max_results = (pass == last_pass).then_some(self.max_events_per_query);
529 let query_count = self.filter_for(
530 indent,
531 filter_item,
532 &mut filter_res,
533 filter_res_prev.as_ref(),
534 max_results,
535 &mut scan_budget,
536 )?;
537 query_counts.push(query_count);
538 filter_res_prev = Some(filter_res);
539 }
540
541 let multi_args = filter_res_prev
544 .unwrap()
545 .into_iter()
546 .take(self.max_events_per_query)
547 .collect::<Vec<Vec<u8>>>();
548
549 let res = self.db.multi_get(multi_args);
550 debug!(
551 "Filter will try to deserialize to SCOutputEvent {} values",
552 res.len()
553 );
554
555 let events = res
556 .into_iter()
557 .filter_map(|value| match value {
558 Ok(Some(value)) => self
559 .event_deser
560 .deserialize::<DeserializeError>(&value)
561 .map(|(_, event)| event)
562 .map_err(|e| warn!("Event cache: cannot deserialize event: {}", e))
563 .ok(),
564 Ok(None) => {
567 warn!("Event cache: index key points to a missing event, skipping");
568 None
569 }
570 Err(e) => {
571 warn!("Event cache: cannot read event: {}", e);
572 None
573 }
574 })
575 .collect::<Vec<SCOutputEvent>>();
576
577 Ok((query_counts, events))
578 }
579
580 fn filter_for(
585 &self,
586 indent: &KeyIndent,
587 filter_item: &FilterItem,
588 result: &mut BTreeSet<Vec<u8>>,
589 seen: Option<&BTreeSet<Vec<u8>>>,
590 max_results: Option<usize>,
591 scan_budget: &mut u64,
592 ) -> Result<u64, String> {
593 let mut query_count: u64 = 0;
594
595 if *indent == KeyIndent::Event {
596 let opts = match filter_item {
597 FilterItem::SlotStart(_start) => {
598 let key_start = self
599 .key_builder
600 .prefix_key_from_filter_item(filter_item, indent);
601 let mut options = rocksdb::ReadOptions::default();
602 options.set_iterate_lower_bound(key_start);
603 options
604 }
605 FilterItem::SlotEnd(_end) => {
606 let key_end = self
607 .key_builder
608 .prefix_key_from_filter_item(filter_item, indent);
609 let mut options = rocksdb::ReadOptions::default();
610 options.set_iterate_upper_bound(key_end);
611 options
612 }
613 FilterItem::SlotStartEnd(start, end) => {
614 let key_start = self
615 .key_builder
616 .prefix_key_from_filter_item(&FilterItem::SlotStart(*start), indent);
617 let key_end = self
618 .key_builder
619 .prefix_key_from_filter_item(&FilterItem::SlotEnd(*end), indent);
620 let mut options = rocksdb::ReadOptions::default();
621 options.set_iterate_range(key_start..key_end);
622 options
623 }
624 _ => unreachable!(),
625 };
626
627 #[allow(clippy::manual_flatten)]
628 for kvb in self.db.iterator_opt(IteratorMode::Start, opts) {
629 if let Ok(kvb) = kvb {
630 if !kvb.0.starts_with(&[*indent as u8]) {
631 break;
633 }
634
635 let found = kvb.0.to_vec();
636 query_count = query_count.saturating_add(1);
637 if *scan_budget == 0 {
638 return Err(format!(
639 "event query scans more than {} index keys: narrow the filter (slot range, emitter, caller or operation id)",
640 self.max_scanned_keys_per_query
641 ));
642 }
643 *scan_budget -= 1;
644
645 if let Some(filter_set_seen) = seen {
646 if filter_set_seen.contains(&found) {
647 result.insert(found);
648 }
649
650 if filter_set_seen.len() == result.len() {
654 break;
655 }
656 } else {
657 result.insert(found);
658 }
659
660 if max_results.is_some_and(|max| result.len() >= max) {
661 break;
662 }
663 }
664 }
665 } else {
666 let prefix_filter = match filter_item {
667 FilterItem::EmitterAddress(_addr) => self
668 .key_builder
669 .prefix_key_from_filter_item(filter_item, indent),
670 FilterItem::OriginalCallerAddress(_addr) => self
671 .key_builder
672 .prefix_key_from_filter_item(filter_item, indent),
673 FilterItem::OriginalOperationId(_op_id) => self
674 .key_builder
675 .prefix_key_from_filter_item(filter_item, indent),
676 FilterItem::IsError(_is_error) => self
677 .key_builder
678 .prefix_key_from_filter_item(filter_item, indent),
679 _ => unreachable!(),
680 };
681
682 #[allow(clippy::manual_flatten)]
683 for kvb in self.db.prefix_iterator(prefix_filter.as_slice()) {
684 if let Ok(kvb) = kvb {
685 if !kvb.0.starts_with(&[*indent as u8]) {
686 break;
688 }
689
690 if !kvb.0.starts_with(prefix_filter.as_slice()) {
691 break;
693 }
694
695 let found = kvb
696 .0
697 .strip_prefix(prefix_filter.as_slice())
698 .unwrap() .to_vec();
700
701 query_count = query_count.saturating_add(1);
702 if *scan_budget == 0 {
703 return Err(format!(
704 "event query scans more than {} index keys: narrow the filter (slot range, emitter, caller or operation id)",
705 self.max_scanned_keys_per_query
706 ));
707 }
708 *scan_budget -= 1;
709
710 if let Some(filter_set_seen) = seen {
711 if filter_set_seen.contains(&found) {
712 result.insert(found);
713 }
714
715 if filter_set_seen.len() == result.len() {
719 break;
720 }
721 } else {
722 result.insert(found);
723 }
724
725 if max_results.is_some_and(|max| result.len() >= max) {
726 break;
727 }
728 }
729 }
730 }
731
732 Ok(query_count)
733 }
734
735 fn filter_item_estimate_count(
737 &self,
738 key_indent: &KeyIndent,
739 filter_item: &FilterItem,
740 ) -> Result<u64, ModelsError> {
741 match filter_item {
742 FilterItem::SlotStart(start) => {
743 let diff = self
744 .last_slot
745 .slots_since(start, self.thread_count)
746 .unwrap_or(0); Ok(diff
750 .saturating_mul(self.max_events_per_operation)
751 .saturating_mul(self.max_operations_per_block))
752 }
753 FilterItem::SlotStartEnd(start, end) => {
754 let diff = end.slots_since(start, self.thread_count).unwrap_or(0);
756 Ok(diff
757 .saturating_mul(self.max_events_per_operation)
758 .saturating_mul(self.max_operations_per_block))
759 }
760 FilterItem::SlotEnd(end) => {
761 let diff = end
763 .slots_since(&self.first_slot, self.thread_count)
764 .unwrap_or(0);
765 Ok(diff
766 .saturating_mul(self.max_events_per_operation)
767 .saturating_mul(self.max_operations_per_block))
768 }
769 FilterItem::EmitterAddress(_addr) => {
770 let counter_key = self
771 .key_builder
772 .counter_key_from_filter_item(filter_item, key_indent);
773 let counter = self.db.get(counter_key).expect(COUNTER_ERROR);
774 let counter_value = counter
775 .map(|b| u64::from_be_bytes(b.try_into().unwrap()))
776 .unwrap_or(0);
777 Ok(counter_value)
778 }
779 FilterItem::OriginalCallerAddress(_addr) => {
780 let counter_key = self
781 .key_builder
782 .counter_key_from_filter_item(filter_item, key_indent);
783 let counter = self.db.get(counter_key).expect(COUNTER_ERROR);
784 let counter_value = counter
785 .map(|b| u64::from_be_bytes(b.try_into().unwrap()))
786 .unwrap_or(0);
787 Ok(counter_value)
788 }
789 FilterItem::OriginalOperationId(_op_id) => {
790 let counter_key = self
791 .key_builder
792 .counter_key_from_filter_item(filter_item, key_indent);
793 let counter = self.db.get(counter_key).expect(COUNTER_ERROR);
794 let counter_value = counter
795 .map(|b| u64::from_be_bytes(b.try_into().unwrap()))
796 .unwrap_or(0);
797 Ok(counter_value)
798 }
799 FilterItem::IsError(_is_error) => {
800 let counter_key = self
801 .key_builder
802 .counter_key_from_filter_item(filter_item, key_indent);
803 let counter = self.db.get(counter_key).expect(COUNTER_ERROR);
804 let counter_value = counter
805 .map(|b| u64::from_be_bytes(b.try_into().unwrap()))
806 .unwrap_or(0);
807 Ok(counter_value)
808 }
809 }
810 }
811
812 fn snip(&mut self, snip_amount: Option<usize>) {
814 let mut iter = self.db.iterator(IteratorMode::Start);
815 let mut batch = WriteBatch::default();
816 let mut snipped_count: usize = 0;
817 let snip_amount = snip_amount.unwrap_or(self.snip_amount);
818
819 let mut counter_keys = vec![];
820
821 while snipped_count < snip_amount {
822 let key_value = iter.next();
823 if key_value.is_none() {
824 break;
825 }
826 let kvb = key_value
827 .unwrap() .expect(EVENT_DESER_ERROR);
829
830 let key = kvb.0;
831 if !key.starts_with(&[u8::from(KeyIndent::Event)]) {
832 continue;
833 }
834
835 let (_, event) = self
836 .event_deser
837 .deserialize::<DeserializeError>(&kvb.1)
838 .unwrap();
839
840 if let Some(key) = self.key_builder.key_from_event(
842 &event,
843 &KeyIndent::EmitterAddress,
844 &KeyKind::Regular,
845 ) {
846 let key_counter = self
847 .key_builder
848 .key_from_event(&event, &KeyIndent::EmitterAddress, &KeyKind::Counter)
849 .expect(COUNTER_ERROR);
850 batch.delete(key);
851 counter_keys.push(key_counter.clone());
852 batch.merge(key_counter, (-1i64).to_be_bytes());
853 }
854 if let Some(key) = self.key_builder.key_from_event(
855 &event,
856 &KeyIndent::OriginalCallerAddress,
857 &KeyKind::Regular,
858 ) {
859 let key_counter = self
860 .key_builder
861 .key_from_event(&event, &KeyIndent::OriginalCallerAddress, &KeyKind::Counter)
862 .expect(COUNTER_ERROR);
863 batch.delete(key);
864 counter_keys.push(key_counter.clone());
865 batch.merge(key_counter, (-1i64).to_be_bytes());
866 }
867
868 if let Some(key) = self.key_builder.key_from_event(
869 &event,
870 &KeyIndent::OriginalOperationId,
871 &KeyKind::Regular,
872 ) {
873 let key_counter = self
874 .key_builder
875 .key_from_event(&event, &KeyIndent::OriginalOperationId, &KeyKind::Counter)
876 .expect(COUNTER_ERROR);
877 batch.delete(key);
878 counter_keys.push(key_counter.clone());
879 batch.merge(key_counter, (-1i64).to_be_bytes());
880 }
881 if let Some(key) =
882 self.key_builder
883 .key_from_event(&event, &KeyIndent::IsError, &KeyKind::Regular)
884 {
885 let key_counter = self
886 .key_builder
887 .key_from_event(&event, &KeyIndent::IsError, &KeyKind::Counter)
888 .expect(COUNTER_ERROR);
889 batch.delete(key);
890 counter_keys.push(key_counter.clone());
891 batch.merge(key_counter, (-1i64).to_be_bytes());
892 }
893
894 batch.delete(key);
895 snipped_count += 1;
896 }
897
898 self.db.write(&batch).expect(CRUD_ERROR);
900 self.entry_count = self.entry_count.saturating_sub(snipped_count);
901
902 let mut batch_counters = WriteBatch::default();
904 const U64_ZERO_BYTES: [u8; 8] = 0u64.to_be_bytes();
905 for (value, key) in self.db.multi_get(&counter_keys).iter().zip(counter_keys) {
906 if let Ok(Some(value)) = value {
907 if *value == U64_ZERO_BYTES.to_vec() {
908 batch_counters.delete(key);
909 }
910 }
911 }
912 self.db.write(&batch_counters).expect(CRUD_ERROR);
913
914 if self.entry_count == 0 {
916 self.first_slot = Slot::new(0, 0);
918 self.last_slot = Slot::new(0, 0);
919 } else {
920 let key_prefix = self.key_builder.prefix_key_from_indent(&KeyIndent::Event);
924 let mut it_slot = self.db.prefix_iterator(key_prefix);
925
926 let key_value = it_slot.next();
927 let kvb = key_value.unwrap().expect(EVENT_DESER_ERROR);
928
929 let (_, event) = self
930 .event_deser
931 .deserialize::<DeserializeError>(&kvb.1)
932 .unwrap();
933 self.first_slot = event.context.slot;
934 }
935 }
936}
937
938#[derive(Debug)]
940enum FilterItem {
941 SlotStart(Slot),
942 SlotStartEnd(Slot, Slot),
943 SlotEnd(Slot),
944 EmitterAddress(Address),
945 OriginalCallerAddress(Address),
946 OriginalOperationId(OperationId),
947 IsError(bool),
948}
949
950fn from_event_filter(event_filter: &EventFilter) -> Vec<(KeyIndent, FilterItem)> {
952 let mut filter_items = vec![];
953 if event_filter.start.is_some() && event_filter.end.is_some() {
954 let start = event_filter.start.unwrap();
955 let end = event_filter.end.unwrap();
956 filter_items.push((KeyIndent::Event, FilterItem::SlotStartEnd(start, end)));
957 } else if event_filter.start.is_some() {
958 let start = event_filter.start.unwrap();
959 filter_items.push((KeyIndent::Event, FilterItem::SlotStart(start)));
960 } else if event_filter.end.is_some() {
961 let end = event_filter.end.unwrap();
962 filter_items.push((KeyIndent::Event, FilterItem::SlotEnd(end)));
963 }
964
965 if let Some(addr) = event_filter.emitter_address {
966 filter_items.push((KeyIndent::EmitterAddress, FilterItem::EmitterAddress(addr)));
967 }
968
969 if let Some(addr) = event_filter.original_caller_address {
970 filter_items.push((
971 KeyIndent::OriginalCallerAddress,
972 FilterItem::OriginalCallerAddress(addr),
973 ));
974 }
975
976 if let Some(op_id) = event_filter.original_operation_id {
977 filter_items.push((
978 KeyIndent::OriginalOperationId,
979 FilterItem::OriginalOperationId(op_id),
980 ));
981 }
982
983 if let Some(is_error) = event_filter.is_error {
984 filter_items.push((KeyIndent::IsError, FilterItem::IsError(is_error)));
985 }
986
987 filter_items
988}
989
990#[cfg(test)]
991impl EventCache {
992 fn iter_all(
994 &self,
995 mode: Option<IteratorMode>,
996 ) -> impl Iterator<Item = (Box<[u8]>, Box<[u8]>)> + '_ {
997 self.db
998 .iterator(mode.unwrap_or(IteratorMode::Start))
999 .flatten()
1000 }
1001}
1002
1003#[cfg(test)]
1004mod tests {
1005 use super::*;
1006 use std::collections::VecDeque;
1008 use std::str::FromStr;
1009 use more_asserts::assert_gt;
1011 use rand::seq::SliceRandom;
1012 use rand::thread_rng;
1013 use serial_test::serial;
1014 use tempfile::TempDir;
1015 use massa_models::config::{
1017 MAX_EVENT_DATA_SIZE, MAX_EVENT_PER_OPERATION, MAX_OPERATIONS_PER_BLOCK,
1018 MAX_RECURSIVE_CALLS_DEPTH, THREAD_COUNT,
1019 };
1020 use massa_models::operation::OperationId;
1021 use massa_models::output_event::EventExecutionContext;
1022 use massa_models::slot::Slot;
1023
1024 fn setup() -> EventCache {
1025 let tmp_path = TempDir::new().unwrap().path().to_path_buf();
1026 EventCache::new(
1027 &tmp_path,
1028 1000,
1029 300,
1030 THREAD_COUNT,
1031 MAX_RECURSIVE_CALLS_DEPTH,
1032 MAX_EVENT_DATA_SIZE as u64,
1033 MAX_EVENT_PER_OPERATION as u64,
1034 MAX_OPERATIONS_PER_BLOCK as u64,
1035 5000, 1_000_000, )
1038 }
1039
1040 #[test]
1041 #[serial]
1042 fn test_db_insert_order() {
1043 let mut cache = setup();
1046 let slot_1 = Slot::new(1, 0);
1047 let index_1_0 = 0;
1048 let event = SCOutputEvent {
1049 context: EventExecutionContext {
1050 slot: slot_1,
1051 block: None,
1052 read_only: false,
1053 index_in_slot: index_1_0,
1054 call_stack: Default::default(),
1055 origin_operation_id: None,
1056 is_final: true,
1057 is_error: false,
1058 deferred_call_id: None,
1059 async_msg_id: None,
1060 },
1061 data: "message foo bar".to_string(),
1062 };
1063
1064 let max_entry_count = cache.max_entry_count - 5;
1065 let mut events = (0..max_entry_count)
1066 .map(|i| {
1067 let mut event = event.clone();
1068 event.context.index_in_slot = i as u64;
1069 event
1070 })
1071 .collect::<Vec<SCOutputEvent>>();
1072
1073 let slot_2 = Slot::new(2, 0);
1074 let event_slot_2 = {
1075 let mut event = event.clone();
1076 event.context.slot = slot_2;
1077 event.context.index_in_slot = 0u64;
1078 event
1079 };
1080 let index_2_2 = 256u64;
1081 let event_slot_2_2 = {
1082 let mut event = event.clone();
1083 event.context.slot = slot_2;
1084 event.context.index_in_slot = index_2_2;
1085 event
1086 };
1087 events.push(event_slot_2.clone());
1088 events.push(event_slot_2_2.clone());
1089 events.shuffle(&mut thread_rng());
1091
1092 for event in events {
1093 cache.insert(event);
1094 }
1095
1096 let mut prev_slot = None;
1099 let mut prev_event_index = None;
1100 for kvb in cache.iter_all(None) {
1101 let bytes = kvb.0.iter().as_slice();
1102
1103 if bytes[0] != u8::from(KeyIndent::Event) {
1104 continue;
1105 }
1106
1107 let slot = Slot::from_bytes_key(&bytes[1..=9].try_into().unwrap());
1108 let event_index = u64::from_be_bytes(bytes[10..].try_into().unwrap());
1109 if prev_slot.is_some() && prev_event_index.is_some() {
1110 assert_gt!(
1111 (slot, event_index),
1112 (prev_slot.unwrap(), prev_event_index.unwrap())
1113 );
1114 } else {
1115 assert_eq!(slot, slot_1);
1116 assert_eq!(event_index, index_1_0);
1117 }
1118 prev_slot = Some(slot);
1119 prev_event_index = Some(event_index);
1120 }
1121
1122 assert_eq!(prev_slot, Some(slot_2));
1123 assert_eq!(prev_event_index, Some(index_2_2));
1124 }
1125
1126 #[test]
1127 #[serial]
1128 fn test_insert_more_than_max_entry() {
1129 let mut cache = setup();
1132 let event = SCOutputEvent {
1133 context: EventExecutionContext {
1134 slot: Slot::new(1, 0),
1135 block: None,
1136 read_only: false,
1137 index_in_slot: 0,
1138 call_stack: Default::default(),
1139 origin_operation_id: None,
1140 is_final: true,
1141 is_error: false,
1142 deferred_call_id: None,
1143 async_msg_id: None,
1144 },
1145 data: "message foo bar".to_string(),
1146 };
1147
1148 for count in 0..cache.max_entry_count {
1150 let mut event = event.clone();
1151 event.context.index_in_slot = count as u64;
1152 cache.insert(event.clone());
1153 }
1154 assert_eq!(cache.entry_count, cache.max_entry_count);
1155
1156 let mut event_last = event.clone();
1158 event_last.context.index_in_slot = u64::MAX;
1159 cache.insert(event_last);
1160 assert_eq!(
1161 cache.entry_count,
1162 cache.max_entry_count - cache.snip_amount + 1
1163 );
1164 dbg!(cache.entry_count);
1165 }
1166
1167 #[test]
1168 #[serial]
1169 fn test_insert_more_than_max_entry_2() {
1170 let mut cache = setup();
1173 let event = SCOutputEvent {
1174 context: EventExecutionContext {
1175 slot: Slot::new(1, 0),
1176 block: None,
1177 read_only: false,
1178 index_in_slot: 0,
1179 call_stack: Default::default(),
1180 origin_operation_id: None,
1181 is_final: true,
1182 is_error: false,
1183 deferred_call_id: None,
1184 async_msg_id: None,
1185 },
1186 data: "message foo bar".to_string(),
1187 };
1188
1189 let it = (0..cache.max_entry_count).map(|i| {
1190 let mut event = event.clone();
1191 event.context.index_in_slot = i as u64;
1192 event
1193 });
1194 cache.insert_multi_it(it);
1195
1196 assert_eq!(cache.entry_count, cache.max_entry_count);
1197
1198 let mut event_last = event.clone();
1200 event_last.context.index_in_slot = u64::MAX;
1201 cache.insert(event_last);
1202 assert_eq!(
1203 cache.entry_count,
1204 cache.max_entry_count - cache.snip_amount + 1
1205 );
1206 dbg!(cache.entry_count);
1207 }
1208
1209 #[test]
1210 #[serial]
1211 fn test_snip() {
1212 let mut cache = setup();
1215 cache.max_entry_count = 10;
1216
1217 let event = SCOutputEvent {
1218 context: EventExecutionContext {
1219 slot: Slot::new(1, 0),
1220 block: None,
1221 read_only: false,
1222 index_in_slot: 0,
1223 call_stack: Default::default(),
1224 origin_operation_id: None,
1225 is_final: true,
1226 is_error: false,
1227 deferred_call_id: None,
1228 async_msg_id: None,
1229 },
1230 data: "message foo bar".to_string(),
1231 };
1232
1233 let it = (0..cache.max_entry_count).map(|i| {
1234 let mut event = event.clone();
1235 event.context.index_in_slot = i as u64;
1236 event
1237 });
1238 cache.insert_multi_it(it);
1239
1240 assert_eq!(cache.entry_count, cache.max_entry_count);
1241
1242 cache.snip(Some(cache.entry_count));
1243
1244 assert_eq!(cache.entry_count, 0);
1245 assert_eq!(cache.iter_all(None).count(), 0);
1246 }
1247
1248 #[test]
1249 #[serial]
1250 fn test_counter_0() {
1251 let mut cache = setup();
1254 cache.max_entry_count = 10;
1255
1256 let dummy_addr =
1257 Address::from_str("AU12qePoXhNbYWE1jZuafqJong7bbq1jw3k89RgbMawbrdZpaasoA").unwrap();
1258 let emit_addr_1 =
1259 Address::from_str("AU122Em8qkqegdLb1eyH8rdkSCNEf7RZLeTJve4Q2inRPGiTJ2xNv").unwrap();
1260 let emit_addr_2 =
1261 Address::from_str("AU12WuVR1Td74q9eAbtYZUnk5jnRbUuUacyhQFwm217bV5v1mNqTZ").unwrap();
1262
1263 let event = SCOutputEvent {
1264 context: EventExecutionContext {
1265 slot: Slot::new(1, 0),
1266 block: None,
1267 read_only: false,
1268 index_in_slot: 0,
1269 call_stack: VecDeque::from(vec![dummy_addr, emit_addr_1]),
1270 origin_operation_id: None,
1271 is_final: true,
1272 is_error: false,
1273 deferred_call_id: None,
1274 async_msg_id: None,
1275 },
1276 data: "message foo bar".to_string(),
1277 };
1278
1279 let event_2 = {
1280 let mut evt = event.clone();
1281 evt.context.slot = Slot::new(2, 0);
1282 evt.context.call_stack = VecDeque::from(vec![dummy_addr, emit_addr_2]);
1283 evt
1284 };
1285
1286 cache.insert_multi_it([event, event_2].into_iter());
1287
1288 let key_counters = cache
1290 .key_builder
1291 .prefix_key_from_indent(&KeyIndent::Counter);
1292 let kvbs: Result<Vec<(_, _)>, _> = cache
1293 .db
1294 .prefix_iterator(key_counters)
1295 .take_while(|kvb| {
1296 kvb.as_ref()
1297 .unwrap()
1298 .0
1299 .starts_with(&[u8::from(KeyIndent::Counter)])
1300 })
1301 .collect();
1302 assert_eq!(kvbs.unwrap().len(), 4);
1309
1310 let key_counter_1 = cache.key_builder.counter_key_from_filter_item(
1311 &FilterItem::EmitterAddress(emit_addr_1),
1312 &KeyIndent::EmitterAddress,
1313 );
1314 let key_counter_2 = cache.key_builder.counter_key_from_filter_item(
1315 &FilterItem::EmitterAddress(emit_addr_2),
1316 &KeyIndent::EmitterAddress,
1317 );
1318
1319 let v1 = cache.db.get(key_counter_1.clone());
1320 let v2 = cache.db.get(key_counter_2.clone());
1321
1322 assert_eq!(v1, Ok(Some(1u64.to_be_bytes().to_vec())));
1324 assert_eq!(v2, Ok(Some(1u64.to_be_bytes().to_vec())));
1325
1326 cache.snip(Some(1));
1327
1328 let v1 = cache.db.get(key_counter_1);
1329 let v2 = cache.db.get(key_counter_2);
1330
1331 assert_eq!(v1, Ok(None)); assert_eq!(v2, Ok(Some(1u64.to_be_bytes().to_vec())));
1334 }
1335
1336 #[test]
1337 #[serial]
1338 fn test_event_filter() {
1339 let mut cache = setup();
1342 let slot_1 = Slot::new(1, 0);
1343 let index_1_0 = 0;
1344 let event = SCOutputEvent {
1345 context: EventExecutionContext {
1346 slot: slot_1,
1347 block: None,
1348 read_only: false,
1349 index_in_slot: index_1_0,
1350 call_stack: Default::default(),
1351 origin_operation_id: None,
1352 is_final: true,
1353 is_error: false,
1354 deferred_call_id: None,
1355 async_msg_id: None,
1356 },
1357 data: "message foo bar".to_string(),
1358 };
1359
1360 let mut events = (0..cache.max_entry_count - 5)
1361 .map(|i| {
1362 let mut event = event.clone();
1363 event.context.index_in_slot = i as u64;
1364 event
1365 })
1366 .collect::<Vec<SCOutputEvent>>();
1367
1368 let slot_2 = Slot::new(2, 0);
1369 let index_2_1 = 0u64;
1370 let event_slot_2 = {
1371 let mut event = event.clone();
1372 event.context.slot = slot_2;
1373 event.context.index_in_slot = index_2_1;
1374 event
1375 };
1376 let index_2_2 = 256u64;
1377 let event_slot_2_2 = {
1378 let mut event = event.clone();
1379 event.context.slot = slot_2;
1380 event.context.index_in_slot = index_2_2;
1381 event
1382 };
1383 events.push(event_slot_2.clone());
1384 events.push(event_slot_2_2.clone());
1385 events.shuffle(&mut thread_rng());
1387
1388 cache.insert_multi_it(events.into_iter());
1389
1390 let filter_1 = EventFilter {
1391 start: Some(Slot::new(2, 0)),
1392 ..Default::default()
1393 };
1394
1395 let (_, filtered_events_1) = cache.get_filtered_sc_output_events(&filter_1).unwrap();
1396
1397 assert_eq!(filtered_events_1.len(), 2);
1398 assert_eq!(filtered_events_1[0].context.slot, slot_2);
1399 assert_eq!(filtered_events_1[0].context.index_in_slot, index_2_1);
1400 assert_eq!(filtered_events_1[1].context.slot, slot_2);
1401 assert_eq!(filtered_events_1[1].context.index_in_slot, index_2_2);
1402 }
1403
1404 #[test]
1405 #[serial]
1406 fn test_event_filter_2() {
1407 let mut cache = setup();
1410 cache.max_entry_count = 10;
1411
1412 let slot_1 = Slot::new(1, 0);
1413 let index_1_0 = 0;
1414 let op_id_1 =
1415 OperationId::from_str("O12n1vt8uTLh3H65J4TVuztaWfBh3oumjjVtRCkke7Ba5qWdXdjD").unwrap();
1416 let op_id_2 =
1417 OperationId::from_str("O1p5P691KF672fQ8tQougxzSERBwDKZF8FwtkifMSJbP14sEuGc").unwrap();
1418 let op_id_unknown =
1419 OperationId::from_str("O1kvXTfsnVbQcmDERkC89vqAd2xRTLCb3q5b2E5WaVPHwFd7Qth").unwrap();
1420
1421 let event = SCOutputEvent {
1422 context: EventExecutionContext {
1423 slot: slot_1,
1424 block: None,
1425 read_only: false,
1426 index_in_slot: index_1_0,
1427 call_stack: Default::default(),
1428 origin_operation_id: Some(op_id_1),
1429 is_final: true,
1430 is_error: false,
1431 deferred_call_id: None,
1432 async_msg_id: None,
1433 },
1434 data: "message foo bar".to_string(),
1435 };
1436
1437 let mut events = (0..cache.max_entry_count - 5)
1438 .map(|i| {
1439 let mut event = event.clone();
1440 event.context.index_in_slot = i as u64;
1441 event
1442 })
1443 .collect::<Vec<SCOutputEvent>>();
1444
1445 let slot_2 = Slot::new(2, 0);
1446 let index_2_1 = 0u64;
1447 let event_slot_2 = {
1448 let mut event = event.clone();
1449 event.context.slot = slot_2;
1450 event.context.index_in_slot = index_2_1;
1451 event.context.origin_operation_id = Some(op_id_2);
1452 event
1453 };
1454 let index_2_2 = 256u64;
1455 let event_slot_2_2 = {
1456 let mut event = event.clone();
1457 event.context.slot = slot_2;
1458 event.context.index_in_slot = index_2_2;
1459 event.context.origin_operation_id = Some(op_id_2);
1460 event
1461 };
1462 events.push(event_slot_2.clone());
1463 events.push(event_slot_2_2.clone());
1464 events.shuffle(&mut thread_rng());
1466 cache.insert_multi_it(events.into_iter());
1467
1468 let mut filter_1 = EventFilter {
1469 original_operation_id: Some(op_id_1),
1470 ..Default::default()
1471 };
1472
1473 let (_, filtered_events_1) = cache.get_filtered_sc_output_events(&filter_1).unwrap();
1474
1475 assert_eq!(filtered_events_1.len(), cache.max_entry_count - 5);
1476 filtered_events_1.iter().enumerate().for_each(|(i, event)| {
1477 assert_eq!(event.context.slot, slot_1);
1478 assert_eq!(event.context.index_in_slot, i as u64);
1479 });
1480
1481 {
1482 filter_1.original_operation_id = Some(op_id_2);
1483 let (_, filtered_events_2) = cache.get_filtered_sc_output_events(&filter_1).unwrap();
1484 assert_eq!(filtered_events_2.len(), 2);
1485 filtered_events_2.iter().enumerate().for_each(|(i, event)| {
1486 assert_eq!(event.context.slot, slot_2);
1487 if i == 0 {
1488 assert_eq!(event.context.index_in_slot, i as u64);
1489 } else {
1490 assert_eq!(event.context.index_in_slot, 256u64);
1491 }
1492 });
1493 }
1494
1495 {
1496 filter_1.original_operation_id = Some(op_id_unknown);
1497 let (_, filtered_events_2) = cache.get_filtered_sc_output_events(&filter_1).unwrap();
1498 assert_eq!(filtered_events_2.len(), 0);
1499 }
1500 }
1501
1502 #[test]
1503 #[serial]
1504 fn test_event_filter_3() {
1505 let mut cache = setup();
1508 cache.max_entry_count = 10;
1509
1510 let slot_1 = Slot::new(1, 0);
1511 let index_1_0 = 0;
1512
1513 let dummy_addr =
1514 Address::from_str("AU12qePoXhNbYWE1jZuafqJong7bbq1jw3k89RgbMawbrdZpaasoA").unwrap();
1515 let emit_addr_1 =
1516 Address::from_str("AU122Em8qkqegdLb1eyH8rdkSCNEf7RZLeTJve4Q2inRPGiTJ2xNv").unwrap();
1517 let emit_addr_2 =
1518 Address::from_str("AU12WuVR1Td74q9eAbtYZUnk5jnRbUuUacyhQFwm217bV5v1mNqTZ").unwrap();
1519 let emit_addr_unknown =
1520 Address::from_str("AU1zLC4TFUiaKDg7quQyusMPQcHT4ykWVs3FsFpuhdNSmowUG2As").unwrap();
1521
1522 let event = SCOutputEvent {
1523 context: EventExecutionContext {
1524 slot: slot_1,
1525 block: None,
1526 read_only: false,
1527 index_in_slot: index_1_0,
1528 call_stack: Default::default(),
1529 origin_operation_id: None,
1530 is_final: true,
1531 is_error: false,
1532 deferred_call_id: None,
1533 async_msg_id: None,
1534 },
1535 data: "message foo bar".to_string(),
1536 };
1537
1538 let to_insert_count = cache.max_entry_count - 5;
1539 let threshold = to_insert_count / 2;
1540 let mut events = (0..cache.max_entry_count - 5)
1541 .map(|i| {
1542 let mut event = event.clone();
1543 event.context.index_in_slot = i as u64;
1544 if i < threshold {
1545 event.context.call_stack = VecDeque::from(vec![dummy_addr, emit_addr_1]);
1546 } else {
1547 event.context.call_stack = VecDeque::from(vec![dummy_addr, emit_addr_2]);
1548 }
1549 event
1550 })
1551 .collect::<Vec<SCOutputEvent>>();
1552
1553 let slot_2 = Slot::new(2, 0);
1554 let index_2_1 = 0u64;
1555 let event_slot_2 = {
1556 let mut event = event.clone();
1557 event.context.slot = slot_2;
1558 event.context.index_in_slot = index_2_1;
1559 event.context.call_stack = VecDeque::from(vec![dummy_addr, emit_addr_2]);
1560 event
1561 };
1562 let index_2_2 = 256u64;
1563 let event_slot_2_2 = {
1564 let mut event = event.clone();
1565 event.context.slot = slot_2;
1566 event.context.index_in_slot = index_2_2;
1567 event.context.call_stack = VecDeque::from(vec![dummy_addr, emit_addr_2]);
1568 event
1569 };
1570 events.push(event_slot_2.clone());
1571 events.push(event_slot_2_2.clone());
1572 events.shuffle(&mut thread_rng());
1574
1575 cache.insert_multi_it(events.into_iter());
1576
1577 let mut filter_1 = EventFilter {
1578 emitter_address: Some(emit_addr_1),
1579 ..Default::default()
1580 };
1581
1582 let (_, filtered_events_1) = cache.get_filtered_sc_output_events(&filter_1).unwrap();
1583
1584 assert_eq!(filtered_events_1.len(), threshold);
1585 filtered_events_1.iter().for_each(|event| {
1586 assert_eq!(event.context.slot, slot_1);
1587 assert_eq!(*event.context.call_stack.back().unwrap(), emit_addr_1)
1588 });
1589
1590 {
1592 filter_1.emitter_address = Some(emit_addr_2);
1593 let (_, filtered_events_2) = cache.get_filtered_sc_output_events(&filter_1).unwrap();
1594 assert_eq!(filtered_events_2.len(), threshold + 1 + 2);
1595 filtered_events_2.iter().for_each(|event| {
1596 assert_eq!(*event.context.call_stack.back().unwrap(), emit_addr_2)
1597 });
1598 }
1599 {
1601 filter_1.emitter_address = Some(dummy_addr);
1602 let (_, filtered_events_2) = cache.get_filtered_sc_output_events(&filter_1).unwrap();
1603 assert_eq!(filtered_events_2.len(), 0);
1604 }
1605 {
1607 filter_1.emitter_address = Some(emit_addr_unknown);
1608 let (_, filtered_events_2) = cache.get_filtered_sc_output_events(&filter_1).unwrap();
1609 assert_eq!(filtered_events_2.len(), 0);
1610 }
1611 }
1612
1613 #[test]
1614 #[serial]
1615 fn test_event_filter_4() {
1616 let mut cache = setup();
1619 cache.max_entry_count = 10;
1620
1621 let slot_1 = Slot::new(1, 0);
1622 let index_1_0 = 0;
1623
1624 let dummy_addr =
1625 Address::from_str("AU12qePoXhNbYWE1jZuafqJong7bbq1jw3k89RgbMawbrdZpaasoA").unwrap();
1626 let caller_addr_1 =
1627 Address::from_str("AU122Em8qkqegdLb1eyH8rdkSCNEf7RZLeTJve4Q2inRPGiTJ2xNv").unwrap();
1628 let caller_addr_2 =
1629 Address::from_str("AU12WuVR1Td74q9eAbtYZUnk5jnRbUuUacyhQFwm217bV5v1mNqTZ").unwrap();
1630 let caller_addr_unknown =
1631 Address::from_str("AU1zLC4TFUiaKDg7quQyusMPQcHT4ykWVs3FsFpuhdNSmowUG2As").unwrap();
1632
1633 let event = SCOutputEvent {
1634 context: EventExecutionContext {
1635 slot: slot_1,
1636 block: None,
1637 read_only: false,
1638 index_in_slot: index_1_0,
1639 call_stack: Default::default(),
1640 origin_operation_id: None,
1641 is_final: true,
1642 is_error: false,
1643 deferred_call_id: None,
1644 async_msg_id: None,
1645 },
1646 data: "message foo bar".to_string(),
1647 };
1648
1649 let to_insert_count = cache.max_entry_count - 5;
1650 let threshold = to_insert_count / 2;
1651 let mut events = (0..cache.max_entry_count - 5)
1652 .map(|i| {
1653 let mut event = event.clone();
1654 event.context.index_in_slot = i as u64;
1655 if i < threshold {
1656 event.context.call_stack = VecDeque::from(vec![caller_addr_1, dummy_addr]);
1657 } else {
1658 event.context.call_stack = VecDeque::from(vec![caller_addr_2, dummy_addr]);
1659 }
1660 event
1661 })
1662 .collect::<Vec<SCOutputEvent>>();
1663
1664 let slot_2 = Slot::new(2, 0);
1665 let index_2_1 = 0u64;
1666 let event_slot_2 = {
1667 let mut event = event.clone();
1668 event.context.slot = slot_2;
1669 event.context.index_in_slot = index_2_1;
1670 event.context.call_stack = VecDeque::from(vec![caller_addr_2, dummy_addr]);
1671 event
1672 };
1673 let index_2_2 = 256u64;
1674 let event_slot_2_2 = {
1675 let mut event = event.clone();
1676 event.context.slot = slot_2;
1677 event.context.index_in_slot = index_2_2;
1678 event.context.call_stack = VecDeque::from(vec![caller_addr_2, dummy_addr]);
1679 event
1680 };
1681 events.push(event_slot_2.clone());
1682 events.push(event_slot_2_2.clone());
1683 events.shuffle(&mut thread_rng());
1685 cache.insert_multi_it(events.into_iter());
1686
1687 let mut filter_1 = EventFilter {
1688 original_caller_address: Some(caller_addr_1),
1689 ..Default::default()
1690 };
1691
1692 let (_, filtered_events_1) = cache.get_filtered_sc_output_events(&filter_1).unwrap();
1693
1694 assert_eq!(filtered_events_1.len(), threshold);
1695 filtered_events_1.iter().for_each(|event| {
1696 assert_eq!(event.context.slot, slot_1);
1697 assert_eq!(*event.context.call_stack.front().unwrap(), caller_addr_1);
1698 });
1699
1700 {
1701 filter_1.original_caller_address = Some(caller_addr_2);
1702 let (_, filtered_events_2) = cache.get_filtered_sc_output_events(&filter_1).unwrap();
1703 assert_eq!(filtered_events_2.len(), threshold + 1 + 2);
1704 filtered_events_2.iter().for_each(|event| {
1705 assert_eq!(*event.context.call_stack.front().unwrap(), caller_addr_2);
1706 });
1707 }
1708 {
1709 filter_1.original_caller_address = Some(dummy_addr);
1710 let (_, filtered_events_2) = cache.get_filtered_sc_output_events(&filter_1).unwrap();
1711 assert_eq!(filtered_events_2.len(), 0);
1712 }
1713 {
1714 filter_1.original_caller_address = Some(caller_addr_unknown);
1715 let (_, filtered_events_2) = cache.get_filtered_sc_output_events(&filter_1).unwrap();
1716 assert_eq!(filtered_events_2.len(), 0);
1717 }
1718 }
1719
1720 #[test]
1721 #[serial]
1722 fn test_event_filter_5() {
1723 let mut cache = setup();
1726 cache.max_entry_count = 10;
1727
1728 let slot_1 = Slot::new(1, 0);
1729 let index_1_0 = 0;
1730
1731 let dummy_addr =
1732 Address::from_str("AU12qePoXhNbYWE1jZuafqJong7bbq1jw3k89RgbMawbrdZpaasoA").unwrap();
1733 let emit_addr_1 =
1734 Address::from_str("AU122Em8qkqegdLb1eyH8rdkSCNEf7RZLeTJve4Q2inRPGiTJ2xNv").unwrap();
1735 let emit_addr_2 =
1736 Address::from_str("AU12WuVR1Td74q9eAbtYZUnk5jnRbUuUacyhQFwm217bV5v1mNqTZ").unwrap();
1737
1738 let event = SCOutputEvent {
1739 context: EventExecutionContext {
1740 slot: slot_1,
1741 block: None,
1742 read_only: false,
1743 index_in_slot: index_1_0,
1744 call_stack: Default::default(),
1745 origin_operation_id: None,
1746 is_final: true,
1747 is_error: false,
1748 deferred_call_id: None,
1749 async_msg_id: None,
1750 },
1751 data: "message foo bar".to_string(),
1752 };
1753
1754 let to_insert_count = cache.max_entry_count - 5;
1755 let threshold = to_insert_count / 2;
1756 let mut events = (0..cache.max_entry_count - 5)
1757 .map(|i| {
1758 let mut event = event.clone();
1759 event.context.index_in_slot = i as u64;
1760 if i < threshold {
1761 event.context.call_stack = VecDeque::from(vec![dummy_addr, emit_addr_1]);
1762 } else {
1763 event.context.call_stack = VecDeque::from(vec![dummy_addr, emit_addr_2]);
1764 }
1765 event
1766 })
1767 .collect::<Vec<SCOutputEvent>>();
1768
1769 let slot_2 = Slot::new(2, 0);
1770 let index_2_1 = 0u64;
1771 let event_slot_2 = {
1772 let mut event = event.clone();
1773 event.context.slot = slot_2;
1774 event.context.index_in_slot = index_2_1;
1775 event.context.call_stack = VecDeque::from(vec![dummy_addr, emit_addr_2]);
1776 event
1777 };
1778 let index_2_2 = 256u64;
1779 let event_slot_2_2 = {
1780 let mut event = event.clone();
1781 event.context.slot = slot_2;
1782 event.context.index_in_slot = index_2_2;
1783 event.context.call_stack = VecDeque::from(vec![dummy_addr, emit_addr_2]);
1784 event.context.is_error = true;
1785 event
1786 };
1787 events.push(event_slot_2.clone());
1788 events.push(event_slot_2_2.clone());
1789 events.shuffle(&mut thread_rng());
1791 cache.insert_multi_it(events.into_iter());
1792
1793 let filter_1 = EventFilter {
1794 is_error: Some(true),
1795 ..Default::default()
1796 };
1797
1798 let (_, filtered_events_1) = cache.get_filtered_sc_output_events(&filter_1).unwrap();
1799 assert_eq!(filtered_events_1.len(), 1);
1800 assert!(filtered_events_1[0].context.is_error);
1801 assert_eq!(filtered_events_1[0].context.slot, slot_2);
1802 assert_eq!(filtered_events_1[0].context.index_in_slot, index_2_2);
1803 }
1804
1805 #[test]
1806 #[serial]
1807 fn test_orphan_index_key_does_not_panic() {
1808 let mut cache = setup();
1812 let emitter =
1813 Address::from_str("AU12qePoXhNbYWE1jZuafqJong7bbq1jw3k89RgbMawbrdZpaasoA").unwrap();
1814 let event = SCOutputEvent {
1815 context: EventExecutionContext {
1816 slot: Slot::new(1, 0),
1817 block: None,
1818 read_only: false,
1819 index_in_slot: 0,
1820 call_stack: VecDeque::from(vec![emitter]),
1821 origin_operation_id: None,
1822 is_final: true,
1823 is_error: false,
1824 deferred_call_id: None,
1825 async_msg_id: None,
1826 },
1827 data: "message foo bar".to_string(),
1828 };
1829 cache.insert(event.clone());
1830
1831 let event_key = cache
1833 .key_builder
1834 .key_from_event(&event, &KeyIndent::Event, &KeyKind::Regular)
1835 .unwrap();
1836 cache.db.delete(event_key).unwrap();
1837
1838 let filter = EventFilter {
1839 emitter_address: Some(emitter),
1840 ..Default::default()
1841 };
1842 let (_, events) = cache.get_filtered_sc_output_events(&filter).unwrap();
1843 assert!(events.is_empty());
1844 }
1845
1846 #[test]
1847 #[serial]
1848 fn test_filter_optimisations() {
1849 let mut cache = setup();
1852 cache.max_entry_count = 10;
1853
1854 let slot_1 = Slot::new(1, 0);
1855 let index_1_0 = 0;
1856
1857 let dummy_addr =
1858 Address::from_str("AU12qePoXhNbYWE1jZuafqJong7bbq1jw3k89RgbMawbrdZpaasoA").unwrap();
1859 let emit_addr_1 =
1860 Address::from_str("AU122Em8qkqegdLb1eyH8rdkSCNEf7RZLeTJve4Q2inRPGiTJ2xNv").unwrap();
1861 let emit_addr_2 =
1862 Address::from_str("AU12WuVR1Td74q9eAbtYZUnk5jnRbUuUacyhQFwm217bV5v1mNqTZ").unwrap();
1863
1864 let event = SCOutputEvent {
1865 context: EventExecutionContext {
1866 slot: slot_1,
1867 block: None,
1868 read_only: false,
1869 index_in_slot: index_1_0,
1870 call_stack: Default::default(),
1871 origin_operation_id: None,
1872 is_final: true,
1873 is_error: true,
1874 deferred_call_id: None,
1875 async_msg_id: None,
1876 },
1877 data: "error foo bar".to_string(),
1878 };
1879
1880 let to_insert_count = cache.max_entry_count - 5;
1881 let threshold = to_insert_count / 2;
1882 let mut events = (0..cache.max_entry_count - 5)
1883 .map(|i| {
1884 let mut event = event.clone();
1885 event.context.index_in_slot = i as u64;
1886 if i < threshold {
1887 event.context.call_stack = VecDeque::from(vec![dummy_addr, emit_addr_1]);
1888 } else {
1889 event.context.call_stack = VecDeque::from(vec![dummy_addr, emit_addr_2]);
1890 }
1891 event
1892 })
1893 .collect::<Vec<SCOutputEvent>>();
1894
1895 let slot_2 = Slot::new(2, 0);
1896 let index_2_1 = 0u64;
1897 let event_slot_2 = {
1898 let mut event = event.clone();
1899 event.context.slot = slot_2;
1900 event.context.index_in_slot = index_2_1;
1901 event.context.call_stack = VecDeque::from(vec![dummy_addr, emit_addr_2]);
1902 event
1903 };
1904 let index_2_2 = 256u64;
1905 let event_slot_2_2 = {
1906 let mut event = event.clone();
1907 event.context.slot = slot_2;
1908 event.context.index_in_slot = index_2_2;
1909 event.context.call_stack = VecDeque::from(vec![dummy_addr, emit_addr_2]);
1910 event
1912 };
1913 events.push(event_slot_2.clone());
1914 events.push(event_slot_2_2.clone());
1915 events.shuffle(&mut thread_rng());
1917 cache.insert_multi_it(events.into_iter());
1918
1919 let emit_addr_1_count = cache
1921 .filter_item_estimate_count(
1922 &KeyIndent::EmitterAddress,
1923 &FilterItem::EmitterAddress(emit_addr_1),
1924 )
1925 .unwrap();
1926 let emit_addr_2_count = cache
1927 .filter_item_estimate_count(
1928 &KeyIndent::EmitterAddress,
1929 &FilterItem::EmitterAddress(emit_addr_2),
1930 )
1931 .unwrap();
1932
1933 assert_eq!(emit_addr_1_count, (threshold) as u64);
1934 assert_eq!(emit_addr_2_count, (threshold + 1 + 2) as u64);
1935
1936 let slot_start_1 = Slot::new(3, 0); let slot_end_1 = Slot::new(0, 0); let slot_start_end_1 = Slot::new(2, 0)..Slot::new(1, 0); let slot_start_2 = Slot::new(1, 0);
1945 let slot_end_2 = Slot::new(2, 0);
1946 let slot_start_end_2 = Slot::new(1, 0)..Slot::new(2, 0);
1947
1948 let slot_start_count_1 = cache
1949 .filter_item_estimate_count(&KeyIndent::Event, &FilterItem::SlotStart(slot_start_1))
1950 .unwrap();
1951 let slot_end_count_1 = cache
1952 .filter_item_estimate_count(&KeyIndent::Event, &FilterItem::SlotEnd(slot_end_1))
1953 .unwrap();
1954 let slot_start_end_count_1 = cache
1955 .filter_item_estimate_count(
1956 &KeyIndent::Event,
1957 &FilterItem::SlotStartEnd(slot_start_end_1.start, slot_start_end_1.end),
1958 )
1959 .unwrap();
1960
1961 let slot_start_count_2 = cache
1962 .filter_item_estimate_count(&KeyIndent::Event, &FilterItem::SlotStart(slot_start_2))
1963 .unwrap();
1964 let slot_end_count_2 = cache
1965 .filter_item_estimate_count(&KeyIndent::Event, &FilterItem::SlotEnd(slot_end_2))
1966 .unwrap();
1967 let slot_start_end_count_2 = cache
1968 .filter_item_estimate_count(
1969 &KeyIndent::Event,
1970 &FilterItem::SlotStartEnd(slot_start_end_2.start, slot_start_end_2.end),
1971 )
1972 .unwrap();
1973
1974 assert_eq!(slot_start_count_1, 0);
1975 assert_eq!(slot_end_count_1, 0);
1976 assert_eq!(slot_start_end_count_1, 0);
1977
1978 let expected_count_per_period =
1979 THREAD_COUNT as u64 * MAX_EVENT_PER_OPERATION as u64 * MAX_OPERATIONS_PER_BLOCK as u64;
1980 assert_eq!(slot_start_count_2, expected_count_per_period);
1981 assert_eq!(slot_end_count_2, 2 * expected_count_per_period); assert_eq!(slot_start_end_count_2, expected_count_per_period);
1983
1984 let filter_1 = EventFilter {
1987 emitter_address: Some(emit_addr_1),
1988 is_error: Some(true),
1989 ..Default::default()
1990 };
1991
1992 let (query_counts, _filtered_events_1) =
1993 cache.get_filtered_sc_output_events(&filter_1).unwrap();
1994 println!("threshold: {:?}", threshold);
1995 println!("query_counts: {:?}", query_counts);
1996
1997 assert_eq!(query_counts[0], threshold as u64);
1999 assert_eq!(query_counts[1], threshold as u64);
2002 }
2003
2004 fn cache_with_events(
2008 max_events_per_query: usize,
2009 max_scanned_keys_per_query: u64,
2010 count: u64,
2011 emitter_every: u64,
2012 ) -> (EventCache, Address, TempDir) {
2013 let tmp = TempDir::new().unwrap();
2014 let mut cache = EventCache::new(
2015 tmp.path(),
2016 1000,
2017 300,
2018 THREAD_COUNT,
2019 MAX_RECURSIVE_CALLS_DEPTH,
2020 MAX_EVENT_DATA_SIZE as u64,
2021 MAX_EVENT_PER_OPERATION as u64,
2022 MAX_OPERATIONS_PER_BLOCK as u64,
2023 max_events_per_query,
2024 max_scanned_keys_per_query,
2025 );
2026 let emitter =
2027 Address::from_str("AU122Em8qkqegdLb1eyH8rdkSCNEf7RZLeTJve4Q2inRPGiTJ2xNv").unwrap();
2028 let other =
2029 Address::from_str("AU12WuVR1Td74q9eAbtYZUnk5jnRbUuUacyhQFwm217bV5v1mNqTZ").unwrap();
2030 let mut events: Vec<SCOutputEvent> = (0..count)
2031 .map(|i| SCOutputEvent {
2032 context: EventExecutionContext {
2033 slot: Slot::new(i + 1, 0),
2034 block: None,
2035 read_only: false,
2036 index_in_slot: 0,
2037 call_stack: VecDeque::from(vec![if i % emitter_every == 0 {
2038 emitter
2039 } else {
2040 other
2041 }]),
2042 origin_operation_id: None,
2043 is_final: true,
2044 is_error: false,
2045 deferred_call_id: None,
2046 async_msg_id: None,
2047 },
2048 data: format!("event {}", i),
2049 })
2050 .collect();
2051 events.shuffle(&mut thread_rng());
2052 cache.insert_multi_it(events.into_iter());
2053 (cache, emitter, tmp)
2054 }
2055
2056 #[test]
2059 #[serial]
2060 fn test_query_stops_scanning_at_max_events_per_query() {
2061 let (capped, emitter, _tmp_1) = cache_with_events(10, 1_000_000, 300, 2);
2062 let (full, _, _tmp_2) = cache_with_events(1_000, 1_000_000, 300, 2);
2063
2064 let filters = [
2065 EventFilter::default(),
2067 EventFilter {
2069 emitter_address: Some(emitter),
2070 ..Default::default()
2071 },
2072 EventFilter {
2074 start: Some(Slot::new(50, 0)),
2075 emitter_address: Some(emitter),
2076 ..Default::default()
2077 },
2078 ];
2079 for filter in filters {
2080 let (counts, events) = capped.get_filtered_sc_output_events(&filter).unwrap();
2081 let (full_counts, all_events) = full.get_filtered_sc_output_events(&filter).unwrap();
2082 assert_eq!(events.len(), 10, "{filter:?}");
2083 assert_eq!(events, all_events[..10], "{filter:?}");
2084 assert!(
2086 counts.last().unwrap() < full_counts.last().unwrap(),
2087 "{filter:?}: {counts:?} vs {full_counts:?}"
2088 );
2089 if counts.len() == 1 {
2090 assert_eq!(counts[0], 10, "{filter:?}");
2092 }
2093 }
2094 }
2095
2096 #[test]
2099 #[serial]
2100 fn test_query_fails_past_the_scan_budget() {
2101 let (cache, emitter, _tmp) = cache_with_events(1_000, 50, 300, 100);
2103
2104 let err = cache
2106 .get_filtered_sc_output_events(&EventFilter::default())
2107 .unwrap_err();
2108 assert!(err.contains("narrow the filter"), "{err}");
2109
2110 let (_, events) = cache
2112 .get_filtered_sc_output_events(&EventFilter {
2113 emitter_address: Some(emitter),
2114 ..Default::default()
2115 })
2116 .unwrap();
2117 assert_eq!(events.len(), 3);
2118
2119 let err = cache
2122 .get_filtered_sc_output_events(&EventFilter {
2123 start: Some(Slot::new(150, 0)),
2124 emitter_address: Some(emitter),
2125 ..Default::default()
2126 })
2127 .unwrap_err();
2128 assert!(err.contains("narrow the filter"), "{err}");
2129 }
2130}