massa_event_cache/
event_cache.rs

1// std
2use std::cmp::max;
3use std::collections::{BTreeMap, BTreeSet};
4use std::path::Path;
5// third-party
6use num_enum::IntoPrimitive;
7use rocksdb::{IteratorMode, Options, WriteBatch, DB};
8use tracing::{debug, warn};
9// internal
10use crate::rocksdb_operator::counter_merge;
11use crate::ser_deser::{
12    SCOutputEventDeserializer, SCOutputEventDeserializerArgs, SCOutputEventSerializer,
13};
14use massa_models::address::Address;
15use massa_models::error::ModelsError;
16use massa_models::execution::EventFilter;
17use massa_models::operation::{OperationId, OperationIdSerializer};
18use massa_models::output_event::SCOutputEvent;
19use massa_models::slot::Slot;
20use massa_serialization::{DeserializeError, Deserializer, Serializer};
21
22const OPEN_ERROR: &str = "critical: rocksdb open operation failed";
23// const COUNTER_INIT_ERROR: &str = "critical: cannot init rocksdb counters";
24const DESTROY_ERROR: &str = "critical: rocksdb delete operation failed";
25const CRUD_ERROR: &str = "critical: rocksdb crud operation failed";
26const EVENT_DESER_ERROR: &str = "critical: event deserialization failed";
27const OPERATION_ID_DESER_ERROR: &str = "critical: deserialization failed for op id in rocksdb";
28const COUNTER_ERROR: &str = "critical: cannot get counter";
29const COUNTER_KEY_CREATION_ERROR: &str = "critical: cannot create counter key";
30
31#[allow(dead_code)]
32/// Prefix u8 used to identify rocksdb keys
33#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, IntoPrimitive)]
34#[repr(u8)]
35enum KeyIndent {
36    Counter = 0,
37    Event,
38    EmitterAddress,
39    OriginalCallerAddress,
40    OriginalOperationId,
41    IsError,
42    IsFinal,
43}
44
45/// Key for this type of data that we want to get
46enum KeyBuilderType<'a> {
47    Slot(&'a Slot),
48    Event(&'a Slot, u64),
49    Address(&'a Address),
50    OperationId(&'a OperationId),
51    Bool(bool),
52    None,
53}
54
55enum KeyKind {
56    Regular,
57    Prefix,
58    Counter,
59}
60
61struct DbKeyBuilder {
62    /// Operation Id Serializer
63    op_id_ser: OperationIdSerializer,
64}
65
66impl DbKeyBuilder {
67    fn new() -> Self {
68        Self {
69            op_id_ser: OperationIdSerializer::new(),
70        }
71    }
72
73    /// Low level key builder function
74    /// There is no guarantees that the key will be unique
75    /// Recommended to use high level function like:
76    /// `key_from_event`, `prefix_key_from_indent`,
77    /// `prefix_key_from_filter_item` or `counter_key_from_filter_item`
78    fn key(&self, indent: &KeyIndent, key_type: KeyBuilderType, key_kind: &KeyKind) -> Vec<u8> {
79        let mut key_base = if matches!(key_kind, KeyKind::Counter) {
80            vec![u8::from(KeyIndent::Counter), u8::from(*indent)]
81        } else {
82            vec![u8::from(*indent)]
83        };
84
85        match key_type {
86            KeyBuilderType::Slot(slot) => {
87                key_base.extend(slot.to_bytes_key());
88            }
89            KeyBuilderType::Event(slot, index) => {
90                key_base.extend(slot.to_bytes_key());
91                key_base.extend(index.to_be_bytes());
92            }
93            KeyBuilderType::Address(addr) => {
94                let addr_bytes = addr.to_prefixed_bytes();
95                let addr_bytes_len = addr_bytes.len();
96                key_base.extend(addr_bytes);
97                key_base.push(addr_bytes_len as u8);
98            }
99            KeyBuilderType::OperationId(op_id) => {
100                let mut buffer = Vec::new();
101                self.op_id_ser
102                    .serialize(op_id, &mut buffer)
103                    .expect(OPERATION_ID_DESER_ERROR);
104                key_base.extend(&buffer);
105                key_base.extend(u32::to_be_bytes(buffer.len() as u32));
106            }
107            KeyBuilderType::Bool(value) => {
108                key_base.push(u8::from(value));
109            }
110            KeyBuilderType::None => {}
111        }
112
113        key_base
114    }
115
116    /// Key usually used to populate the DB
117    fn key_from_event(
118        &self,
119        event: &SCOutputEvent,
120        indent: &KeyIndent,
121        key_kind: &KeyKind,
122    ) -> Option<Vec<u8>> {
123        // Db format:
124        // * Regular keys:
125        //   * Event key: [Event Indent][Slot][Index] -> Event value: Event serialized
126        //   * Emitter address key: [Emitter Address Indent][Addr][Addr len][Event key] -> []
127        // * Prefix keys:
128        //   * Emitter address prefix key: [Emitter Address Indent][Addr][Addr len]
129        // * Counter keys:
130        //   * Emitter address counter key: [Counter indent][Emitter Address Indent][Addr][Addr len][Event key] -> u64
131
132        let key = match indent {
133            KeyIndent::Event => {
134                let item = KeyBuilderType::Event(&event.context.slot, event.context.index_in_slot);
135                Some(self.key(indent, item, key_kind))
136            }
137            KeyIndent::EmitterAddress => {
138                if let Some(addr) = event.context.call_stack.back() {
139                    let item = KeyBuilderType::Address(addr);
140                    let mut key = self.key(indent, item, key_kind);
141                    let item =
142                        KeyBuilderType::Event(&event.context.slot, event.context.index_in_slot);
143                    if matches!(key_kind, KeyKind::Regular) {
144                        key.extend(self.key(&KeyIndent::Event, item, &KeyKind::Regular));
145                    }
146                    Some(key)
147                } else {
148                    None
149                }
150            }
151            KeyIndent::OriginalCallerAddress => {
152                if let Some(addr) = event.context.call_stack.front() {
153                    let item = KeyBuilderType::Address(addr);
154                    let mut key = self.key(indent, item, key_kind);
155                    let item =
156                        KeyBuilderType::Event(&event.context.slot, event.context.index_in_slot);
157                    if matches!(key_kind, KeyKind::Regular) {
158                        key.extend(self.key(&KeyIndent::Event, item, &KeyKind::Regular));
159                    }
160                    Some(key)
161                } else {
162                    None
163                }
164            }
165            KeyIndent::OriginalOperationId => {
166                if let Some(op_id) = event.context.origin_operation_id.as_ref() {
167                    let item = KeyBuilderType::OperationId(op_id);
168                    let mut key = self.key(indent, item, key_kind);
169                    let item =
170                        KeyBuilderType::Event(&event.context.slot, event.context.index_in_slot);
171                    if matches!(key_kind, KeyKind::Regular) {
172                        key.extend(self.key(&KeyIndent::Event, item, &KeyKind::Regular));
173                    }
174                    Some(key)
175                } else {
176                    None
177                }
178            }
179            KeyIndent::IsError => {
180                let item = KeyBuilderType::Bool(event.context.is_error);
181                let mut key = self.key(indent, item, key_kind);
182                let item = KeyBuilderType::Event(&event.context.slot, event.context.index_in_slot);
183                if matches!(key_kind, KeyKind::Regular) {
184                    key.extend(self.key(&KeyIndent::Event, item, &KeyKind::Regular));
185                }
186                Some(key)
187            }
188            _ => unreachable!(),
189        };
190
191        key
192    }
193
194    /// Prefix key to iterate over all events / emitter_address / ...
195    fn prefix_key_from_indent(&self, indent: &KeyIndent) -> Vec<u8> {
196        self.key(indent, KeyBuilderType::None, &KeyKind::Regular)
197    }
198
199    /// Prefix key to iterate over specific emitter_address / operation_id / ...
200    fn prefix_key_from_filter_item(&self, filter_item: &FilterItem, indent: &KeyIndent) -> Vec<u8> {
201        match (indent, filter_item) {
202            (KeyIndent::Event, FilterItem::SlotStartEnd(_start, _end)) => {
203                unimplemented!()
204            }
205            (KeyIndent::Event, FilterItem::SlotStart(start)) => {
206                self.key(indent, KeyBuilderType::Slot(start), &KeyKind::Prefix)
207            }
208            (KeyIndent::Event, FilterItem::SlotEnd(end)) => {
209                self.key(indent, KeyBuilderType::Slot(end), &KeyKind::Prefix)
210            }
211            (KeyIndent::EmitterAddress, FilterItem::EmitterAddress(addr)) => {
212                self.key(indent, KeyBuilderType::Address(addr), &KeyKind::Prefix)
213            }
214            (KeyIndent::OriginalCallerAddress, FilterItem::OriginalCallerAddress(addr)) => {
215                self.key(indent, KeyBuilderType::Address(addr), &KeyKind::Prefix)
216            }
217            (KeyIndent::OriginalOperationId, FilterItem::OriginalOperationId(op_id)) => {
218                self.key(indent, KeyBuilderType::OperationId(op_id), &KeyKind::Prefix)
219            }
220            (KeyIndent::IsError, FilterItem::IsError(v)) => {
221                self.key(indent, KeyBuilderType::Bool(*v), &KeyKind::Prefix)
222            }
223            _ => {
224                unreachable!()
225            }
226        }
227    }
228
229    /// Counter key for specific emitter_address / operation_id / ...
230    fn counter_key_from_filter_item(
231        &self,
232        filter_item: &FilterItem,
233        indent: &KeyIndent,
234    ) -> Vec<u8> {
235        // High level key builder function
236        match (indent, filter_item) {
237            (KeyIndent::Event, FilterItem::SlotStartEnd(_start, _end)) => {
238                unimplemented!()
239            }
240            (KeyIndent::Event, FilterItem::SlotStart(start)) => {
241                self.key(indent, KeyBuilderType::Slot(start), &KeyKind::Counter)
242            }
243            (KeyIndent::Event, FilterItem::SlotEnd(end)) => {
244                self.key(indent, KeyBuilderType::Slot(end), &KeyKind::Counter)
245            }
246            (KeyIndent::EmitterAddress, FilterItem::EmitterAddress(addr)) => {
247                self.key(indent, KeyBuilderType::Address(addr), &KeyKind::Counter)
248            }
249            (KeyIndent::OriginalCallerAddress, FilterItem::OriginalCallerAddress(addr)) => {
250                self.key(indent, KeyBuilderType::Address(addr), &KeyKind::Counter)
251            }
252            (KeyIndent::OriginalOperationId, FilterItem::OriginalOperationId(op_id)) => self.key(
253                indent,
254                KeyBuilderType::OperationId(op_id),
255                &KeyKind::Counter,
256            ),
257            (KeyIndent::IsError, FilterItem::IsError(v)) => {
258                self.key(indent, KeyBuilderType::Bool(*v), &KeyKind::Counter)
259            }
260            _ => {
261                unreachable!()
262            }
263        }
264    }
265}
266
267/// Disk based event cache db (rocksdb based)
268pub(crate) struct EventCache {
269    /// RocksDB database
270    db: DB,
271    /// How many entries are in the db. Count is initialized at creation time by iterating
272    /// over all the entries in the db then it is maintained in memory
273    entry_count: usize,
274    /// Maximum number of entries we want to keep in the db.
275    /// When this maximum is reached `snip_amount` entries are removed
276    max_entry_count: usize,
277    /// How many entries are removed when `entry_count` reaches `max_entry_count`
278    snip_amount: usize,
279    /// Event serializer
280    event_ser: SCOutputEventSerializer,
281    /// Event deserializer
282    event_deser: SCOutputEventDeserializer,
283    /// Key builder
284    key_builder: DbKeyBuilder,
285    /// First event slot in db
286    first_slot: Slot,
287    /// Last event slot in db
288    last_slot: Slot,
289    /// Thread count
290    thread_count: u8,
291    /// Maximum number of events per operation
292    max_events_per_operation: u64,
293    /// Maximum number of operations per block
294    max_operations_per_block: u64,
295    /// Max number of events returned by a query
296    max_events_per_query: usize,
297    /// Max number of index keys a query may scan, over all its filter criteria.
298    /// Bounds the work (and the memory of the intermediate key sets) of one query.
299    max_scanned_keys_per_query: u64,
300}
301
302impl EventCache {
303    #[allow(clippy::too_many_arguments)]
304    /// Create a new EventCache
305    pub fn new(
306        path: &Path,
307        max_entry_count: usize,
308        snip_amount: usize,
309        thread_count: u8,
310        max_recursive_call_depth: u16,
311        max_event_data_length: u64,
312        max_events_per_operation: u64,
313        max_operations_per_block: u64,
314        max_events_per_query: usize,
315        max_scanned_keys_per_query: u64,
316    ) -> Self {
317        // Clear the db
318        if path.exists() {
319            DB::destroy(&Options::default(), path).expect(DESTROY_ERROR);
320        }
321        let options = {
322            let mut opts = Options::default();
323            opts.create_if_missing(true);
324            opts.set_merge_operator_associative("counter merge operator", counter_merge);
325            opts.set_max_open_files(128);
326            opts
327        };
328        let db = DB::open(&options, path).expect(OPEN_ERROR);
329
330        let key_builder_2 = DbKeyBuilder::new();
331
332        Self {
333            db,
334            entry_count: 0,
335            max_entry_count,
336            snip_amount,
337            event_ser: SCOutputEventSerializer::new(),
338            event_deser: SCOutputEventDeserializer::new(SCOutputEventDeserializerArgs {
339                thread_count,
340                max_call_stack_length: max_recursive_call_depth,
341                max_event_data_length,
342            }),
343            key_builder: key_builder_2,
344            first_slot: Slot::new(0, 0),
345            last_slot: Slot::new(0, 0),
346            thread_count,
347            max_events_per_operation,
348            max_operations_per_block,
349            max_events_per_query,
350            max_scanned_keys_per_query,
351        }
352    }
353
354    /// From an event add keys & values into a rocksdb batch
355    fn insert_into_batch(&mut self, event: SCOutputEvent, batch: &mut WriteBatch) {
356        let mut event_buffer = Vec::new();
357        self.event_ser.serialize(&event, &mut event_buffer).unwrap();
358
359        batch.put(
360            self.key_builder
361                .key_from_event(&event, &KeyIndent::Event, &KeyKind::Regular)
362                .unwrap(),
363            event_buffer,
364        );
365
366        if let Some(key) =
367            self.key_builder
368                .key_from_event(&event, &KeyIndent::EmitterAddress, &KeyKind::Regular)
369        {
370            let key_counter = self.key_builder.key_from_event(
371                &event,
372                &KeyIndent::EmitterAddress,
373                &KeyKind::Counter,
374            );
375            batch.put(key, vec![]);
376            let key_counter = key_counter.expect(COUNTER_KEY_CREATION_ERROR);
377            batch.merge(key_counter, 1i64.to_be_bytes());
378        }
379
380        if let Some(key) = self.key_builder.key_from_event(
381            &event,
382            &KeyIndent::OriginalCallerAddress,
383            &KeyKind::Regular,
384        ) {
385            let key_counter = self.key_builder.key_from_event(
386                &event,
387                &KeyIndent::OriginalCallerAddress,
388                &KeyKind::Counter,
389            );
390            batch.put(key, vec![]);
391            let key_counter = key_counter.expect(COUNTER_KEY_CREATION_ERROR);
392            batch.merge(key_counter, 1i64.to_be_bytes());
393        }
394
395        if let Some(key) = self.key_builder.key_from_event(
396            &event,
397            &KeyIndent::OriginalOperationId,
398            &KeyKind::Regular,
399        ) {
400            let key_counter = self.key_builder.key_from_event(
401                &event,
402                &KeyIndent::OriginalOperationId,
403                &KeyKind::Counter,
404            );
405            batch.put(key, vec![]);
406            let key_counter = key_counter.expect(COUNTER_KEY_CREATION_ERROR);
407            batch.merge(key_counter, 1i64.to_be_bytes());
408        }
409
410        {
411            if let Some(key) =
412                self.key_builder
413                    .key_from_event(&event, &KeyIndent::IsError, &KeyKind::Regular)
414            {
415                let key_counter =
416                    self.key_builder
417                        .key_from_event(&event, &KeyIndent::IsError, &KeyKind::Counter);
418                let key_counter = key_counter.expect(COUNTER_KEY_CREATION_ERROR);
419                batch.put(key, vec![]);
420                batch.merge(key_counter, 1i64.to_be_bytes());
421            }
422        }
423
424        // Keep track of last slot (and start slot) of events in the DB
425        // Help for event filtering
426        self.last_slot = max(self.last_slot, event.context.slot);
427    }
428
429    #[allow(dead_code)]
430    /// Insert a new event in the cache
431    pub fn insert(&mut self, event: SCOutputEvent) {
432        if self.entry_count >= self.max_entry_count {
433            self.snip(None);
434        }
435
436        let mut batch = WriteBatch::default();
437        self.insert_into_batch(event, &mut batch);
438        self.db.write(&batch).expect(CRUD_ERROR);
439
440        // Note:
441        // This assumes that events are always added, never overwritten
442        self.entry_count = self.entry_count.saturating_add(1);
443
444        debug!("(Event insert) entry_count is: {}", self.entry_count);
445    }
446
447    /// Insert new events in the cache
448    pub fn insert_multi_it(
449        &mut self,
450        events: impl ExactSizeIterator<Item = SCOutputEvent> + Clone,
451    ) {
452        let events_len = events.len();
453
454        if self.entry_count + events_len >= self.max_entry_count {
455            let snip_amount = max(self.snip_amount, events_len);
456            self.snip(Some(snip_amount));
457        }
458
459        let mut batch = WriteBatch::default();
460        for event in events {
461            self.insert_into_batch(event, &mut batch);
462        }
463        self.db.write(&batch).expect(CRUD_ERROR);
464        // Note:
465        // This assumes that events are always added, never overwritten
466        self.entry_count = self.entry_count.saturating_add(events_len);
467
468        debug!("(Events insert) entry_count is: {}", self.entry_count);
469    }
470
471    /// Get events filtered by the given argument
472    /// Returns the events matching `filter` (at most `max_events_per_query`), with
473    /// the number of index keys scanned for each filter criterion.
474    ///
475    /// Fails without returning anything if the query would scan more than
476    /// `max_scanned_keys_per_query` index keys: the client must narrow its filter.
477    pub(crate) fn get_filtered_sc_output_events(
478        &self,
479        filter: &EventFilter,
480    ) -> Result<(Vec<u64>, Vec<SCOutputEvent>), String> {
481        // All events stored in this cache are final: a query for non-final
482        // events can never match anything here
483        if filter.is_final == Some(false) {
484            return Ok((vec![], vec![]));
485        }
486
487        // Step 1
488        // Build a (sorted) map with key: (counter value, indent), value: filter
489        // Will be used to iterate from the lower count index to the highest count index
490        // e.g. if index for emitter address is 10 (index count), and origin operation id is 20
491        //      iter over emitter address index then origin operation id index
492
493        let mut filter_items = from_event_filter(filter);
494
495        if filter_items.is_empty() {
496            // No criteria (is_final is None or Some(true)): scan the whole event
497            // column, bounded below by max_events_per_query
498            filter_items.push((KeyIndent::Event, FilterItem::SlotStart(Slot::new(0, 0))));
499        }
500
501        let it = filter_items.iter().map(|(key_indent, filter_item)| {
502            let count = self
503                .filter_item_estimate_count(key_indent, filter_item)
504                .unwrap_or_else(|e| {
505                    warn!(
506                        "Could not estimate count for key indent: {:?} - filter_item: {:?}: {}",
507                        key_indent, filter_item, e
508                    );
509                    self.max_entry_count as u64
510                });
511            ((count, key_indent), filter_item)
512        });
513
514        let map = BTreeMap::from_iter(it);
515        debug!("Filter items map: {:?}", map);
516
517        // Step 2: apply filter from the lowest counter to the highest counter
518
519        let mut query_counts = Vec::with_capacity(map.len());
520        let mut filter_res_prev = None;
521        let mut scan_budget = self.max_scanned_keys_per_query;
522        let last_pass = map.len().saturating_sub(1);
523        for (pass, ((_counter, indent), filter_item)) in map.iter().enumerate() {
524            let mut filter_res = BTreeSet::new();
525            // Only the first `max_events_per_query` keys of the last pass are kept (step 3).
526            // Keys are scanned in increasing order, so that pass can stop once it has them,
527            // with the same result. Earlier passes are intersected later, so they can't.
528            let max_results = (pass == last_pass).then_some(self.max_events_per_query);
529            let query_count = self.filter_for(
530                indent,
531                filter_item,
532                &mut filter_res,
533                filter_res_prev.as_ref(),
534                max_results,
535                &mut scan_budget,
536            )?;
537            query_counts.push(query_count);
538            filter_res_prev = Some(filter_res);
539        }
540
541        // Step 3: get values & deserialize
542
543        let multi_args = filter_res_prev
544            .unwrap()
545            .into_iter()
546            .take(self.max_events_per_query)
547            .collect::<Vec<Vec<u8>>>();
548
549        let res = self.db.multi_get(multi_args);
550        debug!(
551            "Filter will try to deserialize to SCOutputEvent {} values",
552            res.len()
553        );
554
555        let events = res
556            .into_iter()
557            .filter_map(|value| match value {
558                Ok(Some(value)) => self
559                    .event_deser
560                    .deserialize::<DeserializeError>(&value)
561                    .map(|(_, event)| event)
562                    .map_err(|e| warn!("Event cache: cannot deserialize event: {}", e))
563                    .ok(),
564                // An index key without its event (orphan index) must not kill
565                // the node: skip it
566                Ok(None) => {
567                    warn!("Event cache: index key points to a missing event, skipping");
568                    None
569                }
570                Err(e) => {
571                    warn!("Event cache: cannot read event: {}", e);
572                    None
573                }
574            })
575            .collect::<Vec<SCOutputEvent>>();
576
577        Ok((query_counts, events))
578    }
579
580    /// Collects into `result` the keys matching `filter_item` (and present in `seen` if given),
581    /// stopping once `result` holds `max_results` keys.
582    ///
583    /// Every scanned key consumes one unit of `scan_budget`; fails once it is exhausted.
584    fn filter_for(
585        &self,
586        indent: &KeyIndent,
587        filter_item: &FilterItem,
588        result: &mut BTreeSet<Vec<u8>>,
589        seen: Option<&BTreeSet<Vec<u8>>>,
590        max_results: Option<usize>,
591        scan_budget: &mut u64,
592    ) -> Result<u64, String> {
593        let mut query_count: u64 = 0;
594
595        if *indent == KeyIndent::Event {
596            let opts = match filter_item {
597                FilterItem::SlotStart(_start) => {
598                    let key_start = self
599                        .key_builder
600                        .prefix_key_from_filter_item(filter_item, indent);
601                    let mut options = rocksdb::ReadOptions::default();
602                    options.set_iterate_lower_bound(key_start);
603                    options
604                }
605                FilterItem::SlotEnd(_end) => {
606                    let key_end = self
607                        .key_builder
608                        .prefix_key_from_filter_item(filter_item, indent);
609                    let mut options = rocksdb::ReadOptions::default();
610                    options.set_iterate_upper_bound(key_end);
611                    options
612                }
613                FilterItem::SlotStartEnd(start, end) => {
614                    let key_start = self
615                        .key_builder
616                        .prefix_key_from_filter_item(&FilterItem::SlotStart(*start), indent);
617                    let key_end = self
618                        .key_builder
619                        .prefix_key_from_filter_item(&FilterItem::SlotEnd(*end), indent);
620                    let mut options = rocksdb::ReadOptions::default();
621                    options.set_iterate_range(key_start..key_end);
622                    options
623                }
624                _ => unreachable!(),
625            };
626
627            #[allow(clippy::manual_flatten)]
628            for kvb in self.db.iterator_opt(IteratorMode::Start, opts) {
629                if let Ok(kvb) = kvb {
630                    if !kvb.0.starts_with(&[*indent as u8]) {
631                        // Stop as soon as our key does not start with the right indent
632                        break;
633                    }
634
635                    let found = kvb.0.to_vec();
636                    query_count = query_count.saturating_add(1);
637                    if *scan_budget == 0 {
638                        return Err(format!(
639                            "event query scans more than {} index keys: narrow the filter (slot range, emitter, caller or operation id)",
640                            self.max_scanned_keys_per_query
641                        ));
642                    }
643                    *scan_budget -= 1;
644
645                    if let Some(filter_set_seen) = seen {
646                        if filter_set_seen.contains(&found) {
647                            result.insert(found);
648                        }
649
650                        // We have already found as many items as in a previous search
651                        // As we search from the lowest count to the highest count, we will never add new items
652                        // in our result, so we can break here
653                        if filter_set_seen.len() == result.len() {
654                            break;
655                        }
656                    } else {
657                        result.insert(found);
658                    }
659
660                    if max_results.is_some_and(|max| result.len() >= max) {
661                        break;
662                    }
663                }
664            }
665        } else {
666            let prefix_filter = match filter_item {
667                FilterItem::EmitterAddress(_addr) => self
668                    .key_builder
669                    .prefix_key_from_filter_item(filter_item, indent),
670                FilterItem::OriginalCallerAddress(_addr) => self
671                    .key_builder
672                    .prefix_key_from_filter_item(filter_item, indent),
673                FilterItem::OriginalOperationId(_op_id) => self
674                    .key_builder
675                    .prefix_key_from_filter_item(filter_item, indent),
676                FilterItem::IsError(_is_error) => self
677                    .key_builder
678                    .prefix_key_from_filter_item(filter_item, indent),
679                _ => unreachable!(),
680            };
681
682            #[allow(clippy::manual_flatten)]
683            for kvb in self.db.prefix_iterator(prefix_filter.as_slice()) {
684                if let Ok(kvb) = kvb {
685                    if !kvb.0.starts_with(&[*indent as u8]) {
686                        // Stop as soon as our key does not start with the right indent
687                        break;
688                    }
689
690                    if !kvb.0.starts_with(prefix_filter.as_slice()) {
691                        // Stop as soon as our key does not start with our current prefix
692                        break;
693                    }
694
695                    let found = kvb
696                        .0
697                        .strip_prefix(prefix_filter.as_slice())
698                        .unwrap() // safe to unwrap() - already tested
699                        .to_vec();
700
701                    query_count = query_count.saturating_add(1);
702                    if *scan_budget == 0 {
703                        return Err(format!(
704                            "event query scans more than {} index keys: narrow the filter (slot range, emitter, caller or operation id)",
705                            self.max_scanned_keys_per_query
706                        ));
707                    }
708                    *scan_budget -= 1;
709
710                    if let Some(filter_set_seen) = seen {
711                        if filter_set_seen.contains(&found) {
712                            result.insert(found);
713                        }
714
715                        // We have already found as many items as in a previous search
716                        // As we search from the lowest count to the highest count, we will never add new items
717                        // in our result, so we can break here
718                        if filter_set_seen.len() == result.len() {
719                            break;
720                        }
721                    } else {
722                        result.insert(found);
723                    }
724
725                    if max_results.is_some_and(|max| result.len() >= max) {
726                        break;
727                    }
728                }
729            }
730        }
731
732        Ok(query_count)
733    }
734
735    /// Estimate for a given KeyIndent & FilterItem the number of row to iterate
736    fn filter_item_estimate_count(
737        &self,
738        key_indent: &KeyIndent,
739        filter_item: &FilterItem,
740    ) -> Result<u64, ModelsError> {
741        match filter_item {
742            FilterItem::SlotStart(start) => {
743                let diff = self
744                    .last_slot
745                    .slots_since(start, self.thread_count)
746                    .unwrap_or(0); // If start is after self.last_slot, we have no events in the filter
747                                   // Note: Pessimistic estimation - should we keep an average count of events per slot
748                                   //       and use that instead?
749                Ok(diff
750                    .saturating_mul(self.max_events_per_operation)
751                    .saturating_mul(self.max_operations_per_block))
752            }
753            FilterItem::SlotStartEnd(start, end) => {
754                // If start is after end, we have no events in the filter as it is inconsistent
755                let diff = end.slots_since(start, self.thread_count).unwrap_or(0);
756                Ok(diff
757                    .saturating_mul(self.max_events_per_operation)
758                    .saturating_mul(self.max_operations_per_block))
759            }
760            FilterItem::SlotEnd(end) => {
761                // If end is after self.first_slot, we have no events in the filter
762                let diff = end
763                    .slots_since(&self.first_slot, self.thread_count)
764                    .unwrap_or(0);
765                Ok(diff
766                    .saturating_mul(self.max_events_per_operation)
767                    .saturating_mul(self.max_operations_per_block))
768            }
769            FilterItem::EmitterAddress(_addr) => {
770                let counter_key = self
771                    .key_builder
772                    .counter_key_from_filter_item(filter_item, key_indent);
773                let counter = self.db.get(counter_key).expect(COUNTER_ERROR);
774                let counter_value = counter
775                    .map(|b| u64::from_be_bytes(b.try_into().unwrap()))
776                    .unwrap_or(0);
777                Ok(counter_value)
778            }
779            FilterItem::OriginalCallerAddress(_addr) => {
780                let counter_key = self
781                    .key_builder
782                    .counter_key_from_filter_item(filter_item, key_indent);
783                let counter = self.db.get(counter_key).expect(COUNTER_ERROR);
784                let counter_value = counter
785                    .map(|b| u64::from_be_bytes(b.try_into().unwrap()))
786                    .unwrap_or(0);
787                Ok(counter_value)
788            }
789            FilterItem::OriginalOperationId(_op_id) => {
790                let counter_key = self
791                    .key_builder
792                    .counter_key_from_filter_item(filter_item, key_indent);
793                let counter = self.db.get(counter_key).expect(COUNTER_ERROR);
794                let counter_value = counter
795                    .map(|b| u64::from_be_bytes(b.try_into().unwrap()))
796                    .unwrap_or(0);
797                Ok(counter_value)
798            }
799            FilterItem::IsError(_is_error) => {
800                let counter_key = self
801                    .key_builder
802                    .counter_key_from_filter_item(filter_item, key_indent);
803                let counter = self.db.get(counter_key).expect(COUNTER_ERROR);
804                let counter_value = counter
805                    .map(|b| u64::from_be_bytes(b.try_into().unwrap()))
806                    .unwrap_or(0);
807                Ok(counter_value)
808            }
809        }
810    }
811
812    /// Try to remove some entries from the db
813    fn snip(&mut self, snip_amount: Option<usize>) {
814        let mut iter = self.db.iterator(IteratorMode::Start);
815        let mut batch = WriteBatch::default();
816        let mut snipped_count: usize = 0;
817        let snip_amount = snip_amount.unwrap_or(self.snip_amount);
818
819        let mut counter_keys = vec![];
820
821        while snipped_count < snip_amount {
822            let key_value = iter.next();
823            if key_value.is_none() {
824                break;
825            }
826            let kvb = key_value
827                .unwrap() // safe to unwrap - just tested it
828                .expect(EVENT_DESER_ERROR);
829
830            let key = kvb.0;
831            if !key.starts_with(&[u8::from(KeyIndent::Event)]) {
832                continue;
833            }
834
835            let (_, event) = self
836                .event_deser
837                .deserialize::<DeserializeError>(&kvb.1)
838                .unwrap();
839
840            // delete all associated key
841            if let Some(key) = self.key_builder.key_from_event(
842                &event,
843                &KeyIndent::EmitterAddress,
844                &KeyKind::Regular,
845            ) {
846                let key_counter = self
847                    .key_builder
848                    .key_from_event(&event, &KeyIndent::EmitterAddress, &KeyKind::Counter)
849                    .expect(COUNTER_ERROR);
850                batch.delete(key);
851                counter_keys.push(key_counter.clone());
852                batch.merge(key_counter, (-1i64).to_be_bytes());
853            }
854            if let Some(key) = self.key_builder.key_from_event(
855                &event,
856                &KeyIndent::OriginalCallerAddress,
857                &KeyKind::Regular,
858            ) {
859                let key_counter = self
860                    .key_builder
861                    .key_from_event(&event, &KeyIndent::OriginalCallerAddress, &KeyKind::Counter)
862                    .expect(COUNTER_ERROR);
863                batch.delete(key);
864                counter_keys.push(key_counter.clone());
865                batch.merge(key_counter, (-1i64).to_be_bytes());
866            }
867
868            if let Some(key) = self.key_builder.key_from_event(
869                &event,
870                &KeyIndent::OriginalOperationId,
871                &KeyKind::Regular,
872            ) {
873                let key_counter = self
874                    .key_builder
875                    .key_from_event(&event, &KeyIndent::OriginalOperationId, &KeyKind::Counter)
876                    .expect(COUNTER_ERROR);
877                batch.delete(key);
878                counter_keys.push(key_counter.clone());
879                batch.merge(key_counter, (-1i64).to_be_bytes());
880            }
881            if let Some(key) =
882                self.key_builder
883                    .key_from_event(&event, &KeyIndent::IsError, &KeyKind::Regular)
884            {
885                let key_counter = self
886                    .key_builder
887                    .key_from_event(&event, &KeyIndent::IsError, &KeyKind::Counter)
888                    .expect(COUNTER_ERROR);
889                batch.delete(key);
890                counter_keys.push(key_counter.clone());
891                batch.merge(key_counter, (-1i64).to_be_bytes());
892            }
893
894            batch.delete(key);
895            snipped_count += 1;
896        }
897
898        // delete the key and reduce entry_count
899        self.db.write(&batch).expect(CRUD_ERROR);
900        self.entry_count = self.entry_count.saturating_sub(snipped_count);
901
902        // delete key counters where value == 0
903        let mut batch_counters = WriteBatch::default();
904        const U64_ZERO_BYTES: [u8; 8] = 0u64.to_be_bytes();
905        for (value, key) in self.db.multi_get(&counter_keys).iter().zip(counter_keys) {
906            if let Ok(Some(value)) = value {
907                if *value == U64_ZERO_BYTES.to_vec() {
908                    batch_counters.delete(key);
909                }
910            }
911        }
912        self.db.write(&batch_counters).expect(CRUD_ERROR);
913
914        // Update first_slot / last_slot in the DB
915        if self.entry_count == 0 {
916            // Reset
917            self.first_slot = Slot::new(0, 0);
918            self.last_slot = Slot::new(0, 0);
919        } else {
920            // Get the first event in the db
921            // By using a prefix iterator this should be fast
922
923            let key_prefix = self.key_builder.prefix_key_from_indent(&KeyIndent::Event);
924            let mut it_slot = self.db.prefix_iterator(key_prefix);
925
926            let key_value = it_slot.next();
927            let kvb = key_value.unwrap().expect(EVENT_DESER_ERROR);
928
929            let (_, event) = self
930                .event_deser
931                .deserialize::<DeserializeError>(&kvb.1)
932                .unwrap();
933            self.first_slot = event.context.slot;
934        }
935    }
936}
937
938/// A filter parameter - used to decompose an EventFilter in multiple filters
939#[derive(Debug)]
940enum FilterItem {
941    SlotStart(Slot),
942    SlotStartEnd(Slot, Slot),
943    SlotEnd(Slot),
944    EmitterAddress(Address),
945    OriginalCallerAddress(Address),
946    OriginalOperationId(OperationId),
947    IsError(bool),
948}
949
950/// Convert a EventFilter into a list of (KeyIndent, FilterItem)
951fn from_event_filter(event_filter: &EventFilter) -> Vec<(KeyIndent, FilterItem)> {
952    let mut filter_items = vec![];
953    if event_filter.start.is_some() && event_filter.end.is_some() {
954        let start = event_filter.start.unwrap();
955        let end = event_filter.end.unwrap();
956        filter_items.push((KeyIndent::Event, FilterItem::SlotStartEnd(start, end)));
957    } else if event_filter.start.is_some() {
958        let start = event_filter.start.unwrap();
959        filter_items.push((KeyIndent::Event, FilterItem::SlotStart(start)));
960    } else if event_filter.end.is_some() {
961        let end = event_filter.end.unwrap();
962        filter_items.push((KeyIndent::Event, FilterItem::SlotEnd(end)));
963    }
964
965    if let Some(addr) = event_filter.emitter_address {
966        filter_items.push((KeyIndent::EmitterAddress, FilterItem::EmitterAddress(addr)));
967    }
968
969    if let Some(addr) = event_filter.original_caller_address {
970        filter_items.push((
971            KeyIndent::OriginalCallerAddress,
972            FilterItem::OriginalCallerAddress(addr),
973        ));
974    }
975
976    if let Some(op_id) = event_filter.original_operation_id {
977        filter_items.push((
978            KeyIndent::OriginalOperationId,
979            FilterItem::OriginalOperationId(op_id),
980        ));
981    }
982
983    if let Some(is_error) = event_filter.is_error {
984        filter_items.push((KeyIndent::IsError, FilterItem::IsError(is_error)));
985    }
986
987    filter_items
988}
989
990#[cfg(test)]
991impl EventCache {
992    /// Iterate over all keys & values in the db - test only
993    fn iter_all(
994        &self,
995        mode: Option<IteratorMode>,
996    ) -> impl Iterator<Item = (Box<[u8]>, Box<[u8]>)> + '_ {
997        self.db
998            .iterator(mode.unwrap_or(IteratorMode::Start))
999            .flatten()
1000    }
1001}
1002
1003#[cfg(test)]
1004mod tests {
1005    use super::*;
1006    // std
1007    use std::collections::VecDeque;
1008    use std::str::FromStr;
1009    // third-party
1010    use more_asserts::assert_gt;
1011    use rand::seq::SliceRandom;
1012    use rand::thread_rng;
1013    use serial_test::serial;
1014    use tempfile::TempDir;
1015    // internal
1016    use massa_models::config::{
1017        MAX_EVENT_DATA_SIZE, MAX_EVENT_PER_OPERATION, MAX_OPERATIONS_PER_BLOCK,
1018        MAX_RECURSIVE_CALLS_DEPTH, THREAD_COUNT,
1019    };
1020    use massa_models::operation::OperationId;
1021    use massa_models::output_event::EventExecutionContext;
1022    use massa_models::slot::Slot;
1023
1024    fn setup() -> EventCache {
1025        let tmp_path = TempDir::new().unwrap().path().to_path_buf();
1026        EventCache::new(
1027            &tmp_path,
1028            1000,
1029            300,
1030            THREAD_COUNT,
1031            MAX_RECURSIVE_CALLS_DEPTH,
1032            MAX_EVENT_DATA_SIZE as u64,
1033            MAX_EVENT_PER_OPERATION as u64,
1034            MAX_OPERATIONS_PER_BLOCK as u64,
1035            5000,      // MAX_EVENTS_PER_QUERY,
1036            1_000_000, // MAX_SCANNED_KEYS_PER_QUERY
1037        )
1038    }
1039
1040    #[test]
1041    #[serial]
1042    fn test_db_insert_order() {
1043        // Test that the data will be correctly ordered (when iterated from start) in db
1044
1045        let mut cache = setup();
1046        let slot_1 = Slot::new(1, 0);
1047        let index_1_0 = 0;
1048        let event = SCOutputEvent {
1049            context: EventExecutionContext {
1050                slot: slot_1,
1051                block: None,
1052                read_only: false,
1053                index_in_slot: index_1_0,
1054                call_stack: Default::default(),
1055                origin_operation_id: None,
1056                is_final: true,
1057                is_error: false,
1058                deferred_call_id: None,
1059                async_msg_id: None,
1060            },
1061            data: "message foo bar".to_string(),
1062        };
1063
1064        let max_entry_count = cache.max_entry_count - 5;
1065        let mut events = (0..max_entry_count)
1066            .map(|i| {
1067                let mut event = event.clone();
1068                event.context.index_in_slot = i as u64;
1069                event
1070            })
1071            .collect::<Vec<SCOutputEvent>>();
1072
1073        let slot_2 = Slot::new(2, 0);
1074        let event_slot_2 = {
1075            let mut event = event.clone();
1076            event.context.slot = slot_2;
1077            event.context.index_in_slot = 0u64;
1078            event
1079        };
1080        let index_2_2 = 256u64;
1081        let event_slot_2_2 = {
1082            let mut event = event.clone();
1083            event.context.slot = slot_2;
1084            event.context.index_in_slot = index_2_2;
1085            event
1086        };
1087        events.push(event_slot_2.clone());
1088        events.push(event_slot_2_2.clone());
1089        // Randomize the events so we insert in random orders in DB
1090        events.shuffle(&mut thread_rng());
1091
1092        for event in events {
1093            cache.insert(event);
1094        }
1095
1096        // Now check that we are going to iter in correct order
1097        // let db_it = cache.db_iter(Some(IteratorMode::Start));
1098        let mut prev_slot = None;
1099        let mut prev_event_index = None;
1100        for kvb in cache.iter_all(None) {
1101            let bytes = kvb.0.iter().as_slice();
1102
1103            if bytes[0] != u8::from(KeyIndent::Event) {
1104                continue;
1105            }
1106
1107            let slot = Slot::from_bytes_key(&bytes[1..=9].try_into().unwrap());
1108            let event_index = u64::from_be_bytes(bytes[10..].try_into().unwrap());
1109            if prev_slot.is_some() && prev_event_index.is_some() {
1110                assert_gt!(
1111                    (slot, event_index),
1112                    (prev_slot.unwrap(), prev_event_index.unwrap())
1113                );
1114            } else {
1115                assert_eq!(slot, slot_1);
1116                assert_eq!(event_index, index_1_0);
1117            }
1118            prev_slot = Some(slot);
1119            prev_event_index = Some(event_index);
1120        }
1121
1122        assert_eq!(prev_slot, Some(slot_2));
1123        assert_eq!(prev_event_index, Some(index_2_2));
1124    }
1125
1126    #[test]
1127    #[serial]
1128    fn test_insert_more_than_max_entry() {
1129        // Test insert (and snip) so we do not store too many event in the cache
1130
1131        let mut cache = setup();
1132        let event = SCOutputEvent {
1133            context: EventExecutionContext {
1134                slot: Slot::new(1, 0),
1135                block: None,
1136                read_only: false,
1137                index_in_slot: 0,
1138                call_stack: Default::default(),
1139                origin_operation_id: None,
1140                is_final: true,
1141                is_error: false,
1142                deferred_call_id: None,
1143                async_msg_id: None,
1144            },
1145            data: "message foo bar".to_string(),
1146        };
1147
1148        // fill the db: add cache.max_entry_count entries
1149        for count in 0..cache.max_entry_count {
1150            let mut event = event.clone();
1151            event.context.index_in_slot = count as u64;
1152            cache.insert(event.clone());
1153        }
1154        assert_eq!(cache.entry_count, cache.max_entry_count);
1155
1156        // insert one more entry
1157        let mut event_last = event.clone();
1158        event_last.context.index_in_slot = u64::MAX;
1159        cache.insert(event_last);
1160        assert_eq!(
1161            cache.entry_count,
1162            cache.max_entry_count - cache.snip_amount + 1
1163        );
1164        dbg!(cache.entry_count);
1165    }
1166
1167    #[test]
1168    #[serial]
1169    fn test_insert_more_than_max_entry_2() {
1170        // Test insert_multi_it (and snip) so we do not store too many event in the cache
1171
1172        let mut cache = setup();
1173        let event = SCOutputEvent {
1174            context: EventExecutionContext {
1175                slot: Slot::new(1, 0),
1176                block: None,
1177                read_only: false,
1178                index_in_slot: 0,
1179                call_stack: Default::default(),
1180                origin_operation_id: None,
1181                is_final: true,
1182                is_error: false,
1183                deferred_call_id: None,
1184                async_msg_id: None,
1185            },
1186            data: "message foo bar".to_string(),
1187        };
1188
1189        let it = (0..cache.max_entry_count).map(|i| {
1190            let mut event = event.clone();
1191            event.context.index_in_slot = i as u64;
1192            event
1193        });
1194        cache.insert_multi_it(it);
1195
1196        assert_eq!(cache.entry_count, cache.max_entry_count);
1197
1198        // insert one more entry
1199        let mut event_last = event.clone();
1200        event_last.context.index_in_slot = u64::MAX;
1201        cache.insert(event_last);
1202        assert_eq!(
1203            cache.entry_count,
1204            cache.max_entry_count - cache.snip_amount + 1
1205        );
1206        dbg!(cache.entry_count);
1207    }
1208
1209    #[test]
1210    #[serial]
1211    fn test_snip() {
1212        // Test snip so we enforce that all db keys are removed
1213
1214        let mut cache = setup();
1215        cache.max_entry_count = 10;
1216
1217        let event = SCOutputEvent {
1218            context: EventExecutionContext {
1219                slot: Slot::new(1, 0),
1220                block: None,
1221                read_only: false,
1222                index_in_slot: 0,
1223                call_stack: Default::default(),
1224                origin_operation_id: None,
1225                is_final: true,
1226                is_error: false,
1227                deferred_call_id: None,
1228                async_msg_id: None,
1229            },
1230            data: "message foo bar".to_string(),
1231        };
1232
1233        let it = (0..cache.max_entry_count).map(|i| {
1234            let mut event = event.clone();
1235            event.context.index_in_slot = i as u64;
1236            event
1237        });
1238        cache.insert_multi_it(it);
1239
1240        assert_eq!(cache.entry_count, cache.max_entry_count);
1241
1242        cache.snip(Some(cache.entry_count));
1243
1244        assert_eq!(cache.entry_count, 0);
1245        assert_eq!(cache.iter_all(None).count(), 0);
1246    }
1247
1248    #[test]
1249    #[serial]
1250    fn test_counter_0() {
1251        // Test snip so we enforce that all db keys are removed
1252
1253        let mut cache = setup();
1254        cache.max_entry_count = 10;
1255
1256        let dummy_addr =
1257            Address::from_str("AU12qePoXhNbYWE1jZuafqJong7bbq1jw3k89RgbMawbrdZpaasoA").unwrap();
1258        let emit_addr_1 =
1259            Address::from_str("AU122Em8qkqegdLb1eyH8rdkSCNEf7RZLeTJve4Q2inRPGiTJ2xNv").unwrap();
1260        let emit_addr_2 =
1261            Address::from_str("AU12WuVR1Td74q9eAbtYZUnk5jnRbUuUacyhQFwm217bV5v1mNqTZ").unwrap();
1262
1263        let event = SCOutputEvent {
1264            context: EventExecutionContext {
1265                slot: Slot::new(1, 0),
1266                block: None,
1267                read_only: false,
1268                index_in_slot: 0,
1269                call_stack: VecDeque::from(vec![dummy_addr, emit_addr_1]),
1270                origin_operation_id: None,
1271                is_final: true,
1272                is_error: false,
1273                deferred_call_id: None,
1274                async_msg_id: None,
1275            },
1276            data: "message foo bar".to_string(),
1277        };
1278
1279        let event_2 = {
1280            let mut evt = event.clone();
1281            evt.context.slot = Slot::new(2, 0);
1282            evt.context.call_stack = VecDeque::from(vec![dummy_addr, emit_addr_2]);
1283            evt
1284        };
1285
1286        cache.insert_multi_it([event, event_2].into_iter());
1287
1288        // Check counters key length
1289        let key_counters = cache
1290            .key_builder
1291            .prefix_key_from_indent(&KeyIndent::Counter);
1292        let kvbs: Result<Vec<(_, _)>, _> = cache
1293            .db
1294            .prefix_iterator(key_counters)
1295            .take_while(|kvb| {
1296                kvb.as_ref()
1297                    .unwrap()
1298                    .0
1299                    .starts_with(&[u8::from(KeyIndent::Counter)])
1300            })
1301            .collect();
1302        // println!("kvbs: {:#?}", kvbs);
1303
1304        // Expected 4 counters:
1305        // 2 for emitter address (emit_addr_1 & emit_addr_2)
1306        // 1 for original caller address (dummy_addr)
1307        // 1 for is_error(false)
1308        assert_eq!(kvbs.unwrap().len(), 4);
1309
1310        let key_counter_1 = cache.key_builder.counter_key_from_filter_item(
1311            &FilterItem::EmitterAddress(emit_addr_1),
1312            &KeyIndent::EmitterAddress,
1313        );
1314        let key_counter_2 = cache.key_builder.counter_key_from_filter_item(
1315            &FilterItem::EmitterAddress(emit_addr_2),
1316            &KeyIndent::EmitterAddress,
1317        );
1318
1319        let v1 = cache.db.get(key_counter_1.clone());
1320        let v2 = cache.db.get(key_counter_2.clone());
1321
1322        // println!("v1: {:?} - v2: {:?}", v1, v2);
1323        assert_eq!(v1, Ok(Some(1u64.to_be_bytes().to_vec())));
1324        assert_eq!(v2, Ok(Some(1u64.to_be_bytes().to_vec())));
1325
1326        cache.snip(Some(1));
1327
1328        let v1 = cache.db.get(key_counter_1);
1329        let v2 = cache.db.get(key_counter_2);
1330
1331        // println!("v1: {:?} - v2: {:?}", v1, v2);
1332        assert_eq!(v1, Ok(None)); // counter has been removed
1333        assert_eq!(v2, Ok(Some(1u64.to_be_bytes().to_vec())));
1334    }
1335
1336    #[test]
1337    #[serial]
1338    fn test_event_filter() {
1339        // Test that the data will be correctly ordered (when filtered) in db
1340
1341        let mut cache = setup();
1342        let slot_1 = Slot::new(1, 0);
1343        let index_1_0 = 0;
1344        let event = SCOutputEvent {
1345            context: EventExecutionContext {
1346                slot: slot_1,
1347                block: None,
1348                read_only: false,
1349                index_in_slot: index_1_0,
1350                call_stack: Default::default(),
1351                origin_operation_id: None,
1352                is_final: true,
1353                is_error: false,
1354                deferred_call_id: None,
1355                async_msg_id: None,
1356            },
1357            data: "message foo bar".to_string(),
1358        };
1359
1360        let mut events = (0..cache.max_entry_count - 5)
1361            .map(|i| {
1362                let mut event = event.clone();
1363                event.context.index_in_slot = i as u64;
1364                event
1365            })
1366            .collect::<Vec<SCOutputEvent>>();
1367
1368        let slot_2 = Slot::new(2, 0);
1369        let index_2_1 = 0u64;
1370        let event_slot_2 = {
1371            let mut event = event.clone();
1372            event.context.slot = slot_2;
1373            event.context.index_in_slot = index_2_1;
1374            event
1375        };
1376        let index_2_2 = 256u64;
1377        let event_slot_2_2 = {
1378            let mut event = event.clone();
1379            event.context.slot = slot_2;
1380            event.context.index_in_slot = index_2_2;
1381            event
1382        };
1383        events.push(event_slot_2.clone());
1384        events.push(event_slot_2_2.clone());
1385        // Randomize the events so we insert in random orders in the DB
1386        events.shuffle(&mut thread_rng());
1387
1388        cache.insert_multi_it(events.into_iter());
1389
1390        let filter_1 = EventFilter {
1391            start: Some(Slot::new(2, 0)),
1392            ..Default::default()
1393        };
1394
1395        let (_, filtered_events_1) = cache.get_filtered_sc_output_events(&filter_1).unwrap();
1396
1397        assert_eq!(filtered_events_1.len(), 2);
1398        assert_eq!(filtered_events_1[0].context.slot, slot_2);
1399        assert_eq!(filtered_events_1[0].context.index_in_slot, index_2_1);
1400        assert_eq!(filtered_events_1[1].context.slot, slot_2);
1401        assert_eq!(filtered_events_1[1].context.index_in_slot, index_2_2);
1402    }
1403
1404    #[test]
1405    #[serial]
1406    fn test_event_filter_2() {
1407        // Test get_filtered_sc_output_events + op id
1408
1409        let mut cache = setup();
1410        cache.max_entry_count = 10;
1411
1412        let slot_1 = Slot::new(1, 0);
1413        let index_1_0 = 0;
1414        let op_id_1 =
1415            OperationId::from_str("O12n1vt8uTLh3H65J4TVuztaWfBh3oumjjVtRCkke7Ba5qWdXdjD").unwrap();
1416        let op_id_2 =
1417            OperationId::from_str("O1p5P691KF672fQ8tQougxzSERBwDKZF8FwtkifMSJbP14sEuGc").unwrap();
1418        let op_id_unknown =
1419            OperationId::from_str("O1kvXTfsnVbQcmDERkC89vqAd2xRTLCb3q5b2E5WaVPHwFd7Qth").unwrap();
1420
1421        let event = SCOutputEvent {
1422            context: EventExecutionContext {
1423                slot: slot_1,
1424                block: None,
1425                read_only: false,
1426                index_in_slot: index_1_0,
1427                call_stack: Default::default(),
1428                origin_operation_id: Some(op_id_1),
1429                is_final: true,
1430                is_error: false,
1431                deferred_call_id: None,
1432                async_msg_id: None,
1433            },
1434            data: "message foo bar".to_string(),
1435        };
1436
1437        let mut events = (0..cache.max_entry_count - 5)
1438            .map(|i| {
1439                let mut event = event.clone();
1440                event.context.index_in_slot = i as u64;
1441                event
1442            })
1443            .collect::<Vec<SCOutputEvent>>();
1444
1445        let slot_2 = Slot::new(2, 0);
1446        let index_2_1 = 0u64;
1447        let event_slot_2 = {
1448            let mut event = event.clone();
1449            event.context.slot = slot_2;
1450            event.context.index_in_slot = index_2_1;
1451            event.context.origin_operation_id = Some(op_id_2);
1452            event
1453        };
1454        let index_2_2 = 256u64;
1455        let event_slot_2_2 = {
1456            let mut event = event.clone();
1457            event.context.slot = slot_2;
1458            event.context.index_in_slot = index_2_2;
1459            event.context.origin_operation_id = Some(op_id_2);
1460            event
1461        };
1462        events.push(event_slot_2.clone());
1463        events.push(event_slot_2_2.clone());
1464        // Randomize the events so we insert in random orders in the DB
1465        events.shuffle(&mut thread_rng());
1466        cache.insert_multi_it(events.into_iter());
1467
1468        let mut filter_1 = EventFilter {
1469            original_operation_id: Some(op_id_1),
1470            ..Default::default()
1471        };
1472
1473        let (_, filtered_events_1) = cache.get_filtered_sc_output_events(&filter_1).unwrap();
1474
1475        assert_eq!(filtered_events_1.len(), cache.max_entry_count - 5);
1476        filtered_events_1.iter().enumerate().for_each(|(i, event)| {
1477            assert_eq!(event.context.slot, slot_1);
1478            assert_eq!(event.context.index_in_slot, i as u64);
1479        });
1480
1481        {
1482            filter_1.original_operation_id = Some(op_id_2);
1483            let (_, filtered_events_2) = cache.get_filtered_sc_output_events(&filter_1).unwrap();
1484            assert_eq!(filtered_events_2.len(), 2);
1485            filtered_events_2.iter().enumerate().for_each(|(i, event)| {
1486                assert_eq!(event.context.slot, slot_2);
1487                if i == 0 {
1488                    assert_eq!(event.context.index_in_slot, i as u64);
1489                } else {
1490                    assert_eq!(event.context.index_in_slot, 256u64);
1491                }
1492            });
1493        }
1494
1495        {
1496            filter_1.original_operation_id = Some(op_id_unknown);
1497            let (_, filtered_events_2) = cache.get_filtered_sc_output_events(&filter_1).unwrap();
1498            assert_eq!(filtered_events_2.len(), 0);
1499        }
1500    }
1501
1502    #[test]
1503    #[serial]
1504    fn test_event_filter_3() {
1505        // Test get_filtered_sc_output_events + emitter address
1506
1507        let mut cache = setup();
1508        cache.max_entry_count = 10;
1509
1510        let slot_1 = Slot::new(1, 0);
1511        let index_1_0 = 0;
1512
1513        let dummy_addr =
1514            Address::from_str("AU12qePoXhNbYWE1jZuafqJong7bbq1jw3k89RgbMawbrdZpaasoA").unwrap();
1515        let emit_addr_1 =
1516            Address::from_str("AU122Em8qkqegdLb1eyH8rdkSCNEf7RZLeTJve4Q2inRPGiTJ2xNv").unwrap();
1517        let emit_addr_2 =
1518            Address::from_str("AU12WuVR1Td74q9eAbtYZUnk5jnRbUuUacyhQFwm217bV5v1mNqTZ").unwrap();
1519        let emit_addr_unknown =
1520            Address::from_str("AU1zLC4TFUiaKDg7quQyusMPQcHT4ykWVs3FsFpuhdNSmowUG2As").unwrap();
1521
1522        let event = SCOutputEvent {
1523            context: EventExecutionContext {
1524                slot: slot_1,
1525                block: None,
1526                read_only: false,
1527                index_in_slot: index_1_0,
1528                call_stack: Default::default(),
1529                origin_operation_id: None,
1530                is_final: true,
1531                is_error: false,
1532                deferred_call_id: None,
1533                async_msg_id: None,
1534            },
1535            data: "message foo bar".to_string(),
1536        };
1537
1538        let to_insert_count = cache.max_entry_count - 5;
1539        let threshold = to_insert_count / 2;
1540        let mut events = (0..cache.max_entry_count - 5)
1541            .map(|i| {
1542                let mut event = event.clone();
1543                event.context.index_in_slot = i as u64;
1544                if i < threshold {
1545                    event.context.call_stack = VecDeque::from(vec![dummy_addr, emit_addr_1]);
1546                } else {
1547                    event.context.call_stack = VecDeque::from(vec![dummy_addr, emit_addr_2]);
1548                }
1549                event
1550            })
1551            .collect::<Vec<SCOutputEvent>>();
1552
1553        let slot_2 = Slot::new(2, 0);
1554        let index_2_1 = 0u64;
1555        let event_slot_2 = {
1556            let mut event = event.clone();
1557            event.context.slot = slot_2;
1558            event.context.index_in_slot = index_2_1;
1559            event.context.call_stack = VecDeque::from(vec![dummy_addr, emit_addr_2]);
1560            event
1561        };
1562        let index_2_2 = 256u64;
1563        let event_slot_2_2 = {
1564            let mut event = event.clone();
1565            event.context.slot = slot_2;
1566            event.context.index_in_slot = index_2_2;
1567            event.context.call_stack = VecDeque::from(vec![dummy_addr, emit_addr_2]);
1568            event
1569        };
1570        events.push(event_slot_2.clone());
1571        events.push(event_slot_2_2.clone());
1572        // Randomize the events so we insert in random orders in the DB
1573        events.shuffle(&mut thread_rng());
1574
1575        cache.insert_multi_it(events.into_iter());
1576
1577        let mut filter_1 = EventFilter {
1578            emitter_address: Some(emit_addr_1),
1579            ..Default::default()
1580        };
1581
1582        let (_, filtered_events_1) = cache.get_filtered_sc_output_events(&filter_1).unwrap();
1583
1584        assert_eq!(filtered_events_1.len(), threshold);
1585        filtered_events_1.iter().for_each(|event| {
1586            assert_eq!(event.context.slot, slot_1);
1587            assert_eq!(*event.context.call_stack.back().unwrap(), emit_addr_1)
1588        });
1589
1590        // filter with emit_addr_2
1591        {
1592            filter_1.emitter_address = Some(emit_addr_2);
1593            let (_, filtered_events_2) = cache.get_filtered_sc_output_events(&filter_1).unwrap();
1594            assert_eq!(filtered_events_2.len(), threshold + 1 + 2);
1595            filtered_events_2.iter().for_each(|event| {
1596                assert_eq!(*event.context.call_stack.back().unwrap(), emit_addr_2)
1597            });
1598        }
1599        // filter with dummy_addr
1600        {
1601            filter_1.emitter_address = Some(dummy_addr);
1602            let (_, filtered_events_2) = cache.get_filtered_sc_output_events(&filter_1).unwrap();
1603            assert_eq!(filtered_events_2.len(), 0);
1604        }
1605        // filter with address that is not in the DB
1606        {
1607            filter_1.emitter_address = Some(emit_addr_unknown);
1608            let (_, filtered_events_2) = cache.get_filtered_sc_output_events(&filter_1).unwrap();
1609            assert_eq!(filtered_events_2.len(), 0);
1610        }
1611    }
1612
1613    #[test]
1614    #[serial]
1615    fn test_event_filter_4() {
1616        // Test get_filtered_sc_output_events + original caller addr
1617
1618        let mut cache = setup();
1619        cache.max_entry_count = 10;
1620
1621        let slot_1 = Slot::new(1, 0);
1622        let index_1_0 = 0;
1623
1624        let dummy_addr =
1625            Address::from_str("AU12qePoXhNbYWE1jZuafqJong7bbq1jw3k89RgbMawbrdZpaasoA").unwrap();
1626        let caller_addr_1 =
1627            Address::from_str("AU122Em8qkqegdLb1eyH8rdkSCNEf7RZLeTJve4Q2inRPGiTJ2xNv").unwrap();
1628        let caller_addr_2 =
1629            Address::from_str("AU12WuVR1Td74q9eAbtYZUnk5jnRbUuUacyhQFwm217bV5v1mNqTZ").unwrap();
1630        let caller_addr_unknown =
1631            Address::from_str("AU1zLC4TFUiaKDg7quQyusMPQcHT4ykWVs3FsFpuhdNSmowUG2As").unwrap();
1632
1633        let event = SCOutputEvent {
1634            context: EventExecutionContext {
1635                slot: slot_1,
1636                block: None,
1637                read_only: false,
1638                index_in_slot: index_1_0,
1639                call_stack: Default::default(),
1640                origin_operation_id: None,
1641                is_final: true,
1642                is_error: false,
1643                deferred_call_id: None,
1644                async_msg_id: None,
1645            },
1646            data: "message foo bar".to_string(),
1647        };
1648
1649        let to_insert_count = cache.max_entry_count - 5;
1650        let threshold = to_insert_count / 2;
1651        let mut events = (0..cache.max_entry_count - 5)
1652            .map(|i| {
1653                let mut event = event.clone();
1654                event.context.index_in_slot = i as u64;
1655                if i < threshold {
1656                    event.context.call_stack = VecDeque::from(vec![caller_addr_1, dummy_addr]);
1657                } else {
1658                    event.context.call_stack = VecDeque::from(vec![caller_addr_2, dummy_addr]);
1659                }
1660                event
1661            })
1662            .collect::<Vec<SCOutputEvent>>();
1663
1664        let slot_2 = Slot::new(2, 0);
1665        let index_2_1 = 0u64;
1666        let event_slot_2 = {
1667            let mut event = event.clone();
1668            event.context.slot = slot_2;
1669            event.context.index_in_slot = index_2_1;
1670            event.context.call_stack = VecDeque::from(vec![caller_addr_2, dummy_addr]);
1671            event
1672        };
1673        let index_2_2 = 256u64;
1674        let event_slot_2_2 = {
1675            let mut event = event.clone();
1676            event.context.slot = slot_2;
1677            event.context.index_in_slot = index_2_2;
1678            event.context.call_stack = VecDeque::from(vec![caller_addr_2, dummy_addr]);
1679            event
1680        };
1681        events.push(event_slot_2.clone());
1682        events.push(event_slot_2_2.clone());
1683        // Randomize the events so we insert in random orders in the DB
1684        events.shuffle(&mut thread_rng());
1685        cache.insert_multi_it(events.into_iter());
1686
1687        let mut filter_1 = EventFilter {
1688            original_caller_address: Some(caller_addr_1),
1689            ..Default::default()
1690        };
1691
1692        let (_, filtered_events_1) = cache.get_filtered_sc_output_events(&filter_1).unwrap();
1693
1694        assert_eq!(filtered_events_1.len(), threshold);
1695        filtered_events_1.iter().for_each(|event| {
1696            assert_eq!(event.context.slot, slot_1);
1697            assert_eq!(*event.context.call_stack.front().unwrap(), caller_addr_1);
1698        });
1699
1700        {
1701            filter_1.original_caller_address = Some(caller_addr_2);
1702            let (_, filtered_events_2) = cache.get_filtered_sc_output_events(&filter_1).unwrap();
1703            assert_eq!(filtered_events_2.len(), threshold + 1 + 2);
1704            filtered_events_2.iter().for_each(|event| {
1705                assert_eq!(*event.context.call_stack.front().unwrap(), caller_addr_2);
1706            });
1707        }
1708        {
1709            filter_1.original_caller_address = Some(dummy_addr);
1710            let (_, filtered_events_2) = cache.get_filtered_sc_output_events(&filter_1).unwrap();
1711            assert_eq!(filtered_events_2.len(), 0);
1712        }
1713        {
1714            filter_1.original_caller_address = Some(caller_addr_unknown);
1715            let (_, filtered_events_2) = cache.get_filtered_sc_output_events(&filter_1).unwrap();
1716            assert_eq!(filtered_events_2.len(), 0);
1717        }
1718    }
1719
1720    #[test]
1721    #[serial]
1722    fn test_event_filter_5() {
1723        // Test get_filtered_sc_output_events + is error
1724
1725        let mut cache = setup();
1726        cache.max_entry_count = 10;
1727
1728        let slot_1 = Slot::new(1, 0);
1729        let index_1_0 = 0;
1730
1731        let dummy_addr =
1732            Address::from_str("AU12qePoXhNbYWE1jZuafqJong7bbq1jw3k89RgbMawbrdZpaasoA").unwrap();
1733        let emit_addr_1 =
1734            Address::from_str("AU122Em8qkqegdLb1eyH8rdkSCNEf7RZLeTJve4Q2inRPGiTJ2xNv").unwrap();
1735        let emit_addr_2 =
1736            Address::from_str("AU12WuVR1Td74q9eAbtYZUnk5jnRbUuUacyhQFwm217bV5v1mNqTZ").unwrap();
1737
1738        let event = SCOutputEvent {
1739            context: EventExecutionContext {
1740                slot: slot_1,
1741                block: None,
1742                read_only: false,
1743                index_in_slot: index_1_0,
1744                call_stack: Default::default(),
1745                origin_operation_id: None,
1746                is_final: true,
1747                is_error: false,
1748                deferred_call_id: None,
1749                async_msg_id: None,
1750            },
1751            data: "message foo bar".to_string(),
1752        };
1753
1754        let to_insert_count = cache.max_entry_count - 5;
1755        let threshold = to_insert_count / 2;
1756        let mut events = (0..cache.max_entry_count - 5)
1757            .map(|i| {
1758                let mut event = event.clone();
1759                event.context.index_in_slot = i as u64;
1760                if i < threshold {
1761                    event.context.call_stack = VecDeque::from(vec![dummy_addr, emit_addr_1]);
1762                } else {
1763                    event.context.call_stack = VecDeque::from(vec![dummy_addr, emit_addr_2]);
1764                }
1765                event
1766            })
1767            .collect::<Vec<SCOutputEvent>>();
1768
1769        let slot_2 = Slot::new(2, 0);
1770        let index_2_1 = 0u64;
1771        let event_slot_2 = {
1772            let mut event = event.clone();
1773            event.context.slot = slot_2;
1774            event.context.index_in_slot = index_2_1;
1775            event.context.call_stack = VecDeque::from(vec![dummy_addr, emit_addr_2]);
1776            event
1777        };
1778        let index_2_2 = 256u64;
1779        let event_slot_2_2 = {
1780            let mut event = event.clone();
1781            event.context.slot = slot_2;
1782            event.context.index_in_slot = index_2_2;
1783            event.context.call_stack = VecDeque::from(vec![dummy_addr, emit_addr_2]);
1784            event.context.is_error = true;
1785            event
1786        };
1787        events.push(event_slot_2.clone());
1788        events.push(event_slot_2_2.clone());
1789        // Randomize the events so we insert in random orders in the DB
1790        events.shuffle(&mut thread_rng());
1791        cache.insert_multi_it(events.into_iter());
1792
1793        let filter_1 = EventFilter {
1794            is_error: Some(true),
1795            ..Default::default()
1796        };
1797
1798        let (_, filtered_events_1) = cache.get_filtered_sc_output_events(&filter_1).unwrap();
1799        assert_eq!(filtered_events_1.len(), 1);
1800        assert!(filtered_events_1[0].context.is_error);
1801        assert_eq!(filtered_events_1[0].context.slot, slot_2);
1802        assert_eq!(filtered_events_1[0].context.index_in_slot, index_2_2);
1803    }
1804
1805    #[test]
1806    #[serial]
1807    fn test_orphan_index_key_does_not_panic() {
1808        // An index key whose event is missing (e.g. event key overwritten by a
1809        // duplicate (slot, index) then snipped) used to panic on unwrap()
1810
1811        let mut cache = setup();
1812        let emitter =
1813            Address::from_str("AU12qePoXhNbYWE1jZuafqJong7bbq1jw3k89RgbMawbrdZpaasoA").unwrap();
1814        let event = SCOutputEvent {
1815            context: EventExecutionContext {
1816                slot: Slot::new(1, 0),
1817                block: None,
1818                read_only: false,
1819                index_in_slot: 0,
1820                call_stack: VecDeque::from(vec![emitter]),
1821                origin_operation_id: None,
1822                is_final: true,
1823                is_error: false,
1824                deferred_call_id: None,
1825                async_msg_id: None,
1826            },
1827            data: "message foo bar".to_string(),
1828        };
1829        cache.insert(event.clone());
1830
1831        // Remove the event value but keep its index keys
1832        let event_key = cache
1833            .key_builder
1834            .key_from_event(&event, &KeyIndent::Event, &KeyKind::Regular)
1835            .unwrap();
1836        cache.db.delete(event_key).unwrap();
1837
1838        let filter = EventFilter {
1839            emitter_address: Some(emitter),
1840            ..Default::default()
1841        };
1842        let (_, events) = cache.get_filtered_sc_output_events(&filter).unwrap();
1843        assert!(events.is_empty());
1844    }
1845
1846    #[test]
1847    #[serial]
1848    fn test_filter_optimisations() {
1849        // Test we iterate over the right number of rows when filtering
1850
1851        let mut cache = setup();
1852        cache.max_entry_count = 10;
1853
1854        let slot_1 = Slot::new(1, 0);
1855        let index_1_0 = 0;
1856
1857        let dummy_addr =
1858            Address::from_str("AU12qePoXhNbYWE1jZuafqJong7bbq1jw3k89RgbMawbrdZpaasoA").unwrap();
1859        let emit_addr_1 =
1860            Address::from_str("AU122Em8qkqegdLb1eyH8rdkSCNEf7RZLeTJve4Q2inRPGiTJ2xNv").unwrap();
1861        let emit_addr_2 =
1862            Address::from_str("AU12WuVR1Td74q9eAbtYZUnk5jnRbUuUacyhQFwm217bV5v1mNqTZ").unwrap();
1863
1864        let event = SCOutputEvent {
1865            context: EventExecutionContext {
1866                slot: slot_1,
1867                block: None,
1868                read_only: false,
1869                index_in_slot: index_1_0,
1870                call_stack: Default::default(),
1871                origin_operation_id: None,
1872                is_final: true,
1873                is_error: true,
1874                deferred_call_id: None,
1875                async_msg_id: None,
1876            },
1877            data: "error foo bar".to_string(),
1878        };
1879
1880        let to_insert_count = cache.max_entry_count - 5;
1881        let threshold = to_insert_count / 2;
1882        let mut events = (0..cache.max_entry_count - 5)
1883            .map(|i| {
1884                let mut event = event.clone();
1885                event.context.index_in_slot = i as u64;
1886                if i < threshold {
1887                    event.context.call_stack = VecDeque::from(vec![dummy_addr, emit_addr_1]);
1888                } else {
1889                    event.context.call_stack = VecDeque::from(vec![dummy_addr, emit_addr_2]);
1890                }
1891                event
1892            })
1893            .collect::<Vec<SCOutputEvent>>();
1894
1895        let slot_2 = Slot::new(2, 0);
1896        let index_2_1 = 0u64;
1897        let event_slot_2 = {
1898            let mut event = event.clone();
1899            event.context.slot = slot_2;
1900            event.context.index_in_slot = index_2_1;
1901            event.context.call_stack = VecDeque::from(vec![dummy_addr, emit_addr_2]);
1902            event
1903        };
1904        let index_2_2 = 256u64;
1905        let event_slot_2_2 = {
1906            let mut event = event.clone();
1907            event.context.slot = slot_2;
1908            event.context.index_in_slot = index_2_2;
1909            event.context.call_stack = VecDeque::from(vec![dummy_addr, emit_addr_2]);
1910            // event.context.is_error = true;
1911            event
1912        };
1913        events.push(event_slot_2.clone());
1914        events.push(event_slot_2_2.clone());
1915        // Randomize the events so we insert in random orders in the DB
1916        events.shuffle(&mut thread_rng());
1917        cache.insert_multi_it(events.into_iter());
1918
1919        // Check if we correctly count the number of events in the DB with emit_addr_1 & emit_addr_2
1920        let emit_addr_1_count = cache
1921            .filter_item_estimate_count(
1922                &KeyIndent::EmitterAddress,
1923                &FilterItem::EmitterAddress(emit_addr_1),
1924            )
1925            .unwrap();
1926        let emit_addr_2_count = cache
1927            .filter_item_estimate_count(
1928                &KeyIndent::EmitterAddress,
1929                &FilterItem::EmitterAddress(emit_addr_2),
1930            )
1931            .unwrap();
1932
1933        assert_eq!(emit_addr_1_count, (threshold) as u64);
1934        assert_eq!(emit_addr_2_count, (threshold + 1 + 2) as u64);
1935
1936        // Check if we correctly count the number of events in the DB with slot related filters
1937        // First with filters that should give a count of 0 (too early, too late, or inconsistent start/end)
1938        let slot_start_1 = Slot::new(3, 0); // start is after the last slot
1939        let slot_end_1 = Slot::new(0, 0); // end is before the first slot
1940        let slot_start_end_1 = Slot::new(2, 0)..Slot::new(1, 0); // start is after end
1941
1942        // Then with filters that should give a positive count
1943        // num_thread * max_events_per_operation * max_operations_per_block
1944        let slot_start_2 = Slot::new(1, 0);
1945        let slot_end_2 = Slot::new(2, 0);
1946        let slot_start_end_2 = Slot::new(1, 0)..Slot::new(2, 0);
1947
1948        let slot_start_count_1 = cache
1949            .filter_item_estimate_count(&KeyIndent::Event, &FilterItem::SlotStart(slot_start_1))
1950            .unwrap();
1951        let slot_end_count_1 = cache
1952            .filter_item_estimate_count(&KeyIndent::Event, &FilterItem::SlotEnd(slot_end_1))
1953            .unwrap();
1954        let slot_start_end_count_1 = cache
1955            .filter_item_estimate_count(
1956                &KeyIndent::Event,
1957                &FilterItem::SlotStartEnd(slot_start_end_1.start, slot_start_end_1.end),
1958            )
1959            .unwrap();
1960
1961        let slot_start_count_2 = cache
1962            .filter_item_estimate_count(&KeyIndent::Event, &FilterItem::SlotStart(slot_start_2))
1963            .unwrap();
1964        let slot_end_count_2 = cache
1965            .filter_item_estimate_count(&KeyIndent::Event, &FilterItem::SlotEnd(slot_end_2))
1966            .unwrap();
1967        let slot_start_end_count_2 = cache
1968            .filter_item_estimate_count(
1969                &KeyIndent::Event,
1970                &FilterItem::SlotStartEnd(slot_start_end_2.start, slot_start_end_2.end),
1971            )
1972            .unwrap();
1973
1974        assert_eq!(slot_start_count_1, 0);
1975        assert_eq!(slot_end_count_1, 0);
1976        assert_eq!(slot_start_end_count_1, 0);
1977
1978        let expected_count_per_period =
1979            THREAD_COUNT as u64 * MAX_EVENT_PER_OPERATION as u64 * MAX_OPERATIONS_PER_BLOCK as u64;
1980        assert_eq!(slot_start_count_2, expected_count_per_period);
1981        assert_eq!(slot_end_count_2, 2 * expected_count_per_period); // 2 periods as first_slot is (0,0)
1982        assert_eq!(slot_start_end_count_2, expected_count_per_period);
1983
1984        // Check if we query first by emitter address then is_error
1985
1986        let filter_1 = EventFilter {
1987            emitter_address: Some(emit_addr_1),
1988            is_error: Some(true),
1989            ..Default::default()
1990        };
1991
1992        let (query_counts, _filtered_events_1) =
1993            cache.get_filtered_sc_output_events(&filter_1).unwrap();
1994        println!("threshold: {:?}", threshold);
1995        println!("query_counts: {:?}", query_counts);
1996
1997        // Check that we iter no more than needed (here: only 2 (== threshold) event with emit addr 1)
1998        assert_eq!(query_counts[0], threshold as u64);
1999        // For second filter (is_error) we could have iter more (all events have is_error = true)
2000        // but as soon as we found 2 items we could return (as the previous filter already limit the final count)
2001        assert_eq!(query_counts[1], threshold as u64);
2002    }
2003
2004    /// Cache with custom query limits, filled with `count` final events over slots
2005    /// (1, 0), (2, 0), ... emitted by `emitter` every `emitter_every` events
2006    /// (others by a second address).
2007    fn cache_with_events(
2008        max_events_per_query: usize,
2009        max_scanned_keys_per_query: u64,
2010        count: u64,
2011        emitter_every: u64,
2012    ) -> (EventCache, Address, TempDir) {
2013        let tmp = TempDir::new().unwrap();
2014        let mut cache = EventCache::new(
2015            tmp.path(),
2016            1000,
2017            300,
2018            THREAD_COUNT,
2019            MAX_RECURSIVE_CALLS_DEPTH,
2020            MAX_EVENT_DATA_SIZE as u64,
2021            MAX_EVENT_PER_OPERATION as u64,
2022            MAX_OPERATIONS_PER_BLOCK as u64,
2023            max_events_per_query,
2024            max_scanned_keys_per_query,
2025        );
2026        let emitter =
2027            Address::from_str("AU122Em8qkqegdLb1eyH8rdkSCNEf7RZLeTJve4Q2inRPGiTJ2xNv").unwrap();
2028        let other =
2029            Address::from_str("AU12WuVR1Td74q9eAbtYZUnk5jnRbUuUacyhQFwm217bV5v1mNqTZ").unwrap();
2030        let mut events: Vec<SCOutputEvent> = (0..count)
2031            .map(|i| SCOutputEvent {
2032                context: EventExecutionContext {
2033                    slot: Slot::new(i + 1, 0),
2034                    block: None,
2035                    read_only: false,
2036                    index_in_slot: 0,
2037                    call_stack: VecDeque::from(vec![if i % emitter_every == 0 {
2038                        emitter
2039                    } else {
2040                        other
2041                    }]),
2042                    origin_operation_id: None,
2043                    is_final: true,
2044                    is_error: false,
2045                    deferred_call_id: None,
2046                    async_msg_id: None,
2047                },
2048                data: format!("event {}", i),
2049            })
2050            .collect();
2051        events.shuffle(&mut thread_rng());
2052        cache.insert_multi_it(events.into_iter());
2053        (cache, emitter, tmp)
2054    }
2055
2056    /// MASSA-012: the last filter pass stops once it has `max_events_per_query` keys,
2057    /// and returns exactly the events a full scan would.
2058    #[test]
2059    #[serial]
2060    fn test_query_stops_scanning_at_max_events_per_query() {
2061        let (capped, emitter, _tmp_1) = cache_with_events(10, 1_000_000, 300, 2);
2062        let (full, _, _tmp_2) = cache_with_events(1_000, 1_000_000, 300, 2);
2063
2064        let filters = [
2065            // no criteria: one pass over the whole event column
2066            EventFilter::default(),
2067            // one criterion
2068            EventFilter {
2069                emitter_address: Some(emitter),
2070                ..Default::default()
2071            },
2072            // two criteria: only the last pass can stop early
2073            EventFilter {
2074                start: Some(Slot::new(50, 0)),
2075                emitter_address: Some(emitter),
2076                ..Default::default()
2077            },
2078        ];
2079        for filter in filters {
2080            let (counts, events) = capped.get_filtered_sc_output_events(&filter).unwrap();
2081            let (full_counts, all_events) = full.get_filtered_sc_output_events(&filter).unwrap();
2082            assert_eq!(events.len(), 10, "{filter:?}");
2083            assert_eq!(events, all_events[..10], "{filter:?}");
2084            // the last pass stopped early
2085            assert!(
2086                counts.last().unwrap() < full_counts.last().unwrap(),
2087                "{filter:?}: {counts:?} vs {full_counts:?}"
2088            );
2089            if counts.len() == 1 {
2090                // a single pass scans exactly the keys it keeps
2091                assert_eq!(counts[0], 10, "{filter:?}");
2092            }
2093        }
2094    }
2095
2096    /// MASSA-012: a query that would scan more index keys than allowed fails instead of
2097    /// scanning the whole cache, while narrower queries still succeed.
2098    #[test]
2099    #[serial]
2100    fn test_query_fails_past_the_scan_budget() {
2101        // 300 events, 1 in 100 emitted by `emitter`
2102        let (cache, emitter, _tmp) = cache_with_events(1_000, 50, 300, 100);
2103
2104        // a query with no criteria would scan all 300 event keys
2105        let err = cache
2106            .get_filtered_sc_output_events(&EventFilter::default())
2107            .unwrap_err();
2108        assert!(err.contains("narrow the filter"), "{err}");
2109
2110        // the emitter index only holds 3 keys: well within the budget
2111        let (_, events) = cache
2112            .get_filtered_sc_output_events(&EventFilter {
2113                emitter_address: Some(emitter),
2114                ..Default::default()
2115            })
2116            .unwrap();
2117        assert_eq!(events.len(), 3);
2118
2119        // the budget covers every pass: 3 keys for the emitter, then a slot range
2120        // whose scan does not find them all and goes past the budget
2121        let err = cache
2122            .get_filtered_sc_output_events(&EventFilter {
2123                start: Some(Slot::new(150, 0)),
2124                emitter_address: Some(emitter),
2125                ..Default::default()
2126            })
2127            .unwrap_err();
2128        assert!(err.contains("narrow the filter"), "{err}");
2129    }
2130}