massa_metrics/
lib.rs

1//! this library is used to collect metrics from the node and expose them to the prometheus server
2//!
3//! the metrics are collected from the node and from the survey
4//! the survey is a separate thread that is used to collect metrics from the network (active connections)
5//!
6
7use std::{
8    collections::HashMap,
9    net::SocketAddr,
10    sync::{Arc, RwLock},
11    thread::JoinHandle,
12    time::Duration,
13};
14
15use lazy_static::lazy_static;
16use prometheus::{
17    register_int_counter, register_int_gauge, Gauge, Histogram, IntCounter, IntGauge,
18};
19use tokio::sync::oneshot::Sender;
20use tracing::warn;
21
22mod server;
23
24lazy_static! {
25    // use lazy_static for these metrics because they are used in storage which implement default
26    static ref OPERATIONS_COUNTER: IntGauge = register_int_gauge!(
27        "operations_storage_counter",
28        "operations storage counter len"
29    )
30    .unwrap();
31    static ref BLOCKS_COUNTER: IntGauge =
32        register_int_gauge!("blocks_storage_counter", "blocks storage counter len").unwrap();
33    static ref ENDORSEMENTS_COUNTER: IntGauge =
34        register_int_gauge!("endorsements_storage_counter", "endorsements storage counter len").unwrap();
35
36        static ref DEFERRED_CALL_REGISTERED: IntGauge = register_int_gauge!(
37        "deferred_calls_registered", "number of deferred calls registered" ).unwrap();
38
39        static ref DEFERRED_CALL_CANCELLED: IntGauge = register_int_gauge!(
40            "deferred_calls_cancelled", "number of deferred calls cancelled" ).unwrap();
41
42        static ref DEFERRED_CALL_EXECUTED: IntGauge = register_int_gauge!(
43            "deferred_calls_executed", "number of deferred calls executed" ).unwrap();
44
45        static ref DEFERRED_CALL_FAILED: IntGauge = register_int_gauge!(
46            "deferred_calls_failed", "number of deferred calls failed" ).unwrap();
47
48        static ref DEFERRED_CALLS_TOTAL_GAS: IntGauge = register_int_gauge!(
49            "deferred_calls_total_gas", "total gas used by deferred calls" ).unwrap();
50
51        static ref EVENT_CACHE_VEC: IntGauge = register_int_gauge!(
52            "event_cache_vec_len", "vector len for events" ).unwrap();
53
54        static ref FACTORY_POOL_TIMEOUT_ENDORSEMENTS: IntCounter = register_int_counter!(
55            "factory_pool_timeout_endorsements",
56            "number of times block factory timed out while fetching endorsements from the pool"
57        ).unwrap();
58
59        static ref FACTORY_POOL_TIMEOUT_OPERATIONS: IntCounter = register_int_counter!(
60            "factory_pool_timeout_operations",
61            "number of times block factory timed out while fetching operations from the pool"
62        ).unwrap();
63
64        static ref FACTORY_POOL_TIMEOUT_DENUNCIATIONS: IntCounter = register_int_counter!(
65            "factory_pool_timeout_denunciations",
66            "number of times block factory timed out while fetching denunciations from the pool"
67        ).unwrap();
68
69        static ref FACTORY_BLOCKS_PRODUCED: IntCounter = register_int_counter!(
70            "factory_blocks_produced",
71            "number of blocks produced by the local block factory"
72        ).unwrap();
73
74        static ref FACTORY_BLOCKS_PRODUCED_WITH_POOL_TIMEOUT: IntCounter = register_int_counter!(
75            "factory_blocks_produced_with_pool_timeout",
76            "number of locally produced blocks where at least one pool read timed out during assembly"
77        ).unwrap();
78
79}
80
81pub fn inc_deferred_calls_registered() {
82    DEFERRED_CALL_REGISTERED.inc();
83}
84
85pub fn set_event_cache_vec_len(val: usize) {
86    EVENT_CACHE_VEC.set(val as i64);
87}
88
89pub fn set_deferred_calls_total_gas(val: u128) {
90    DEFERRED_CALLS_TOTAL_GAS.set(val as i64);
91}
92
93pub fn inc_deferred_calls_executed_by(val: u64) {
94    DEFERRED_CALL_EXECUTED.set(DEFERRED_CALL_EXECUTED.get().saturating_add(val as i64));
95}
96
97pub fn inc_deferred_calls_failed_by(val: u64) {
98    DEFERRED_CALL_FAILED.set(DEFERRED_CALL_FAILED.get().saturating_add(val as i64));
99}
100
101pub fn dec_deferred_calls_cancelled_by(val: u64) {
102    DEFERRED_CALL_CANCELLED.set(
103        DEFERRED_CALL_CANCELLED
104            .get()
105            .saturating_sub(val as i64)
106            .max(0),
107    );
108}
109
110pub fn dec_deferred_calls_registered_by(val: u64) {
111    DEFERRED_CALL_REGISTERED.set(
112        DEFERRED_CALL_REGISTERED
113            .get()
114            .saturating_sub(val as i64)
115            .max(0),
116    );
117}
118
119pub fn inc_deferred_calls_cancelled() {
120    DEFERRED_CALL_CANCELLED.inc();
121}
122
123pub fn set_blocks_counter(val: usize) {
124    BLOCKS_COUNTER.set(val as i64);
125}
126
127pub fn set_endorsements_counter(val: usize) {
128    ENDORSEMENTS_COUNTER.set(val as i64);
129}
130
131pub fn set_operations_counter(val: usize) {
132    OPERATIONS_COUNTER.set(val as i64);
133}
134
135pub fn inc_factory_pool_timeout_endorsements() {
136    FACTORY_POOL_TIMEOUT_ENDORSEMENTS.inc();
137}
138
139pub fn inc_factory_pool_timeout_operations() {
140    FACTORY_POOL_TIMEOUT_OPERATIONS.inc();
141}
142
143pub fn inc_factory_pool_timeout_denunciations() {
144    FACTORY_POOL_TIMEOUT_DENUNCIATIONS.inc();
145}
146
147pub fn inc_factory_blocks_produced(had_pool_timeout: bool) {
148    FACTORY_BLOCKS_PRODUCED.inc();
149    if had_pool_timeout {
150        FACTORY_BLOCKS_PRODUCED_WITH_POOL_TIMEOUT.inc();
151    }
152}
153
154/// Eagerly register lazy_static metrics so they appear on `/metrics` even before
155/// the first increment (e.g. factory pool timeouts that have not occurred yet).
156#[cfg(not(feature = "test-exports"))]
157fn register_lazy_static_metrics() {
158    lazy_static::initialize(&FACTORY_POOL_TIMEOUT_ENDORSEMENTS);
159    lazy_static::initialize(&FACTORY_POOL_TIMEOUT_OPERATIONS);
160    lazy_static::initialize(&FACTORY_POOL_TIMEOUT_DENUNCIATIONS);
161    lazy_static::initialize(&FACTORY_BLOCKS_PRODUCED);
162    lazy_static::initialize(&FACTORY_BLOCKS_PRODUCED_WITH_POOL_TIMEOUT);
163}
164
165#[derive(Default)]
166pub struct MetricsStopper {
167    pub(crate) stopper: Option<Sender<()>>,
168    pub(crate) stop_handle: Option<JoinHandle<()>>,
169}
170
171impl MetricsStopper {
172    pub fn stop(&mut self) {
173        if let Some(stopper) = self.stopper.take() {
174            if stopper.send(()).is_err() {
175                warn!("failed to send stop signal to metrics server");
176            }
177
178            if let Some(handle) = self.stop_handle.take() {
179                if let Err(_e) = handle.join() {
180                    warn!("failed to join metrics server thread");
181                }
182            }
183        }
184    }
185}
186
187/// Registers `collector` in the default registry, replacing any collector already registered under
188/// the same name.
189///
190/// The node creates a new `MassaMetrics` on every in-process restart (re-bootstrap after a desync).
191/// Without the replacement the registration fails, and the endpoint keeps exporting the collectors
192/// of the previous instance, frozen at their last value.
193fn register<C: prometheus::core::Collector + Clone + 'static>(collector: &C) {
194    let _ = prometheus::unregister(Box::new(collector.clone()));
195    if let Err(e) = prometheus::register(Box::new(collector.clone())) {
196        warn!("failed to register metric: {}", e);
197    }
198}
199
200#[derive(Clone)]
201pub struct MassaMetrics {
202    /// enable metrics
203    enabled: bool,
204
205    /// number of processors
206    process_available_processors: IntGauge,
207
208    /// consensus period for each thread
209    /// index 0 = thread 0 ...
210    consensus_vec: Vec<Gauge>,
211
212    /// number of stakers
213    stakers: IntGauge,
214    /// number of rolls
215    rolls: IntGauge,
216
217    // thread of actual slot
218    current_time_thread: IntGauge,
219    // period of actual slot
220    current_time_period: IntGauge,
221
222    /// number of elements in the active_history of execution
223    active_history: IntGauge,
224
225    /// number of operations in the operation pool
226    operations_pool: IntGauge,
227    /// number of endorsements in the endorsement pool
228    endorsements_pool: IntGauge,
229    /// number of elements in the denunciation pool
230    denunciations_pool: IntGauge,
231
232    // number of autonomous SCs messages in pool
233    async_message_pool_size: IntGauge,
234
235    // number of autonomous SC messages executed as final
236    sc_messages_final: IntCounter,
237
238    /// number of times our node (re-)bootstrapped
239    bootstrap_counter: IntCounter,
240    /// number of times we successfully bootstrapped someone
241    bootstrap_peers_success: IntCounter,
242    /// number of times we failed/refused to bootstrap someone
243    bootstrap_peers_failed: IntCounter,
244
245    /// number of times we successfully tested someone
246    protocol_tester_success: IntCounter,
247    /// number of times we failed to test someone
248    protocol_tester_failed: IntCounter,
249
250    /// know peers in protocol
251    protocol_known_peers: IntGauge,
252    /// banned peers in protocol
253    protocol_banned_peers: IntGauge,
254
255    /// executed final slot
256    executed_final_slot: IntCounter,
257    /// executed final slot with block (not miss)
258    executed_final_slot_with_block: IntCounter,
259
260    /// total bytes receive by peernet manager
261    peernet_total_bytes_received: IntCounter,
262    /// total bytes sent by peernet manager
263    peernet_total_bytes_sent: IntCounter,
264
265    /// block slot delay
266    block_slot_delay: Histogram,
267
268    /// active in connections peer
269    active_in_connections: IntGauge,
270    /// active out connections peer
271    active_out_connections: IntGauge,
272
273    /// counter of operations for final slot
274    operations_final_counter: IntCounter,
275
276    // block_cache
277    block_cache_checked_headers_size: IntGauge,
278    block_cache_blocks_known_by_peer: IntGauge,
279
280    // Operation cache
281    operation_cache_checked_operations: IntGauge,
282    operation_cache_checked_operations_prefix: IntGauge,
283    operation_cache_ops_know_by_peer: IntGauge,
284
285    // Consensus state
286    consensus_state_active_index: IntGauge,
287    consensus_state_active_index_without_ops: IntGauge,
288    consensus_state_incoming_index: IntGauge,
289    consensus_state_discarded_index: IntGauge,
290    consensus_state_block_statuses: IntGauge,
291
292    // endorsement cache
293    endorsement_cache_checked_endorsements: IntGauge,
294    endorsement_cache_known_by_peer: IntGauge,
295
296    // cursor
297    active_cursor_thread: IntGauge,
298    active_cursor_period: IntGauge,
299
300    final_cursor_thread: IntGauge,
301    final_cursor_period: IntGauge,
302
303    // peer bandwidth (bytes sent, bytes received)
304    peers_bandwidth: Arc<RwLock<HashMap<String, (IntCounter, IntCounter)>>>,
305
306    // network versions votes <version, votes>
307    network_versions_votes: Arc<RwLock<HashMap<u32, IntGauge>>>,
308    network_current_version: IntGauge,
309
310    // massa-db
311    db_change_history_size: IntGauge,
312    db_change_versioning_history_size: IntGauge,
313
314    // module cache
315    module_lru_cache_memory_usage: IntGauge,
316
317    // active history
318    active_history_total_event_len: IntGauge,
319
320    // read-only execution queue
321    /// number of read-only requests waiting in the execution worker queue
322    readonly_queue_length: IntGauge,
323    /// number of read-only requests admitted but not yet taken in by the execution worker
324    readonly_input_queue_length: IntGauge,
325    /// number of read-only requests answered with the queue timeout error
326    readonly_requests_timed_out: IntCounter,
327    /// number of read-only requests refused because the queue was full
328    readonly_requests_refused: IntCounter,
329
330    pub tick_delay: Duration,
331}
332
333impl MassaMetrics {
334    #[allow(unused_variables)]
335    #[allow(unused_mut)]
336    pub fn new(
337        enabled: bool,
338        addr: SocketAddr,
339        nb_thread: u8,
340        tick_delay: Duration,
341    ) -> (Self, MetricsStopper) {
342        let mut consensus_vec = vec![];
343        for i in 0..nb_thread {
344            let gauge = Gauge::new(
345                format!("consensus_thread_{}", i),
346                "consensus thread actual period",
347            )
348            .expect("Failed to create gauge");
349            #[cfg(not(feature = "test-exports"))]
350            {
351                register(&gauge);
352            }
353
354            consensus_vec.push(gauge);
355        }
356
357        // set available processors
358        let process_available_processors =
359            IntGauge::new("process_available_processors", "number of processors")
360                .expect("Failed to create available_processors counter");
361
362        // stakers
363        let stakers = IntGauge::new("stakers", "number of stakers").unwrap();
364        let rolls = IntGauge::new("rolls", "number of rolls").unwrap();
365
366        let current_time_period =
367            IntGauge::new("current_time_period", "period of actual slot").unwrap();
368
369        let current_time_thread =
370            IntGauge::new("current_time_thread", "thread of actual slot").unwrap();
371
372        let executed_final_slot =
373            IntCounter::new("executed_final_slot", "number of executed final slot").unwrap();
374        let executed_final_slot_with_block = IntCounter::new(
375            "executed_final_slot_with_block",
376            "number of executed final slot with block (not miss)",
377        )
378        .unwrap();
379
380        let protocol_tester_success = IntCounter::new(
381            "protocol_tester_success",
382            "number of times we successfully tested someone",
383        )
384        .unwrap();
385        let protocol_tester_failed = IntCounter::new(
386            "protocol_tester_failed",
387            "number of times we failed to test someone",
388        )
389        .unwrap();
390
391        // pool
392        let operations_pool = IntGauge::new(
393            "operations_pool",
394            "number of operations in the operation pool",
395        )
396        .unwrap();
397        let endorsements_pool = IntGauge::new(
398            "endorsements_pool",
399            "number of endorsements in the endorsement pool",
400        )
401        .unwrap();
402        let denunciations_pool = IntGauge::new(
403            "denunciations_pool",
404            "number of elements in the denunciation pool",
405        )
406        .unwrap();
407
408        let async_message_pool_size = IntGauge::new(
409            "async_message_pool_size",
410            "number of autonomous SCs messages in pool",
411        )
412        .unwrap();
413
414        let sc_messages_final = IntCounter::new(
415            "sc_messages_final",
416            "number of autonomous SC messages executed as final",
417        )
418        .unwrap();
419
420        let bootstrap_counter = IntCounter::new(
421            "bootstrap_counter",
422            "number of times our node (re-)bootstrapped",
423        )
424        .unwrap();
425        let bootstrap_success = IntCounter::new(
426            "bootstrap_peers_success",
427            "number of times we successfully bootstrapped someone",
428        )
429        .unwrap();
430        let bootstrap_failed = IntCounter::new(
431            "bootstrap_peers_failed",
432            "number of times we failed/refused to bootstrap someone",
433        )
434        .unwrap();
435
436        let active_history = IntGauge::new(
437            "active_history",
438            "number of elements in the active_history of execution",
439        )
440        .unwrap();
441
442        let know_peers =
443            IntGauge::new("protocol_known_peers", "number of known peers in protocol").unwrap();
444        let banned_peers = IntGauge::new(
445            "protocol_banned_peers",
446            "number of banned peers in protocol",
447        )
448        .unwrap();
449
450        // active cursor
451        let active_cursor_thread =
452            IntGauge::new("active_cursor_thread", "execution active cursor thread").unwrap();
453        let active_cursor_period =
454            IntGauge::new("active_cursor_period", "execution active cursor period").unwrap();
455
456        // final cursor
457        let final_cursor_thread =
458            IntGauge::new("final_cursor_thread", "execution final cursor thread").unwrap();
459        let final_cursor_period =
460            IntGauge::new("final_cursor_period", "execution final cursor period").unwrap();
461
462        // active connections IN
463        let active_in_connections =
464            IntGauge::new("active_in_connections", "active connections IN len").unwrap();
465
466        // active connections OUT
467        let active_out_connections =
468            IntGauge::new("active_out_connections", "active connections OUT len").unwrap();
469
470        // block cache
471        let block_cache_checked_headers_size = IntGauge::new(
472            "block_cache_checked_headers_size",
473            "size of BlockCache checked_headers",
474        )
475        .unwrap();
476
477        let block_cache_blocks_known_by_peer = IntGauge::new(
478            "block_cache_blocks_known_by_peer_size",
479            "size of BlockCache blocks_known_by_peer",
480        )
481        .unwrap();
482
483        // operation cache
484        let operation_cache_checked_operations = IntGauge::new(
485            "operation_cache_checked_operations",
486            "size of OperationCache checked_operations",
487        )
488        .unwrap();
489
490        let operation_cache_checked_operations_prefix = IntGauge::new(
491            "operation_cache_checked_operations_prefix",
492            "size of OperationCache checked_operations_prefix",
493        )
494        .unwrap();
495
496        let operation_cache_ops_know_by_peer = IntGauge::new(
497            "operation_cache_ops_know_by_peer",
498            "size of OperationCache operation_cache_ops_know_by_peer",
499        )
500        .unwrap();
501
502        // consensus state from tick.rs
503        let consensus_state_active_index = IntGauge::new(
504            "consensus_state_active_index",
505            "consensus state active index size",
506        )
507        .unwrap();
508
509        let consensus_state_active_index_without_ops = IntGauge::new(
510            "consensus_state_active_index_without_ops",
511            "consensus state active index without ops size",
512        )
513        .unwrap();
514
515        let consensus_state_incoming_index = IntGauge::new(
516            "consensus_state_incoming_index",
517            "consensus state incoming index size",
518        )
519        .unwrap();
520
521        let consensus_state_discarded_index = IntGauge::new(
522            "consensus_state_discarded_index",
523            "consensus state discarded index size",
524        )
525        .unwrap();
526
527        let consensus_state_block_statuses = IntGauge::new(
528            "consensus_state_block_statuses",
529            "consensus state block statuses size",
530        )
531        .unwrap();
532
533        let endorsement_cache_checked_endorsements = IntGauge::new(
534            "endorsement_cache_checked_endorsements",
535            "endorsement cache checked endorsements size",
536        )
537        .unwrap();
538
539        let endorsement_cache_known_by_peer = IntGauge::new(
540            "endorsement_cache_known_by_peer",
541            "endorsement cache know by peer size",
542        )
543        .unwrap();
544
545        let peernet_total_bytes_received = IntCounter::new(
546            "peernet_total_bytes_received",
547            "total byte received by peernet",
548        )
549        .unwrap();
550
551        let peernet_total_bytes_sent =
552            IntCounter::new("peernet_total_bytes_sent", "total byte sent by peernet").unwrap();
553
554        let operations_final_counter =
555            IntCounter::new("operations_final_counter", "total final operations").unwrap();
556
557        let block_slot_delay = Histogram::with_opts(
558            prometheus::HistogramOpts::new("block_slot_delay", "block slot delay").buckets(vec![
559                0.100, 0.250, 0.500, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0,
560            ]),
561        )
562        .unwrap();
563
564        let network_current_version =
565            IntGauge::new("network_current_version", "current version of network").unwrap();
566
567        let db_change_history_size = IntGauge::new(
568            "db_change_history_size",
569            "total size of change history in DB",
570        )
571        .unwrap();
572
573        let db_change_versioning_history_size = IntGauge::new(
574            "db_change_versioning_history_size",
575            "total size of change versioning history in DB",
576        )
577        .unwrap();
578
579        let module_lru_cache_memory_usage = IntGauge::new(
580            "module_lru_cache_memory_usage",
581            "total size of the lru module cache in bytes",
582        )
583        .unwrap();
584
585        let active_history_total_event_len = IntGauge::new(
586            "active_history_total_event_len",
587            "total events len currently in the active history (1 event: max 1kB)",
588        )
589        .unwrap();
590
591        let readonly_queue_length = IntGauge::new(
592            "readonly_queue_length",
593            "number of read-only requests waiting in the execution worker queue",
594        )
595        .unwrap();
596        let readonly_input_queue_length = IntGauge::new(
597            "readonly_input_queue_length",
598            "number of read-only requests admitted but not yet taken in by the execution worker",
599        )
600        .unwrap();
601        let readonly_requests_timed_out = IntCounter::new(
602            "readonly_requests_timed_out",
603            "number of read-only requests answered with a timeout because they waited too long in the queue",
604        )
605        .unwrap();
606        let readonly_requests_refused = IntCounter::new(
607            "readonly_requests_refused",
608            "number of read-only requests refused because the queue was full",
609        )
610        .unwrap();
611
612        let mut stopper = MetricsStopper::default();
613
614        if enabled {
615            #[cfg(not(feature = "test-exports"))]
616            {
617                register_lazy_static_metrics();
618                register(&final_cursor_thread);
619                register(&final_cursor_period);
620                register(&active_cursor_thread);
621                register(&active_cursor_period);
622                register(&active_out_connections);
623                register(&block_cache_blocks_known_by_peer);
624                register(&block_cache_checked_headers_size);
625                register(&operation_cache_checked_operations);
626                register(&active_in_connections);
627                register(&operation_cache_ops_know_by_peer);
628                register(&consensus_state_active_index);
629                register(&consensus_state_active_index_without_ops);
630                register(&consensus_state_incoming_index);
631                register(&consensus_state_discarded_index);
632                register(&consensus_state_block_statuses);
633                register(&operation_cache_checked_operations_prefix);
634                register(&endorsement_cache_checked_endorsements);
635                register(&endorsement_cache_known_by_peer);
636                register(&peernet_total_bytes_received);
637                register(&peernet_total_bytes_sent);
638                register(&operations_final_counter);
639                register(&stakers);
640                register(&rolls);
641                register(&know_peers);
642                register(&banned_peers);
643                register(&executed_final_slot);
644                register(&executed_final_slot_with_block);
645                register(&active_history);
646                register(&bootstrap_counter);
647                register(&bootstrap_success);
648                register(&bootstrap_failed);
649                register(&process_available_processors);
650                register(&operations_pool);
651                register(&endorsements_pool);
652                register(&denunciations_pool);
653                register(&protocol_tester_success);
654                register(&protocol_tester_failed);
655                register(&sc_messages_final);
656                register(&async_message_pool_size);
657                register(&current_time_period);
658                register(&current_time_thread);
659                register(&block_slot_delay);
660                register(&network_current_version);
661                register(&db_change_history_size);
662                register(&db_change_versioning_history_size);
663                register(&module_lru_cache_memory_usage);
664                register(&active_history_total_event_len);
665                register(&readonly_queue_length);
666                register(&readonly_input_queue_length);
667                register(&readonly_requests_timed_out);
668                register(&readonly_requests_refused);
669
670                stopper = server::bind_metrics(addr);
671            }
672        }
673
674        (
675            MassaMetrics {
676                enabled,
677                process_available_processors,
678                consensus_vec,
679                stakers,
680                rolls,
681                current_time_thread,
682                current_time_period,
683                active_history,
684                operations_pool,
685                endorsements_pool,
686                denunciations_pool,
687                async_message_pool_size,
688                sc_messages_final,
689                bootstrap_counter,
690                bootstrap_peers_success: bootstrap_success,
691                bootstrap_peers_failed: bootstrap_failed,
692                protocol_tester_success,
693                protocol_tester_failed,
694                protocol_known_peers: know_peers,
695                protocol_banned_peers: banned_peers,
696                executed_final_slot,
697                executed_final_slot_with_block,
698                peernet_total_bytes_received,
699                peernet_total_bytes_sent,
700                block_slot_delay,
701                active_in_connections,
702                active_out_connections,
703                operations_final_counter,
704                block_cache_checked_headers_size,
705                block_cache_blocks_known_by_peer,
706                operation_cache_checked_operations,
707                operation_cache_checked_operations_prefix,
708                operation_cache_ops_know_by_peer,
709                consensus_state_active_index,
710                consensus_state_active_index_without_ops,
711                consensus_state_incoming_index,
712                consensus_state_discarded_index,
713                consensus_state_block_statuses,
714                endorsement_cache_checked_endorsements,
715                endorsement_cache_known_by_peer,
716                // blocks_counter,
717                // endorsements_counter,
718                // operations_counter,
719                active_cursor_thread,
720                active_cursor_period,
721                final_cursor_thread,
722                final_cursor_period,
723                peers_bandwidth: Arc::new(RwLock::new(HashMap::new())),
724                network_versions_votes: Arc::new(RwLock::new(HashMap::new())),
725                network_current_version,
726                db_change_history_size,
727                db_change_versioning_history_size,
728                module_lru_cache_memory_usage,
729                active_history_total_event_len,
730                readonly_queue_length,
731                readonly_input_queue_length,
732                readonly_requests_timed_out,
733                readonly_requests_refused,
734                tick_delay,
735            },
736            stopper,
737        )
738    }
739
740    pub fn is_enabled(&self) -> bool {
741        self.enabled
742    }
743
744    pub fn get_metrics_for_survey_thread(&self) -> (i64, i64, u64, u64) {
745        (
746            self.active_in_connections.clone().get(),
747            self.active_out_connections.clone().get(),
748            self.peernet_total_bytes_sent.clone().get(),
749            self.peernet_total_bytes_received.clone().get(),
750        )
751    }
752
753    pub fn set_active_connections(&self, in_connections: usize, out_connections: usize) {
754        self.active_in_connections.set(in_connections as i64);
755        self.active_out_connections.set(out_connections as i64);
756    }
757
758    pub fn set_active_cursor(&self, period: u64, thread: u8) {
759        self.active_cursor_thread.set(thread as i64);
760        self.active_cursor_period.set(period as i64);
761    }
762
763    pub fn set_final_cursor(&self, period: u64, thread: u8) {
764        self.final_cursor_thread.set(thread as i64);
765        self.final_cursor_period.set(period as i64);
766    }
767
768    pub fn set_consensus_period(&self, thread: usize, period: u64) {
769        if let Some(g) = self.consensus_vec.get(thread) {
770            g.set(period as f64);
771        }
772    }
773
774    pub fn set_consensus_state(
775        &self,
776        active_index: usize,
777        incoming_index: usize,
778        discarded_index: usize,
779        block_statuses: usize,
780        active_index_without_ops: usize,
781    ) {
782        self.consensus_state_active_index.set(active_index as i64);
783        self.consensus_state_incoming_index
784            .set(incoming_index as i64);
785        self.consensus_state_discarded_index
786            .set(discarded_index as i64);
787        self.consensus_state_block_statuses
788            .set(block_statuses as i64);
789        self.consensus_state_active_index_without_ops
790            .set(active_index_without_ops as i64);
791    }
792
793    pub fn set_block_cache_metrics(&self, checked_header_size: usize, blocks_known_by_peer: usize) {
794        self.block_cache_checked_headers_size
795            .set(checked_header_size as i64);
796        self.block_cache_blocks_known_by_peer
797            .set(blocks_known_by_peer as i64);
798    }
799
800    pub fn set_operations_cache_metrics(
801        &self,
802        checked_operations: usize,
803        checked_operations_prefix: usize,
804        ops_know_by_peer: usize,
805    ) {
806        self.operation_cache_checked_operations
807            .set(checked_operations as i64);
808        self.operation_cache_checked_operations_prefix
809            .set(checked_operations_prefix as i64);
810        self.operation_cache_ops_know_by_peer
811            .set(ops_know_by_peer as i64);
812    }
813
814    pub fn set_endorsements_cache_metrics(
815        &self,
816        checked_endorsements: usize,
817        known_by_peer: usize,
818    ) {
819        self.endorsement_cache_checked_endorsements
820            .set(checked_endorsements as i64);
821        self.endorsement_cache_known_by_peer
822            .set(known_by_peer as i64);
823    }
824
825    pub fn set_peernet_total_bytes_received(&self, new_value: u64) {
826        let diff = new_value.saturating_sub(self.peernet_total_bytes_received.get());
827        self.peernet_total_bytes_received.inc_by(diff);
828    }
829
830    pub fn set_peernet_total_bytes_sent(&self, new_value: u64) {
831        let diff = new_value.saturating_sub(self.peernet_total_bytes_sent.get());
832        self.peernet_total_bytes_sent.inc_by(diff);
833    }
834
835    pub fn inc_operations_final_counter(&self, diff: u64) {
836        self.operations_final_counter.inc_by(diff);
837    }
838
839    pub fn set_known_peers(&self, nb: usize) {
840        self.protocol_known_peers.set(nb as i64);
841    }
842
843    pub fn set_banned_peers(&self, nb: usize) {
844        self.protocol_banned_peers.set(nb as i64);
845    }
846
847    pub fn inc_executed_final_slot(&self) {
848        self.executed_final_slot.inc();
849    }
850
851    pub fn inc_executed_final_slot_with_block(&self) {
852        self.executed_final_slot_with_block.inc();
853    }
854
855    pub fn set_active_history(&self, nb: usize) {
856        self.active_history.set(nb as i64);
857    }
858
859    pub fn inc_bootstrap_counter(&self) {
860        self.bootstrap_counter.inc();
861    }
862
863    pub fn inc_bootstrap_peers_success(&self) {
864        self.bootstrap_peers_success.inc();
865    }
866
867    pub fn inc_bootstrap_peers_failed(&self) {
868        self.bootstrap_peers_failed.inc();
869    }
870
871    pub fn set_operations_pool(&self, nb: usize) {
872        self.operations_pool.set(nb as i64);
873    }
874
875    pub fn set_endorsements_pool(&self, nb: usize) {
876        self.endorsements_pool.set(nb as i64);
877    }
878
879    pub fn set_denunciations_pool(&self, nb: usize) {
880        self.denunciations_pool.set(nb as i64);
881    }
882
883    pub fn inc_protocol_tester_success(&self) {
884        self.protocol_tester_success.inc();
885    }
886
887    pub fn inc_protocol_tester_failed(&self) {
888        self.protocol_tester_failed.inc();
889    }
890
891    pub fn set_stakers(&self, nb: usize) {
892        self.stakers.set(nb as i64);
893    }
894
895    pub fn set_rolls(&self, nb: usize) {
896        self.rolls.set(nb as i64);
897    }
898
899    pub fn inc_sc_messages_final_by(&self, diff: usize) {
900        self.sc_messages_final.inc_by(diff as u64);
901    }
902
903    pub fn set_async_message_pool_size(&self, nb: usize) {
904        self.async_message_pool_size.set(nb as i64);
905    }
906
907    pub fn set_available_processors(&self, nb: usize) {
908        self.process_available_processors.set(nb as i64);
909    }
910
911    pub fn set_current_time_period(&self, period: u64) {
912        self.current_time_period.set(period as i64);
913    }
914
915    pub fn set_current_time_thread(&self, thread: u8) {
916        self.current_time_thread.set(thread as i64);
917    }
918
919    pub fn set_block_slot_delay(&self, delay: f64) {
920        self.block_slot_delay.observe(delay);
921    }
922
923    pub fn set_network_current_version(&self, version: u32) {
924        self.network_current_version.set(version as i64);
925    }
926
927    pub fn set_db_change_history_size(&self, size: usize) {
928        self.db_change_history_size.set(size as i64);
929    }
930
931    pub fn set_db_change_versioning_history_size(&self, size: usize) {
932        self.db_change_versioning_history_size.set(size as i64);
933    }
934
935    pub fn set_module_lru_cache_memory_usage(&self, size: usize) {
936        self.module_lru_cache_memory_usage.set(size as i64);
937    }
938
939    pub fn set_active_history_total_event_len(&self, size: usize) {
940        self.active_history_total_event_len.set(size as i64);
941    }
942
943    pub fn set_readonly_queue_length(&self, len: usize) {
944        self.readonly_queue_length.set(len as i64);
945    }
946
947    pub fn set_readonly_input_queue_length(&self, len: usize) {
948        self.readonly_input_queue_length.set(len as i64);
949    }
950
951    pub fn inc_readonly_requests_timed_out_by(&self, count: usize) {
952        self.readonly_requests_timed_out.inc_by(count as u64);
953    }
954
955    pub fn inc_readonly_requests_refused_by(&self, count: usize) {
956        self.readonly_requests_refused.inc_by(count as u64);
957    }
958
959    // Update the network version vote metrics
960    pub fn update_network_version_vote(&self, data: HashMap<u32, u64>) {
961        if self.enabled {
962            let mut write = self.network_versions_votes.write().unwrap();
963
964            let current_version: u32 = self.network_current_version.get() as u32;
965
966            {
967                let missing_version = write
968                    .keys()
969                    .filter(|key| !data.contains_key(key))
970                    .cloned()
971                    .collect::<Vec<u32>>();
972
973                for key in missing_version {
974                    if let Some(counter) = write.remove(&key) {
975                        if let Err(e) = prometheus::unregister(Box::new(counter)) {
976                            warn!("Failed to unregister network_version_vote_{} : {}", key, e);
977                        }
978                    }
979                }
980
981                if current_version > 0 {
982                    // remove metrics for version 0 if we have a current version > 0
983                    // in this case 0 means no vote
984                    if let Some(counter) = write.remove(&0) {
985                        if let Err(e) = prometheus::unregister(Box::new(counter)) {
986                            warn!("Failed to unregister network_version_vote_0 : {}", e);
987                        }
988                    }
989                }
990            }
991
992            for (version, count) in data.into_iter() {
993                if version.eq(&0) && current_version > 0 {
994                    // skip version 0 if we have a current version
995                    continue;
996                }
997                if let Some(actual_counter) = write.get_mut(&version) {
998                    actual_counter.set(count as i64);
999                } else {
1000                    let label = format!("network_version_votes_{}", version);
1001                    let counter = IntGauge::new(label, "vote counter for network version").unwrap();
1002                    counter.set(count as i64);
1003                    register(&counter);
1004                    write.insert(version, counter);
1005                }
1006            }
1007        }
1008    }
1009
1010    /// Update the bandwidth metrics for all peers
1011    /// HashMap<peer_id, (tx, rx)>
1012    pub fn update_peers_tx_rx(&self, data: HashMap<String, (u64, u64)>) {
1013        if self.enabled {
1014            let mut write = self.peers_bandwidth.write().unwrap();
1015
1016            // metrics of peers that are not in the data HashMap are removed
1017            let missing_peer: Vec<String> = write
1018                .keys()
1019                .filter(|key| !data.contains_key(key.as_str()))
1020                .cloned()
1021                .collect();
1022
1023            for key in missing_peer {
1024                // remove peer and unregister metrics
1025                if let Some((tx, rx)) = write.remove(&key) {
1026                    if let Err(e) = prometheus::unregister(Box::new(tx)) {
1027                        warn!("Failed to unregister tx metricfor peer {} : {}", key, e);
1028                    }
1029
1030                    if let Err(e) = prometheus::unregister(Box::new(rx)) {
1031                        warn!("Failed to unregister rx metric for peer {} : {}", key, e);
1032                    }
1033                }
1034            }
1035
1036            for (k, (tx_peernet, rx_peernet)) in data {
1037                if let Some((tx_metric, rx_metric)) = write.get_mut(&k) {
1038                    // peer metrics exist
1039                    // update tx and rx
1040
1041                    let to_add = tx_peernet.saturating_sub(tx_metric.get());
1042                    tx_metric.inc_by(to_add);
1043
1044                    let to_add = rx_peernet.saturating_sub(rx_metric.get());
1045                    rx_metric.inc_by(to_add);
1046                } else {
1047                    // peer metrics does not exist
1048                    let label_rx = format!("peer_total_bytes_receive_{}", k);
1049                    let label_tx = format!("peer_total_bytes_sent_{}", k);
1050
1051                    let peer_total_bytes_receive =
1052                        IntCounter::new(label_rx, "total byte received by the peer").unwrap();
1053
1054                    let peer_total_bytes_sent =
1055                        IntCounter::new(label_tx, "total byte sent by the peer").unwrap();
1056
1057                    peer_total_bytes_sent.inc_by(tx_peernet);
1058                    peer_total_bytes_receive.inc_by(rx_peernet);
1059
1060                    register(&peer_total_bytes_receive);
1061                    register(&peer_total_bytes_sent);
1062
1063                    write.insert(k, (peer_total_bytes_sent, peer_total_bytes_receive));
1064                }
1065            }
1066        }
1067    }
1068}
1069
1070#[cfg(all(test, not(feature = "test-exports")))]
1071mod tests {
1072    use super::*;
1073
1074    /// The node creates a new `MassaMetrics` on every in-process restart (re-bootstrap after a
1075    /// desync): the endpoint must then export the new collectors, not the previous ones.
1076    #[test]
1077    fn new_instance_replaces_the_previous_one_in_the_registry() {
1078        let addr = "127.0.0.1:0".parse().unwrap();
1079        let tick = Duration::from_secs(5);
1080
1081        let (first, mut first_stopper) = MassaMetrics::new(true, addr, 2, tick);
1082        first.inc_executed_final_slot();
1083        first.update_peers_tx_rx(HashMap::from([("peer".to_string(), (10, 10))]));
1084        first_stopper.stop();
1085
1086        let (second, mut second_stopper) = MassaMetrics::new(true, addr, 2, tick);
1087        for _ in 0..3 {
1088            second.inc_executed_final_slot();
1089        }
1090        second.update_peers_tx_rx(HashMap::from([("peer".to_string(), (70, 70))]));
1091        second_stopper.stop();
1092
1093        let exported = |name: &str| {
1094            prometheus::gather()
1095                .into_iter()
1096                .find(|family| family.get_name() == name)
1097                .unwrap_or_else(|| panic!("{} is not exported", name))
1098                .get_metric()[0]
1099                .get_counter()
1100                .get_value()
1101        };
1102        assert_eq!(exported("executed_final_slot"), 3.0);
1103        assert_eq!(exported("peer_total_bytes_sent_peer"), 70.0);
1104    }
1105}