massa_node/
main.rs

1// Copyright (c) 2022 MASSA LABS <info@massa.net>
2
3#![doc = include_str!("../../README.md")]
4#![warn(missing_docs)]
5#![warn(unused_crate_dependencies)]
6
7#[cfg(unix)]
8mod jemalloc_init;
9
10extern crate massa_logging;
11
12#[cfg(feature = "op_spammer")]
13use crate::operation_injector::start_operation_injector;
14use crate::settings::SETTINGS;
15use crate::survey::MassaSurvey;
16
17use cfg_if::cfg_if;
18use clap::{crate_version, Parser};
19use crossbeam_channel::TryRecvError;
20use dialoguer::Password;
21use massa_api::{ApiServer, ApiV2, Private, Public, RpcServer, StopHandle, API};
22use massa_api_exports::config::APIConfig;
23use massa_async_pool::AsyncPoolConfig;
24use massa_bootstrap::BootstrapError;
25use massa_bootstrap::{
26    get_state, start_bootstrap_server, BootstrapConfig, BootstrapManager, BootstrapTcpListener,
27    DefaultConnector,
28};
29use massa_channel::receiver::MassaReceiver;
30use massa_channel::MassaChannel;
31use massa_consensus_exports::events::ConsensusEvent;
32use massa_consensus_exports::{
33    ConsensusBroadcasts, ConsensusChannels, ConsensusConfig, ConsensusManager,
34};
35use massa_consensus_worker::start_consensus_worker;
36use massa_db_exports::{MassaDBConfig, MassaDBController};
37use massa_db_worker::MassaDB;
38use massa_deferred_calls::config::DeferredCallsConfig;
39use massa_executed_ops::{ExecutedDenunciationsConfig, ExecutedOpsConfig};
40use massa_execution_exports::{
41    CondomLimits, ExecutionChannels, ExecutionConfig, ExecutionManager, GasCosts,
42    StorageCostsConstants,
43};
44use massa_execution_worker::start_execution_worker;
45#[cfg(all(
46    feature = "dump-block",
47    feature = "file_storage_backend",
48    not(feature = "db_storage_backend")
49))]
50use massa_execution_worker::storage_backend::FileStorageBackend;
51#[cfg(all(feature = "dump-block", feature = "db_storage_backend"))]
52use massa_execution_worker::storage_backend::RocksDBStorageBackend;
53
54use massa_factory_exports::{FactoryChannels, FactoryConfig, FactoryManager};
55use massa_factory_worker::start_factory;
56use massa_final_state::{FinalState, FinalStateConfig, FinalStateController};
57use massa_grpc::config::{GrpcConfig, ServiceName};
58use massa_grpc::server::{MassaPrivateGrpc, MassaPublicGrpc};
59use massa_ledger_exports::LedgerConfig;
60use massa_ledger_worker::FinalLedger;
61use massa_logging::massa_trace;
62use massa_metrics::{MassaMetrics, MetricsStopper};
63use massa_models::address::Address;
64use massa_models::amount::Amount;
65use massa_models::config::constants::{
66    ASYNC_MSG_CST_GAS_COST, BLOCK_REWARD, BOOTSTRAP_RANDOMNESS_SIZE_BYTES, CHANNEL_SIZE,
67    CONSENSUS_BOOTSTRAP_PART_SIZE, DEFERRED_CALL_MAX_FUTURE_SLOTS, DELTA_F0,
68    DENUNCIATION_EXPIRE_PERIODS, ENDORSEMENT_COUNT, END_TIMESTAMP, GENESIS_KEY, GENESIS_TIMESTAMP,
69    INITIAL_DRAW_SEED, LEDGER_COST_PER_BYTE, LEDGER_ENTRY_BASE_COST,
70    LEDGER_ENTRY_DATASTORE_BASE_SIZE, MAX_ADVERTISE_LENGTH, MAX_ASYNC_GAS, MAX_ASYNC_POOL_LENGTH,
71    MAX_BLOCK_SIZE, MAX_BOOTSTRAP_BLOCKS, MAX_BOOTSTRAP_ERROR_LENGTH, MAX_BYTECODE_LENGTH,
72    MAX_CONSENSUS_BLOCKS_IDS, MAX_DATASTORE_ENTRY_COUNT, MAX_DATASTORE_KEY_LENGTH,
73    MAX_DATASTORE_VALUE_LENGTH, MAX_DEFERRED_CREDITS_LENGTH, MAX_DENUNCIATIONS_PER_BLOCK_HEADER,
74    MAX_DENUNCIATION_CHANGES_LENGTH, MAX_ENDORSEMENTS_PER_MESSAGE, MAX_EXECUTED_OPS_CHANGES_LENGTH,
75    MAX_EXECUTED_OPS_LENGTH, MAX_FUNCTION_NAME_LENGTH, MAX_GAS_PER_BLOCK, MAX_LEDGER_CHANGES_COUNT,
76    MAX_LISTENERS_PER_PEER, MAX_OPERATIONS_PER_BLOCK, MAX_OPERATIONS_PER_MESSAGE,
77    MAX_OPERATION_DATASTORE_ENTRY_COUNT, MAX_OPERATION_DATASTORE_KEY_LENGTH,
78    MAX_OPERATION_DATASTORE_VALUE_LENGTH, MAX_OPERATION_STORAGE_TIME, MAX_PARAMETERS_SIZE,
79    MAX_PEERS_IN_ANNOUNCEMENT_LIST, MAX_PRODUCTION_STATS_LENGTH, MAX_ROLLS_COUNT_LENGTH,
80    MAX_SIZE_CHANNEL_COMMANDS_CONNECTIVITY, MAX_SIZE_CHANNEL_COMMANDS_PEERS,
81    MAX_SIZE_CHANNEL_COMMANDS_PEER_TESTERS, MAX_SIZE_CHANNEL_COMMANDS_PROPAGATION_BLOCKS,
82    MAX_SIZE_CHANNEL_COMMANDS_PROPAGATION_ENDORSEMENTS,
83    MAX_SIZE_CHANNEL_COMMANDS_PROPAGATION_OPERATIONS, MAX_SIZE_CHANNEL_COMMANDS_RETRIEVAL_BLOCKS,
84    MAX_SIZE_CHANNEL_COMMANDS_RETRIEVAL_ENDORSEMENTS,
85    MAX_SIZE_CHANNEL_COMMANDS_RETRIEVAL_OPERATIONS, MAX_SIZE_CHANNEL_NETWORK_TO_BLOCK_HANDLER,
86    MAX_SIZE_CHANNEL_NETWORK_TO_ENDORSEMENT_HANDLER, MAX_SIZE_CHANNEL_NETWORK_TO_OPERATION_HANDLER,
87    MAX_SIZE_CHANNEL_NETWORK_TO_PEER_HANDLER, MIP_STORE_STATS_BLOCK_CONSIDERED,
88    OPERATION_VALIDITY_PERIODS, PERIODS_PER_CYCLE, POS_MISS_RATE_DEACTIVATION_THRESHOLD,
89    POS_SAVED_CYCLES, PROTOCOL_CONTROLLER_CHANNEL_SIZE, PROTOCOL_EVENT_CHANNEL_SIZE,
90    ROLL_COUNT_TO_SLASH_ON_DENUNCIATION, ROLL_PRICE, SELECTOR_DRAW_CACHE_SIZE, T0, THREAD_COUNT,
91    VERSION,
92};
93use massa_models::config::{
94    bootstrap_batch_allocation_budget, handle_disclaimer, BASE_OPERATION_GAS_COST, CHAINID,
95    DEFERRED_CALL_BASE_FEE_MAX_CHANGE_DENOMINATOR, DEFERRED_CALL_CST_GAS_COST,
96    DEFERRED_CALL_GLOBAL_OVERBOOKING_PENALTY, DEFERRED_CALL_MAX_ASYNC_GAS,
97    DEFERRED_CALL_MAX_POOL_CHANGES, DEFERRED_CALL_MIN_GAS_COST, DEFERRED_CALL_MIN_GAS_INCREMENT,
98    DEFERRED_CALL_SLOT_OVERBOOKING_PENALTY, KEEP_EXECUTED_HISTORY_EXTRA_PERIODS,
99    MAX_BOOTSTRAP_FINAL_STATE_PARTS_SIZE, MAX_BOOTSTRAP_VERSIONING_ELEMENTS_SIZE,
100    MAX_EVENT_DATA_SIZE, MAX_EVENT_DATA_SIZE_V0, MAX_EVENT_PER_OPERATION, MAX_MESSAGE_SIZE,
101    MAX_RECURSIVE_CALLS_DEPTH, MAX_RUNTIME_MODULE_CODE_SECTION_SIZE,
102    MAX_RUNTIME_MODULE_CUSTOM_SECTION_DATA_LEN, MAX_RUNTIME_MODULE_CUSTOM_SECTION_LEN,
103    MAX_RUNTIME_MODULE_EXPORTS, MAX_RUNTIME_MODULE_FUNCTIONS, MAX_RUNTIME_MODULE_FUNCTION_NAME_LEN,
104    MAX_RUNTIME_MODULE_GLOBAL_INITIALIZER, MAX_RUNTIME_MODULE_IMPORTS, MAX_RUNTIME_MODULE_LOCALS,
105    MAX_RUNTIME_MODULE_MEMORIES, MAX_RUNTIME_MODULE_NAME_LEN, MAX_RUNTIME_MODULE_PASSIVE_DATA,
106    MAX_RUNTIME_MODULE_PASSIVE_ELEMENT, MAX_RUNTIME_MODULE_SIGNATURE_LEN, MAX_RUNTIME_MODULE_TABLE,
107    MAX_RUNTIME_MODULE_TABLE_ELEMENTS, MAX_RUNTIME_MODULE_TABLE_INITIALIZER,
108};
109use massa_models::slot::Slot;
110use massa_models::timeslots::get_block_slot_timestamp;
111use massa_pool_exports::{PoolBroadcasts, PoolChannels, PoolConfig, PoolManager};
112use massa_pool_worker::start_pool_controller;
113use massa_pos_exports::{PoSConfig, SelectorConfig, SelectorManager};
114use massa_pos_worker::start_selector_worker;
115use massa_protocol_exports::{ProtocolConfig, ProtocolManager, TransportType};
116use massa_protocol_worker::{create_protocol_controller, start_protocol_controller};
117use massa_signature::KeyPair;
118use massa_storage::Storage;
119use massa_time::MassaTime;
120use massa_versioning::keypair_factory::KeyPairFactory;
121use massa_versioning::mips::get_mip_list;
122use massa_versioning::versioning::{MipStatsConfig, MipStore};
123use massa_wallet::Wallet;
124use num::rational::Ratio;
125use parking_lot::RwLock;
126use settings::GrpcSettings;
127use std::collections::HashMap;
128use std::path::PathBuf;
129use std::sync::atomic::{AtomicUsize, Ordering};
130use std::sync::{Condvar, Mutex};
131use std::time::Duration;
132use std::{path::Path, process, sync::Arc};
133
134use massa_event_cache::config::EventCacheConfig;
135use massa_event_cache::worker::{start_event_cache_writer_worker, EventCacheManager};
136use survey::MassaSurveyStopper;
137use tokio::sync::broadcast;
138use tracing::{debug, error, info, warn};
139use tracing_subscriber::filter::{filter_fn, LevelFilter};
140
141#[cfg(feature = "op_spammer")]
142mod operation_injector;
143mod settings;
144mod survey;
145
146async fn launch(
147    args: &Args,
148    node_wallet: Arc<RwLock<Wallet>>,
149    sig_int_toggled: Arc<(Mutex<bool>, Condvar)>,
150) -> (
151    MassaReceiver<ConsensusEvent>,
152    Option<BootstrapManager>,
153    Box<dyn ConsensusManager>,
154    Box<dyn ExecutionManager>,
155    Box<dyn SelectorManager>,
156    Box<dyn PoolManager>,
157    Box<dyn ProtocolManager>,
158    Box<dyn FactoryManager>,
159    Box<dyn EventCacheManager>,
160    StopHandle,
161    StopHandle,
162    StopHandle,
163    Option<massa_grpc::server::StopHandle>,
164    Option<massa_grpc::server::StopHandle>,
165    MetricsStopper,
166    MassaSurveyStopper,
167) {
168    let now = MassaTime::now();
169
170    if let Some(end) = *END_TIMESTAMP {
171        if now > end {
172            panic!("This episode has come to an end, please get the latest testnet node version to continue");
173        }
174    }
175
176    // Storage shared by multiple components.
177    let shared_storage: Storage = Storage::create_root();
178
179    // init final state
180    let ledger_config = LedgerConfig {
181        thread_count: THREAD_COUNT,
182        initial_ledger_path: SETTINGS.ledger.initial_ledger_path.clone(),
183        max_key_length: MAX_DATASTORE_KEY_LENGTH,
184        max_datastore_value_length: MAX_DATASTORE_VALUE_LENGTH,
185        max_bytecode_size: MAX_BYTECODE_LENGTH,
186    };
187    let async_pool_config = AsyncPoolConfig {
188        max_length: MAX_ASYNC_POOL_LENGTH,
189        thread_count: THREAD_COUNT,
190        max_function_length: MAX_FUNCTION_NAME_LENGTH,
191        max_function_params_length: MAX_PARAMETERS_SIZE as u64,
192        max_key_length: MAX_DATASTORE_KEY_LENGTH as u32,
193    };
194    let pos_config = PoSConfig {
195        periods_per_cycle: PERIODS_PER_CYCLE,
196        thread_count: THREAD_COUNT,
197        cycle_history_length: POS_SAVED_CYCLES,
198        max_rolls_length: MAX_ROLLS_COUNT_LENGTH,
199        max_production_stats_length: MAX_PRODUCTION_STATS_LENGTH,
200        max_credit_length: MAX_DEFERRED_CREDITS_LENGTH,
201        initial_deferred_credits_path: SETTINGS.ledger.initial_deferred_credits_path.clone(),
202    };
203    let executed_ops_config = ExecutedOpsConfig {
204        thread_count: THREAD_COUNT,
205        keep_executed_history_extra_periods: KEEP_EXECUTED_HISTORY_EXTRA_PERIODS,
206    };
207    let executed_denunciations_config = ExecutedDenunciationsConfig {
208        denunciation_expire_periods: DENUNCIATION_EXPIRE_PERIODS,
209        thread_count: THREAD_COUNT,
210        endorsement_count: ENDORSEMENT_COUNT,
211        keep_executed_history_extra_periods: KEEP_EXECUTED_HISTORY_EXTRA_PERIODS,
212    };
213    let deferred_calls_config = DeferredCallsConfig {
214        thread_count: THREAD_COUNT,
215        max_function_name_length: MAX_FUNCTION_NAME_LENGTH,
216        max_parameter_size: MAX_PARAMETERS_SIZE,
217        max_pool_changes: DEFERRED_CALL_MAX_POOL_CHANGES,
218        max_gas: DEFERRED_CALL_MAX_ASYNC_GAS,
219        max_future_slots: DEFERRED_CALL_MAX_FUTURE_SLOTS,
220        base_fee_max_max_change_denominator: DEFERRED_CALL_BASE_FEE_MAX_CHANGE_DENOMINATOR,
221        min_gas_increment: DEFERRED_CALL_MIN_GAS_INCREMENT,
222        min_gas_cost: DEFERRED_CALL_MIN_GAS_COST,
223        global_overbooking_penalty: DEFERRED_CALL_GLOBAL_OVERBOOKING_PENALTY,
224        slot_overbooking_penalty: DEFERRED_CALL_SLOT_OVERBOOKING_PENALTY,
225        call_cst_gas_cost: DEFERRED_CALL_CST_GAS_COST,
226        ledger_cost_per_byte: LEDGER_COST_PER_BYTE,
227    };
228    let final_state_config = FinalStateConfig {
229        ledger_config: ledger_config.clone(),
230        async_pool_config,
231        deferred_calls_config,
232        pos_config,
233        executed_ops_config,
234        executed_denunciations_config,
235        final_history_length: SETTINGS.ledger.final_history_length,
236        thread_count: THREAD_COUNT,
237        periods_per_cycle: PERIODS_PER_CYCLE,
238        initial_seed_string: INITIAL_DRAW_SEED.into(),
239        initial_rolls_path: SETTINGS.selector.initial_rolls_path.clone(),
240        endorsement_count: ENDORSEMENT_COUNT,
241        max_executed_denunciations_length: MAX_DENUNCIATION_CHANGES_LENGTH,
242        max_denunciations_per_block_header: MAX_DENUNCIATIONS_PER_BLOCK_HEADER,
243        ledger_backup_periods_interval: SETTINGS.ledger.ledger_backup_periods_interval,
244        t0: T0,
245        genesis_timestamp: *GENESIS_TIMESTAMP,
246    };
247
248    // Start massa metrics
249    let (massa_metrics, metrics_stopper) = MassaMetrics::new(
250        SETTINGS.metrics.enabled,
251        SETTINGS.metrics.bind,
252        THREAD_COUNT,
253        SETTINGS.metrics.tick_delay.to_duration(),
254    );
255
256    // Remove current disk ledger if there is one and we don't want to restart from snapshot
257    // NOTE: this is temporary, since we cannot currently handle bootstrap from remaining ledger
258    if args.keep_ledger || args.restart_from_snapshot_at_period.is_some() {
259        info!("Loading old ledger for next episode");
260    } else {
261        if SETTINGS.ledger.disk_ledger_path.exists() {
262            std::fs::remove_dir_all(SETTINGS.ledger.disk_ledger_path.clone())
263                .expect("disk ledger delete failed");
264        }
265        if SETTINGS.execution.hd_cache_path.exists() {
266            std::fs::remove_dir_all(SETTINGS.execution.hd_cache_path.clone())
267                .expect("disk hd cache delete failed");
268        }
269    }
270
271    let db_config = MassaDBConfig {
272        path: SETTINGS.ledger.disk_ledger_path.clone(),
273        max_history_length: SETTINGS.ledger.final_history_length,
274        max_final_state_elements_size: MAX_BOOTSTRAP_FINAL_STATE_PARTS_SIZE.try_into().unwrap(),
275        max_versioning_elements_size: MAX_BOOTSTRAP_VERSIONING_ELEMENTS_SIZE.try_into().unwrap(),
276        thread_count: THREAD_COUNT,
277        max_ledger_backups: SETTINGS.ledger.max_ledger_backups,
278        enable_metrics: SETTINGS.metrics.enabled,
279    };
280    let db = Arc::new(RwLock::new(
281        Box::new(MassaDB::new(db_config)) as Box<dyn MassaDBController + 'static>
282    ));
283
284    // Create final ledger
285    let ledger = FinalLedger::new(ledger_config.clone(), db.clone());
286
287    // launch selector worker
288    let (selector_manager, selector_controller) = start_selector_worker(SelectorConfig {
289        max_draw_cache: SELECTOR_DRAW_CACHE_SIZE,
290        channel_size: CHANNEL_SIZE,
291        thread_count: THREAD_COUNT,
292        endorsement_count: ENDORSEMENT_COUNT,
293        periods_per_cycle: PERIODS_PER_CYCLE,
294        genesis_address: Address::from_public_key(&GENESIS_KEY.get_public_key()),
295    })
296    .expect("could not start selector worker");
297
298    // Creates an empty default store
299    let mip_stats_config = MipStatsConfig {
300        block_count_considered: MIP_STORE_STATS_BLOCK_CONSIDERED,
301        warn_announced_version_ratio: Ratio::new(
302            u64::from(SETTINGS.versioning.mip_stats_warn_announced_version),
303            100,
304        ),
305    };
306    // Ratio::new_raw(*SETTINGS.versioning.warn_announced_version_ratio, 100),
307
308    // Create final state, either from a snapshot, or from scratch
309    let final_state: Arc<RwLock<dyn FinalStateController>> = Arc::new(parking_lot::RwLock::new(
310        match args.restart_from_snapshot_at_period {
311            Some(last_start_period) => {
312                // The node is restarted from a snapshot:
313                // MIP store by reading from the db as it must have been updated by the massa ledger editor
314                // (to shift transitions that might have happened during the network shutdown)
315                // Note that FinalState::new_derived_from_snapshot will check if MIP store is consistent
316                // No Bootstrap are expected after this
317                let mip_store: MipStore = MipStore::try_from_db(db.clone(), mip_stats_config)
318                    .expect("MIP store creation failed");
319                debug!("After read from db, Mip store: {:?}", mip_store);
320
321                FinalState::new_derived_from_snapshot(
322                    db.clone(),
323                    final_state_config,
324                    Box::new(ledger),
325                    selector_controller.clone(),
326                    mip_store,
327                    last_start_period,
328                )
329                .expect("could not init final state")
330            }
331            None => {
332                // The node is started in a normal way
333                // Read the mip list supported by the current software
334                // The resulting MIP store will likely be updated by the bootstrap process in order
335                // to get the latest information for the MIP store (new states, votes...)
336
337                let mip_list = get_mip_list();
338                debug!("MIP list: {:?}", mip_list);
339                let mip_store = MipStore::try_from((mip_list, mip_stats_config))
340                    .expect("mip store creation failed");
341
342                FinalState::new(
343                    db.clone(),
344                    final_state_config,
345                    Box::new(ledger),
346                    selector_controller.clone(),
347                    mip_store,
348                    true,
349                )
350                .expect("could not init final state")
351            }
352        },
353    ));
354
355    let mip_store = final_state.read().get_mip_store().clone();
356
357    let bootstrap_config: BootstrapConfig = BootstrapConfig {
358        bootstrap_list: SETTINGS.bootstrap.bootstrap_list.clone(),
359        bootstrap_protocol: SETTINGS.bootstrap.bootstrap_protocol,
360        bootstrap_whitelist_path: SETTINGS.bootstrap.bootstrap_whitelist_path.clone(),
361        bootstrap_blacklist_path: SETTINGS.bootstrap.bootstrap_blacklist_path.clone(),
362        listen_addr: SETTINGS.bootstrap.bind,
363        connect_timeout: SETTINGS.bootstrap.connect_timeout,
364        bootstrap_timeout: SETTINGS.bootstrap.bootstrap_timeout,
365        read_timeout: SETTINGS.bootstrap.read_timeout,
366        write_timeout: SETTINGS.bootstrap.write_timeout,
367        read_error_timeout: SETTINGS.bootstrap.read_error_timeout,
368        write_error_timeout: SETTINGS.bootstrap.write_error_timeout,
369        retry_delay: SETTINGS.bootstrap.retry_delay,
370        max_ping: SETTINGS.bootstrap.max_ping,
371        max_clock_delta: SETTINGS.bootstrap.max_clock_delta,
372        cache_duration: SETTINGS.bootstrap.cache_duration,
373        keep_ledger: args.keep_ledger,
374        max_listeners_per_peer: MAX_LISTENERS_PER_PEER as u32,
375        max_simultaneous_bootstraps: SETTINGS.bootstrap.max_simultaneous_bootstraps,
376        per_ip_min_interval: SETTINGS.bootstrap.per_ip_min_interval,
377        ip_list_max_size: SETTINGS.bootstrap.ip_list_max_size,
378        rate_limit: SETTINGS.bootstrap.rate_limit,
379        max_datastore_key_length: MAX_DATASTORE_KEY_LENGTH,
380        randomness_size_bytes: BOOTSTRAP_RANDOMNESS_SIZE_BYTES,
381        thread_count: THREAD_COUNT,
382        t0: T0,
383        genesis_timestamp: *GENESIS_TIMESTAMP,
384        periods_per_cycle: PERIODS_PER_CYCLE,
385        endorsement_count: ENDORSEMENT_COUNT,
386        max_advertise_length: MAX_ADVERTISE_LENGTH,
387        max_bootstrap_blocks_length: MAX_BOOTSTRAP_BLOCKS,
388        max_bootstrap_error_length: MAX_BOOTSTRAP_ERROR_LENGTH,
389        max_final_state_elements_size: MAX_BOOTSTRAP_FINAL_STATE_PARTS_SIZE,
390        max_versioning_elements_size: MAX_BOOTSTRAP_VERSIONING_ELEMENTS_SIZE,
391        // Bound the in-memory footprint of a batch we *receive*. The sending side applies the
392        // same accounting to the batches it builds (see `get_batch_to_stream`), so a batch any
393        // peer of this version sends fits the budget a peer of this version parses with.
394        max_final_state_batch_allocation: bootstrap_batch_allocation_budget(
395            MAX_BOOTSTRAP_FINAL_STATE_PARTS_SIZE as usize,
396        ) as u64,
397        max_versioning_batch_allocation: bootstrap_batch_allocation_budget(
398            MAX_BOOTSTRAP_VERSIONING_ELEMENTS_SIZE as usize,
399        ) as u64,
400        max_operations_per_block: MAX_OPERATIONS_PER_BLOCK,
401        max_datastore_entry_count: MAX_DATASTORE_ENTRY_COUNT,
402        max_datastore_value_length: MAX_DATASTORE_VALUE_LENGTH,
403        max_function_name_length: MAX_FUNCTION_NAME_LENGTH,
404        max_ledger_changes_count: MAX_LEDGER_CHANGES_COUNT,
405        max_parameters_size: MAX_PARAMETERS_SIZE,
406        max_op_datastore_entry_count: MAX_OPERATION_DATASTORE_ENTRY_COUNT,
407        max_op_datastore_key_length: MAX_OPERATION_DATASTORE_KEY_LENGTH,
408        max_op_datastore_value_length: MAX_OPERATION_DATASTORE_VALUE_LENGTH,
409        max_changes_slot_count: SETTINGS.ledger.final_history_length as u64,
410        max_rolls_length: MAX_ROLLS_COUNT_LENGTH,
411        max_production_stats_length: MAX_PRODUCTION_STATS_LENGTH,
412        max_credits_length: MAX_DEFERRED_CREDITS_LENGTH,
413        max_executed_ops_length: MAX_EXECUTED_OPS_LENGTH,
414        max_ops_changes_length: MAX_EXECUTED_OPS_CHANGES_LENGTH,
415        consensus_bootstrap_part_size: CONSENSUS_BOOTSTRAP_PART_SIZE,
416        max_consensus_block_ids: MAX_CONSENSUS_BLOCKS_IDS,
417        mip_store_stats_block_considered: MIP_STORE_STATS_BLOCK_CONSIDERED,
418        max_denunciations_per_block_header: MAX_DENUNCIATIONS_PER_BLOCK_HEADER,
419        max_denunciation_changes_length: MAX_DENUNCIATION_CHANGES_LENGTH,
420        chain_id: *CHAINID,
421    };
422
423    let bootstrap_state = match get_state(
424        &bootstrap_config,
425        final_state.clone(),
426        DefaultConnector,
427        *VERSION,
428        *GENESIS_TIMESTAMP,
429        *END_TIMESTAMP,
430        args.restart_from_snapshot_at_period,
431        sig_int_toggled.clone(),
432        massa_metrics.clone(),
433    ) {
434        Ok(vals) => vals,
435        Err(BootstrapError::Interrupted(msg)) => {
436            info!("{}", msg);
437            process::exit(0);
438        }
439        Err(err) => panic!("critical error detected in the bootstrap process: {}", err),
440    };
441
442    if !final_state.read().is_db_valid() {
443        // TODO: Bootstrap again instead of panicking
444        panic!("critical: db is not valid after bootstrap");
445    }
446
447    if args.restart_from_snapshot_at_period.is_none() {
448        final_state.write().recompute_caches();
449
450        // give the controller to final state in order for it to feed the cycles
451        final_state
452            .write()
453            .compute_initial_draws()
454            .expect("could not compute initial draws"); // TODO: this might just mean a bad bootstrap, no need to panic, just reboot
455    }
456
457    // Note: the consistency of the network restart metadata with the MIP store has already been
458    // checked, either by the bootstrap client before storing what the server sent, or by
459    // `FinalState::new_derived_from_snapshot` when restarting from a local snapshot.
460    if final_state.read().get_last_slot_before_downtime().is_some() {
461        // If we are before a network restart, print the hash to make it easier to debug bootstrapping issues
462        let now = MassaTime::now();
463        let last_start_slot = Slot::new(
464            final_state.read().get_last_start_period(),
465            THREAD_COUNT.saturating_sub(1),
466        );
467        let last_start_slot_timestamp =
468            get_block_slot_timestamp(THREAD_COUNT, T0, *GENESIS_TIMESTAMP, last_start_slot)
469                .expect("Can't get timestamp for last_start_slot");
470        if now < last_start_slot_timestamp {
471            let final_state_hash = final_state.read().get_fingerprint();
472            info!(
473                "final_state hash before network restarts at slot {}: {}",
474                last_start_slot, final_state_hash
475            );
476        }
477    }
478
479    // Event cache thread
480    let event_cache_config = EventCacheConfig {
481        event_cache_path: SETTINGS.execution.event_cache_path.clone(),
482        max_event_cache_length: SETTINGS.execution.event_cache_size,
483        snip_amount: SETTINGS.execution.event_snip_amount,
484        // Note: we still use the v0 event data size for the event cache to be able to deserialize
485        // events that bypass the v1 event limitation
486        max_event_data_length: MAX_EVENT_DATA_SIZE_V0 as u64,
487        thread_count: THREAD_COUNT,
488        // Note: SCOutputEvent call stack comes from the execution module, and we assume
489        //       this should return a limited call stack length
490        //       The value remains for future use & limitations
491        max_call_stack_length: u16::MAX,
492
493        max_events_per_operation: MAX_EVENT_PER_OPERATION as u64,
494        max_operations_per_block: MAX_OPERATIONS_PER_BLOCK as u64,
495        max_events_per_query: SETTINGS.execution.max_event_per_query,
496        max_scanned_keys_per_query: SETTINGS.execution.max_event_keys_scanned_per_query,
497    };
498    let (event_cache_manager, event_cache_controller) =
499        start_event_cache_writer_worker(event_cache_config);
500
501    // Storage costs constants
502    let storage_costs_constants = StorageCostsConstants {
503        ledger_cost_per_byte: LEDGER_COST_PER_BYTE,
504        ledger_entry_base_cost: LEDGER_ENTRY_BASE_COST,
505        ledger_entry_datastore_base_cost: LEDGER_COST_PER_BYTE
506            .checked_mul_u64(LEDGER_ENTRY_DATASTORE_BASE_SIZE as u64)
507            .expect("Overflow when creating constant ledger_entry_datastore_base_size"),
508    };
509
510    // gas costs
511    let gas_costs = GasCosts::new(SETTINGS.execution.abi_gas_costs_file.clone())
512        .expect("Failed to load gas costs");
513
514    // Limits imposed to wasm files so the compilation phase is smooth
515    let condom_limits = CondomLimits {
516        max_exports: Some(MAX_RUNTIME_MODULE_EXPORTS),
517        max_functions: Some(MAX_RUNTIME_MODULE_FUNCTIONS),
518        max_signature_len: Some(MAX_RUNTIME_MODULE_SIGNATURE_LEN),
519        max_name_len: Some(MAX_RUNTIME_MODULE_NAME_LEN),
520        max_imports_len: Some(MAX_RUNTIME_MODULE_IMPORTS),
521        max_table_initializers_len: Some(MAX_RUNTIME_MODULE_TABLE_INITIALIZER),
522        max_passive_elements_len: Some(MAX_RUNTIME_MODULE_PASSIVE_ELEMENT),
523        max_passive_data_len: Some(MAX_RUNTIME_MODULE_PASSIVE_DATA),
524        max_global_initializers_len: Some(MAX_RUNTIME_MODULE_GLOBAL_INITIALIZER),
525        max_function_names_len: Some(MAX_RUNTIME_MODULE_FUNCTION_NAME_LEN),
526        max_tables_count: Some(MAX_RUNTIME_MODULE_TABLE),
527        max_table_elements_count: Some(MAX_RUNTIME_MODULE_TABLE_ELEMENTS),
528        max_memories_len: Some(MAX_RUNTIME_MODULE_MEMORIES),
529        max_globals_len: Some(MAX_RUNTIME_MODULE_GLOBAL_INITIALIZER),
530        max_custom_sections_len: Some(MAX_RUNTIME_MODULE_CUSTOM_SECTION_LEN),
531        max_custom_sections_data_len: Some(MAX_RUNTIME_MODULE_CUSTOM_SECTION_DATA_LEN),
532        max_locals_count: Some(MAX_RUNTIME_MODULE_LOCALS),
533        max_code_section_size: Some(MAX_RUNTIME_MODULE_CODE_SECTION_SIZE),
534    };
535
536    let block_dump_folder_path = SETTINGS.block_dump.block_dump_folder_path.clone();
537    if !block_dump_folder_path.exists() {
538        info!("Current folder: {:?}", std::env::current_dir().unwrap());
539        info!("Creating dump folder: {:?}", block_dump_folder_path);
540        std::fs::create_dir_all(block_dump_folder_path.clone())
541            .expect("Cannot create dump block folder");
542    }
543
544    // launch execution module
545    let execution_config = ExecutionConfig {
546        max_final_events: SETTINGS.execution.max_final_events,
547        readonly_queue_length: SETTINGS.execution.readonly_queue_length,
548        readonly_starvation_timeout: SETTINGS.execution.readonly_starvation_timeout,
549        readonly_queue_timeout: SETTINGS.execution.readonly_queue_timeout,
550        cursor_delay: SETTINGS.execution.cursor_delay,
551        max_async_gas: MAX_ASYNC_GAS,
552        async_msg_cst_gas_cost: ASYNC_MSG_CST_GAS_COST,
553        max_gas_per_block: MAX_GAS_PER_BLOCK,
554        roll_price: ROLL_PRICE,
555        thread_count: THREAD_COUNT,
556        t0: T0,
557        genesis_timestamp: *GENESIS_TIMESTAMP,
558        block_reward: BLOCK_REWARD,
559        endorsement_count: ENDORSEMENT_COUNT as u64,
560        operation_validity_period: OPERATION_VALIDITY_PERIODS,
561        periods_per_cycle: PERIODS_PER_CYCLE,
562        stats_time_window_duration: SETTINGS.execution.stats_time_window_duration,
563        max_miss_ratio: *POS_MISS_RATE_DEACTIVATION_THRESHOLD,
564        max_datastore_key_length: MAX_DATASTORE_KEY_LENGTH,
565        max_bytecode_size: MAX_BYTECODE_LENGTH,
566        max_datastore_value_size: MAX_DATASTORE_VALUE_LENGTH,
567        storage_costs_constants,
568        max_read_only_gas: SETTINGS.execution.max_read_only_gas,
569        gas_costs: gas_costs.clone(),
570        base_operation_gas_cost: BASE_OPERATION_GAS_COST,
571        last_start_period: final_state.read().get_last_start_period(),
572        hd_cache_path: SETTINGS.execution.hd_cache_path.clone(),
573        lru_cache_size: SETTINGS.execution.lru_cache_size,
574        hd_cache_size: SETTINGS.execution.hd_cache_size,
575        snip_amount: SETTINGS.execution.snip_amount,
576        roll_count_to_slash_on_denunciation: ROLL_COUNT_TO_SLASH_ON_DENUNCIATION,
577        denunciation_expire_periods: DENUNCIATION_EXPIRE_PERIODS,
578        broadcast_enabled: SETTINGS.api.enable_broadcast,
579        broadcast_slot_execution_output_channel_capacity: SETTINGS
580            .execution
581            .broadcast_slot_execution_output_channel_capacity,
582        max_event_size_v0: MAX_EVENT_DATA_SIZE_V0,
583        max_event_size: MAX_EVENT_DATA_SIZE,
584        max_function_length: MAX_FUNCTION_NAME_LENGTH,
585        max_parameter_length: MAX_PARAMETERS_SIZE,
586        chain_id: *CHAINID,
587        #[cfg(feature = "execution-trace")]
588        broadcast_traces_enabled: true,
589        #[cfg(not(feature = "execution-trace"))]
590        broadcast_traces_enabled: false,
591        broadcast_slot_execution_traces_channel_capacity: SETTINGS
592            .execution
593            .broadcast_slot_execution_traces_channel_capacity,
594        max_execution_traces_slot_limit: SETTINGS.execution.execution_traces_limit,
595        block_dump_folder_path,
596        max_recursive_calls_depth: MAX_RECURSIVE_CALLS_DEPTH,
597        condom_limits,
598        deferred_calls_config,
599        max_event_per_operation: MAX_EVENT_PER_OPERATION,
600        event_cache_path: SETTINGS.execution.event_cache_path.clone(),
601        event_cache_size: SETTINGS.execution.event_cache_size,
602        event_snip_amount: SETTINGS.execution.event_snip_amount,
603        broadcast_slot_execution_info_channel_capacity: SETTINGS
604            .execution
605            .broadcast_slot_execution_info_channel_capacity,
606    };
607
608    let execution_channels = ExecutionChannels {
609        slot_execution_output_sender: broadcast::channel(
610            execution_config.broadcast_slot_execution_output_channel_capacity,
611        )
612        .0,
613        #[cfg(feature = "execution-trace")]
614        slot_execution_traces_sender: broadcast::channel(
615            execution_config.broadcast_slot_execution_traces_channel_capacity,
616        )
617        .0,
618        #[cfg(feature = "execution-info")]
619        slot_execution_info_sender: broadcast::channel(
620            execution_config.broadcast_slot_execution_info_channel_capacity,
621        )
622        .0,
623    };
624
625    cfg_if! {
626        if #[cfg(all(feature = "dump-block", feature = "db_storage_backend"))] {
627            let block_storage_backend = Arc::new(RwLock::new(
628                RocksDBStorageBackend::new(
629                execution_config.block_dump_folder_path.clone(), SETTINGS.block_dump.max_blocks),
630            ));
631        } else if #[cfg(all(feature = "dump-block", feature = "file_storage_backend"))] {
632            let block_storage_backend = Arc::new(RwLock::new(
633                FileStorageBackend::new(
634                execution_config.block_dump_folder_path.clone(), SETTINGS.block_dump.max_blocks),
635            ));
636        } else if #[cfg(feature = "dump-block")] {
637            compile_error!("feature dump-block requise either db_storage_backend or file_storage_backend");
638        }
639    }
640
641    let (execution_manager, execution_controller) = start_execution_worker(
642        execution_config,
643        final_state.clone(),
644        selector_controller.clone(),
645        mip_store.clone(),
646        execution_channels.clone(),
647        node_wallet.clone(),
648        massa_metrics.clone(),
649        event_cache_controller,
650        #[cfg(feature = "dump-block")]
651        block_storage_backend.clone(),
652    );
653
654    // launch pool controller
655    let pool_config = PoolConfig {
656        thread_count: THREAD_COUNT,
657        max_block_size: MAX_BLOCK_SIZE,
658        max_block_gas: MAX_GAS_PER_BLOCK,
659        base_operation_gas_cost: BASE_OPERATION_GAS_COST,
660        sp_compilation_cost: gas_costs.sp_compilation_cost,
661        roll_price: ROLL_PRICE,
662        max_block_endorsement_count: ENDORSEMENT_COUNT,
663        operation_validity_periods: OPERATION_VALIDITY_PERIODS,
664        max_operations_per_block: MAX_OPERATIONS_PER_BLOCK,
665        max_operation_pool_size: SETTINGS.pool.max_operation_pool_size,
666        max_operation_pool_excess_items: SETTINGS.pool.max_operation_pool_excess_items,
667        operation_pool_refresh_interval: SETTINGS.pool.operation_pool_refresh_interval,
668        operation_pool_swap_interval: SETTINGS.pool.operation_pool_swap_interval,
669        endorsement_pool_swap_interval: SETTINGS.pool.endorsement_pool_swap_interval,
670        denunciation_pool_refresh_interval: SETTINGS.pool.denunciation_pool_refresh_interval,
671        denunciation_pool_swap_interval: SETTINGS.pool.denunciation_pool_swap_interval,
672        operation_max_future_start_delay: SETTINGS.pool.operation_max_future_start_delay,
673        max_endorsements_pool_size_per_thread: SETTINGS.pool.max_endorsements_pool_size_per_thread,
674        operations_channel_size: SETTINGS.pool.operations_channel_capacity,
675        endorsements_channel_size: SETTINGS.pool.endorsements_channel_capacity,
676        denunciations_channel_size: SETTINGS.pool.denunciations_channel_capacity,
677        broadcast_enabled: SETTINGS.api.enable_broadcast,
678        broadcast_endorsements_channel_capacity: SETTINGS
679            .pool
680            .broadcast_endorsements_channel_capacity,
681        broadcast_operations_channel_capacity: SETTINGS.pool.broadcast_operations_channel_capacity,
682        genesis_timestamp: *GENESIS_TIMESTAMP,
683        t0: T0,
684        periods_per_cycle: PERIODS_PER_CYCLE,
685        denunciation_expire_periods: DENUNCIATION_EXPIRE_PERIODS,
686        max_denunciations_per_block_header: MAX_DENUNCIATIONS_PER_BLOCK_HEADER,
687        minimal_fees: SETTINGS.pool.minimal_fees,
688        last_start_period: final_state.read().get_last_start_period(),
689    };
690
691    let pool_channels = PoolChannels {
692        broadcasts: PoolBroadcasts {
693            endorsement_sender: broadcast::channel(
694                pool_config.broadcast_endorsements_channel_capacity,
695            )
696            .0,
697            operation_sender: broadcast::channel(pool_config.broadcast_operations_channel_capacity)
698                .0,
699        },
700        selector: selector_controller.clone(),
701        execution_controller: execution_controller.clone(),
702        chain_id: *CHAINID,
703    };
704
705    let (pool_manager, pool_controller) = start_pool_controller(
706        pool_config,
707        &shared_storage,
708        pool_channels.clone(),
709        node_wallet.clone(),
710    );
711
712    // launch protocol controller
713    let mut listeners = HashMap::default();
714    listeners.insert(SETTINGS.protocol.bind, TransportType::Tcp);
715    let protocol_config = ProtocolConfig {
716        thread_count: THREAD_COUNT,
717        ask_block_timeout: SETTINGS.protocol.ask_block_timeout,
718        max_known_blocks_size: SETTINGS.protocol.max_known_blocks_size,
719        max_node_known_blocks_size: SETTINGS.protocol.max_node_known_blocks_size,
720        max_block_propagation_time: SETTINGS.protocol.max_block_propagation_time,
721        max_node_wanted_blocks_size: SETTINGS.protocol.max_node_wanted_blocks_size,
722        max_known_ops_size: SETTINGS.protocol.max_known_ops_size,
723        max_node_known_ops_size: SETTINGS.protocol.max_node_known_ops_size,
724        max_known_endorsements_size: SETTINGS.protocol.max_known_endorsements_size,
725        max_node_known_endorsements_size: SETTINGS.protocol.max_node_known_endorsements_size,
726        max_simultaneous_ask_blocks_per_node: SETTINGS
727            .protocol
728            .max_simultaneous_ask_blocks_per_node,
729        max_send_wait: SETTINGS.protocol.max_send_wait,
730        operation_batch_buffer_capacity: SETTINGS.protocol.operation_batch_buffer_capacity,
731        operation_announcement_buffer_capacity: SETTINGS
732            .protocol
733            .operation_announcement_buffer_capacity,
734        operation_batch_proc_period: SETTINGS.protocol.operation_batch_proc_period,
735        operation_announcement_interval: SETTINGS.protocol.operation_announcement_interval,
736        max_operations_per_message: SETTINGS.protocol.max_operations_per_message,
737        max_serialized_operations_size_per_block: MAX_BLOCK_SIZE as usize,
738        max_block_gas: MAX_GAS_PER_BLOCK,
739        base_operation_gas_cost: BASE_OPERATION_GAS_COST,
740        sp_compilation_cost: gas_costs.sp_compilation_cost,
741        max_operations_per_block: MAX_OPERATIONS_PER_BLOCK,
742        controller_channel_size: PROTOCOL_CONTROLLER_CHANNEL_SIZE,
743        event_channel_size: PROTOCOL_EVENT_CHANNEL_SIZE,
744        genesis_timestamp: *GENESIS_TIMESTAMP,
745        t0: T0,
746        endorsement_count: ENDORSEMENT_COUNT,
747        max_message_size: MAX_MESSAGE_SIZE as usize,
748        max_ops_kept_for_propagation: SETTINGS.protocol.max_ops_kept_for_propagation,
749        max_endorsements_per_propagation_round: SETTINGS
750            .protocol
751            .max_endorsements_per_propagation_round,
752        max_operations_propagation_time: SETTINGS.protocol.max_operations_propagation_time,
753        max_endorsements_propagation_time: SETTINGS.protocol.max_endorsements_propagation_time,
754        last_start_period: final_state.read().get_last_start_period(),
755        max_endorsements_per_message: MAX_ENDORSEMENTS_PER_MESSAGE as u64,
756        max_denunciations_in_block_header: MAX_DENUNCIATIONS_PER_BLOCK_HEADER,
757        initial_peers: SETTINGS.protocol.initial_peers_file.clone(),
758        listeners,
759        keypair_file: SETTINGS.protocol.keypair_file.clone(),
760        max_blocks_kept_for_propagation: SETTINGS.protocol.max_blocks_kept_for_propagation,
761        block_propagation_tick: SETTINGS.protocol.block_propagation_tick,
762        asked_operations_buffer_capacity: SETTINGS.protocol.asked_operations_buffer_capacity,
763        thread_tester_count: SETTINGS.protocol.thread_tester_count,
764        max_operation_storage_time: MAX_OPERATION_STORAGE_TIME,
765        max_size_channel_commands_propagation_blocks: MAX_SIZE_CHANNEL_COMMANDS_PROPAGATION_BLOCKS,
766        max_size_channel_commands_propagation_operations:
767            MAX_SIZE_CHANNEL_COMMANDS_PROPAGATION_OPERATIONS,
768        max_size_channel_commands_propagation_endorsements:
769            MAX_SIZE_CHANNEL_COMMANDS_PROPAGATION_ENDORSEMENTS,
770        max_size_channel_commands_retrieval_blocks: MAX_SIZE_CHANNEL_COMMANDS_RETRIEVAL_BLOCKS,
771        max_size_channel_commands_retrieval_operations:
772            MAX_SIZE_CHANNEL_COMMANDS_RETRIEVAL_OPERATIONS,
773        max_size_channel_commands_retrieval_endorsements:
774            MAX_SIZE_CHANNEL_COMMANDS_RETRIEVAL_ENDORSEMENTS,
775        max_size_channel_commands_connectivity: MAX_SIZE_CHANNEL_COMMANDS_CONNECTIVITY,
776        max_size_channel_commands_peers: MAX_SIZE_CHANNEL_COMMANDS_PEERS,
777        max_size_channel_commands_peer_testers: MAX_SIZE_CHANNEL_COMMANDS_PEER_TESTERS,
778        max_size_channel_network_to_block_handler: MAX_SIZE_CHANNEL_NETWORK_TO_BLOCK_HANDLER,
779        max_size_channel_network_to_operation_handler:
780            MAX_SIZE_CHANNEL_NETWORK_TO_OPERATION_HANDLER,
781        max_size_channel_network_to_endorsement_handler:
782            MAX_SIZE_CHANNEL_NETWORK_TO_ENDORSEMENT_HANDLER,
783        max_size_channel_network_to_peer_handler: MAX_SIZE_CHANNEL_NETWORK_TO_PEER_HANDLER,
784        max_bytecode_size: MAX_BYTECODE_LENGTH,
785        max_op_datastore_entry_count: MAX_OPERATION_DATASTORE_ENTRY_COUNT,
786        max_op_datastore_key_length: MAX_OPERATION_DATASTORE_KEY_LENGTH,
787        max_op_datastore_value_length: MAX_OPERATION_DATASTORE_VALUE_LENGTH,
788        max_size_function_name: MAX_FUNCTION_NAME_LENGTH,
789        max_size_call_sc_parameter: MAX_PARAMETERS_SIZE,
790        max_size_listeners_per_peer: MAX_LISTENERS_PER_PEER,
791        max_size_peers_announcement: MAX_PEERS_IN_ANNOUNCEMENT_LIST,
792        read_write_limit_bytes_per_second: SETTINGS.protocol.read_write_limit_bytes_per_second
793            as u128,
794        try_connection_timer: SETTINGS.protocol.try_connection_timer,
795        unban_everyone_timer: SETTINGS.protocol.unban_everyone_timer,
796        max_in_connections: SETTINGS.protocol.max_in_connections,
797        timeout_connection: SETTINGS.protocol.timeout_connection,
798        message_timeout: SETTINGS.protocol.message_timeout,
799        tester_timeout: SETTINGS.protocol.tester_timeout,
800        routable_ip: SETTINGS
801            .protocol
802            .routable_ip
803            .or(SETTINGS.network.routable_ip),
804        debug: false,
805        peers_categories: SETTINGS.protocol.peers_categories.clone(),
806        default_category_info: SETTINGS.protocol.default_category_info,
807        version: *VERSION,
808        try_connection_timer_same_peer: SETTINGS.protocol.try_connection_timer_same_peer,
809        test_oldest_peer_cooldown: SETTINGS.protocol.test_oldest_peer_cooldown,
810        rate_limit: SETTINGS.protocol.rate_limit,
811        chain_id: *CHAINID,
812    };
813
814    let (protocol_controller, protocol_channels) =
815        create_protocol_controller(protocol_config.clone());
816
817    let consensus_config = ConsensusConfig {
818        genesis_timestamp: *GENESIS_TIMESTAMP,
819        end_timestamp: *END_TIMESTAMP,
820        thread_count: THREAD_COUNT,
821        t0: T0,
822        genesis_key: GENESIS_KEY.clone(),
823        max_discarded_blocks: SETTINGS.consensus.max_discarded_blocks,
824        max_future_processing_blocks: SETTINGS.consensus.max_future_processing_blocks,
825        max_dependency_blocks: SETTINGS.consensus.max_dependency_blocks,
826        delta_f0: DELTA_F0,
827        operation_validity_periods: OPERATION_VALIDITY_PERIODS,
828        periods_per_cycle: PERIODS_PER_CYCLE,
829        stats_timespan: SETTINGS.consensus.stats_timespan,
830        force_keep_final_periods: SETTINGS.consensus.force_keep_final_periods,
831        endorsement_count: ENDORSEMENT_COUNT,
832        block_db_prune_interval: SETTINGS.consensus.block_db_prune_interval,
833        max_gas_per_block: MAX_GAS_PER_BLOCK,
834        channel_size: CHANNEL_SIZE,
835        bootstrap_part_size: CONSENSUS_BOOTSTRAP_PART_SIZE,
836        broadcast_enabled: SETTINGS.api.enable_broadcast,
837        broadcast_blocks_headers_channel_capacity: SETTINGS
838            .consensus
839            .broadcast_blocks_headers_channel_capacity,
840        broadcast_blocks_channel_capacity: SETTINGS.consensus.broadcast_blocks_channel_capacity,
841        broadcast_filled_blocks_channel_capacity: SETTINGS
842            .consensus
843            .broadcast_filled_blocks_channel_capacity,
844        last_start_period: final_state.read().get_last_start_period(),
845        force_keep_final_periods_without_ops: SETTINGS
846            .consensus
847            .force_keep_final_periods_without_ops,
848        chain_id: *CHAINID,
849    };
850
851    // Blocks are kept in the shared storage by consensus until they are older than
852    // `force_keep_final_periods` (past that point `strip_to_block` drops the storage
853    // reference, so the block is no longer servable to peers). That retention is what
854    // covers the tail of the propagation window when the propagation cache has to evict
855    // an entry early because of its `max_blocks_kept_for_propagation` size cap.
856    let consensus_block_retention = consensus_config
857        .t0
858        .saturating_mul(consensus_config.force_keep_final_periods);
859    if consensus_block_retention < protocol_config.max_block_propagation_time {
860        warn!(
861            "consensus only keeps blocks for {:?} (force_keep_final_periods={} * t0={:?}) which is shorter than max_block_propagation_time={:?}: a block evicted early by the max_blocks_kept_for_propagation cap will not be retrievable by peers anymore, consider raising force_keep_final_periods or lowering max_block_propagation_time",
862            consensus_block_retention.to_duration(),
863            consensus_config.force_keep_final_periods,
864            consensus_config.t0.to_duration(),
865            protocol_config.max_block_propagation_time.to_duration()
866        );
867    }
868
869    let (consensus_event_sender, consensus_event_receiver) =
870        MassaChannel::new("consensus_event".to_string(), Some(CHANNEL_SIZE));
871    let consensus_channels = ConsensusChannels {
872        execution_controller: execution_controller.clone(),
873        selector_controller: selector_controller.clone(),
874        pool_controller: pool_controller.clone(),
875        controller_event_tx: consensus_event_sender,
876        protocol_controller: protocol_controller.clone(),
877        broadcasts: ConsensusBroadcasts {
878            block_header_sender: broadcast::channel(
879                consensus_config.broadcast_blocks_headers_channel_capacity,
880            )
881            .0,
882            block_sender: broadcast::channel(consensus_config.broadcast_blocks_channel_capacity).0,
883            filled_block_sender: broadcast::channel(
884                consensus_config.broadcast_filled_blocks_channel_capacity,
885            )
886            .0,
887        },
888    };
889
890    let (consensus_controller, consensus_manager) = start_consensus_worker(
891        consensus_config,
892        consensus_channels.clone(),
893        bootstrap_state.graph,
894        shared_storage.clone(),
895        massa_metrics.clone(),
896    );
897
898    let (protocol_manager, keypair, node_id) = start_protocol_controller(
899        protocol_config.clone(),
900        selector_controller.clone(),
901        consensus_controller.clone(),
902        bootstrap_state.peers,
903        pool_controller.clone(),
904        shared_storage.clone(),
905        protocol_channels,
906        mip_store.clone(),
907        massa_metrics.clone(),
908    )
909    .expect("could not start protocol controller");
910
911    // launch factory
912    let factory_config = FactoryConfig {
913        thread_count: THREAD_COUNT,
914        genesis_timestamp: *GENESIS_TIMESTAMP,
915        t0: T0,
916        initial_delay: SETTINGS.factory.initial_delay,
917        max_block_size: MAX_BLOCK_SIZE as u64,
918        max_block_gas: MAX_GAS_PER_BLOCK,
919        max_operations_per_block: MAX_OPERATIONS_PER_BLOCK,
920        endorsement_count: ENDORSEMENT_COUNT,
921        last_start_period: final_state.read().get_last_start_period(),
922        periods_per_cycle: PERIODS_PER_CYCLE,
923        denunciation_expire_periods: DENUNCIATION_EXPIRE_PERIODS,
924        stop_production_when_zero_connections: SETTINGS
925            .factory
926            .stop_production_when_zero_connections,
927        chain_id: *CHAINID,
928        block_delay_warn: SETTINGS.factory.block_delay_warn_threshold,
929        block_opt_channel_timeout: SETTINGS.factory.block_opt_channel_timeout,
930    };
931    let factory_channels = FactoryChannels {
932        selector: selector_controller.clone(),
933        consensus: consensus_controller.clone(),
934        pool: pool_controller.clone(),
935        protocol: protocol_controller.clone(),
936        storage: shared_storage.clone(),
937    };
938    let factory_manager = start_factory(
939        factory_config,
940        node_wallet.clone(),
941        factory_channels,
942        mip_store.clone(),
943    );
944
945    let bootstrap_manager = bootstrap_config.listen_addr.map(|addr| {
946        let (listener_stopper, listener) =
947            BootstrapTcpListener::create(&addr).unwrap_or_else(|_| {
948                panic!(
949                    "{}",
950                    format!("Could not bind to address: {}", addr).as_str()
951                )
952            });
953
954        start_bootstrap_server(
955            listener,
956            listener_stopper,
957            consensus_controller.clone(),
958            protocol_controller.clone(),
959            final_state.clone(),
960            bootstrap_config,
961            keypair.clone(),
962            *VERSION,
963            massa_metrics.clone(),
964        )
965        .expect("Could not start bootstrap server")
966    });
967
968    let api_config: APIConfig = APIConfig {
969        bind_private: SETTINGS.api.bind_private,
970        bind_public: SETTINGS.api.bind_public,
971        bind_api: SETTINGS.api.bind_api,
972        draw_lookahead_period_count: SETTINGS.api.draw_lookahead_period_count,
973        max_arguments: SETTINGS.api.max_arguments,
974        openrpc_spec_path: SETTINGS.api.openrpc_spec_path.clone(),
975        bootstrap_whitelist_path: SETTINGS.bootstrap.bootstrap_whitelist_path.clone(),
976        bootstrap_blacklist_path: SETTINGS.bootstrap.bootstrap_blacklist_path.clone(),
977        max_request_body_size: SETTINGS.api.max_request_body_size,
978        max_response_body_size: SETTINGS.api.max_response_body_size,
979        max_connections: SETTINGS.api.max_connections,
980        max_subscriptions_per_connection: SETTINGS.api.max_subscriptions_per_connection,
981        max_log_length: SETTINGS.api.max_log_length,
982        allow_hosts: SETTINGS.api.allow_hosts.clone(),
983        batch_request_limit: SETTINGS.api.batch_request_limit,
984        ping_interval: SETTINGS.api.ping_interval,
985        enable_http: SETTINGS.api.enable_http,
986        enable_ws: SETTINGS.api.enable_ws,
987        max_bytecode_size: MAX_BYTECODE_LENGTH,
988        max_op_datastore_entry_count: MAX_OPERATION_DATASTORE_ENTRY_COUNT,
989        max_op_datastore_key_length: MAX_OPERATION_DATASTORE_KEY_LENGTH,
990        max_op_datastore_value_length: MAX_OPERATION_DATASTORE_VALUE_LENGTH,
991        max_gas_per_block: MAX_GAS_PER_BLOCK,
992        max_serialized_operation_size: MAX_BLOCK_SIZE as usize,
993        base_operation_gas_cost: BASE_OPERATION_GAS_COST,
994        sp_compilation_cost: gas_costs.sp_compilation_cost,
995        max_function_name_length: MAX_FUNCTION_NAME_LENGTH,
996        max_parameter_size: MAX_PARAMETERS_SIZE,
997        thread_count: THREAD_COUNT,
998        keypair: keypair.clone(),
999        genesis_timestamp: *GENESIS_TIMESTAMP,
1000        t0: T0,
1001        periods_per_cycle: PERIODS_PER_CYCLE,
1002        last_start_period: final_state.read().get_last_start_period(),
1003        chain_id: *CHAINID,
1004        deferred_credits_delta: SETTINGS.api.deferred_credits_delta,
1005        minimal_fees: SETTINGS.pool.minimal_fees,
1006        deferred_calls_config,
1007        max_datastore_keys_queries: SETTINGS.api.max_datastore_keys_query,
1008        max_event_per_query: SETTINGS.execution.max_event_per_query as u32,
1009        query_state_deadline_ms: SETTINGS.execution.query_state_deadline_ms,
1010        max_datastore_key_length: MAX_DATASTORE_KEY_LENGTH,
1011        max_addresses_datastore_keys_query: SETTINGS.api.max_addresses_datastore_keys_query,
1012        pool_api_timeout: SETTINGS.api.pool_api_timeout,
1013    };
1014
1015    // spawn Massa API
1016    let api = API::<ApiV2>::new(
1017        consensus_controller.clone(),
1018        consensus_channels.broadcasts.clone(),
1019        execution_controller.clone(),
1020        pool_channels.broadcasts.clone(),
1021        api_config.clone(),
1022        *VERSION,
1023    );
1024    let api_handle = api
1025        .serve(&SETTINGS.api.bind_api, &api_config)
1026        .await
1027        .expect("failed to start MASSA API");
1028
1029    info!(
1030        "API | EXPERIMENTAL JsonRPC | listening on: {}",
1031        &SETTINGS.api.bind_api
1032    );
1033
1034    // Disable WebSockets for Private and Public API's
1035    let mut api_config = api_config.clone();
1036    api_config.enable_ws = false;
1037
1038    // Whether to spawn gRPC PUBLIC API
1039    let grpc_public_handle = if SETTINGS.grpc.public.enabled {
1040        let grpc_public_config = configure_grpc(
1041            ServiceName::Public,
1042            &SETTINGS.grpc.public,
1043            keypair.clone(),
1044            &final_state,
1045            SETTINGS.pool.minimal_fees,
1046            gas_costs.sp_compilation_cost,
1047            SETTINGS.execution.max_event_per_query,
1048            SETTINGS.execution.query_state_deadline_ms,
1049        );
1050
1051        let grpc_public_api = MassaPublicGrpc {
1052            consensus_controller: consensus_controller.clone(),
1053            consensus_broadcasts: consensus_channels.broadcasts.clone(),
1054            execution_controller: execution_controller.clone(),
1055            execution_channels,
1056            pool_broadcasts: pool_channels.broadcasts.clone(),
1057            pool_controller: pool_controller.clone(),
1058            protocol_controller: protocol_controller.clone(),
1059            selector_controller: selector_controller.clone(),
1060            storage: shared_storage.clone(),
1061            grpc_config: grpc_public_config.clone(),
1062            protocol_config: protocol_config.clone(),
1063            node_id,
1064            version: *VERSION,
1065            keypair_factory: KeyPairFactory {
1066                mip_store: mip_store.clone(),
1067            },
1068        };
1069
1070        // Spawn gRPC PUBLIC API
1071        let grpc_public_stop_handle = grpc_public_api
1072            .serve(&grpc_public_config)
1073            .await
1074            .expect("failed to start gRPC PUBLIC API");
1075        info!("gRPC | PUBLIC | listening on: {}", grpc_public_config.bind);
1076
1077        Some(grpc_public_stop_handle)
1078    } else {
1079        None
1080    };
1081
1082    // Whether to spawn gRPC PRIVATE API
1083    let grpc_private_handle = if SETTINGS.grpc.private.enabled {
1084        let grpc_private_config = configure_grpc(
1085            ServiceName::Private,
1086            &SETTINGS.grpc.private,
1087            keypair.clone(),
1088            &final_state,
1089            SETTINGS.pool.minimal_fees,
1090            gas_costs.sp_compilation_cost,
1091            SETTINGS.execution.max_event_per_query,
1092            SETTINGS.execution.query_state_deadline_ms,
1093        );
1094
1095        let bs_white_black_list = bootstrap_manager
1096            .as_ref()
1097            .map(|manager| manager.white_black_list.clone());
1098
1099        let grpc_private_api = MassaPrivateGrpc {
1100            consensus_controller: consensus_controller.clone(),
1101            execution_controller: execution_controller.clone(),
1102            pool_controller: pool_controller.clone(),
1103            protocol_controller: protocol_controller.clone(),
1104            grpc_config: grpc_private_config.clone(),
1105            protocol_config: protocol_config.clone(),
1106            node_id,
1107            mip_store: mip_store.clone(),
1108            version: *VERSION,
1109            stop_cv: sig_int_toggled.clone(),
1110            node_wallet: node_wallet.clone(),
1111            bs_white_black_list,
1112        };
1113
1114        // Spawn gRPC PRIVATE API
1115        let grpc_private_stop_handle = grpc_private_api
1116            .serve(&grpc_private_config)
1117            .await
1118            .expect("failed to start gRPC PRIVATE API");
1119        info!(
1120            "gRPC | PRIVATE | listening on: {}",
1121            grpc_private_config.bind
1122        );
1123
1124        Some(grpc_private_stop_handle)
1125    } else {
1126        None
1127    };
1128
1129    #[cfg(feature = "op_spammer")]
1130    start_operation_injector(
1131        *GENESIS_TIMESTAMP,
1132        shared_storage.clone_without_refs(),
1133        node_wallet.read().clone(),
1134        pool_controller.clone(),
1135        protocol_controller.clone(),
1136        args.nb_op,
1137    );
1138
1139    // spawn private API
1140    let api_private = API::<Private>::new(
1141        protocol_controller.clone(),
1142        execution_controller.clone(),
1143        api_config.clone(),
1144        sig_int_toggled,
1145        node_wallet,
1146    );
1147    let api_private_handle = api_private
1148        .serve(&SETTINGS.api.bind_private, &api_config)
1149        .await
1150        .expect("failed to start PRIVATE API");
1151    info!(
1152        "API | PRIVATE JsonRPC | listening on: {}",
1153        api_config.bind_private
1154    );
1155
1156    // spawn public API
1157    let api_public = API::<Public>::new(
1158        consensus_controller.clone(),
1159        execution_controller.clone(),
1160        api_config.clone(),
1161        selector_controller.clone(),
1162        pool_controller.clone(),
1163        protocol_controller.clone(),
1164        protocol_config.clone(),
1165        *VERSION,
1166        node_id,
1167        shared_storage.clone(),
1168        mip_store.clone(),
1169    );
1170    let api_public_handle = api_public
1171        .serve(&SETTINGS.api.bind_public, &api_config)
1172        .await
1173        .expect("failed to start PUBLIC API");
1174    info!(
1175        "API | PUBLIC JsonRPC | listening on: {}",
1176        api_config.bind_public
1177    );
1178
1179    let massa_survey_stopper = MassaSurvey::run(
1180        SETTINGS.metrics.tick_delay.to_duration(),
1181        execution_controller,
1182        pool_controller,
1183        db.clone(),
1184        massa_metrics,
1185        (
1186            api_config.thread_count,
1187            api_config.t0,
1188            api_config.genesis_timestamp,
1189            api_config.periods_per_cycle,
1190            api_config.last_start_period,
1191        ),
1192        mip_store,
1193    );
1194
1195    #[cfg(feature = "deadlock_detection")]
1196    {
1197        // only for #[cfg]
1198        use parking_lot::deadlock;
1199        use std::thread;
1200
1201        let interval = Duration::from_secs(args.dl_interval);
1202        warn!("deadlocks detector will run every {:?}", interval);
1203
1204        // Create a background thread which checks for deadlocks at the defined interval
1205        let thread_builder = thread::Builder::new().name("deadlock-detection".into());
1206        thread_builder
1207            .spawn(move || loop {
1208                thread::sleep(interval);
1209                let deadlocks = deadlock::check_deadlock();
1210                if deadlocks.is_empty() {
1211                    continue;
1212                }
1213                warn!("{} deadlocks detected", deadlocks.len());
1214                for (i, threads) in deadlocks.iter().enumerate() {
1215                    warn!("Deadlock #{}", i);
1216                    for t in threads {
1217                        warn!("Thread Id {:#?}", t.thread_id());
1218                        warn!("{:#?}", t.backtrace());
1219                    }
1220                }
1221            })
1222            .expect("failed to spawn thread : deadlock-detection");
1223    }
1224    (
1225        consensus_event_receiver,
1226        bootstrap_manager,
1227        consensus_manager,
1228        execution_manager,
1229        selector_manager,
1230        pool_manager,
1231        protocol_manager,
1232        factory_manager,
1233        event_cache_manager,
1234        api_private_handle,
1235        api_public_handle,
1236        api_handle,
1237        grpc_private_handle,
1238        grpc_public_handle,
1239        metrics_stopper,
1240        massa_survey_stopper,
1241    )
1242}
1243
1244// Get the configuration of the gRPC server
1245#[allow(clippy::too_many_arguments)]
1246fn configure_grpc(
1247    name: ServiceName,
1248    settings: &GrpcSettings,
1249    keypair: KeyPair,
1250    final_state: &Arc<RwLock<dyn FinalStateController>>,
1251    minimal_fees: Amount,
1252    sp_compilation_cost: u64,
1253    max_event_per_query: usize,
1254    query_state_deadline_ms: Option<u64>,
1255) -> GrpcConfig {
1256    GrpcConfig {
1257        name,
1258        enabled: settings.enabled,
1259        accept_http1: settings.accept_http1,
1260        enable_cors: settings.enable_cors,
1261        enable_health: settings.enable_health,
1262        enable_reflection: settings.enable_reflection,
1263        enable_tls: settings.enable_tls,
1264        enable_mtls: settings.enable_mtls,
1265        generate_self_signed_certificates: settings.generate_self_signed_certificates,
1266        subject_alt_names: settings.subject_alt_names.clone(),
1267        bind: settings.bind,
1268        accept_compressed: settings.accept_compressed.clone(),
1269        send_compressed: settings.send_compressed.clone(),
1270        max_decoding_message_size: settings.max_decoding_message_size,
1271        max_encoding_message_size: settings.max_encoding_message_size,
1272        concurrency_limit_per_connection: settings.concurrency_limit_per_connection,
1273        timeout: settings.timeout.to_duration(),
1274        initial_stream_window_size: settings.initial_stream_window_size,
1275        initial_connection_window_size: settings.initial_connection_window_size,
1276        max_concurrent_streams: settings.max_concurrent_streams,
1277        max_arguments: settings.max_arguments,
1278        tcp_keepalive: settings.tcp_keepalive.map(|t| t.to_duration()),
1279        tcp_nodelay: settings.tcp_nodelay,
1280        http2_keepalive_interval: settings.http2_keepalive_interval.map(|t| t.to_duration()),
1281        http2_keepalive_timeout: settings.http2_keepalive_timeout.map(|t| t.to_duration()),
1282        http2_adaptive_window: settings.http2_adaptive_window,
1283        max_frame_size: settings.max_frame_size,
1284        thread_count: THREAD_COUNT,
1285        max_operations_per_block: MAX_OPERATIONS_PER_BLOCK,
1286        endorsement_count: ENDORSEMENT_COUNT,
1287        max_endorsements_per_message: MAX_ENDORSEMENTS_PER_MESSAGE,
1288        max_bytecode_size: MAX_BYTECODE_LENGTH,
1289        max_op_datastore_entry_count: MAX_OPERATION_DATASTORE_ENTRY_COUNT,
1290        max_datastore_entries_per_request: settings.max_datastore_entries_per_request,
1291        max_op_datastore_key_length: MAX_OPERATION_DATASTORE_KEY_LENGTH,
1292        max_op_datastore_value_length: MAX_OPERATION_DATASTORE_VALUE_LENGTH,
1293        max_function_name_length: MAX_FUNCTION_NAME_LENGTH,
1294        max_parameter_size: MAX_PARAMETERS_SIZE,
1295        max_operations_per_message: MAX_OPERATIONS_PER_MESSAGE,
1296        max_gas_per_block: MAX_GAS_PER_BLOCK,
1297        max_serialized_operation_size: MAX_BLOCK_SIZE as usize,
1298        base_operation_gas_cost: BASE_OPERATION_GAS_COST,
1299        sp_compilation_cost,
1300        genesis_timestamp: *GENESIS_TIMESTAMP,
1301        t0: T0,
1302        periods_per_cycle: PERIODS_PER_CYCLE,
1303        keypair,
1304        max_channel_size: settings.max_channel_size,
1305        draw_lookahead_period_count: settings.draw_lookahead_period_count,
1306        last_start_period: final_state.read().get_last_start_period(),
1307        max_denunciations_per_block_header: MAX_DENUNCIATIONS_PER_BLOCK_HEADER,
1308        max_addresses_per_request: settings.max_addresses_per_request,
1309        max_slot_ranges_per_request: settings.max_slot_ranges_per_request,
1310        max_block_ids_per_request: settings.max_block_ids_per_request,
1311        max_endorsement_ids_per_request: settings.max_endorsement_ids_per_request,
1312        max_operation_ids_per_request: settings.max_operation_ids_per_request,
1313        max_filters_per_request: settings.max_filters_per_request,
1314        max_query_items_per_request: settings.max_query_items_per_request,
1315        max_event_per_query: max_event_per_query as u32,
1316        query_state_deadline_ms,
1317        certificate_authority_root_path: settings.certificate_authority_root_path.clone(),
1318        server_certificate_path: settings.server_certificate_path.clone(),
1319        server_private_key_path: settings.server_private_key_path.clone(),
1320        client_certificate_authority_root_path: settings
1321            .client_certificate_authority_root_path
1322            .clone(),
1323        client_certificate_path: settings.client_certificate_path.clone(),
1324        client_private_key_path: settings.client_private_key_path.clone(),
1325        chain_id: *CHAINID,
1326        minimal_fees,
1327        max_datastore_keys_queries: settings.max_datastore_keys_query,
1328        max_datastore_key_length: MAX_DATASTORE_KEY_LENGTH,
1329        unidirectional_stream_interval_check: settings.unidirectional_stream_interval_check,
1330    }
1331}
1332
1333struct Managers {
1334    bootstrap_manager: Option<BootstrapManager>,
1335    consensus_manager: Box<dyn ConsensusManager>,
1336    execution_manager: Box<dyn ExecutionManager>,
1337    selector_manager: Box<dyn SelectorManager>,
1338    pool_manager: Box<dyn PoolManager>,
1339    protocol_manager: Box<dyn ProtocolManager>,
1340    factory_manager: Box<dyn FactoryManager>,
1341    event_cache_manager: Box<dyn EventCacheManager>,
1342}
1343
1344#[allow(clippy::too_many_arguments)]
1345async fn stop(
1346    _consensus_event_receiver: MassaReceiver<ConsensusEvent>,
1347    Managers {
1348        bootstrap_manager,
1349        mut execution_manager,
1350        mut consensus_manager,
1351        mut selector_manager,
1352        mut pool_manager,
1353        mut protocol_manager,
1354        mut factory_manager,
1355        mut event_cache_manager,
1356    }: Managers,
1357    api_private_handle: StopHandle,
1358    api_public_handle: StopHandle,
1359    api_handle: StopHandle,
1360    grpc_private_handle: Option<massa_grpc::server::StopHandle>,
1361    grpc_public_handle: Option<massa_grpc::server::StopHandle>,
1362    mut metrics_stopper: MetricsStopper,
1363    mut massa_survey_stopper: MassaSurveyStopper,
1364) {
1365    // stop bootstrap
1366    if let Some(bootstrap_manager) = bootstrap_manager {
1367        bootstrap_manager
1368            .stop()
1369            .expect("bootstrap server shutdown failed")
1370    }
1371
1372    info!("Start stopping API's: gRPC(PUBLIC, PRIVATE), EXPERIMENTAL, PUBLIC, PRIVATE");
1373
1374    // stop Massa gRPC PUBLIC API
1375    if let Some(handle) = grpc_public_handle {
1376        handle.stop();
1377    }
1378    info!("API | PUBLIC gRPC | stopped");
1379
1380    // stop Massa gRPC PRIVATE API
1381    if let Some(handle) = grpc_private_handle {
1382        handle.stop();
1383    }
1384    info!("API | PRIVATE gRPC | stopped");
1385
1386    // stop Massa API
1387    api_handle.stop().await;
1388    info!("API | EXPERIMENTAL JsonRPC | stopped");
1389
1390    // stop public API
1391    api_public_handle.stop().await;
1392    info!("API | PUBLIC JsonRPC | stopped");
1393
1394    // stop private API
1395    api_private_handle.stop().await;
1396    info!("API | PRIVATE JsonRPC | stopped");
1397
1398    // stop metrics
1399    metrics_stopper.stop();
1400
1401    // stop massa survey thread
1402    massa_survey_stopper.stop();
1403
1404    // stop factory
1405    factory_manager.stop();
1406
1407    // stop protocol controller
1408    protocol_manager.stop();
1409
1410    // stop consensus
1411    consensus_manager.stop();
1412
1413    // stop pool
1414    pool_manager.stop();
1415
1416    // stop execution controller
1417    execution_manager.stop();
1418
1419    // stop selector controller
1420    selector_manager.stop();
1421
1422    // stop pool controller
1423    // TODO
1424    //let protocol_pool_event_receiver = pool_manager.stop().await.expect("pool shutdown failed");
1425
1426    // note that FinalLedger gets destroyed as soon as its Arc count goes to zero
1427
1428    event_cache_manager.stop();
1429}
1430
1431#[derive(Parser)]
1432#[command(version = crate_version!())]
1433struct Args {
1434    #[arg(long = "keep-ledger")]
1435    keep_ledger: bool,
1436    /// Wallet password
1437    // #[arg(short = "p", long = "pwd")]
1438    #[arg(short = 'p', long = "pwd")]
1439    password: Option<String>,
1440
1441    /// restart_from_snapshot_at_period
1442    #[arg(long = "restart-from-snapshot-at-period")]
1443    restart_from_snapshot_at_period: Option<u64>,
1444
1445    #[arg(short = 'a', long = "accept-community-charter")]
1446    accept_community_charter: bool,
1447
1448    #[cfg(feature = "op_spammer")]
1449    /// number of operations
1450    #[arg(
1451        name = "number of operations",
1452        long_help = "Define the number of operations the node can spam.",
1453        short = 'n',
1454        long = "number-operations"
1455    )]
1456    nb_op: u64,
1457
1458    #[cfg(feature = "deadlock_detection")]
1459    /// Deadlocks detector
1460    #[structopt(
1461        name = "deadlocks interval",
1462        long_help = "Define the interval of launching a deadlocks checking.",
1463        short = 'i',
1464        long = "dli",
1465        default_value = "10"
1466    )]
1467    dl_interval: u64,
1468}
1469
1470/// Load wallet, asking for passwords if necessary
1471fn load_wallet(
1472    password: Option<String>,
1473    path: &Path,
1474    chain_id: u64,
1475) -> anyhow::Result<Arc<RwLock<Wallet>>> {
1476    let password = if path.is_dir() {
1477        password.unwrap_or_else(|| {
1478            Password::new()
1479                .with_prompt("Enter staking keys file password")
1480                .interact()
1481                .expect("IO error: Password reading failed, staking keys file couldn't be unlocked")
1482        })
1483    } else {
1484        password.unwrap_or_else(|| {
1485            Password::new()
1486                .with_prompt("Enter new password for staking keys file")
1487                .with_confirmation("Confirm password", "Passwords mismatching")
1488                .interact()
1489                .expect("IO error: Password reading failed, staking keys file couldn't be created")
1490        })
1491    };
1492    Ok(Arc::new(RwLock::new(Wallet::new(
1493        PathBuf::from(path),
1494        password,
1495        chain_id,
1496    )?)))
1497}
1498
1499fn main() -> anyhow::Result<()> {
1500    let args = Args::parse();
1501
1502    let tokio_rt = tokio::runtime::Builder::new_multi_thread()
1503        .thread_name_fn(|| {
1504            static ATOMIC_ID: AtomicUsize = AtomicUsize::new(0);
1505            let id = ATOMIC_ID.fetch_add(1, Ordering::SeqCst);
1506            format!("tokio-node-{}", id)
1507        })
1508        .enable_all()
1509        .build()
1510        .unwrap();
1511
1512    tokio_rt.block_on(run(args))
1513}
1514
1515async fn run(args: Args) -> anyhow::Result<()> {
1516    let mut cur_args = args;
1517    use tracing_subscriber::prelude::*;
1518    // spawn the console server in the background, returning a `Layer`:
1519    let tracing_layer = tracing_subscriber::fmt::layer()
1520        .with_filter(match SETTINGS.logging.level {
1521            4 => LevelFilter::TRACE,
1522            3 => LevelFilter::DEBUG,
1523            2 => LevelFilter::INFO,
1524            1 => LevelFilter::WARN,
1525            _ => LevelFilter::ERROR,
1526        })
1527        .with_filter(filter_fn(|metadata| {
1528            metadata.target().starts_with("massa") // ignore non-massa logs
1529        }));
1530    // build a `Subscriber` by combining layers with a `tracing_subscriber::Registry`:
1531    tracing_subscriber::registry()
1532        // add the console layer to the subscriber or default layers...
1533        .with(tracing_layer)
1534        .init();
1535
1536    // Setup panic handlers,
1537    // and when a panic occurs,
1538    // run default handler,
1539    // and then shutdown.
1540    let default_panic = std::panic::take_hook();
1541    std::panic::set_hook(Box::new(move |info| {
1542        default_panic(info);
1543        std::process::exit(1);
1544    }));
1545
1546    info!("Node version : {}", *VERSION);
1547
1548    handle_disclaimer(
1549        cur_args.accept_community_charter,
1550        &SETTINGS.cli.approved_community_charter_file_path,
1551    );
1552
1553    // load or create wallet, asking for password if necessary
1554    let node_wallet = load_wallet(
1555        cur_args.password.clone(),
1556        &SETTINGS.factory.staking_wallet_path,
1557        *CHAINID,
1558    )?;
1559
1560    // interrupt signal listener
1561    let sig_int_toggled = Arc::new((Mutex::new(false), Condvar::new()));
1562
1563    let sig_int_toggled_clone = Arc::clone(&sig_int_toggled);
1564    ctrlc::set_handler(move || {
1565        *sig_int_toggled_clone
1566            .0
1567            .lock()
1568            .expect("double-lock on interrupt bool in ctrl-c handler") = true;
1569        sig_int_toggled_clone.1.notify_all();
1570    })
1571    .expect("Error setting Ctrl-C handler");
1572
1573    #[cfg(feature = "resync_check")]
1574    let mut resync_check = Some(std::time::Instant::now() + std::time::Duration::from_secs(10));
1575
1576    loop {
1577        let (
1578            consensus_event_receiver,
1579            bootstrap_manager,
1580            consensus_manager,
1581            execution_manager,
1582            selector_manager,
1583            pool_manager,
1584            protocol_manager,
1585            factory_manager,
1586            event_cache_manager,
1587            api_private_handle,
1588            api_public_handle,
1589            api_handle,
1590            grpc_private_handle,
1591            grpc_public_handle,
1592            metrics_stopper,
1593            massa_survey_stopper,
1594        ) = launch(&cur_args, node_wallet.clone(), Arc::clone(&sig_int_toggled)).await;
1595
1596        // loop over messages
1597        let restart = loop {
1598            massa_trace!("massa-node.main.run.select", {});
1599            match consensus_event_receiver.try_recv() {
1600                Ok(evt) => match evt {
1601                    ConsensusEvent::NeedSync => {
1602                        warn!("in response to a desynchronization, the node is going to bootstrap again");
1603                        break true;
1604                    }
1605                    ConsensusEvent::Stop => {
1606                        break false;
1607                    }
1608                },
1609                Err(TryRecvError::Disconnected) => {
1610                    error!("consensus_event_receiver.wait_event disconnected");
1611                    break false;
1612                }
1613                _ => {}
1614            };
1615
1616            // every 100ms/or when alerted, check if sigint toggled
1617            // if toggled, break loop
1618            let int_sig = sig_int_toggled
1619                .0
1620                .lock()
1621                .expect("double-lock() on interrupted signal mutex");
1622            let wake = sig_int_toggled
1623                .1
1624                .wait_timeout(int_sig, Duration::from_millis(100))
1625                .expect("interrupt signal mutex poisoned");
1626            if *wake.0 {
1627                info!("interrupt signal received");
1628                break false;
1629            }
1630
1631            // Elements of the system that involve stopping and restarting should be checked by forcing a relaunch.
1632            // This check allows the system to start up as normal, wait 10s, then force a relaunch. If Things take too long
1633            // to shutdown, or does not allow for a clean relaunch, this feature flag can expose those issues.
1634            #[cfg(feature = "resync_check")]
1635            if let Some(resync_moment) = resync_check {
1636                if resync_moment < std::time::Instant::now() {
1637                    warn!("resync check triggered");
1638                    resync_check = None;
1639                    break true;
1640                }
1641            }
1642        };
1643        stop(
1644            consensus_event_receiver,
1645            Managers {
1646                bootstrap_manager,
1647                consensus_manager,
1648                execution_manager,
1649                selector_manager,
1650                pool_manager,
1651                protocol_manager,
1652                factory_manager,
1653                event_cache_manager,
1654            },
1655            api_private_handle,
1656            api_public_handle,
1657            api_handle,
1658            grpc_private_handle,
1659            grpc_public_handle,
1660            metrics_stopper,
1661            massa_survey_stopper,
1662        )
1663        .await;
1664
1665        if !restart {
1666            break;
1667        }
1668        // If we restart because of a desync, then we do not want to restart from a snapshot
1669        cur_args.restart_from_snapshot_at_period = None;
1670    }
1671    Ok(())
1672}