leo_lang/cli/commands/devnet/
mod.rs

1// Copyright (C) 2019-2025 Provable Inc.
2// This file is part of the Leo library.
3
4// The Leo library is free software: you can redistribute it and/or modify
5// it under the terms of the GNU General Public License as published by
6// the Free Software Foundation, either version 3 of the License, or
7// (at your option) any later version.
8
9// The Leo library is distributed in the hope that it will be useful,
10// but WITHOUT ANY WARRANTY; without even the implied warranty of
11// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12// GNU General Public License for more details.
13
14// You should have received a copy of the GNU General Public License
15// along with the Leo library. If not, see <https://www.gnu.org/licenses/>.
16
17#![forbid(unsafe_op_in_unsafe_fn)]
18
19mod child_manager;
20use child_manager::*;
21
22#[cfg(windows)]
23mod windows_kill_tree;
24
25mod shutdown;
26use shutdown::*;
27
28mod utilities;
29use utilities::*;
30
31use anyhow::{Context as AnyhowContext, Result as AnyhowResult, anyhow, bail, ensure};
32use chrono::Local;
33use clap::Parser;
34use dunce::canonicalize;
35use itertools::Itertools;
36use parking_lot::Mutex;
37use std::{
38    env,
39    ffi::OsStr,
40    path::{Path, PathBuf},
41    process::{Child, Command as StdCommand, Stdio},
42    sync::Arc,
43    time::Duration,
44};
45use tracing::{self, Span};
46
47#[cfg(unix)]
48use {libc::setsid, std::os::unix::process::CommandExt};
49
50use super::*;
51use leo_ast::NetworkName;
52
53/// A high REST RPS (requests per second) for snarkOS devnets.
54const REST_RPS: &str = "999999999";
55
56/// Launch a local devnet (validators + clients) using snarkOS.
57#[derive(Parser, Debug)]
58pub struct LeoDevnet {
59    #[clap(long, help = "Number of validators", default_value = "4")]
60    pub(crate) num_validators: usize,
61    #[clap(long, help = "Number of clients", default_value = "2")]
62    pub(crate) num_clients: usize,
63    #[clap(short = 'n', long, help = "Network (mainnet=0, testnet=1, canary=2)", default_value = "testnet")]
64    pub(crate) network: NetworkName,
65    #[clap(long, help = "Ledger / log root directory", default_value = "./")]
66    pub(crate) storage: String,
67    #[clap(long, help = "Clear existing ledgers before start")]
68    pub(crate) clear_storage: bool,
69    #[clap(long, help = "Path to snarkOS binary. If it does not exist, set `--install` to build it at this path.")]
70    pub(crate) snarkos: PathBuf,
71    #[clap(long, help = "Required features for snarkOS (e.g. `test_network`)", value_delimiter = ',')]
72    pub(crate) snarkos_features: Vec<String>,
73    #[clap(long, help = "Required version for snarkOS (e.g. `4.1.0`). Defaults to latest version on `crates.io`.")]
74    pub(crate) snarkos_version: Option<String>,
75    #[clap(long, help = "(Re)install snarkOS at the provided `--snarkos` path with the given `--snarkos-features`")]
76    pub(crate) install: bool,
77    #[clap(
78        long,
79        help = "Optional consensus heights to use. The `test_network` feature must be enabled for this to work.",
80        value_delimiter = ',',
81        env = "CONSENSUS_VERSION_HEIGHTS"
82    )]
83    pub(crate) consensus_heights: Option<Vec<u32>>,
84    #[clap(long, help = "Run nodes in tmux (only available on Unix)")]
85    pub(crate) tmux: bool,
86    #[clap(long, help = "snarkOS verbosity (0-4)", default_value = "1")]
87    pub(crate) verbosity: u8,
88    #[clap(long, short = 'y', help = "Skip confirmation prompts and proceed with the devnet startup")]
89    pub(crate) yes: bool,
90}
91
92impl Command for LeoDevnet {
93    type Input = ();
94    type Output = ();
95
96    fn log_span(&self) -> Span {
97        tracing::span!(tracing::Level::INFO, "LeoDevnet")
98    }
99
100    fn prelude(&self, _: Context) -> Result<Self::Input> {
101        Ok(())
102    }
103
104    fn apply(self, _cx: Context, _: Self::Input) -> Result<Self::Output> {
105        self.handle_apply().map_err(|e| CliError::custom(format!("Failed to start devnet: {e}")).into())
106    }
107}
108
109impl LeoDevnet {
110    /// Handle the actual devnet startup logic.
111    fn handle_apply(&self) -> AnyhowResult<()> {
112        //───────────────────────────────────────────────────────────────────
113        // 0. Guard rails
114        //───────────────────────────────────────────────────────────────────
115        if cfg!(windows) && self.tmux {
116            bail!("tmux mode is not available on Windows – remove `--tmux`.");
117        }
118        if self.tmux && std::env::var("TMUX").is_ok() {
119            bail!("Nested tmux session detected.  Unset $TMUX and retry.");
120        }
121
122        // If the devnet heights are provided, ensure the `test_network` feature is enabled, and validate the heights.
123        if let Some(ref heights) = self.consensus_heights {
124            if !self.snarkos_features.contains(&"test_network".to_string()) {
125                bail!("The `test_network` feature must be enabled on snarkOS to use `--consensus-heights`.");
126            }
127            validate_consensus_heights(heights.as_slice())?;
128        }
129
130        // Validate the number of validators.
131        if self.num_validators < 4 {
132            bail!("The number of validators must be at least 4.");
133        }
134
135        // Resolve the snarkOS path to its canonical form.
136
137        if self.install {
138            // If installing, make sure we can write to a file at the path.
139            if let Some(parent) = self.snarkos.parent()
140                && !parent.exists()
141            {
142                std::fs::create_dir_all(parent)
143                    .with_context(|| format!("Failed to create directory for binary: {}", parent.display()))?;
144            }
145            std::fs::write(&self.snarkos, [0u8]).with_context(|| {
146                format!("Failed to write to path {} for snarkos installation", self.snarkos.display())
147            })?;
148        } else {
149            // If not installing, ensure the snarkOS binary exists at the provided path.
150            if !self.snarkos.exists() {
151                bail!(
152                    "The snarkOS binary at `{}` does not exist. Please provide a valid path or use `--install`.",
153                    self.snarkos.display()
154                );
155            }
156        };
157        let snarkos = canonicalize(&self.snarkos)
158            .with_context(|| format!("Failed to resolve snarkOS path: {}", self.snarkos.display()))?;
159
160        // Confirm with the user the options they provided.
161        println!("🔧  Starting devnet with the following options:");
162        println!("  • Network: {}", self.network);
163        println!("  • Validators: {}", self.num_validators);
164        println!("  • Clients: {}", self.num_clients);
165        println!("  • Storage: {}", self.storage);
166        if self.install {
167            println!("  • Installing snarkOS at: {}", snarkos.display());
168            if let Some(ref version) = self.snarkos_version {
169                println!("  • version: {version}");
170            }
171            if !self.snarkos_features.is_empty() {
172                println!("  • features: {}", self.snarkos_features.iter().format(","));
173            }
174        } else {
175            println!("  • Using snarkOS binary at: {}", snarkos.display());
176        }
177        if let Some(heights) = &self.consensus_heights {
178            println!("  • Consensus heights: {}", heights.iter().format(","));
179        } else {
180            println!("  • Consensus heights: default (based on your snarkOS binary)");
181        }
182        println!("  • Clear storage: {}", if self.clear_storage { "yes" } else { "no" });
183        println!("  • Verbosity: {}", self.verbosity);
184        println!("  • tmux: {}", if self.tmux { "yes" } else { "no" });
185
186        //───────────────────────────────────────────────────────────────────
187        // 1. Child-manager & shutdown listener (no race!)
188        //───────────────────────────────────────────────────────────────────
189        let manager = Arc::new(Mutex::new(ChildManager::new()));
190
191        // Install the listener to catch any early shutdown signals.
192        let (tx_shutdown, rx_shutdown) = crossbeam_channel::bounded::<()>(1);
193        let _signal_thread =
194            install_shutdown_listener(tx_shutdown.clone()).context("Failed to install shutdown listener")?;
195
196        //───────────────────────────────────────────────────────────────────
197        // 2. snarkOS binary  (+ optional build)
198        //───────────────────────────────────────────────────────────────────
199        let snarkos = if self.install {
200            if !confirm("\nProceed with snarkOS installation?", self.yes)? {
201                println!("❌ Installation aborted.");
202                return Ok(());
203            }
204            install_snarkos(&snarkos, self.snarkos_version.as_deref(), &self.snarkos_features)?
205        } else {
206            snarkos
207        };
208
209        // Run `snarkOS --version` and confirm with the user that they'd like to proceed.
210        let version_output = StdCommand::new(&snarkos)
211            .arg("--version")
212            .output()
213            .context(format!("Failed to run `{}`", snarkos.display()))?;
214        if !version_output.status.success() {
215            bail!("Failed to run `{}`: {}", snarkos.display(), String::from_utf8_lossy(&version_output.stderr));
216        }
217
218        // Print the version output.
219        let version_str = String::from_utf8_lossy(&version_output.stdout);
220        println!("🔍  Detected: {version_str}");
221
222        // The version string has the following form:
223        // "snarkos refs/heads/staging ace765a42551092fbb47799c2651d6b6df30e49a features=[default,snarkos_node_metrics,test_network]"
224        // Parse the features and see if it matches the expected features.
225        let features_str = version_str
226            .trim()
227            .split("features=[")
228            .nth(1)
229            .and_then(|s| s.split(']').next())
230            .ok_or_else(|| anyhow!("Failed to parse snarkOS features from version string: {version_str}"))?;
231        let found_features: Vec<String> = features_str.split(',').map(|s| s.trim().to_string()).collect();
232        for feature in &self.snarkos_features {
233            if !found_features.contains(feature) {
234                println!("⚠️  Warning: snarkOS does not have the required feature `{feature}` enabled.");
235            }
236        }
237
238        if !confirm("\nProceed with devnet startup?", self.yes)? {
239            println!("❌ Devnet aborted.");
240            return Ok(());
241        }
242
243        //───────────────────────────────────────────────────────────────────
244        // 3. Resolve storage & create log dir
245        //───────────────────────────────────────────────────────────────────
246        // Create the storage directory if it does not exist.
247        let storage = PathBuf::from(&self.storage);
248        if !storage.exists() {
249            std::fs::create_dir_all(&storage)
250                .context(format!("Failed to create storage directory: {}", self.storage))?;
251        } else if !storage.is_dir() {
252            bail!("The storage path `{}` is not a directory.", self.storage);
253        }
254        // Resolve the storage directory to its canonical form.
255        let storage =
256            canonicalize(&storage).with_context(|| format!("Failed to resolve storage path: {}", self.storage))?;
257        // Create the log directory inside the storage directory.
258        let log_dir = {
259            let ts = Local::now().format(".logs-%Y-%m-%d-%H-%M-%S").to_string();
260            let p = storage.join(ts);
261            std::fs::create_dir_all(&p)?;
262            p
263        };
264
265        //───────────────────────────────────────────────────────────────────
266        // 4. (Optional) ledger cleanup
267        //───────────────────────────────────────────────────────────────────
268        if self.clear_storage {
269            println!("🧹  Cleaning ledgers …");
270            let mut cleaners = Vec::new();
271            for idx in 0..self.num_validators {
272                cleaners.push(clean_snarkos(&snarkos, self.network as usize, "validator", idx, storage.as_path())?);
273            }
274            for idx in 0..self.num_clients {
275                cleaners.push(clean_snarkos(
276                    &snarkos,
277                    self.network as usize,
278                    "client",
279                    idx + self.num_validators,
280                    storage.as_path(),
281                )?);
282            }
283            for mut c in cleaners {
284                c.wait()?;
285            }
286        }
287
288        //───────────────────────────────────────────────────────────────────
289        // 5. Spawn nodes (tmux **or** background)
290        //───────────────────────────────────────────────────────────────────
291
292        #[allow(clippy::too_many_arguments)]
293        fn build_args(
294            role: &str,
295            verbosity: u8,
296            network: usize,
297            num_validators: usize,
298            idx: usize,
299            log_file: &Path,
300            metrics_port: Option<u16>,
301        ) -> Vec<String> {
302            let mut base = vec![
303                "start".to_string(),
304                "--nodisplay".to_string(),
305                "--network".to_string(),
306                network.to_string(),
307                "--dev".to_string(),
308                idx.to_string(),
309                "--dev-num-validators".to_string(),
310                num_validators.to_string(),
311                "--rest-rps".to_string(),
312                REST_RPS.to_string(),
313                "--logfile".to_string(),
314                log_file.to_str().unwrap().to_string(),
315                "--verbosity".to_string(),
316                verbosity.to_string(),
317            ];
318            match role {
319                "validator" => {
320                    base.extend(
321                        ["--allow-external-peers", "--validator", "--no-dev-txs"].into_iter().map(String::from),
322                    );
323                    if let Some(p) = metrics_port {
324                        base.extend(["--metrics".into(), "--metrics-ip".into(), format!("0.0.0.0:{p}")]);
325                    }
326                }
327                "client" => base.push("--client".into()),
328                _ => unreachable!(),
329            }
330            base
331        }
332
333        // Set the environment variable for the consensus heights if provided.
334        // These are used by all child processes.
335        if let Some(ref heights) = self.consensus_heights {
336            let heights = heights.iter().join(",");
337            println!("🔧  Setting consensus heights: {heights}");
338            #[allow(unsafe_code)]
339            unsafe {
340                // SAFETY:
341                //  - `CONSENSUS_VERSION_HEIGHTS` is only set once and is only read in `snarkvm::prelude::load_consensus_heights`.
342                //  - There are no concurrent threads running at this point in the execution.
343                // WHY:
344                //  - This is needed because there is no way to set the desired consensus heights for a particular `VM` instance in a node
345                //    without using the environment variable `CONSENSUS_VERSION_HEIGHTS`. Which is itself read once, and stored in a `OnceLock`.
346                env::set_var("CONSENSUS_VERSION_HEIGHTS", heights);
347            }
348        }
349
350        //────────────── tmux branch ──────────────
351        if self.tmux {
352            // Create session.
353            let mut args: Vec<String> =
354                vec!["new-session", "-d", "-s", "devnet", "-n", "validator-0"].into_iter().map(Into::into).collect();
355
356            // If a tmux server is already running, the new session will inherit the environment
357            // variables of the server. As such, we need to explicitly set the CONSENSUS_VERSION_HEIGHTS
358            // env var in the new session we are creatomg.
359            if let Some(ref heights) = self.consensus_heights {
360                let heights = heights.iter().join(",");
361                args.push("-e".to_string());
362                args.push(format!("CONSENSUS_VERSION_HEIGHTS={heights}"));
363            }
364
365            ensure!(StdCommand::new("tmux").args(args).status()?.success(), "tmux failed to create session");
366
367            // Determine base-index.
368            let base_index = {
369                let out = StdCommand::new("tmux").args(["show-option", "-gv", "base-index"]).output()?;
370                String::from_utf8_lossy(&out.stdout).trim().parse::<usize>().unwrap_or(0)
371            };
372
373            // Validators
374            for idx in 0..self.num_validators {
375                let win_idx = idx + base_index;
376                let window_name = format!("validator-{idx}");
377                if idx != 0 {
378                    StdCommand::new("tmux")
379                        .args(["new-window", "-t", &format!("devnet:{win_idx}"), "-n", &window_name])
380                        .status()?;
381                }
382                let log_file = log_dir.join(format!("{window_name}.log"));
383                let metrics_port = 9000 + idx as u16;
384                let cmd = std::iter::once(snarkos.to_string_lossy().into_owned())
385                    .chain(build_args(
386                        "validator",
387                        self.verbosity,
388                        self.network as usize,
389                        self.num_validators,
390                        idx,
391                        log_file.as_path(),
392                        Some(metrics_port),
393                    ))
394                    .collect::<Vec<_>>()
395                    .join(" ");
396                StdCommand::new("tmux")
397                    .args(["send-keys", "-t", &format!("devnet:{win_idx}"), &cmd, "C-m"])
398                    .status()?;
399            }
400
401            // Clients
402            for idx in 0..self.num_clients {
403                let dev_idx = idx + self.num_validators;
404                let win_idx = dev_idx + base_index;
405                let window_name = format!("client-{idx}");
406                StdCommand::new("tmux")
407                    .args(["new-window", "-t", &format!("devnet:{win_idx}"), "-n", &window_name])
408                    .status()?;
409                let log_file = log_dir.join(format!("{window_name}.log"));
410                let cmd = std::iter::once(snarkos.to_string_lossy().into_owned())
411                    .chain(build_args(
412                        "client",
413                        self.verbosity,
414                        self.network as usize,
415                        self.num_validators,
416                        dev_idx,
417                        log_file.as_path(),
418                        None,
419                    ))
420                    .collect::<Vec<_>>()
421                    .join(" ");
422                StdCommand::new("tmux")
423                    .args(["send-keys", "-t", &format!("devnet:{win_idx}"), &cmd, "C-m"])
424                    .status()?;
425            }
426
427            println!("✅  tmux session \"devnet\" is ready – attaching …");
428            StdCommand::new("tmux").args(["attach-session", "-t", "devnet"]).status()?;
429            return Ok(()); // tmux will hold the terminal
430        }
431
432        //──────────── background branch ──────────
433        println!("⚙️  Spawning nodes as background tasks …");
434
435        // Helper: setsid() on Unix, Job-object attach on Windows.
436        let spawn_with_group = |mut cmd: StdCommand, log_file: &Path| -> AnyhowResult<Child> {
437            let log_handle = std::fs::OpenOptions::new().create(true).append(true).open(log_file)?;
438            cmd.stdout(Stdio::from(log_handle.try_clone()?));
439            cmd.stderr(Stdio::from(log_handle));
440
441            #[cfg(unix)]
442            #[allow(unsafe_code)]
443            unsafe {
444                // SAFETY: We are in the child just before exec; setsid() only
445                // affects the child and cannot violate Rust invariants.
446                cmd.pre_exec(|| {
447                    setsid();
448                    Ok(())
449                });
450            }
451
452            let child = cmd.spawn().map_err(|e| anyhow!("spawn {e}"))?;
453
454            #[cfg(windows)]
455            windows_kill_tree::attach_to_global_job(child.id())?;
456
457            Ok(child)
458        };
459
460        {
461            // This should be safe since only the current thread will write to the manager.
462            let mut guard = manager.lock();
463
464            // Validators
465            for idx in 0..self.num_validators {
466                let log_file = log_dir.join(format!("validator-{idx}.log"));
467                let child = spawn_with_group(
468                    {
469                        let mut c = StdCommand::new(&snarkos);
470                        c.args(build_args(
471                            "validator",
472                            self.verbosity,
473                            self.network as usize,
474                            self.num_validators,
475                            idx,
476                            &log_file,
477                            Some(9000 + idx as u16),
478                        ));
479                        c
480                    },
481                    &log_file,
482                )?;
483                println!("  • validator {idx}  (pid = {})", child.id());
484                guard.push(child);
485            }
486
487            // Clients
488            for idx in 0..self.num_clients {
489                let dev_idx = idx + self.num_validators;
490                let log_file = log_dir.join(format!("client-{idx}.log"));
491                let child = spawn_with_group(
492                    {
493                        let mut c = StdCommand::new(&snarkos);
494                        c.args(build_args(
495                            "client",
496                            self.verbosity,
497                            self.network as usize,
498                            self.num_validators,
499                            dev_idx,
500                            &log_file,
501                            None,
502                        ));
503                        c
504                    },
505                    &log_file,
506                )?;
507                println!("  • client    {idx}  (pid = {})", child.id());
508                guard.push(child);
509            }
510        }
511
512        // Print the main process ID.
513        println!("📌  Main process ID: {}", std::process::id());
514        println!("\nDevnet running – Ctrl+C, SIGTERM, or terminal close to stop.");
515
516        // Block here until the first (coalesced) shutdown request
517        let _ = rx_shutdown.recv();
518        manager.lock().shutdown_all(Duration::from_secs(30));
519
520        Ok(())
521    }
522}
leo_lang/cli/commands/devnet/mod.rs

leo_lang/cli/commands/devnet/
mod.rs
