leo_lang/cli/commands/devnet/
mod.rs

1// Copyright (C) 2019-2025 Provable Inc.
2// This file is part of the Leo library.
3
4// The Leo library is free software: you can redistribute it and/or modify
5// it under the terms of the GNU General Public License as published by
6// the Free Software Foundation, either version 3 of the License, or
7// (at your option) any later version.
8
9// The Leo library is distributed in the hope that it will be useful,
10// but WITHOUT ANY WARRANTY; without even the implied warranty of
11// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12// GNU General Public License for more details.
13
14// You should have received a copy of the GNU General Public License
15// along with the Leo library. If not, see <https://www.gnu.org/licenses/>.
16
17#![forbid(unsafe_op_in_unsafe_fn)]
18
19mod child_manager;
20use child_manager::*;
21
22#[cfg(windows)]
23mod windows_kill_tree;
24
25mod shutdown;
26use shutdown::*;
27
28mod utilities;
29use utilities::*;
30
31use anyhow::{Context as AnyhowContext, Result as AnyhowResult, anyhow, bail, ensure};
32use chrono::Local;
33use clap::Parser;
34use dunce::canonicalize;
35use itertools::Itertools;
36use parking_lot::Mutex;
37use std::{
38    env,
39    ffi::OsStr,
40    path::{Path, PathBuf},
41    process::{Child, Command as StdCommand, Stdio},
42    sync::Arc,
43    time::Duration,
44};
45use tracing::{self, Span};
46
47#[cfg(unix)]
48use {libc::setsid, std::os::unix::process::CommandExt};
49
50use super::*;
51use leo_ast::NetworkName;
52
53/// A high REST RPS (requests per second) for snarkOS devnets.
54const REST_RPS: &str = "999999999";
55
56/// Launch a local devnet (validators + clients) using snarkOS.
57#[derive(Parser, Debug)]
58pub struct LeoDevnet {
59    #[clap(long, help = "Number of validators", default_value = "4")]
60    pub(crate) num_validators: usize,
61    #[clap(long, help = "Number of clients", default_value = "2")]
62    pub(crate) num_clients: usize,
63    #[clap(short = 'n', long, help = "Network (mainnet=0, testnet=1, canary=2)", default_value = "testnet")]
64    pub(crate) network: NetworkName,
65    #[clap(long, help = "Ledger / log root directory", default_value = "./")]
66    pub(crate) storage: String,
67    #[clap(long, help = "Clear existing ledgers before start")]
68    pub(crate) clear_storage: bool,
69    #[clap(long, help = "Path to snarkOS binary. If it does not exist, set `--install` to build it at this path.")]
70    pub(crate) snarkos: PathBuf,
71    #[clap(long, help = "Required features for snarkOS (e.g. `test_network`)", value_delimiter = ',')]
72    pub(crate) snarkos_features: Vec<String>,
73    #[clap(long, help = "Required version for snarkOS (e.g. `4.1.0`). Defaults to latest version on `crates.io`.")]
74    pub(crate) snarkos_version: Option<String>,
75    #[clap(long, help = "(Re)install snarkOS at the provided `--snarkos` path with the given `--snarkos-features`")]
76    pub(crate) install: bool,
77    #[clap(
78        long,
79        help = "Optional consensus heights to use. The `test_network` feature must be enabled for this to work.",
80        value_delimiter = ','
81    )]
82    pub(crate) consensus_heights: Option<Vec<u32>>,
83    #[clap(long, help = "Run nodes in tmux (only available on Unix)")]
84    pub(crate) tmux: bool,
85    #[clap(long, help = "snarkOS verbosity (0-4)", default_value = "1")]
86    pub(crate) verbosity: u8,
87    #[clap(long, short = 'y', help = "Skip confirmation prompts and proceed with the devnet startup")]
88    pub(crate) yes: bool,
89}
90
91impl Command for LeoDevnet {
92    type Input = ();
93    type Output = ();
94
95    fn log_span(&self) -> Span {
96        tracing::span!(tracing::Level::INFO, "LeoDevnet")
97    }
98
99    fn prelude(&self, _: Context) -> Result<Self::Input> {
100        Ok(())
101    }
102
103    fn apply(self, _cx: Context, _: Self::Input) -> Result<Self::Output> {
104        self.handle_apply().map_err(|e| CliError::custom(format!("Failed to start devnet: {e}")).into())
105    }
106}
107
108impl LeoDevnet {
109    /// Handle the actual devnet startup logic.
110    fn handle_apply(&self) -> AnyhowResult<()> {
111        //───────────────────────────────────────────────────────────────────
112        // 0. Guard rails
113        //───────────────────────────────────────────────────────────────────
114        if cfg!(windows) && self.tmux {
115            bail!("tmux mode is not available on Windows – remove `--tmux`.");
116        }
117        if self.tmux && std::env::var("TMUX").is_ok() {
118            bail!("Nested tmux session detected.  Unset $TMUX and retry.");
119        }
120
121        // If the devnet heights are provided, ensure the `test_network` feature is enabled, and validate the heights.
122        if let Some(ref heights) = self.consensus_heights {
123            if !self.snarkos_features.contains(&"test_network".to_string()) {
124                bail!("The `test_network` feature must be enabled on snarkOS to use `--consensus-heights`.");
125            }
126            validate_consensus_heights(heights.as_slice())?;
127        }
128
129        // Resolve the snarkOS path to its canonical form.
130
131        if self.install {
132            // If installing, make sure we can write to a file at the path.
133            if let Some(parent) = self.snarkos.parent() {
134                if !parent.exists() {
135                    std::fs::create_dir_all(parent)
136                        .with_context(|| format!("Failed to create directory for binary: {}", parent.display()))?;
137                }
138            }
139            std::fs::write(&self.snarkos, [0u8]).with_context(|| {
140                format!("Failed to write to path {} for snarkos installation", self.snarkos.display())
141            })?;
142        } else {
143            // If not installing, ensure the snarkOS binary exists at the provided path.
144            if !self.snarkos.exists() {
145                bail!(
146                    "The snarkOS binary at `{}` does not exist. Please provide a valid path or use `--install`.",
147                    self.snarkos.display()
148                );
149            }
150        };
151        let snarkos = canonicalize(&self.snarkos)
152            .with_context(|| format!("Failed to resolve snarkOS path: {}", self.snarkos.display()))?;
153
154        // Confirm with the user the options they provided.
155        println!("🔧  Starting devnet with the following options:");
156        println!("  • Network: {}", self.network);
157        println!("  • Validators: {}", self.num_validators);
158        println!("  • Clients: {}", self.num_clients);
159        println!("  • Storage: {}", self.storage);
160        if self.install {
161            println!("  • Installing snarkOS at: {}", snarkos.display());
162            if let Some(ref version) = self.snarkos_version {
163                println!("  • version: {version}");
164            }
165            if !self.snarkos_features.is_empty() {
166                println!("  • features: {}", self.snarkos_features.iter().format(","));
167            }
168        } else {
169            println!("  • Using snarkOS binary at: {}", snarkos.display());
170        }
171        if let Some(heights) = &self.consensus_heights {
172            println!("  • Consensus heights: {}", heights.iter().format(","));
173        } else {
174            println!("  • Consensus heights: default (based on your snarkOS binary)");
175        }
176        println!("  • Clear storage: {}", if self.clear_storage { "yes" } else { "no" });
177        println!("  • Verbosity: {}", self.verbosity);
178        println!("  • tmux: {}", if self.tmux { "yes" } else { "no" });
179
180        //───────────────────────────────────────────────────────────────────
181        // 1. Child-manager & shutdown listener (no race!)
182        //───────────────────────────────────────────────────────────────────
183        let manager = Arc::new(Mutex::new(ChildManager::new()));
184
185        // Install the listener to catch any early shutdown signals.
186        let (tx_shutdown, rx_shutdown) = crossbeam_channel::bounded::<()>(1);
187        let _signal_thread =
188            install_shutdown_listener(tx_shutdown.clone()).context("Failed to install shutdown listener")?;
189
190        //───────────────────────────────────────────────────────────────────
191        // 2. snarkOS binary  (+ optional build)
192        //───────────────────────────────────────────────────────────────────
193        let snarkos = if self.install {
194            if !confirm("\nProceed with snarkOS installation?", self.yes)? {
195                println!("❌ Installation aborted.");
196                return Ok(());
197            }
198            install_snarkos(&snarkos, self.snarkos_version.as_deref(), &self.snarkos_features)?
199        } else {
200            snarkos
201        };
202
203        // Run `snarkOS --version` and confirm with the user that they'd like to proceed.
204        let version_output = StdCommand::new(&snarkos)
205            .arg("--version")
206            .output()
207            .context(format!("Failed to run `{}`", snarkos.display()))?;
208        if !version_output.status.success() {
209            bail!("Failed to run `{}`: {}", snarkos.display(), String::from_utf8_lossy(&version_output.stderr));
210        }
211
212        // Print the version output.
213        let version_str = String::from_utf8_lossy(&version_output.stdout);
214        println!("🔍  Detected: {version_str}");
215
216        // The version string has the following form:
217        // "snarkos refs/heads/staging ace765a42551092fbb47799c2651d6b6df30e49a features=[default,snarkos_node_metrics,test_network]"
218        // Parse the features and see if it matches the expected features.
219        let features_str = version_str
220            .trim()
221            .split("features=[")
222            .nth(1)
223            .and_then(|s| s.split(']').next())
224            .ok_or_else(|| anyhow!("Failed to parse snarkOS features from version string: {version_str}"))?;
225        let found_features: Vec<String> = features_str.split(',').map(|s| s.trim().to_string()).collect();
226        for feature in &self.snarkos_features {
227            if !found_features.contains(feature) {
228                println!("⚠️  Warning: snarkOS does not have the required feature `{feature}` enabled.");
229            }
230        }
231
232        if !confirm("\nProceed with devnet startup?", self.yes)? {
233            println!("❌ Devnet aborted.");
234            return Ok(());
235        }
236
237        //───────────────────────────────────────────────────────────────────
238        // 3. Resolve storage & create log dir
239        //───────────────────────────────────────────────────────────────────
240        // Create the storage directory if it does not exist.
241        let storage = PathBuf::from(&self.storage);
242        if !storage.exists() {
243            std::fs::create_dir_all(&storage)
244                .context(format!("Failed to create storage directory: {}", self.storage))?;
245        } else if !storage.is_dir() {
246            bail!("The storage path `{}` is not a directory.", self.storage);
247        }
248        // Resolve the storage directory to its canonical form.
249        let storage =
250            canonicalize(&storage).with_context(|| format!("Failed to resolve storage path: {}", self.storage))?;
251        // Create the log directory inside the storage directory.
252        let log_dir = {
253            let ts = Local::now().format(".logs-%Y-%m-%d-%H-%M-%S").to_string();
254            let p = storage.join(ts);
255            std::fs::create_dir_all(&p)?;
256            p
257        };
258
259        //───────────────────────────────────────────────────────────────────
260        // 4. (Optional) ledger cleanup
261        //───────────────────────────────────────────────────────────────────
262        if self.clear_storage {
263            println!("🧹  Cleaning ledgers …");
264            let mut cleaners = Vec::new();
265            for idx in 0..self.num_validators {
266                cleaners.push(clean_snarkos(&snarkos, self.network as usize, "validator", idx, storage.as_path())?);
267            }
268            for idx in 0..self.num_clients {
269                cleaners.push(clean_snarkos(
270                    &snarkos,
271                    self.network as usize,
272                    "client",
273                    idx + self.num_validators,
274                    storage.as_path(),
275                )?);
276            }
277            for mut c in cleaners {
278                c.wait()?;
279            }
280        }
281
282        //───────────────────────────────────────────────────────────────────
283        // 5. Spawn nodes (tmux **or** background)
284        //───────────────────────────────────────────────────────────────────
285
286        #[allow(clippy::too_many_arguments)]
287        fn build_args(
288            role: &str,
289            verbosity: u8,
290            network: usize,
291            num_validators: usize,
292            idx: usize,
293            log_file: &Path,
294            metrics_port: Option<u16>,
295        ) -> Vec<String> {
296            let mut base = vec![
297                "start".to_string(),
298                "--nodisplay".to_string(),
299                "--network".to_string(),
300                network.to_string(),
301                "--dev".to_string(),
302                idx.to_string(),
303                "--dev-num-validators".to_string(),
304                num_validators.to_string(),
305                "--rest-rps".to_string(),
306                REST_RPS.to_string(),
307                "--logfile".to_string(),
308                log_file.to_str().unwrap().to_string(),
309                "--verbosity".to_string(),
310                verbosity.to_string(),
311            ];
312            match role {
313                "validator" => {
314                    base.extend(
315                        ["--allow-external-peers", "--validator", "--no-dev-txs"].into_iter().map(String::from),
316                    );
317                    if let Some(p) = metrics_port {
318                        base.extend(["--metrics".into(), "--metrics-ip".into(), format!("0.0.0.0:{p}")]);
319                    }
320                }
321                "client" => base.push("--client".into()),
322                _ => unreachable!(),
323            }
324            base
325        }
326
327        // Set the environment variable for the consensus heights if provided.
328        // These are used by all child processes.
329        if let Some(ref heights) = self.consensus_heights {
330            let heights = heights.iter().join(",");
331            println!("🔧  Setting consensus heights: {heights}");
332            #[allow(unsafe_code)]
333            unsafe {
334                // SAFETY:
335                //  - `CONSENSUS_VERSION_HEIGHTS` is only set once and is only read in `snarkvm::prelude::load_consensus_heights`.
336                //  - There are no concurrent threads running at this point in the execution.
337                // WHY:
338                //  - This is needed because there is no way to set the desired consensus heights for a particular `VM` instance in a node
339                //    without using the environment variable `CONSENSUS_VERSION_HEIGHTS`. Which is itself read once, and stored in a `OnceLock`.
340                env::set_var("CONSENSUS_VERSION_HEIGHTS", heights);
341            }
342        }
343
344        //────────────── tmux branch ──────────────
345        if self.tmux {
346            // Create session.
347            ensure!(
348                StdCommand::new("tmux")
349                    .args(["new-session", "-d", "-s", "devnet", "-n", "validator-0"])
350                    .status()?
351                    .success(),
352                "tmux failed to create session"
353            );
354
355            // Determine base-index.
356            let base_index = {
357                let out = StdCommand::new("tmux").args(["show-option", "-gv", "base-index"]).output()?;
358                String::from_utf8_lossy(&out.stdout).trim().parse::<usize>().unwrap_or(0)
359            };
360
361            // Validators
362            for idx in 0..self.num_validators {
363                let win_idx = idx + base_index;
364                let window_name = format!("validator-{idx}");
365                if idx != 0 {
366                    StdCommand::new("tmux")
367                        .args(["new-window", "-t", &format!("devnet:{win_idx}"), "-n", &window_name])
368                        .status()?;
369                }
370                let log_file = log_dir.join(format!("{window_name}.log"));
371                let metrics_port = 9000 + idx as u16;
372                let cmd = std::iter::once(snarkos.to_string_lossy().into_owned())
373                    .chain(build_args(
374                        "validator",
375                        self.verbosity,
376                        self.network as usize,
377                        self.num_validators,
378                        idx,
379                        log_file.as_path(),
380                        Some(metrics_port),
381                    ))
382                    .collect::<Vec<_>>()
383                    .join(" ");
384                StdCommand::new("tmux")
385                    .args(["send-keys", "-t", &format!("devnet:{win_idx}"), &cmd, "C-m"])
386                    .status()?;
387            }
388
389            // Clients
390            for idx in 0..self.num_clients {
391                let dev_idx = idx + self.num_validators;
392                let win_idx = dev_idx + base_index;
393                let window_name = format!("client-{idx}");
394                StdCommand::new("tmux")
395                    .args(["new-window", "-t", &format!("devnet:{win_idx}"), "-n", &window_name])
396                    .status()?;
397                let log_file = log_dir.join(format!("{window_name}.log"));
398                let cmd = std::iter::once(snarkos.to_string_lossy().into_owned())
399                    .chain(build_args(
400                        "client",
401                        self.verbosity,
402                        self.network as usize,
403                        self.num_validators,
404                        dev_idx,
405                        log_file.as_path(),
406                        None,
407                    ))
408                    .collect::<Vec<_>>()
409                    .join(" ");
410                StdCommand::new("tmux")
411                    .args(["send-keys", "-t", &format!("devnet:{win_idx}"), &cmd, "C-m"])
412                    .status()?;
413            }
414
415            println!("✅  tmux session \"devnet\" is ready – attaching …");
416            StdCommand::new("tmux").args(["attach-session", "-t", "devnet"]).status()?;
417            return Ok(()); // tmux will hold the terminal
418        }
419
420        //──────────── background branch ──────────
421        println!("⚙️  Spawning nodes as background tasks …");
422
423        // Helper: setsid() on Unix, Job-object attach on Windows.
424        let spawn_with_group = |mut cmd: StdCommand, log_file: &Path| -> AnyhowResult<Child> {
425            let log_handle = std::fs::OpenOptions::new().create(true).append(true).open(log_file)?;
426            cmd.stdout(Stdio::from(log_handle.try_clone()?));
427            cmd.stderr(Stdio::from(log_handle));
428
429            #[cfg(unix)]
430            #[allow(unsafe_code)]
431            unsafe {
432                // SAFETY: We are in the child just before exec; setsid() only
433                // affects the child and cannot violate Rust invariants.
434                cmd.pre_exec(|| {
435                    setsid();
436                    Ok(())
437                });
438            }
439
440            let child = cmd.spawn().map_err(|e| anyhow!("spawn {e}"))?;
441
442            #[cfg(windows)]
443            windows_kill_tree::attach_to_global_job(child.id())?;
444
445            Ok(child)
446        };
447
448        {
449            // This should be safe since only the current thread will write to the manager.
450            let mut guard = manager.lock();
451
452            // Validators
453            for idx in 0..self.num_validators {
454                let log_file = log_dir.join(format!("validator-{idx}.log"));
455                let child = spawn_with_group(
456                    {
457                        let mut c = StdCommand::new(&snarkos);
458                        c.args(build_args(
459                            "validator",
460                            self.verbosity,
461                            self.network as usize,
462                            self.num_validators,
463                            idx,
464                            &log_file,
465                            Some(9000 + idx as u16),
466                        ));
467                        c
468                    },
469                    &log_file,
470                )?;
471                println!("  • validator {idx}  (pid = {})", child.id());
472                guard.push(child);
473            }
474
475            // Clients
476            for idx in 0..self.num_clients {
477                let dev_idx = idx + self.num_validators;
478                let log_file = log_dir.join(format!("client-{idx}.log"));
479                let child = spawn_with_group(
480                    {
481                        let mut c = StdCommand::new(&snarkos);
482                        c.args(build_args(
483                            "client",
484                            self.verbosity,
485                            self.network as usize,
486                            self.num_validators,
487                            dev_idx,
488                            &log_file,
489                            None,
490                        ));
491                        c
492                    },
493                    &log_file,
494                )?;
495                println!("  • client    {idx}  (pid = {})", child.id());
496                guard.push(child);
497            }
498        }
499
500        // Print the main process ID.
501        println!("📌  Main process ID: {}", std::process::id());
502        println!("\nDevnet running – Ctrl+C, SIGTERM, or terminal close to stop.");
503
504        // Block here until the first (coalesced) shutdown request
505        let _ = rx_shutdown.recv();
506        manager.lock().shutdown_all(Duration::from_secs(30));
507
508        Ok(())
509    }
510}
leo_lang/cli/commands/devnet/mod.rs

leo_lang/cli/commands/devnet/
mod.rs
