sled-agent-sim fails to start with "No control plane disks" error

I worked around #8348 by providing 0 as the listen port, but then startup failed without reporting any error at all.

I added these changes to try to get more data:

dap@ivanova omicron-fix $ git diff
diff --git a/common/src/cmd.rs b/common/src/cmd.rs
index f7ede8d12..0c8576e10 100644
--- a/common/src/cmd.rs
+++ b/common/src/cmd.rs
@@ -26,7 +26,7 @@ pub fn fatal(cmd_error: CmdError) -> ! {
         .unwrap_or("command");
     let (exit_code, message) = match cmd_error {
         CmdError::Usage(m) => (2, m),
-        CmdError::Failure(e) => (1, format!("{e:?}")),
+        CmdError::Failure(e) => (1, format!("{e:#}")),
     };
     eprintln!("{}: {}", arg0, message);
     exit(exit_code);
diff --git a/sled-agent/src/sim/server.rs b/sled-agent/src/sim/server.rs
index 05c75e18c..863f4610b 100644
--- a/sled-agent/src/sim/server.rs
+++ b/sled-agent/src/sim/server.rs
@@ -54,6 +54,7 @@ use omicron_uuid_kinds::ZpoolUuid;
 use oxnet::Ipv6Net;
 use sled_agent_types::rack_init::RecoverySiloConfig;
 use slog::{Drain, Logger, info};
+use slog_error_chain::InlineErrorChain;
 use std::collections::BTreeMap;
 use std::collections::HashMap;
 use std::net::IpAddr;
@@ -175,7 +176,7 @@ impl Server {
             };
             let log_notification_failure = |error, delay| {
                 warn!(log, "failed to contact nexus, will retry in {:?}", delay;
-                    "error" => ?error);
+                    InlineErrorChain::new(&error));
             };
             retry_notify(
                 retry_policy_internal_service_aggressive(),
@@ -542,13 +543,26 @@ pub async fn run_standalone_server(
         }
     }
 
+    eprintln!("\n\n\nDAP: 8\n\n\n");
+
     let certs = match &rss_args.tls_certificate {
         Some(c) => vec![c.clone()],
         None => vec![],
     };
 
     let omicron_physical_disks_config =
-        server.sled_agent.omicron_physical_disks_list()?;
+        server.sled_agent.omicron_physical_disks_list().map_err(|e| {
+            anyhow!(
+                "omicron_physical_disks_list(): {} {} {:?} {:?}",
+                e.status_code,
+                e.error_code
+                    .as_ref()
+                    .map(|s| s.as_str())
+                    .unwrap_or("<no code>"),
+                e.external_message,
+                e.internal_message
+            )
+        })?;
     let mut sled_configs = BTreeMap::new();
     sled_configs.insert(
         config.id,
@@ -568,6 +582,8 @@ pub async fn run_standalone_server(
         },
     );
 
+    eprintln!("\n\n\nDAP: 10\n\n\n");
+
     let blueprint = build_initial_blueprint_from_sled_configs(
         &sled_configs,
         internal_dns_version,

and now I get:

$  cargo run --bin=sled-agent-sim -- d0900913-88c6-4702-8eaf-d2e88e714657 [::1]:0 [::1]:12221 --rss-nexus-external-addr 127.0.0.1:12220 --rss-external-dns-internal-addr [::1]:5353 --rss-internal-dns-dns-addr [::1]:3535 --rss-tls-cert demo-cert.pem --rss-tls-key demo-key.pem 
    Finished `dev` profile [unoptimized + debuginfo] target(s) in 1.06s
     Running `target/debug/sled-agent-sim d0900913-88c6-4702-8eaf-d2e88e714657 '[::1]:0' '[::1]:12221' --rss-nexus-external-addr '127.0.0.1:12220' --rss-external-dns-internal-addr '[::1]:5353' --rss-internal-dns-dns-addr '[::1]:3535' --rss-tls-cert demo-cert.pem --rss-tls-key demo-key.pem`
Jun 17 17:41:38.038 INFO setting up sled agent server, file: sled-agent/src/sim/server.rs:95
Jun 17 17:41:38.086 INFO created simulated sled agent, sim_mode: Auto, server: d0900913-88c6-4702-8eaf-d2e88e714657, component: SledAgent, file: sled-agent/src/sim/sled_agent.rs:124
Jun 17 17:41:38.086 INFO registered sled d0900913-88c6-4702-8eaf-d2e88e714657 storage, file: sled-agent/src/sim/upstairs.rs:48
Jun 17 17:41:38.087 INFO No ledger in /dangerzone/omicron_tmp/.tmppjeJd4/artifact-config.json, component: ArtifactStore, server: d0900913-88c6-4702-8eaf-d2e88e714657, component: SledAgent, file: common/src/ledger.rs:186
Jun 17 17:41:38.087 INFO No ledger in /dangerzone/omicron_tmp/.tmp5s1zym/artifact-config.json, component: ArtifactStore, server: d0900913-88c6-4702-8eaf-d2e88e714657, component: SledAgent, file: common/src/ledger.rs:186
Jun 17 17:41:38.123 INFO listening, local_addr: [::1]:37092, component: dropshot (Repo Depot), server: d0900913-88c6-4702-8eaf-d2e88e714657, component: SledAgent, file: /home/dap/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.2/src/server.rs:328
Jun 17 17:41:38.128 INFO listening, local_addr: [::1]:54743, component: dropshot, file: /home/dap/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.2/src/server.rs:328
Jun 17 17:41:38.142 INFO listening, local_addr: [::1]:51401, component: Simulated CrucibleAgent Dropshot Server, kind: storage, server: d0900913-88c6-4702-8eaf-d2e88e714657, component: SledAgent, file: /home/dap/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.2/src/server.rs:328
Jun 17 17:41:38.142 INFO Created Simulated Crucible Server, address: [::1]:51401, kind: storage, server: d0900913-88c6-4702-8eaf-d2e88e714657, component: SledAgent, file: sled-agent/src/sim/storage.rs:1116
Jun 17 17:41:38.143 INFO listening, local_addr: [::1]:59677, component: Simulated CrucibleAgent Dropshot Server, kind: storage, server: d0900913-88c6-4702-8eaf-d2e88e714657, component: SledAgent, file: /home/dap/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.2/src/server.rs:328
Jun 17 17:41:38.143 INFO Created Simulated Crucible Server, address: [::1]:59677, kind: storage, server: d0900913-88c6-4702-8eaf-d2e88e714657, component: SledAgent, file: sled-agent/src/sim/storage.rs:1116
Jun 17 17:41:38.143 INFO listening, local_addr: [::1]:38977, component: Simulated CrucibleAgent Dropshot Server, kind: storage, server: d0900913-88c6-4702-8eaf-d2e88e714657, component: SledAgent, file: /home/dap/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.2/src/server.rs:328
Jun 17 17:41:38.143 INFO Created Simulated Crucible Server, address: [::1]:38977, kind: storage, server: d0900913-88c6-4702-8eaf-d2e88e714657, component: SledAgent, file: sled-agent/src/sim/storage.rs:1116
Jun 17 17:41:38.143 INFO listening, local_addr: [::1]:39819, component: Simulated CrucibleAgent Dropshot Server, kind: storage, server: d0900913-88c6-4702-8eaf-d2e88e714657, component: SledAgent, file: /home/dap/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.2/src/server.rs:328
Jun 17 17:41:38.144 INFO Created Simulated Crucible Server, address: [::1]:39819, kind: storage, server: d0900913-88c6-4702-8eaf-d2e88e714657, component: SledAgent, file: sled-agent/src/sim/storage.rs:1116
Jun 17 17:41:38.144 INFO listening, local_addr: [::1]:35383, component: Simulated CrucibleAgent Dropshot Server, kind: storage, server: d0900913-88c6-4702-8eaf-d2e88e714657, component: SledAgent, file: /home/dap/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.2/src/server.rs:328
Jun 17 17:41:38.144 INFO Created Simulated Crucible Server, address: [::1]:35383, kind: storage, server: d0900913-88c6-4702-8eaf-d2e88e714657, component: SledAgent, file: sled-agent/src/sim/storage.rs:1116
Jun 17 17:41:38.144 INFO listening, local_addr: [::1]:65247, component: Simulated CrucibleAgent Dropshot Server, kind: storage, server: d0900913-88c6-4702-8eaf-d2e88e714657, component: SledAgent, file: /home/dap/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.2/src/server.rs:328
Jun 17 17:41:38.145 INFO Created Simulated Crucible Server, address: [::1]:65247, kind: storage, server: d0900913-88c6-4702-8eaf-d2e88e714657, component: SledAgent, file: sled-agent/src/sim/storage.rs:1116
Jun 17 17:41:38.145 INFO listening, local_addr: [::1]:53593, component: Simulated CrucibleAgent Dropshot Server, kind: storage, server: d0900913-88c6-4702-8eaf-d2e88e714657, component: SledAgent, file: /home/dap/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.2/src/server.rs:328
Jun 17 17:41:38.145 INFO Created Simulated Crucible Server, address: [::1]:53593, kind: storage, server: d0900913-88c6-4702-8eaf-d2e88e714657, component: SledAgent, file: sled-agent/src/sim/storage.rs:1116
Jun 17 17:41:38.145 INFO listening, local_addr: [::1]:53322, component: Simulated CrucibleAgent Dropshot Server, kind: storage, server: d0900913-88c6-4702-8eaf-d2e88e714657, component: SledAgent, file: /home/dap/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.2/src/server.rs:328
Jun 17 17:41:38.146 INFO Created Simulated Crucible Server, address: [::1]:53322, kind: storage, server: d0900913-88c6-4702-8eaf-d2e88e714657, component: SledAgent, file: sled-agent/src/sim/storage.rs:1116
Jun 17 17:41:38.146 INFO listening, local_addr: [::1]:60283, component: Simulated CrucibleAgent Dropshot Server, kind: storage, server: d0900913-88c6-4702-8eaf-d2e88e714657, component: SledAgent, file: /home/dap/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.2/src/server.rs:328
Jun 17 17:41:38.146 INFO Created Simulated Crucible Server, address: [::1]:60283, kind: storage, server: d0900913-88c6-4702-8eaf-d2e88e714657, component: SledAgent, file: sled-agent/src/sim/storage.rs:1116
Jun 17 17:41:38.146 INFO listening, local_addr: [::1]:62983, component: Simulated CrucibleAgent Dropshot Server, kind: storage, server: d0900913-88c6-4702-8eaf-d2e88e714657, component: SledAgent, file: /home/dap/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.2/src/server.rs:328
Jun 17 17:41:38.147 INFO Created Simulated Crucible Server, address: [::1]:62983, kind: storage, server: d0900913-88c6-4702-8eaf-d2e88e714657, component: SledAgent, file: sled-agent/src/sim/storage.rs:1116
Jun 17 17:41:38.147 INFO sled agent started successfully, file: sled-agent/src/sim/server.rs:341
Jun 17 17:41:38.147 INFO opening sled database, path: /dangerzone/omicron_tmp/.tmpc22ent, component: store, file: dns-server/src/storage.rs:262
Jun 17 17:41:38.194 INFO pruning trees for generations newer than 0, component: store, file: dns-server/src/storage.rs:667
Jun 17 17:41:38.194 INFO pruning trees for generations older than 0, keep: 3, component: store, file: dns-server/src/storage.rs:731
Jun 17 17:41:38.194 INFO DNS server bound to address, local_address: [::1]:3535, component: dns, kind: dns, file: dns-server/src/dns_server.rs:97
Jun 17 17:41:38.194 INFO listening, local_addr: [::1]:47801, component: http, kind: dns, file: /home/dap/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.2/src/server.rs:328
Jun 17 17:41:38.195 INFO listening, local_addr: [::1]:63067, component: dropshot, kind: pantry, file: /home/dap/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.2/src/server.rs:328
Jun 17 17:41:38.195 INFO Started Simulated Crucible Pantry, address: [::1]:63067, kind: pantry, file: sled-agent/src/sim/storage.rs:2178
Jun 17 17:41:38.231 INFO accepted connection, remote_addr: [::1]:52072, local_addr: [::1]:47801, component: http, kind: dns, file: /home/dap/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.2/src/server.rs:1025
Jun 17 17:41:38.233 INFO attempting generation update, new_generation: 1, req_id: b1289f3d-c58e-4165-bb93-572fb7129fa4, component: store, file: dns-server/src/storage.rs:514
Jun 17 17:41:38.233 INFO pruning trees for generations newer than 0, component: store, file: dns-server/src/storage.rs:667
Jun 17 17:41:38.233 INFO pruning trees for generations older than 1, keep: 3, component: store, file: dns-server/src/storage.rs:731
Jun 17 17:41:38.234 INFO updated generation, new_generation: 1, req_id: b1289f3d-c58e-4165-bb93-572fb7129fa4, component: store, file: dns-server/src/storage.rs:517
Jun 17 17:41:38.234 INFO request completed, latency_us: 1587, response_code: 204, uri: /config, method: PUT, req_id: b1289f3d-c58e-4165-bb93-572fb7129fa4, remote_addr: [::1]:52072, local_addr: [::1]:47801, component: http, kind: dns, file: /home/dap/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/dropshot-0.16.2/src/server.rs:867



DAP: 8



sled-agent-sim: omicron_physical_disks_list(): 404 Not Found <no code> "Not Found" "No control plane disks"
Jun 17 17:41:38.236 INFO beginning graceful shutdown, local_addr: [::1]:47801

Reproduce the failure with the cargo run --bin=sled-agent-sim command and inspect sled-agent/src/sim/server.rs, especially run_standalone_server and its omicron_physical_disks_list() call. Compare the error path with common/src/cmd.rs; done means the simulator either starts successfully or reports a clear, actionable startup error instead of failing silently.